tsdb

package
v1.20.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 10, 2025 License: Apache-2.0 Imports: 53 Imported by: 3

Documentation

Index

Constants

View Source 
const (
	CacheBackendMemcached = "memcached"
	CacheBackendRedis     = "redis"
	CacheBackendInMemory  = "inmemory"
)
View Source 
const (
	// TenantIDExternalLabel is the external label containing the tenant ID,
	// set when shipping blocks to the storage.
	TenantIDExternalLabel = "__org_id__"

	// IngesterIDExternalLabel is the external label containing the ingester ID,
	// set when shipping blocks to the storage.
	IngesterIDExternalLabel = "__ingester_id__"

	// How often are open TSDBs checked for being idle and closed.
	DefaultCloseIdleTSDBInterval = 5 * time.Minute

	// How often expired items are cleaned from the PostingsCache
	ExpandedCachingExpireInterval = 5 * time.Minute

	// How often to check for tenant deletion mark.
	DeletionMarkCheckInterval = 1 * time.Hour

	// Default minimum bucket size (bytes) of the chunk pool.
	ChunkPoolDefaultMinBucketSize = store.EstimatedMaxChunkSize

	// Default maximum bucket size (bytes) of the chunk pool.
	ChunkPoolDefaultMaxBucketSize = 50e6
)
View Source 
const (
	// IndexCacheBackendInMemory is the value for the in-memory index cache backend.
	IndexCacheBackendInMemory = "inmemory"

	// IndexCacheBackendMemcached is the value for the memcached index cache backend.
	IndexCacheBackendMemcached = "memcached"

	// IndexCacheBackendRedis is the value for the redis index cache backend.
	IndexCacheBackendRedis = "redis"

	// IndexCacheBackendDefault is the value for the default index cache backend.
	IndexCacheBackendDefault = IndexCacheBackendInMemory
)
View Source 
const (
	UserScanStrategyList      = "list"
	UserScanStrategyUserIndex = "user_index"
)
View Source 
const TenantDeletionMarkFile = "tenant-deletion-mark.json"

Variables

View Source 
var (
	ErrInvalidBucketIndexBlockDiscoveryStrategy         = errors.New("bucket index block discovery strategy can only be enabled when bucket index is enabled")
	ErrBlockDiscoveryStrategy                           = errors.New("invalid block discovery strategy")
	ErrInvalidTokenBucketBytesLimiterMode               = errors.New("invalid token bucket bytes limiter mode")
	ErrInvalidLazyExpandedPostingGroupMaxKeySeriesRatio = errors.New("lazy expanded posting group max key series ratio needs to be equal or greater than 0")
)

Validation errors

View Source 
var (
	ErrInvalidUserScannerStrategy = errors.New("invalid user scanner strategy")
	ErrInvalidMaxStalePeriod      = errors.New("max stale period must be positive")
	ErrInvalidCacheTTL            = errors.New("cache TTL must be >= 0")
)
View Source 
var (
	DefaultPartitionInfo = PartitionInfo{
		PartitionedGroupID:           0,
		PartitionID:                  0,
		PartitionCount:               1,
		PartitionedGroupCreationTime: 0,
	}
)

Functions

func CreateCachingBucket added in v1.2.0 

func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig MetadataCacheConfig, parquetLabelsConfig ParquetLabelsCacheConfig, matchers Matchers, bkt objstore.InstrumentedBucket, logger log.Logger, reg prometheus.Registerer) (objstore.InstrumentedBucket, error)

func CreateCachingBucketForCompactor added in v1.20.0 

func CreateCachingBucketForCompactor(metadataConfig MetadataCacheConfig, cleaner bool, bkt objstore.InstrumentedBucket, logger log.Logger, reg prometheus.Registerer) (objstore.InstrumentedBucket, error)

func DeleteTenantDeletionMark added in v1.17.0 

func DeleteTenantDeletionMark(ctx context.Context, bkt objstore.Bucket, userID string) error

Deletes the tenant deletion mark for given user if it exists.

func GetGlobalDeletionMarkPath added in v1.17.0 

func GetGlobalDeletionMarkPath(userID string) string

func GetLocalDeletionMarkPath added in v1.17.0 

func GetLocalDeletionMarkPath(userID string) string

func HashBlockID added in v1.1.0 

func HashBlockID(id ulid.ULID) uint32

HashBlockID returns a 32-bit hash of the block ID useful for ring-based sharding.

func IsOneOfTheExpectedErrors added in v1.16.0 

func IsOneOfTheExpectedErrors(f ...objstore.IsOpFailureExpectedFunc) objstore.IsOpFailureExpectedFunc

func NewCachedBlockChunkQuerier added in v1.19.0 

func NewCachedBlockChunkQuerier(cache ExpandedPostingsCache, b prom_tsdb.BlockReader, mint, maxt int64) (storage.ChunkQuerier, error)

func NewIndexCache added in v0.7.0 

func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error)

NewIndexCache creates a new index cache based on the input configuration.

func TenantDeletionMarkExists added in v1.6.0 

func TenantDeletionMarkExists(ctx context.Context, bkt objstore.BucketReader, userID string) (bool, error)

Checks for deletion mark for tenant. Errors other than "object not found" are returned.

func WriteTenantDeletionMark added in v1.6.0 

func WriteTenantDeletionMark(ctx context.Context, bkt objstore.InstrumentedBucket, userID string, mark *TenantDeletionMark) error

Uploads deletion mark to the tenant location in the bucket.

Types

type BlockDiscoveryStrategy added in v1.17.0 

type BlockDiscoveryStrategy string

BlockDiscoveryStrategy configures how to list block IDs from object storage. 

const (
	ConcurrentDiscovery  BlockDiscoveryStrategy = "concurrent"
	RecursiveDiscovery   BlockDiscoveryStrategy = "recursive"
	BucketIndexDiscovery BlockDiscoveryStrategy = "bucket_index"
)

type BlocksStorageConfig added in v1.3.0 

type BlocksStorageConfig struct {
	Bucket       bucket.Config      `yaml:",inline"`
	BucketStore  BucketStoreConfig  `` /* 141-byte string literal not displayed */
	TSDB         TSDBConfig         `yaml:"tsdb"`
	UsersScanner UsersScannerConfig `yaml:"users_scanner"`
}

BlocksStorageConfig holds the config information for the blocks storage.

func (*BlocksStorageConfig) RegisterFlags added in v1.3.0 

func (cfg *BlocksStorageConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the block storage flags

func (*BlocksStorageConfig) Validate added in v1.3.0 

func (cfg *BlocksStorageConfig) Validate() error

Validate the config.

type BucketCacheBackend added in v1.20.0 

type BucketCacheBackend struct {
	Backend    string                      `yaml:"backend"`
	InMemory   InMemoryBucketCacheConfig   `yaml:"inmemory"`
	Memcached  MemcachedClientConfig       `yaml:"memcached"`
	Redis      RedisClientConfig           `yaml:"redis"`
	MultiLevel MultiLevelBucketCacheConfig `yaml:"multilevel"`
}

func (*BucketCacheBackend) Validate added in v1.20.0 

func (cfg *BucketCacheBackend) Validate() error

Validate the config.

type BucketIndexConfig added in v1.7.0 

type BucketIndexConfig struct {
	Enabled               bool          `yaml:"enabled"`
	UpdateOnErrorInterval time.Duration `yaml:"update_on_error_interval"`
	IdleTimeout           time.Duration `yaml:"idle_timeout"`
	MaxStalePeriod        time.Duration `yaml:"max_stale_period"`
}

func (*BucketIndexConfig) RegisterFlagsWithPrefix added in v1.7.0 

func (cfg *BucketIndexConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

type BucketStoreConfig added in v0.6.0 

type BucketStoreConfig struct {
	SyncDir                  string                   `yaml:"sync_dir"`
	SyncInterval             time.Duration            `yaml:"sync_interval"`
	MaxConcurrent            int                      `yaml:"max_concurrent"`
	MaxInflightRequests      int                      `yaml:"max_inflight_requests"`
	TenantSyncConcurrency    int                      `yaml:"tenant_sync_concurrency"`
	BlockSyncConcurrency     int                      `yaml:"block_sync_concurrency"`
	MetaSyncConcurrency      int                      `yaml:"meta_sync_concurrency"`
	ConsistencyDelay         time.Duration            `yaml:"consistency_delay"`
	IndexCache               IndexCacheConfig         `yaml:"index_cache"`
	ChunksCache              ChunksCacheConfig        `yaml:"chunks_cache"`
	MetadataCache            MetadataCacheConfig      `yaml:"metadata_cache"`
	ParquetLabelsCache       ParquetLabelsCacheConfig `yaml:"parquet_labels_cache"`
	MatchersCacheMaxItems    int                      `yaml:"matchers_cache_max_items"`
	IgnoreDeletionMarksDelay time.Duration            `yaml:"ignore_deletion_mark_delay"`
	IgnoreBlocksWithin       time.Duration            `yaml:"ignore_blocks_within"`
	IgnoreBlocksBefore       time.Duration            `yaml:"ignore_blocks_before"`
	BucketIndex              BucketIndexConfig        `yaml:"bucket_index"`
	BlockDiscoveryStrategy   string                   `yaml:"block_discovery_strategy"`

	// Chunk pool.
	MaxChunkPoolBytes           uint64 `yaml:"max_chunk_pool_bytes"`
	ChunkPoolMinBucketSizeBytes int    `yaml:"chunk_pool_min_bucket_size_bytes" doc:"hidden"`
	ChunkPoolMaxBucketSizeBytes int    `yaml:"chunk_pool_max_bucket_size_bytes" doc:"hidden"`

	// Controls whether index-header lazy loading is enabled.
	IndexHeaderLazyLoadingEnabled     bool          `yaml:"index_header_lazy_loading_enabled"`
	IndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout"`

	// Controls whether lazy expanded posting optimization is enabled or not.
	LazyExpandedPostingsEnabled bool `yaml:"lazy_expanded_postings_enabled"`

	// Controls whether expanded posting group is marked as lazy or not depending on number of keys to fetch.
	LazyExpandedPostingGroupMaxKeySeriesRatio float64 `yaml:"lazy_expanded_posting_group_max_key_series_ratio"`

	// Controls the partitioner, used to aggregate multiple GET object API requests.
	// The config option is hidden until experimental.
	PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" doc:"hidden"`

	// Controls the estimated size to fetch for series and chunk in Store Gateway. Using
	// a large value might cause data overfetch while a small value might need to refetch.
	EstimatedMaxSeriesSizeBytes uint64 `yaml:"estimated_max_series_size_bytes" doc:"hidden"`
	EstimatedMaxChunkSizeBytes  uint64 `yaml:"estimated_max_chunk_size_bytes" doc:"hidden"`

	// Controls what is the ratio of postings offsets store will hold in memory.
	// Larger value will keep less offsets, which will increase CPU cycles needed for query touching those postings.
	// It's meant for setups that want low baseline memory pressure and where less traffic is expected.
	// On the contrary, smaller value will increase baseline memory usage, but improve latency slightly.
	// 1 will keep all in memory. Default value is the same as in Prometheus which gives a good balance.
	PostingOffsetsInMemSampling int `yaml:"postings_offsets_in_mem_sampling" doc:"hidden"`

	// Controls how many series to fetch per batch in Store Gateway. Default value is 10000.
	SeriesBatchSize int `yaml:"series_batch_size"`

	// Token bucket configs
	TokenBucketBytesLimiter TokenBucketBytesLimiterConfig `yaml:"token_bucket_bytes_limiter"`
}

BucketStoreConfig holds the config information for Bucket Stores used by the querier and store-gateway.

func (*BucketStoreConfig) RegisterFlags added in v0.6.0 

func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the BucketStore flags

func (*BucketStoreConfig) Validate added in v1.0.0 

func (cfg *BucketStoreConfig) Validate() error

Validate the config.

type ChunksCacheConfig added in v1.2.0 

type ChunksCacheConfig struct {
	BucketCacheBackend `yaml:",inline"`

	SubrangeSize        int64         `yaml:"subrange_size"`
	MaxGetRangeRequests int           `yaml:"max_get_range_requests"`
	AttributesTTL       time.Duration `yaml:"attributes_ttl"`
	SubrangeTTL         time.Duration `yaml:"subrange_ttl"`
}

func (*ChunksCacheConfig) RegisterFlagsWithPrefix added in v1.2.0 

func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*ChunksCacheConfig) Validate added in v1.2.0 

func (cfg *ChunksCacheConfig) Validate() error

type CircuitBreakerConfig added in v1.17.0 

type CircuitBreakerConfig struct {
	// Enabled enables circuit breaker.
	Enabled bool `yaml:"enabled"`

	// HalfOpenMaxRequests is the maximum number of requests allowed to pass through
	// when the circuit breaker is half-open.
	// If set to 0, the circuit breaker allows only 1 request.
	HalfOpenMaxRequests uint `yaml:"half_open_max_requests"`
	// OpenDuration is the period of the open state after which the state of the circuit breaker becomes half-open.
	// If set to 0, the circuit breaker utilizes the default value of 60 seconds.
	OpenDuration time.Duration `yaml:"open_duration"`
	// MinRequests is minimal requests to trigger the circuit breaker.
	MinRequests uint `yaml:"min_requests"`
	// ConsecutiveFailures represents consecutive failures based on CircuitBreakerMinRequests to determine if the circuit breaker should open.
	ConsecutiveFailures uint `yaml:"consecutive_failures"`
	// FailurePercent represents the failure percentage, which is based on CircuitBreakerMinRequests, to determine if the circuit breaker should open.
	FailurePercent float64 `yaml:"failure_percent"`
}

CircuitBreakerConfig is the config for the circuit breaker.

func (*CircuitBreakerConfig) RegisterFlagsWithPrefix added in v1.17.0 

func (cfg *CircuitBreakerConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

type CortexMetaExtensions added in v1.19.0 

type CortexMetaExtensions struct {
	PartitionInfo *PartitionInfo `json:"partition_info,omitempty"`
	TimeRange     int64          `json:"time_range,omitempty"`
}

func ConvertToCortexMetaExtensions added in v1.19.0 

func ConvertToCortexMetaExtensions(extensions any) (*CortexMetaExtensions, error)

func GetCortexMetaExtensionsFromMeta added in v1.19.0 

func GetCortexMetaExtensionsFromMeta(meta metadata.Meta) (*CortexMetaExtensions, error)

func (*CortexMetaExtensions) TimeRangeStr added in v1.19.0 

func (c *CortexMetaExtensions) TimeRangeStr() string

type DurationList 

type DurationList []time.Duration

DurationList is the block ranges for a tsdb

func (*DurationList) Set 

func (d *DurationList) Set(s string) error

Set implements the flag.Value interface

func (*DurationList) String 

func (d *DurationList) String() string

String implements the flag.Value interface

func (*DurationList) ToMilliseconds added in v0.6.0 

func (d *DurationList) ToMilliseconds() []int64

ToMilliseconds returns the duration list in milliseconds

type ExpandedPostingsCache added in v1.19.0 

type ExpandedPostingsCache interface {
	PostingsForMatchers(ctx context.Context, blockID ulid.ULID, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error)
	ExpireSeries(metric labels.Labels)
	PurgeExpiredItems()
	Clear()
	Size() int
}

type ExpandedPostingsCacheFactory added in v1.19.0 

type ExpandedPostingsCacheFactory struct {
	// contains filtered or unexported fields
}

func NewExpandedPostingsCacheFactory added in v1.19.0 

func NewExpandedPostingsCacheFactory(cfg TSDBPostingsCacheConfig) *ExpandedPostingsCacheFactory

func (*ExpandedPostingsCacheFactory) NewExpandedPostingsCache added in v1.19.0 

func (f *ExpandedPostingsCacheFactory) NewExpandedPostingsCache(userId string, metrics *ExpandedPostingsCacheMetrics) ExpandedPostingsCache

type ExpandedPostingsCacheMetrics added in v1.19.0 

type ExpandedPostingsCacheMetrics struct {
	CacheRequests       *prometheus.CounterVec
	CacheHits           *prometheus.CounterVec
	CacheEvicts         *prometheus.CounterVec
	CacheMiss           *prometheus.CounterVec
	NonCacheableQueries *prometheus.CounterVec
}

func NewPostingCacheMetrics added in v1.19.0 

func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetrics

type InMemoryBucketCacheConfig added in v1.20.0 

type InMemoryBucketCacheConfig struct {
	MaxSizeBytes uint64 `yaml:"max_size_bytes"`
}

func (*InMemoryBucketCacheConfig) RegisterFlagsWithPrefix added in v1.20.0 

func (cfg *InMemoryBucketCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string, item string)

type InMemoryIndexCache added in v1.17.0 

type InMemoryIndexCache struct {
	// contains filtered or unexported fields
}

func NewInMemoryIndexCacheWithConfig added in v1.17.0 

func NewInMemoryIndexCacheWithConfig(logger log.Logger, commonMetrics *storecache.CommonMetrics, reg prometheus.Registerer, config storecache.InMemoryIndexCacheConfig) (*InMemoryIndexCache, error)

NewInMemoryIndexCacheWithConfig creates a new thread-safe cache for index entries. It relies on the cache library (fastcache) to ensures the total cache size approximately does not exceed maxBytes.

func (*InMemoryIndexCache) FetchExpandedPostings added in v1.17.0 

func (c *InMemoryIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, matchers []*labels.Matcher, tenant string) ([]byte, bool)

FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not.

func (*InMemoryIndexCache) FetchMultiPostings added in v1.17.0 

func (c *InMemoryIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label)

FetchMultiPostings fetches multiple postings - each identified by a label - and returns a map containing cache hits, along with a list of missing keys.

func (*InMemoryIndexCache) FetchMultiSeries added in v1.17.0 

func (c *InMemoryIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef)

FetchMultiSeries fetches multiple series - each identified by ID - from the cache and returns a map containing cache hits, along with a list of missing IDs.

func (*InMemoryIndexCache) StoreExpandedPostings added in v1.17.0 

func (c *InMemoryIndexCache) StoreExpandedPostings(blockID ulid.ULID, matchers []*labels.Matcher, v []byte, tenant string)

StoreExpandedPostings stores expanded postings for a set of label matchers.

func (*InMemoryIndexCache) StorePostings added in v1.17.0 

func (c *InMemoryIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string)

StorePostings sets the postings identified by the ulid and label to the value v, if the postings already exists in the cache it is not mutated.

func (*InMemoryIndexCache) StoreSeries added in v1.17.0 

func (c *InMemoryIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string)

StoreSeries sets the series identified by the ulid and id to the value v, if the series already exists in the cache it is not mutated.

type InMemoryIndexCacheConfig added in v1.0.0 

type InMemoryIndexCacheConfig struct {
	MaxSizeBytes uint64   `yaml:"max_size_bytes"`
	EnabledItems []string `yaml:"enabled_items"`
}

func (*InMemoryIndexCacheConfig) RegisterFlagsWithPrefix added in v1.0.0 

func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*InMemoryIndexCacheConfig) Validate added in v1.16.0 

func (cfg *InMemoryIndexCacheConfig) Validate() error

type IndexCacheConfig added in v1.0.0 

type IndexCacheConfig struct {
	Backend    string                     `yaml:"backend"`
	InMemory   InMemoryIndexCacheConfig   `yaml:"inmemory"`
	Memcached  MemcachedIndexCacheConfig  `yaml:"memcached"`
	Redis      RedisIndexCacheConfig      `yaml:"redis"`
	MultiLevel MultiLevelIndexCacheConfig `yaml:"multilevel"`
}

func (*IndexCacheConfig) RegisterFlags added in v1.0.0 

func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet)

func (*IndexCacheConfig) RegisterFlagsWithPrefix added in v1.0.0 

func (cfg *IndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*IndexCacheConfig) Validate added in v1.0.0 

func (cfg *IndexCacheConfig) Validate() error

Validate the config.

type MatcherCacheMetrics added in v1.19.0 

type MatcherCacheMetrics struct {
	// contains filtered or unexported fields
}

func NewMatchCacheMetrics added in v1.19.0 

func NewMatchCacheMetrics(prefix string, r *prometheus.Registry, l log.Logger) *MatcherCacheMetrics

func (*MatcherCacheMetrics) Collect added in v1.19.0 

func (m *MatcherCacheMetrics) Collect(out chan<- prometheus.Metric)

func (*MatcherCacheMetrics) Describe added in v1.19.0 

func (m *MatcherCacheMetrics) Describe(out chan<- *prometheus.Desc)

type Matchers added in v1.17.0 

type Matchers struct {
	// contains filtered or unexported fields
}

func NewMatchers added in v1.17.0 

func NewMatchers() Matchers

func (*Matchers) GetBlockIndexMatcher added in v1.17.0 

func (m *Matchers) GetBlockIndexMatcher() func(string) bool

func (*Matchers) GetBucketIndexMatcher added in v1.17.0 

func (m *Matchers) GetBucketIndexMatcher() func(string) bool

func (*Matchers) GetChunksIterMatcher added in v1.17.0 

func (m *Matchers) GetChunksIterMatcher() func(string) bool

func (*Matchers) GetChunksMatcher added in v1.17.0 

func (m *Matchers) GetChunksMatcher() func(string) bool

func (*Matchers) GetMetafileMatcher added in v1.17.0 

func (m *Matchers) GetMetafileMatcher() func(string) bool

func (*Matchers) GetParquetChunksMatcher added in v1.20.0 

func (m *Matchers) GetParquetChunksMatcher() func(string) bool

func (*Matchers) GetParquetLabelsMatcher added in v1.20.0 

func (m *Matchers) GetParquetLabelsMatcher() func(string) bool

func (*Matchers) GetPartitionedGroupsIterMatcher added in v1.20.0 

func (m *Matchers) GetPartitionedGroupsIterMatcher() func(string) bool

func (*Matchers) GetTenantBlocksIterMatcher added in v1.17.0 

func (m *Matchers) GetTenantBlocksIterMatcher() func(string) bool

func (*Matchers) GetTenantsIterMatcher added in v1.17.0 

func (m *Matchers) GetTenantsIterMatcher() func(string) bool

func (*Matchers) SetBlockIndexMatcher added in v1.17.0 

func (m *Matchers) SetBlockIndexMatcher(f func(string) bool)

func (*Matchers) SetBucketIndexMatcher added in v1.17.0 

func (m *Matchers) SetBucketIndexMatcher(f func(string) bool)

func (*Matchers) SetChunksIterMatcher added in v1.17.0 

func (m *Matchers) SetChunksIterMatcher(f func(string) bool)

func (*Matchers) SetChunksMatcher added in v1.17.0 

func (m *Matchers) SetChunksMatcher(f func(string) bool)

func (*Matchers) SetMetaFileMatcher added in v1.17.0 

func (m *Matchers) SetMetaFileMatcher(f func(string) bool)

func (*Matchers) SetParquetChunksMatcher added in v1.20.0 

func (m *Matchers) SetParquetChunksMatcher(f func(string) bool)

func (*Matchers) SetParquetLabelsMatcher added in v1.20.0 

func (m *Matchers) SetParquetLabelsMatcher(f func(string) bool)

func (*Matchers) SetPartitionedGroupsIterMatcher added in v1.20.0 

func (m *Matchers) SetPartitionedGroupsIterMatcher(f func(string) bool)

func (*Matchers) SetTenantBlocksIterMatcher added in v1.17.0 

func (m *Matchers) SetTenantBlocksIterMatcher(f func(string) bool)

func (*Matchers) SetTenantsIterMatcher added in v1.17.0 

func (m *Matchers) SetTenantsIterMatcher(f func(string) bool)

type MemcachedClientConfig added in v1.2.0 

type MemcachedClientConfig struct {
	Addresses              string               `yaml:"addresses"`
	Timeout                time.Duration        `yaml:"timeout"`
	MaxIdleConnections     int                  `yaml:"max_idle_connections"`
	MaxAsyncConcurrency    int                  `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize     int                  `yaml:"max_async_buffer_size"`
	MaxGetMultiConcurrency int                  `yaml:"max_get_multi_concurrency"`
	MaxGetMultiBatchSize   int                  `yaml:"max_get_multi_batch_size"`
	MaxItemSize            int                  `yaml:"max_item_size"`
	AutoDiscovery          bool                 `yaml:"auto_discovery"`
	SetAsyncCircuitBreaker CircuitBreakerConfig `yaml:"set_async_circuit_breaker_config"`
}

func (*MemcachedClientConfig) GetAddresses added in v1.2.0 

func (cfg *MemcachedClientConfig) GetAddresses() []string

func (*MemcachedClientConfig) RegisterFlagsWithPrefix added in v1.2.0 

func (cfg *MemcachedClientConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (MemcachedClientConfig) ToMemcachedClientConfig added in v1.2.0 

func (cfg MemcachedClientConfig) ToMemcachedClientConfig() cacheutil.MemcachedClientConfig

func (*MemcachedClientConfig) Validate added in v1.2.0 

func (cfg *MemcachedClientConfig) Validate() error

Validate the config.

type MemcachedIndexCacheConfig added in v1.0.0 

type MemcachedIndexCacheConfig struct {
	ClientConfig MemcachedClientConfig `yaml:",inline"`
	EnabledItems []string              `yaml:"enabled_items"`
}

func (*MemcachedIndexCacheConfig) RegisterFlagsWithPrefix added in v1.0.0 

func (cfg *MemcachedIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*MemcachedIndexCacheConfig) Validate added in v1.0.0 

func (cfg *MemcachedIndexCacheConfig) Validate() error

type MetadataCacheConfig added in v1.2.0 

type MetadataCacheConfig struct {
	BucketCacheBackend `yaml:",inline"`

	TenantsListTTL           time.Duration `yaml:"tenants_list_ttl"`
	TenantBlocksListTTL      time.Duration `yaml:"tenant_blocks_list_ttl"`
	ChunksListTTL            time.Duration `yaml:"chunks_list_ttl"`
	MetafileExistsTTL        time.Duration `yaml:"metafile_exists_ttl"`
	MetafileDoesntExistTTL   time.Duration `yaml:"metafile_doesnt_exist_ttl"`
	MetafileContentTTL       time.Duration `yaml:"metafile_content_ttl"`
	MetafileMaxSize          int           `yaml:"metafile_max_size_bytes"`
	MetafileAttributesTTL    time.Duration `yaml:"metafile_attributes_ttl"`
	BlockIndexAttributesTTL  time.Duration `yaml:"block_index_attributes_ttl"`
	BucketIndexContentTTL    time.Duration `yaml:"bucket_index_content_ttl"`
	BucketIndexMaxSize       int           `yaml:"bucket_index_max_size_bytes"`
	PartitionedGroupsListTTL time.Duration `yaml:"partitioned_groups_list_ttl"`
}

func (*MetadataCacheConfig) RegisterFlagsWithPrefix added in v1.2.0 

func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*MetadataCacheConfig) Validate added in v1.2.0 

func (cfg *MetadataCacheConfig) Validate() error

type MultiLevelBucketCacheConfig added in v1.20.0 

type MultiLevelBucketCacheConfig struct {
	MaxAsyncConcurrency int `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize  int `yaml:"max_async_buffer_size"`
	MaxBackfillItems    int `yaml:"max_backfill_items"`

	BackFillTTL time.Duration `yaml:"-"`
}

func (*MultiLevelBucketCacheConfig) RegisterFlagsWithPrefix added in v1.20.0 

func (cfg *MultiLevelBucketCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*MultiLevelBucketCacheConfig) Validate added in v1.20.0 

func (cfg *MultiLevelBucketCacheConfig) Validate() error

type MultiLevelIndexCacheConfig added in v1.17.0 

type MultiLevelIndexCacheConfig struct {
	MaxAsyncConcurrency int `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize  int `yaml:"max_async_buffer_size"`
	MaxBackfillItems    int `yaml:"max_backfill_items"`
}

func (*MultiLevelIndexCacheConfig) RegisterFlagsWithPrefix added in v1.17.0 

func (cfg *MultiLevelIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*MultiLevelIndexCacheConfig) Validate added in v1.17.0 

func (cfg *MultiLevelIndexCacheConfig) Validate() error

type ParquetLabelsCacheConfig added in v1.20.0 

type ParquetLabelsCacheConfig struct {
	BucketCacheBackend `yaml:",inline"`

	SubrangeSize        int64         `yaml:"subrange_size"`
	MaxGetRangeRequests int           `yaml:"max_get_range_requests"`
	AttributesTTL       time.Duration `yaml:"attributes_ttl"`
	SubrangeTTL         time.Duration `yaml:"subrange_ttl"`
}

func (*ParquetLabelsCacheConfig) RegisterFlagsWithPrefix added in v1.20.0 

func (cfg *ParquetLabelsCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*ParquetLabelsCacheConfig) Validate added in v1.20.0 

func (cfg *ParquetLabelsCacheConfig) Validate() error

type PartitionInfo added in v1.19.0 

type PartitionInfo struct {
	PartitionedGroupID           uint32 `json:"partitioned_group_id"`
	PartitionCount               int    `json:"partition_count"`
	PartitionID                  int    `json:"partition_id"`
	PartitionedGroupCreationTime int64  `json:"partitioned_group_creation_time"`
}

func ConvertToPartitionInfo added in v1.19.0 

func ConvertToPartitionInfo(extensions any) (*PartitionInfo, error)

func GetPartitionInfo added in v1.19.0 

func GetPartitionInfo(meta metadata.Meta) (*PartitionInfo, error)

type PostingsCacheConfig added in v1.19.0 

type PostingsCacheConfig struct {
	Enabled  bool          `yaml:"enabled"`
	MaxBytes int64         `yaml:"max_bytes"`
	Ttl      time.Duration `yaml:"ttl"`
}

func (*PostingsCacheConfig) RegisterFlagsWithPrefix added in v1.19.0 

func (cfg *PostingsCacheConfig) RegisterFlagsWithPrefix(prefix, block string, f *flag.FlagSet)

RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet

type RedisClientConfig added in v1.15.0 

type RedisClientConfig struct {
	Addresses  string `yaml:"addresses"`
	Username   string `yaml:"username"`
	Password   string `yaml:"password"`
	DB         int    `yaml:"db"`
	MasterName string `yaml:"master_name"`

	MaxGetMultiConcurrency int `yaml:"max_get_multi_concurrency"`
	GetMultiBatchSize      int `yaml:"get_multi_batch_size"`
	MaxSetMultiConcurrency int `yaml:"max_set_multi_concurrency"`
	SetMultiBatchSize      int `yaml:"set_multi_batch_size"`
	MaxAsyncConcurrency    int `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize     int `yaml:"max_async_buffer_size"`

	DialTimeout  time.Duration `yaml:"dial_timeout"`
	ReadTimeout  time.Duration `yaml:"read_timeout"`
	WriteTimeout time.Duration `yaml:"write_timeout"`

	TLSEnabled bool             `yaml:"tls_enabled"`
	TLS        tls.ClientConfig `yaml:",inline"`

	// If not zero then client-side caching is enabled.
	// Client-side caching is when data is stored in memory
	// instead of fetching data each time.
	// See https://redis.io/docs/manual/client-side-caching/ for info.
	CacheSize int `yaml:"cache_size"`

	// SetAsyncCircuitBreaker configures the circuit breaker for SetAsync operations.
	SetAsyncCircuitBreaker CircuitBreakerConfig `yaml:"set_async_circuit_breaker_config"`
}

func (*RedisClientConfig) RegisterFlagsWithPrefix added in v1.15.0 

func (cfg *RedisClientConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*RedisClientConfig) ToRedisClientConfig added in v1.15.0 

func (cfg *RedisClientConfig) ToRedisClientConfig() cacheutil.RedisClientConfig

func (*RedisClientConfig) Validate added in v1.15.0 

func (cfg *RedisClientConfig) Validate() error

Validate the config.

type RedisIndexCacheConfig added in v1.16.0 

type RedisIndexCacheConfig struct {
	ClientConfig RedisClientConfig `yaml:",inline"`
	EnabledItems []string          `yaml:"enabled_items"`
}

func (*RedisIndexCacheConfig) RegisterFlagsWithPrefix added in v1.16.0 

func (cfg *RedisIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*RedisIndexCacheConfig) Validate added in v1.16.0 

func (cfg *RedisIndexCacheConfig) Validate() error

type TSDBConfig added in v1.3.0 

type TSDBConfig struct {
	Dir                       string        `yaml:"dir"`
	BlockRanges               DurationList  `yaml:"block_ranges_period"`
	Retention                 time.Duration `yaml:"retention_period"`
	ShipInterval              time.Duration `yaml:"ship_interval"`
	ShipConcurrency           int           `yaml:"ship_concurrency"`
	HeadCompactionInterval    time.Duration `yaml:"head_compaction_interval"`
	HeadCompactionConcurrency int           `yaml:"head_compaction_concurrency"`
	HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout"`
	HeadChunksWriteBufferSize int           `yaml:"head_chunks_write_buffer_size_bytes"`
	StripeSize                int           `yaml:"stripe_size"`
	WALCompressionType        string        `yaml:"wal_compression_type"`
	WALSegmentSizeBytes       int           `yaml:"wal_segment_size_bytes"`
	FlushBlocksOnShutdown     bool          `yaml:"flush_blocks_on_shutdown"`
	CloseIdleTSDBTimeout      time.Duration `yaml:"close_idle_tsdb_timeout"`
	// The size of the in-memory queue used before flushing chunks to the disk.
	HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size"`

	// MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup.
	MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`

	// If true, user TSDBs are not closed on shutdown. Only for testing.
	// If false (default), user TSDBs are closed to make sure all resources are released and closed properly.
	KeepUserTSDBOpenOnShutdown bool `yaml:"-"`

	// How often to check for idle TSDBs for closing. DefaultCloseIdleTSDBInterval is not suitable for testing, so tests can override.
	CloseIdleTSDBInterval time.Duration `yaml:"-"`

	// How often expired items are cleaned from the PostingsCache. ExpandedCachingExpireInterval is not suitable for testing, so tests can override.
	ExpandedCachingExpireInterval time.Duration `yaml:"-"`

	// Positive value enables experimental support for exemplars. 0 or less to disable.
	MaxExemplars int `yaml:"max_exemplars"`

	// Enable snapshotting of in-memory TSDB data on disk when shutting down.
	MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown"`

	// OutOfOrderCapMax is maximum capacity for OOO chunks (in samples).
	OutOfOrderCapMax int64 `yaml:"out_of_order_cap_max"`

	// Posting Cache Configuration for TSDB
	PostingsCache TSDBPostingsCacheConfig `` /* 208-byte string literal not displayed */
}

TSDBConfig holds the config for TSDB opened in the ingesters.

func (*TSDBConfig) BlocksDir added in v1.3.0 

func (cfg *TSDBConfig) BlocksDir(userID string) string

BlocksDir returns the directory path where TSDB blocks and wal should be stored by the ingester

func (*TSDBConfig) IsBlocksShippingEnabled added in v1.8.0 

func (cfg *TSDBConfig) IsBlocksShippingEnabled() bool

IsBlocksShippingEnabled returns whether blocks shipping is enabled.

func (*TSDBConfig) RegisterFlags added in v1.3.0 

func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the TSDBConfig flags.

func (*TSDBConfig) Validate added in v1.3.0 

func (cfg *TSDBConfig) Validate() error

Validate the config.

type TSDBPostingsCacheConfig added in v1.19.0 

type TSDBPostingsCacheConfig struct {
	Head   PostingsCacheConfig `` /* 172-byte string literal not displayed */
	Blocks PostingsCacheConfig `` /* 148-byte string literal not displayed */

	// The configurations below are used only for testing purpose
	PostingsForMatchers func(ctx context.Context, ix tsdb.IndexReader, ms ...*labels.Matcher) (index.Postings, error) `yaml:"-"`
	SeedSize            int                                                                                           `yaml:"-"`
	// contains filtered or unexported fields
}

func (*TSDBPostingsCacheConfig) RegisterFlagsWithPrefix added in v1.19.0 

func (cfg *TSDBPostingsCacheConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type TenantDeletionMark added in v1.6.0 

type TenantDeletionMark struct {
	// Unix timestamp when deletion marker was created.
	DeletionTime int64 `json:"deletion_time"`

	// Unix timestamp when cleanup was finished.
	FinishedTime int64 `json:"finished_time,omitempty"`
}

func NewTenantDeletionMark added in v1.7.0 

func NewTenantDeletionMark(deletionTime time.Time) *TenantDeletionMark

func ReadTenantDeletionMark added in v1.7.0 

func ReadTenantDeletionMark(ctx context.Context, bkt objstore.InstrumentedBucket, userID string) (*TenantDeletionMark, error)

Returns tenant deletion mark for given user, if it exists. If it doesn't exist, returns nil mark, and no error.

type TokenBucketBytesLimiterConfig added in v1.18.0 

type TokenBucketBytesLimiterConfig struct {
	Mode                       string  `yaml:"mode"`
	InstanceTokenBucketSize    int64   `yaml:"instance_token_bucket_size"`
	UserTokenBucketSize        int64   `yaml:"user_token_bucket_size"`
	RequestTokenBucketSize     int64   `yaml:"request_token_bucket_size"`
	FetchedPostingsTokenFactor float64 `yaml:"fetched_postings_token_factor" doc:"hidden"`
	TouchedPostingsTokenFactor float64 `yaml:"touched_postings_token_factor" doc:"hidden"`
	FetchedSeriesTokenFactor   float64 `yaml:"fetched_series_token_factor" doc:"hidden"`
	TouchedSeriesTokenFactor   float64 `yaml:"touched_series_token_factor" doc:"hidden"`
	FetchedChunksTokenFactor   float64 `yaml:"fetched_chunks_token_factor" doc:"hidden"`
	TouchedChunksTokenFactor   float64 `yaml:"touched_chunks_token_factor" doc:"hidden"`
}

type TokenBucketBytesLimiterMode added in v1.18.0 

type TokenBucketBytesLimiterMode string
const (
	TokenBucketBytesLimiterDisabled TokenBucketBytesLimiterMode = "disabled"
	TokenBucketBytesLimiterDryRun   TokenBucketBytesLimiterMode = "dryrun"
	TokenBucketBytesLimiterEnabled  TokenBucketBytesLimiterMode = "enabled"
)

type UsersScannerConfig added in v1.20.0 

type UsersScannerConfig struct {
	Strategy       string        `yaml:"strategy"`
	MaxStalePeriod time.Duration `yaml:"max_stale_period"`
	CacheTTL       time.Duration `yaml:"cache_ttl"`
}

func (*UsersScannerConfig) RegisterFlagsWithPrefix added in v1.20.0 

func (c *UsersScannerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

func (*UsersScannerConfig) Validate added in v1.20.0 

func (c *UsersScannerConfig) Validate() error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.