tsdb

package
v1.3.0-rc.1.0...-65594da Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 29, 2021 License: Apache-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// BackendS3 is the value for the S3 storage backend
	BackendS3 = "s3"

	// BackendGCS is the value for the GCS storage backend
	BackendGCS = "gcs"

	// BackendAzure is the value for the Azure storage backend
	BackendAzure = "azure"

	// BackendSwift is the value for the Openstack Swift storage backend
	BackendSwift = "swift"

	// BackendFilesystem is the value for the filesystem storge backend
	BackendFilesystem = "filesystem"

	// TenantIDExternalLabel is the external label containing the tenant ID,
	// set when shipping blocks to the storage.
	TenantIDExternalLabel = "__org_id__"

	// IngesterIDExternalLabel is the external label containing the ingester ID,
	// set when shipping blocks to the storage.
	IngesterIDExternalLabel = "__ingester_id__"

	// ShardIDExternalLabel is the external label containing the shard ID
	// and can be used to shard blocks.
	ShardIDExternalLabel = "__shard_id__"
)
View Source 
const (
	// IndexCacheBackendInMemory is the value for the in-memory index cache backend.
	IndexCacheBackendInMemory = "inmemory"

	// IndexCacheBackendMemcached is the value for the memcached index cache backend.
	IndexCacheBackendMemcached = "memcached"

	// IndexCacheBackendDefault is the value for the default index cache backend.
	IndexCacheBackendDefault = IndexCacheBackendInMemory
)
View Source 
const (
	CacheBackendMemcached = "memcached"
)
View Source 
const (
	// DefaultRefCacheTTL is the default RefCache purge TTL. We use a reasonable
	// value that should cover most use cases. The cache would be ineffective if
	// the scrape interval of a series is greater than this TTL.
	DefaultRefCacheTTL = 10 * time.Minute
)

Variables

This section is empty.

Functions

func CreateCachingBucket 

func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig MetadataCacheConfig, bkt objstore.Bucket, logger log.Logger, reg prometheus.Registerer) (objstore.Bucket, error)

func HashBlockID 

func HashBlockID(id ulid.ULID) uint32

HashBlockID returns a 32-bit hash of the block ID useful for ring-based sharding.

func NewBucketClient 

func NewBucketClient(ctx context.Context, cfg BucketConfig, name string, logger log.Logger, reg prometheus.Registerer) (client objstore.Bucket, err error)

NewBucketClient creates a new bucket client based on the configured backend

func NewIndexCache 

func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error)

NewIndexCache creates a new index cache based on the input configuration.

Types

type BlocksStorageConfig 

type BlocksStorageConfig struct {
	Bucket      BucketConfig      `yaml:",inline"`
	BucketStore BucketStoreConfig `yaml:"bucket_store" doc:"description=This configures how the store-gateway synchronizes blocks stored in the bucket."`
	TSDB        TSDBConfig        `yaml:"tsdb"`
}

BlocksStorageConfig holds the config information for the blocks storage.

func (*BlocksStorageConfig) RegisterFlags 

func (cfg *BlocksStorageConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the TSDB flags

func (*BlocksStorageConfig) Validate 

func (cfg *BlocksStorageConfig) Validate() error

Validate the config.

type BucketClientMock 

type BucketClientMock struct {
	mock.Mock
}

BucketClientMock mocks objstore.Bucket

func (*BucketClientMock) Attributes 

func (m *BucketClientMock) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

ObjectSize mocks objstore.Bucket.Attributes()

func (*BucketClientMock) Close 

func (m *BucketClientMock) Close() error

Close mocks objstore.Bucket.Close()

func (*BucketClientMock) Delete 

func (m *BucketClientMock) Delete(ctx context.Context, name string) error

Delete mocks objstore.Bucket.Delete()

func (*BucketClientMock) Exists 

func (m *BucketClientMock) Exists(ctx context.Context, name string) (bool, error)

Exists mocks objstore.Bucket.Exists()

func (*BucketClientMock) Get 

func (m *BucketClientMock) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get mocks objstore.Bucket.Get()

func (*BucketClientMock) GetRange 

func (m *BucketClientMock) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange mocks objstore.Bucket.GetRange()

func (*BucketClientMock) IsObjNotFoundErr 

func (m *BucketClientMock) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr mocks objstore.Bucket.IsObjNotFoundErr()

func (*BucketClientMock) Iter 

func (m *BucketClientMock) Iter(ctx context.Context, dir string, f func(string) error) error

Iter mocks objstore.Bucket.Iter()

func (*BucketClientMock) MockDelete 

func (m *BucketClientMock) MockDelete(name string, err error)

func (*BucketClientMock) MockGet 

func (m *BucketClientMock) MockGet(name, content string, err error)

MockGet is a convenient method to mock Get() and Exists()

func (*BucketClientMock) MockIter 

func (m *BucketClientMock) MockIter(prefix string, objects []string, err error)

MockIter is a convenient method to mock Iter()

func (*BucketClientMock) MockIterWithCallback 

func (m *BucketClientMock) MockIterWithCallback(prefix string, objects []string, err error, cb func())

MockIterWithCallback is a convenient method to mock Iter() and get a callback called when the Iter API is called.

func (*BucketClientMock) Name 

func (m *BucketClientMock) Name() string

Name mocks objstore.Bucket.Name()

func (*BucketClientMock) Upload 

func (m *BucketClientMock) Upload(ctx context.Context, name string, r io.Reader) error

Upload mocks objstore.Bucket.Upload()

type BucketConfig 

type BucketConfig struct {
	Backend string `yaml:"backend"`
	// Backends
	S3         s3.Config         `yaml:"s3"`
	GCS        gcs.Config        `yaml:"gcs"`
	Azure      azure.Config      `yaml:"azure"`
	Swift      swift.Config      `yaml:"swift"`
	Filesystem filesystem.Config `yaml:"filesystem"`

	// Not used internally, meant to allow callers to wrap Buckets
	// created using this config
	Middlewares []func(objstore.Bucket) (objstore.Bucket, error) `yaml:"-"`
}

BucketConfig holds configuration for accessing long-term storage.

func (*BucketConfig) RegisterFlags 

func (cfg *BucketConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the TSDB Backend

func (*BucketConfig) Validate 

func (cfg *BucketConfig) Validate() error

type BucketStoreConfig 

type BucketStoreConfig struct {
	SyncDir                  string              `yaml:"sync_dir"`
	SyncInterval             time.Duration       `yaml:"sync_interval"`
	MaxChunkPoolBytes        uint64              `yaml:"max_chunk_pool_bytes"`
	MaxConcurrent            int                 `yaml:"max_concurrent"`
	TenantSyncConcurrency    int                 `yaml:"tenant_sync_concurrency"`
	BlockSyncConcurrency     int                 `yaml:"block_sync_concurrency"`
	MetaSyncConcurrency      int                 `yaml:"meta_sync_concurrency"`
	ConsistencyDelay         time.Duration       `yaml:"consistency_delay"`
	IndexCache               IndexCacheConfig    `yaml:"index_cache"`
	ChunksCache              ChunksCacheConfig   `yaml:"chunks_cache"`
	MetadataCache            MetadataCacheConfig `yaml:"metadata_cache"`
	IgnoreDeletionMarksDelay time.Duration       `yaml:"ignore_deletion_mark_delay"`

	// Controls what is the ratio of postings offsets store will hold in memory.
	// Larger value will keep less offsets, which will increase CPU cycles needed for query touching those postings.
	// It's meant for setups that want low baseline memory pressure and where less traffic is expected.
	// On the contrary, smaller value will increase baseline memory usage, but improve latency slightly.
	// 1 will keep all in memory. Default value is the same as in Prometheus which gives a good balance.
	PostingOffsetsInMemSampling int `yaml:"postings_offsets_in_mem_sampling" doc:"hidden"`
}

BucketStoreConfig holds the config information for Bucket Stores used by the querier

func (*BucketStoreConfig) RegisterFlags 

func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the BucketStore flags

func (*BucketStoreConfig) Validate 

func (cfg *BucketStoreConfig) Validate() error

Validate the config.

type CacheBackend 

type CacheBackend struct {
	Backend   string                `yaml:"backend"`
	Memcached MemcachedClientConfig `yaml:"memcached"`
}

func (*CacheBackend) Validate 

func (cfg *CacheBackend) Validate() error

Validate the config.

type ChunksCacheConfig 

type ChunksCacheConfig struct {
	CacheBackend `yaml:",inline"`

	SubrangeSize        int64         `yaml:"subrange_size"`
	MaxGetRangeRequests int           `yaml:"max_get_range_requests"`
	AttributesTTL       time.Duration `yaml:"attributes_ttl"`
	SubrangeTTL         time.Duration `yaml:"subrange_ttl"`
}

func (*ChunksCacheConfig) RegisterFlagsWithPrefix 

func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*ChunksCacheConfig) Validate 

func (cfg *ChunksCacheConfig) Validate() error

type DurationList 

type DurationList []time.Duration

DurationList is the block ranges for a tsdb

func (*DurationList) Set 

func (d *DurationList) Set(s string) error

Set implements the flag.Value interface

func (*DurationList) String 

func (d *DurationList) String() string

String implements the flag.Value interface

func (*DurationList) ToMilliseconds 

func (d *DurationList) ToMilliseconds() []int64

ToMilliseconds returns the duration list in milliseconds

type InMemoryIndexCacheConfig 

type InMemoryIndexCacheConfig struct {
	MaxSizeBytes uint64 `yaml:"max_size_bytes"`
}

func (*InMemoryIndexCacheConfig) RegisterFlagsWithPrefix 

func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

type IndexCacheConfig 

type IndexCacheConfig struct {
	Backend             string                   `yaml:"backend"`
	InMemory            InMemoryIndexCacheConfig `yaml:"inmemory"`
	Memcached           MemcachedClientConfig    `yaml:"memcached"`
	PostingsCompression bool                     `yaml:"postings_compression_enabled"`
}

func (*IndexCacheConfig) RegisterFlags 

func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet)

func (*IndexCacheConfig) RegisterFlagsWithPrefix 

func (cfg *IndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*IndexCacheConfig) Validate 

func (cfg *IndexCacheConfig) Validate() error

Validate the config.

type MemcachedClientConfig 

type MemcachedClientConfig struct {
	Addresses              string        `yaml:"addresses"`
	Timeout                time.Duration `yaml:"timeout"`
	MaxIdleConnections     int           `yaml:"max_idle_connections"`
	MaxAsyncConcurrency    int           `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize     int           `yaml:"max_async_buffer_size"`
	MaxGetMultiConcurrency int           `yaml:"max_get_multi_concurrency"`
	MaxGetMultiBatchSize   int           `yaml:"max_get_multi_batch_size"`
	MaxItemSize            int           `yaml:"max_item_size"`
}

func (*MemcachedClientConfig) GetAddresses 

func (cfg *MemcachedClientConfig) GetAddresses() []string

func (*MemcachedClientConfig) RegisterFlagsWithPrefix 

func (cfg *MemcachedClientConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (MemcachedClientConfig) ToMemcachedClientConfig 

func (cfg MemcachedClientConfig) ToMemcachedClientConfig() cacheutil.MemcachedClientConfig

func (*MemcachedClientConfig) Validate 

func (cfg *MemcachedClientConfig) Validate() error

Validate the config.

type MetadataCacheConfig 

type MetadataCacheConfig struct {
	CacheBackend `yaml:",inline"`

	TenantsListTTL         time.Duration `yaml:"tenants_list_ttl"`
	TenantBlocksListTTL    time.Duration `yaml:"tenant_blocks_list_ttl"`
	ChunksListTTL          time.Duration `yaml:"chunks_list_ttl"`
	MetafileExistsTTL      time.Duration `yaml:"metafile_exists_ttl"`
	MetafileDoesntExistTTL time.Duration `yaml:"metafile_doesnt_exist_ttl"`
	MetafileContentTTL     time.Duration `yaml:"metafile_content_ttl"`
	MetafileMaxSize        int           `yaml:"metafile_max_size_bytes"`
}

func (*MetadataCacheConfig) RegisterFlagsWithPrefix 

func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*MetadataCacheConfig) Validate 

func (cfg *MetadataCacheConfig) Validate() error

type RefCache 

type RefCache struct {
	// contains filtered or unexported fields
}

RefCache is a single-tenant cache mapping a labels set with the reference ID in TSDB, in order to be able to append samples to the TSDB head without having to copy write request series labels each time (because the memory buffers used to unmarshal the write request is reused).

func NewRefCache 

func NewRefCache() *RefCache

NewRefCache makes a new RefCache.

func (*RefCache) Purge 

func (c *RefCache) Purge(keepUntil time.Time)

Purge removes expired entries from the cache. This function should be called periodically to avoid memory leaks.

func (*RefCache) Ref 

func (c *RefCache) Ref(now time.Time, series labels.Labels) (uint64, bool)

Ref returns the cached series reference, and guarantees the input labels set is NOT retained.

func (*RefCache) SetRef 

func (c *RefCache) SetRef(now time.Time, series labels.Labels, ref uint64)

SetRef sets/updates the cached series reference. The input labels set IS retained.

type TSDBConfig 

type TSDBConfig struct {
	Dir                       string        `yaml:"dir"`
	BlockRanges               DurationList  `yaml:"block_ranges_period"`
	Retention                 time.Duration `yaml:"retention_period"`
	ShipInterval              time.Duration `yaml:"ship_interval"`
	ShipConcurrency           int           `yaml:"ship_concurrency"`
	HeadCompactionInterval    time.Duration `yaml:"head_compaction_interval"`
	HeadCompactionConcurrency int           `yaml:"head_compaction_concurrency"`
	HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout"`
	StripeSize                int           `yaml:"stripe_size"`
	WALCompressionEnabled     bool          `yaml:"wal_compression_enabled"`
	FlushBlocksOnShutdown     bool          `yaml:"flush_blocks_on_shutdown"`

	// MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup.
	MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`

	// If true, user TSDBs are not closed on shutdown. Only for testing.
	// If false (default), user TSDBs are closed to make sure all resources are released and closed properly.
	KeepUserTSDBOpenOnShutdown bool `yaml:"-"`
}

TSDBConfig holds the config for TSDB opened in the ingesters.

func (*TSDBConfig) BlocksDir 

func (cfg *TSDBConfig) BlocksDir(userID string) string

BlocksDir returns the directory path where TSDB blocks and wal should be stored by the ingester

func (*TSDBConfig) RegisterFlags 

func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the TSDBConfig flags.

func (*TSDBConfig) Validate 

func (cfg *TSDBConfig) Validate() error

Validate the config.

type UserBucketClient 

type UserBucketClient struct {
	UserBucketReaderClient
	// contains filtered or unexported fields
}

UserBucketClient is a wrapper around a objstore.Bucket that prepends writes with a userID

func NewUserBucketClient 

func NewUserBucketClient(userID string, bucket objstore.Bucket) *UserBucketClient

func (*UserBucketClient) Close 

func (b *UserBucketClient) Close() error

Close implements io.Closer

func (*UserBucketClient) Delete 

func (b *UserBucketClient) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*UserBucketClient) Name 

func (b *UserBucketClient) Name() string

Name returns the bucket name for the provider.

func (*UserBucketClient) Upload 

func (b *UserBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

func (*UserBucketClient) WithExpectedErrs 

func (b *UserBucketClient) WithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.Bucket

WithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment thanos_objstore_bucket_operation_failures_total metric.

type UserBucketReaderClient 

type UserBucketReaderClient struct {
	// contains filtered or unexported fields
}

UserBucketReaderClient is a wrapper around a objstore.BucketReader that reads from user-specific subfolder.

func (*UserBucketReaderClient) Attributes 

func (b *UserBucketReaderClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns attributes of the specified object.

func (*UserBucketReaderClient) Exists 

func (b *UserBucketReaderClient) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists in the bucket.

func (*UserBucketReaderClient) Get 

func (b *UserBucketReaderClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*UserBucketReaderClient) GetRange 

func (b *UserBucketReaderClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*UserBucketReaderClient) IsObjNotFoundErr 

func (b *UserBucketReaderClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*UserBucketReaderClient) Iter 

func (b *UserBucketReaderClient) Iter(ctx context.Context, dir string, f func(string) error) error

Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full object name including the prefix of the inspected directory.

func (*UserBucketReaderClient) ReaderWithExpectedErrs 

func (b *UserBucketReaderClient) ReaderWithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.BucketReader

ReaderWithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment thanos_objstore_bucket_operation_failures_total metric.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.