storage

package
v2.4.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 21, 2022 License: AGPL-3.0 Imports: 38 Imported by: 0

Documentation

Index

Constants

View Source
const (
	StorageEngineChunks = "chunks"
	StorageEngineBlocks = "blocks"
)

Supported storage engines

View Source
const (
	StorageTypeAWS            = "aws"
	StorageTypeAWSDynamo      = "aws-dynamo"
	StorageTypeAzure          = "azure"
	StorageTypeBoltDB         = "boltdb"
	StorageTypeCassandra      = "cassandra"
	StorageTypeInMemory       = "inmemory"
	StorageTypeBigTable       = "bigtable"
	StorageTypeBigTableHashed = "bigtable-hashed"
	StorageTypeFileSystem     = "filesystem"
	StorageTypeGCP            = "gcp"
	StorageTypeGCPColumnKey   = "gcp-columnkey"
	StorageTypeGCS            = "gcs"
	StorageTypeGrpc           = "grpc-store"
	StorageTypeS3             = "s3"
	StorageTypeSwift          = "swift"
)

Supported storage clients

Variables

View Source
var (
	ErrInvalidLengthCachingIndexClient = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowCachingIndexClient   = fmt.Errorf("proto: integer overflow")
)
View Source
var Fixtures = []testutils.Fixture{
	fixture{gcp.Fixtures[0]},
}

Fixtures for unit testing the caching storage.

Functions

func NewBucketClient

func NewBucketClient(storageConfig Config) (chunk.BucketClient, error)

NewBucketClient makes a new bucket client based on the configuration.

func NewChunkClient

func NewChunkClient(name string, cfg Config, schemaCfg chunk.SchemaConfig, clientMetrics ClientMetrics, registerer prometheus.Registerer) (chunk.Client, error)

NewChunkClient makes a new chunk.Client of the desired types.

func NewIndexClient

func NewIndexClient(name string, cfg Config, schemaCfg chunk.SchemaConfig, limits StoreLimits, registerer prometheus.Registerer) (chunk.IndexClient, error)

NewIndexClient makes a new index client of the desired type.

func NewObjectClient

func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (chunk.ObjectClient, error)

NewObjectClient makes a new StorageClient of the desired types.

func NewStore

func NewStore(
	cfg Config,
	storeCfg chunk.StoreConfig,
	schemaCfg chunk.SchemaConfig,
	limits StoreLimits,
	clientMetrics ClientMetrics,
	reg prometheus.Registerer,
	cacheGenNumLoader chunk.CacheGenNumLoader,
	logger log.Logger,
) (chunk.Store, error)

NewStore makes the storage clients based on the configuration.

func NewTableClient

func NewTableClient(name string, cfg Config, registerer prometheus.Registerer) (chunk.TableClient, error)

NewTableClient makes a new table client based on the configuration.

func RegisterIndexStore

func RegisterIndexStore(name string, indexClientFactory IndexClientFactoryFunc, tableClientFactory TableClientFactoryFunc)

RegisterIndexStore is used for registering a custom index type. When an index type is registered here with same name as existing types, the registered one takes the precedence.

Types

type Bytes

type Bytes []byte

Bytes exists to stop proto copying the byte array

func (*Bytes) Compare

func (bs *Bytes) Compare(other Bytes) int

Compare Bytes to other

func (*Bytes) Equal

func (bs *Bytes) Equal(other Bytes) bool

Equal returns true if other equals Bytes

func (*Bytes) Marshal

func (bs *Bytes) Marshal() ([]byte, error)

Marshal just returns bs

func (*Bytes) MarshalTo

func (bs *Bytes) MarshalTo(data []byte) (n int, err error)

MarshalTo copies Bytes to data

func (*Bytes) Size

func (bs *Bytes) Size() int

Size returns the length of Bytes

func (*Bytes) Unmarshal

func (bs *Bytes) Unmarshal(data []byte) error

Unmarshal updates Bytes to be data, without a copy

type ClientMetrics

type ClientMetrics struct {
	AzureMetrics azure.BlobStorageMetrics
}

func NewClientMetrics

func NewClientMetrics() ClientMetrics

func (*ClientMetrics) Unregister

func (c *ClientMetrics) Unregister()

type Config

type Config struct {
	Engine                 string                  `yaml:"engine"`
	AWSStorageConfig       aws.StorageConfig       `yaml:"aws"`
	AzureStorageConfig     azure.BlobStorageConfig `yaml:"azure"`
	GCPStorageConfig       gcp.Config              `yaml:"bigtable"`
	GCSConfig              gcp.GCSConfig           `yaml:"gcs"`
	CassandraStorageConfig cassandra.Config        `yaml:"cassandra"`
	BoltDBConfig           local.BoltDBConfig      `yaml:"boltdb"`
	FSConfig               local.FSConfig          `yaml:"filesystem"`
	Swift                  openstack.SwiftConfig   `yaml:"swift"`

	IndexCacheValidity time.Duration `yaml:"index_cache_validity"`

	IndexQueriesCacheConfig  cache.Config `yaml:"index_queries_cache_config"`
	DisableBroadIndexQueries bool         `yaml:"disable_broad_index_queries"`
	MaxParallelGetChunk      int          `yaml:"max_parallel_get_chunk"`

	GrpcConfig grpc.Config `yaml:"grpc_store"`

	Hedging hedging.Config `yaml:"hedging"`
}

Config chooses which storage client to use.

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

func (*Config) Validate

func (cfg *Config) Validate() error

Validate config and returns error on failure

type Entry

type Entry struct {
	Column Bytes `protobuf:"bytes,1,opt,name=Column,proto3,customtype=Bytes" json:"Column"`
	Value  Bytes `protobuf:"bytes,2,opt,name=Value,proto3,customtype=Bytes" json:"Value"`
}

func (*Entry) Descriptor

func (*Entry) Descriptor() ([]byte, []int)

func (*Entry) Equal

func (this *Entry) Equal(that interface{}) bool

func (*Entry) GoString

func (this *Entry) GoString() string

func (*Entry) Marshal

func (m *Entry) Marshal() (dAtA []byte, err error)

func (*Entry) MarshalTo

func (m *Entry) MarshalTo(dAtA []byte) (int, error)

func (*Entry) MarshalToSizedBuffer

func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Entry) ProtoMessage

func (*Entry) ProtoMessage()

func (*Entry) Reset

func (m *Entry) Reset()

func (*Entry) Size

func (m *Entry) Size() (n int)

func (*Entry) String

func (this *Entry) String() string

func (*Entry) Unmarshal

func (m *Entry) Unmarshal(dAtA []byte) error

func (*Entry) XXX_DiscardUnknown

func (m *Entry) XXX_DiscardUnknown()

func (*Entry) XXX_Marshal

func (m *Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Entry) XXX_Merge

func (m *Entry) XXX_Merge(src proto.Message)

func (*Entry) XXX_Size

func (m *Entry) XXX_Size() int

func (*Entry) XXX_Unmarshal

func (m *Entry) XXX_Unmarshal(b []byte) error

type IndexClientFactoryFunc

type IndexClientFactoryFunc func(limits StoreLimits) (chunk.IndexClient, error)

IndexClientFactoryFunc defines signature of function which creates chunk.IndexClient for managing index in index store

type ReadBatch

type ReadBatch struct {
	Entries []Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries"`
	Key     string  `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
	// The time at which the key expires.
	Expiry int64 `protobuf:"varint,3,opt,name=expiry,proto3" json:"expiry,omitempty"`
	// The number of entries; used for cardinality limiting.
	// entries will be empty when this is set.
	Cardinality int32 `protobuf:"varint,4,opt,name=cardinality,proto3" json:"cardinality,omitempty"`
}

func (*ReadBatch) Descriptor

func (*ReadBatch) Descriptor() ([]byte, []int)

func (*ReadBatch) Equal

func (this *ReadBatch) Equal(that interface{}) bool

func (*ReadBatch) GetCardinality

func (m *ReadBatch) GetCardinality() int32

func (*ReadBatch) GetEntries

func (m *ReadBatch) GetEntries() []Entry

func (*ReadBatch) GetExpiry

func (m *ReadBatch) GetExpiry() int64

func (*ReadBatch) GetKey

func (m *ReadBatch) GetKey() string

func (*ReadBatch) GoString

func (this *ReadBatch) GoString() string

func (ReadBatch) Iterator

func (b ReadBatch) Iterator() chunk.ReadBatchIterator

Iterator implements chunk.ReadBatch.

func (*ReadBatch) Marshal

func (m *ReadBatch) Marshal() (dAtA []byte, err error)

func (*ReadBatch) MarshalTo

func (m *ReadBatch) MarshalTo(dAtA []byte) (int, error)

func (*ReadBatch) MarshalToSizedBuffer

func (m *ReadBatch) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ReadBatch) ProtoMessage

func (*ReadBatch) ProtoMessage()

func (*ReadBatch) Reset

func (m *ReadBatch) Reset()

func (*ReadBatch) Size

func (m *ReadBatch) Size() (n int)

func (*ReadBatch) String

func (this *ReadBatch) String() string

func (*ReadBatch) Unmarshal

func (m *ReadBatch) Unmarshal(dAtA []byte) error

func (*ReadBatch) XXX_DiscardUnknown

func (m *ReadBatch) XXX_DiscardUnknown()

func (*ReadBatch) XXX_Marshal

func (m *ReadBatch) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ReadBatch) XXX_Merge

func (m *ReadBatch) XXX_Merge(src proto.Message)

func (*ReadBatch) XXX_Size

func (m *ReadBatch) XXX_Size() int

func (*ReadBatch) XXX_Unmarshal

func (m *ReadBatch) XXX_Unmarshal(b []byte) error

type StoreLimits

type StoreLimits interface {
	downloads.Limits
	CardinalityLimit(userID string) int
	MaxChunksPerQueryFromStore(userID string) int
	MaxQueryLength(userID string) time.Duration
}

StoreLimits helps get Limits specific to Queries for Stores

type TableClientFactoryFunc

type TableClientFactoryFunc func() (chunk.TableClient, error)

TableClientFactoryFunc defines signature of function which creates chunk.TableClient for managing tables in index store

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL