encoding

Documentation

Index

• func CopyBlock(ctx context.Context, meta *backend.BlockMeta, src backend.Reader, ...) error
• func ValidateConfig(b *BlockConfig) error
• type Appender
  • func NewAppender(dataWriter common.DataWriter) Appender
  • func NewBufferedAppender(writer common.DataWriter, indexDownsample int, totalObjectsEstimate int) (Appender, error)
  • func NewRecordAppender(records []common.Record) Appender
• type BackendBlock
  • func NewBackendBlock(meta *backend.BlockMeta, r backend.Reader) (*BackendBlock, error)
  • func (b *BackendBlock) BlockMeta() *backend.BlockMeta
  • func (b *BackendBlock) Find(ctx context.Context, id common.ID) ([]byte, error)
  • func (b *BackendBlock) Iterator(chunkSizeBytes uint32) (Iterator, error)
  • func (b *BackendBlock) NewIndexReader() (common.IndexReader, error)
• type BlockConfig
• type Finder
  • func NewPagedFinder(index common.IndexReader, r common.DataReader, combiner common.ObjectCombiner, ...) Finder
• type Iterator
  • func NewDedupingIterator(iter Iterator, combiner common.ObjectCombiner, dataEncoding string) (Iterator, error)
  • func NewIterator(reader io.Reader, o common.ObjectReaderWriter) Iterator
  • func NewRecordIterator(r []common.Record, dataR common.DataReader, objectRW common.ObjectReaderWriter) Iterator
• type StreamingBlock
  • func NewStreamingBlock(cfg *BlockConfig, id uuid.UUID, tenantID string, metas []*backend.BlockMeta, ...) (*StreamingBlock, error)
  • func (c *StreamingBlock) AddObject(id common.ID, object []byte) error
  • func (c *StreamingBlock) BlockMeta() *backend.BlockMeta
  • func (c *StreamingBlock) Complete(ctx context.Context, tracker backend.AppendTracker, w backend.Writer) (int, error)
  • func (c *StreamingBlock) CurrentBufferLength() int
  • func (c *StreamingBlock) CurrentBufferedObjects() int
  • func (c *StreamingBlock) FlushBuffer(ctx context.Context, tracker backend.AppendTracker, w backend.Writer) (backend.AppendTracker, int, error)
  • func (c *StreamingBlock) Length() int
• type VersionedEncoding
  • func FromVersion(v string) (VersionedEncoding, error)
  • func LatestEncoding() VersionedEncoding

Functions

func CopyBlock

func CopyBlock(ctx context.Context, meta *backend.BlockMeta, src backend.Reader, dest backend.Writer) error

CopyBlock copies a block from one backend to another. It is done at a low level, all encoding/formatting is preserved.

func ValidateConfig

func ValidateConfig(b *BlockConfig) error

ValidateConfig returns true if the config is valid

Types

type Appender

type Appender interface {
	Append(common.ID, []byte) error
	Complete() error
	Records() []common.Record
	RecordsForID(common.ID) []common.Record
	Length() int
	DataLength() uint64
}

Appender is capable of tracking objects and ids that are added to it

func NewAppender

func NewAppender(dataWriter common.DataWriter) Appender

NewAppender returns an appender. This appender simply appends new objects

to the provided dataWriter.

func NewBufferedAppender

func NewBufferedAppender(writer common.DataWriter, indexDownsample int, totalObjectsEstimate int) (Appender, error)

NewBufferedAppender returns an bufferedAppender. This appender builds a writes to

the provided writer and also builds a downsampled records slice.

func NewRecordAppender

func NewRecordAppender(records []common.Record) Appender

NewRecordAppender returns an appender that stores records only.

type BackendBlock

type BackendBlock struct {
	// contains filtered or unexported fields
}

BackendBlock represents a block already in the backend.

func NewBackendBlock

func NewBackendBlock(meta *backend.BlockMeta, r backend.Reader) (*BackendBlock, error)

NewBackendBlock returns a BackendBlock for the given backend.BlockMeta

It is version aware.

func (*BackendBlock) BlockMeta

func (b *BackendBlock) BlockMeta() *backend.BlockMeta

func (*BackendBlock) Find

func (b *BackendBlock) Find(ctx context.Context, id common.ID) ([]byte, error)

Find searches a block for the ID and returns an object if found.

func (*BackendBlock) Iterator

func (b *BackendBlock) Iterator(chunkSizeBytes uint32) (Iterator, error)

Iterator returns an Iterator that iterates over the objects in the block from the backend

func (*BackendBlock) NewIndexReader

func (b *BackendBlock) NewIndexReader() (common.IndexReader, error)

type BlockConfig

type BlockConfig struct {
	IndexDownsampleBytes int              `yaml:"index_downsample_bytes"`
	IndexPageSizeBytes   int              `yaml:"index_page_size_bytes"`
	BloomFP              float64          `yaml:"bloom_filter_false_positive"`
	BloomShardSizeBytes  int              `yaml:"bloom_filter_shard_size_bytes"`
	Encoding             backend.Encoding `yaml:"encoding"`
}

BlockConfig holds configuration options for newly created blocks

type Finder

type Finder interface {
	Find(context.Context, common.ID) ([]byte, error)
}

Finder is capable of finding the requested ID

func NewPagedFinder

func NewPagedFinder(index common.IndexReader, r common.DataReader, combiner common.ObjectCombiner, objectRW common.ObjectReaderWriter, dataEncoding string) Finder

NewPagedFinder returns a paged. This finder is used for searching

a set of records and returning an object. If a set of consecutive records has
matching ids they will be combined using the ObjectCombiner.

type Iterator

type Iterator interface {
	Next(context.Context) (common.ID, []byte, error)
	Close()
}

Iterator is capable of iterating through a set of objects

func NewDedupingIterator

func NewDedupingIterator(iter Iterator, combiner common.ObjectCombiner, dataEncoding string) (Iterator, error)

NewDedupingIterator returns a dedupingIterator. This iterator is used to wrap another

iterator.  It will dedupe consecutive objects with the same id using the ObjectCombiner.

func NewIterator

func NewIterator(reader io.Reader, o common.ObjectReaderWriter) Iterator

NewIterator returns the most basic iterator. It iterates over raw objects.

func NewRecordIterator

func NewRecordIterator(r []common.Record, dataR common.DataReader, objectRW common.ObjectReaderWriter) Iterator

NewRecordIterator returns a recordIterator. This iterator is used for iterating through

a series of objects by reading them one at a time from Records.

type StreamingBlock

type StreamingBlock struct {
	// contains filtered or unexported fields
}

func NewStreamingBlock

func NewStreamingBlock(cfg *BlockConfig, id uuid.UUID, tenantID string, metas []*backend.BlockMeta, estimatedObjects int) (*StreamingBlock, error)

NewStreamingBlock creates a ... new streaming block. Objects are appended one at a time to the backend.

func (*StreamingBlock) AddObject

func (c *StreamingBlock) AddObject(id common.ID, object []byte) error

func (*StreamingBlock) BlockMeta

func (c *StreamingBlock) BlockMeta() *backend.BlockMeta

func (*StreamingBlock) Complete

func (c *StreamingBlock) Complete(ctx context.Context, tracker backend.AppendTracker, w backend.Writer) (int, error)

Complete finishes writes the compactor metadata and closes all buffers and appenders

func (*StreamingBlock) CurrentBufferLength

func (c *StreamingBlock) CurrentBufferLength() int

func (*StreamingBlock) CurrentBufferedObjects

func (c *StreamingBlock) CurrentBufferedObjects() int

func (*StreamingBlock) FlushBuffer

func (c *StreamingBlock) FlushBuffer(ctx context.Context, tracker backend.AppendTracker, w backend.Writer) (backend.AppendTracker, int, error)

FlushBuffer flushes any existing objects to the backend

func (*StreamingBlock) Length

func (c *StreamingBlock) Length() int

type VersionedEncoding

type VersionedEncoding interface {
	Version() string

	NewDataWriter(writer io.Writer, encoding backend.Encoding) (common.DataWriter, error)
	NewIndexWriter(pageSizeBytes int) common.IndexWriter

	NewDataReader(ra backend.ContextReader, encoding backend.Encoding) (common.DataReader, error)
	NewIndexReader(ra backend.ContextReader, pageSizeBytes int, totalPages int) (common.IndexReader, error)

	NewObjectReaderWriter() common.ObjectReaderWriter
	NewRecordReaderWriter() common.RecordReaderWriter
}

VersionedEncoding has a whole bunch of versioned functionality. This is

currently quite sloppy and could easily be tightened up to just a few methods
but it is what it is for now!

func FromVersion

func FromVersion(v string) (VersionedEncoding, error)

FromVersion returns a versioned encoding for the provided string

func LatestEncoding

func LatestEncoding() VersionedEncoding

LatestEncoding is used by Compactor and Complete block