tsm1

package
v0.0.0-...-d500d3c Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 17, 2019 License: MIT Imports: 48 Imported by: 0

Documentation

Overview

Package tsm1 provides a TSDB in the Time Structured Merge tree format.

Index

Constants

View Source 
const (
	// CompactionTempExtension is the extension used for temporary files created during compaction.
	CompactionTempExtension = "tmp"

	// TSMFileExtension is the extension used for TSM files.
	TSMFileExtension = "tsm"

	// TSSFileExtension is the extension used for TSM stats files.
	TSSFileExtension = "tss"
)
View Source 
const (
	DefaultCompactFullWriteColdDuration = time.Duration(4 * time.Hour)
	DefaultCompactThroughput            = 48 * 1024 * 1024
	DefaultCompactThroughputBurst       = 48 * 1024 * 1024
	DefaultCompactMaxConcurrent         = 0
)
View Source 
const (
	DefaultCacheMaxMemorySize             = 1024 * 1024 * 1024 // 1GB
	DefaultCacheSnapshotMemorySize        = 25 * 1024 * 1024   // 25MB
	DefaultCacheSnapshotWriteColdDuration = time.Duration(10 * time.Minute)
)
View Source 
const (
	DefaultWALEnabled    = true
	DefaultWALFsyncDelay = time.Duration(0)
)
View Source 
const (
	// BlockFloat64 designates a block encodes float64 values.
	BlockFloat64 = byte(0)

	// BlockInteger designates a block encodes int64 values.
	BlockInteger = byte(1)

	// BlockBoolean designates a block encodes boolean values.
	BlockBoolean = byte(2)

	// BlockString designates a block encodes string values.
	BlockString = byte(3)

	// BlockUnsigned designates a block encodes uint64 values.
	BlockUnsigned = byte(4)
)
View Source 
const (
	// The extension used to describe temporary snapshot files.
	TmpTSMFileExtension = "tmp"

	// The extension used to describe corrupt snapshot files.
	BadTSMFileExtension = "bad"
)
View Source 
const (
	// MeasurementMagicNumber is written as the first 4 bytes of a data file to
	// identify the file as a tsm1 stats file.
	MeasurementStatsMagicNumber string = "TSS1"

	// MeasurementStatsVersion indicates the version of the TSS1 file format.
	MeasurementStatsVersion byte = 1
)
View Source 
const (
	// DefaultSegmentSize of 10MB is the size at which segment files will be rolled over.
	DefaultSegmentSize = 10 * 1024 * 1024

	// WALFileExtension is the file extension we expect for wal segments.
	WALFileExtension = "wal"

	// WALFilePrefix is the prefix on all wal segment files.
	WALFilePrefix = "_"
)
View Source 
const (
	// MagicNumber is written as the first 4 bytes of a data file to
	// identify the file as a tsm1 formatted file
	MagicNumber uint32 = 0x16D116D1

	// Version indicates the version of the TSM file format.
	Version byte = 1
)
View Source 
const (
	DefaultMADVWillNeed = false
)
View Source 
const (

	// MaxPointsPerBlock is the maximum number of points in an encoded block in a TSM file
	MaxPointsPerBlock = 1000
)

Variables

View Source 
var (
	// ErrWALClosed is returned when attempting to write to a closed WAL file.
	ErrWALClosed = fmt.Errorf("WAL closed")

	// ErrWALCorrupt is returned when reading a corrupt WAL entry.
	ErrWALCorrupt = fmt.Errorf("corrupted WAL entry")
)
View Source 
var (
	//ErrNoValues is returned when TSMWriter.WriteIndex is called and there are no values to write.
	ErrNoValues = fmt.Errorf("no values written")

	// ErrTSMClosed is returned when performing an operation against a closed TSM file.
	ErrTSMClosed = fmt.Errorf("tsm file closed")

	// ErrMaxKeyLengthExceeded is returned when attempting to write a key that is too long.
	ErrMaxKeyLengthExceeded = fmt.Errorf("max key length exceeded")

	// ErrMaxBlocksExceeded is returned when attempting to write a block past the allowed number.
	ErrMaxBlocksExceeded = fmt.Errorf("max blocks exceeded")
)
View Source 
var DefaultMaxConcurrentOpens = runtime.GOMAXPROCS(0)
View Source 
var ErrFileInUse = fmt.Errorf("file still in use")

ErrFileInUse is returned when attempting to remove or close a TSM file that is still being used.

View Source 
var (
	// ErrSnapshotInProgress is returned if a snapshot is attempted while one is already running.
	ErrSnapshotInProgress = fmt.Errorf("snapshot in progress")
)
View Source 
var WithCompactionPlanner = func(planner CompactionPlanner) EngineOption {
	return func(e *Engine) {
		planner.SetFileStore(e.FileStore)
		e.CompactionPlan = planner
	}
}

WithCompactionPlanner sets the compaction planner for the engine.

View Source 
var WithMadviseWillNeed = func(willNeed bool) tsmReaderOption {
	return func(r *TSMReader) {
		r.madviseWillNeed = willNeed
	}
}

WithMadviseWillNeed is an option for specifying whether to provide a MADV_WILL need hint to the kernel.

View Source 
var WithTSMReaderLogger = func(logger *zap.Logger) tsmReaderOption {
	return func(r *TSMReader) {
		r.logger = logger
	}
}
View Source 
var WithTraceLogging = func(logging bool) EngineOption {
	return func(e *Engine) {
		e.FileStore.enableTraceLogging(logging)
	}
}

WithTraceLogging sets if trace logging is enabled for the engine.

View Source 
var WithWAL = func(wal Log) EngineOption {

	if pwal, _ := wal.(*WAL); pwal == nil {
		wal = NopWAL{}
	}

	return func(e *Engine) {
		e.WAL = wal
	}
}

WithWAL sets the WAL for the Engine

Functions

func BlockCount 

func BlockCount(block []byte) int

BlockCount returns the number of timestamps encoded in block.

func BlockType 

func BlockType(block []byte) (byte, error)

BlockType returns the type of value encoded in a block or an error if the block type is unknown.

func BlockTypeToInfluxQLDataType 

func BlockTypeToInfluxQLDataType(typ byte) influxql.DataType

func BooleanArrayDecodeAll 

func BooleanArrayDecodeAll(b []byte, dst []bool) ([]bool, error)

func BooleanArrayEncodeAll 

func BooleanArrayEncodeAll(src []bool, b []byte) ([]byte, error)

BooleanArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.

func CountTimestamps 

func CountTimestamps(b []byte) int

func DecodeBooleanArrayBlock 

func DecodeBooleanArrayBlock(block []byte, a *tsdb.BooleanArray) error

DecodeBooleanArrayBlock decodes the boolean block from the byte slice and writes the values to a.

func DecodeFloatArrayBlock 

func DecodeFloatArrayBlock(block []byte, a *tsdb.FloatArray) error

DecodeFloatArrayBlock decodes the float block from the byte slice and writes the values to a.

func DecodeIntegerArrayBlock 

func DecodeIntegerArrayBlock(block []byte, a *tsdb.IntegerArray) error

DecodeIntegerArrayBlock decodes the integer block from the byte slice and writes the values to a.

func DecodeStringArrayBlock 

func DecodeStringArrayBlock(block []byte, a *tsdb.StringArray) error

DecodeStringArrayBlock decodes the string block from the byte slice and writes the values to a.

func DecodeUnsignedArrayBlock 

func DecodeUnsignedArrayBlock(block []byte, a *tsdb.UnsignedArray) error

DecodeUnsignedArrayBlock decodes the unsigned integer block from the byte slice and writes the values to a.

func DefaultFormatFileName 

func DefaultFormatFileName(generation, sequence int) string

DefaultFormatFileName is the default implementation to format TSM filenames.

func DefaultParseFileName 

func DefaultParseFileName(name string) (int, int, error)

DefaultParseFileName is used to parse the filenames of TSM files.

func EncodeBooleanArrayBlock 

func EncodeBooleanArrayBlock(a *tsdb.BooleanArray, b []byte) ([]byte, error)

func EncodeFloatArrayBlock 

func EncodeFloatArrayBlock(a *tsdb.FloatArray, b []byte) ([]byte, error)

func EncodeIntegerArrayBlock 

func EncodeIntegerArrayBlock(a *tsdb.IntegerArray, b []byte) ([]byte, error)

func EncodeStringArrayBlock 

func EncodeStringArrayBlock(a *tsdb.StringArray, b []byte) ([]byte, error)

func EncodeUnsignedArrayBlock 

func EncodeUnsignedArrayBlock(a *tsdb.UnsignedArray, b []byte) ([]byte, error)

func ErrCacheMemorySizeLimitExceeded 

func ErrCacheMemorySizeLimitExceeded(n, limit uint64) error

ErrCacheMemorySizeLimitExceeded returns an error indicating an operation could not be completed due to exceeding the cache-max-memory-size setting.

func FloatArrayDecodeAll 

func FloatArrayDecodeAll(b []byte, buf []float64) ([]float64, error)

func FloatArrayEncodeAll 

func FloatArrayEncodeAll(src []float64, b []byte) ([]byte, error)

FloatArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.

Currently only the float compression scheme used in Facebook's Gorilla is supported, so this method implements a batch oriented version of that.

func IntegerArrayDecodeAll 

func IntegerArrayDecodeAll(b []byte, dst []int64) ([]int64, error)

func IntegerArrayEncodeAll 

func IntegerArrayEncodeAll(src []int64, b []byte) ([]byte, error)

IntegerArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.

IntegerArrayEncodeAll implements batch oriented versions of the three integer encoding types we support: uncompressed, simple8b and RLE.

Important: IntegerArrayEncodeAll modifies the contents of src by using it as scratch space for delta encoded values. It is NOT SAFE to use src after passing it into IntegerArrayEncodeAll.

func MetricsGroupFromContext 

func MetricsGroupFromContext(ctx context.Context) *metrics.Group

MetricsGroupFromContext returns the tsm1 metrics.Group associated with the context or nil if no group has been assigned.

func NewBooleanArrayFromValues 

func NewBooleanArrayFromValues(v BooleanValues) *tsdb.BooleanArray

func NewContextWithMetricsGroup 

func NewContextWithMetricsGroup(ctx context.Context) context.Context

NewContextWithMetricsGroup creates a new context with a tsm1 metrics.Group for tracking various metrics when accessing TSM data.

func NewFloatArrayFromValues 

func NewFloatArrayFromValues(v FloatValues) *tsdb.FloatArray

func NewIndirectIndex 

func NewIndirectIndex() *indirectIndex

NewIndirectIndex returns a new indirect index.

func NewIntegerArrayFromValues 

func NewIntegerArrayFromValues(v IntegerValues) *tsdb.IntegerArray

func NewStringArrayFromValues 

func NewStringArrayFromValues(v StringValues) *tsdb.StringArray

func NewUnsignedArrayFromValues 

func NewUnsignedArrayFromValues(v UnsignedValues) *tsdb.UnsignedArray

func PrometheusCollectors 

func PrometheusCollectors() []prometheus.Collector

PrometheusCollectors returns all prometheus metrics for the tsm1 package.

func SeriesAndFieldFromCompositeKey 

func SeriesAndFieldFromCompositeKey(key []byte) ([]byte, []byte)

SeriesAndFieldFromCompositeKey returns the series key and the field key extracted from the composite key.

func SeriesFieldKey 

func SeriesFieldKey(seriesKey, field string) string

SeriesFieldKey combine a series key and field name for a unique string to be hashed to a numeric ID.

func SeriesFieldKeyBytes 

func SeriesFieldKeyBytes(seriesKey, field string) []byte

func StatsFilename 

func StatsFilename(tsmPath string) string

StatsFilename returns the path to the stats file for a given TSM file path.

func StringArrayDecodeAll 

func StringArrayDecodeAll(b []byte, dst []string) ([]string, error)

func StringArrayEncodeAll 

func StringArrayEncodeAll(src []string, b []byte) ([]byte, error)

StringArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.

Currently only the string compression scheme used snappy.

func TimeArrayDecodeAll 

func TimeArrayDecodeAll(b []byte, dst []int64) ([]int64, error)

func TimeArrayEncodeAll 

func TimeArrayEncodeAll(src []int64, b []byte) ([]byte, error)

TimeArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capacity to b.

TimeArrayEncodeAll implements batch oriented versions of the three integer encoding types we support: uncompressed, simple8b and RLE.

Timestamp values to be encoded should be sorted before encoding. When encoded, the values are first delta-encoded. The first value is the starting timestamp, subsequent values are the difference from the prior value.

Important: TimeArrayEncodeAll modifies the contents of src by using it as scratch space for delta encoded values. It is NOT SAFE to use src after passing it into TimeArrayEncodeAll.

func UnsignedArrayDecodeAll 

func UnsignedArrayDecodeAll(b []byte, dst []uint64) ([]uint64, error)

func UnsignedArrayEncodeAll 

func UnsignedArrayEncodeAll(src []uint64, b []byte) ([]byte, error)

UnsignedArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.

UnsignedArrayEncodeAll implements batch oriented versions of the three integer encoding types we support: uncompressed, simple8b and RLE.

Important: IntegerArrayEncodeAll modifies the contents of src by using it as scratch space for delta encoded values. It is NOT SAFE to use src after passing it into IntegerArrayEncodeAll.

func ZigZagDecode 

func ZigZagDecode(v uint64) int64

ZigZagDecode converts a previously zigzag encoded uint64 back to a int64.

func ZigZagEncode 

func ZigZagEncode(x int64) uint64

ZigZagEncode converts a int64 to a uint64 by zig zagging negative and positive values across even and odd numbers. Eg. [0,-1,1,-2] becomes [0, 1, 2, 3].

Types

type BatchDeleter 

type BatchDeleter interface {
	DeleteRange(keys [][]byte, min, max int64) error
	Commit() error
	Rollback() error
}

type BatchDeleters 

type BatchDeleters []BatchDeleter

func (BatchDeleters) Commit 

func (a BatchDeleters) Commit() error

func (BatchDeleters) DeleteRange 

func (a BatchDeleters) DeleteRange(keys [][]byte, min, max int64) error

func (BatchDeleters) Rollback 

func (a BatchDeleters) Rollback() error

type BitReader 

type BitReader struct {
	// contains filtered or unexported fields
}

BitReader reads bits from an io.Reader.

func NewBitReader 

func NewBitReader(data []byte) *BitReader

NewBitReader returns a new instance of BitReader that reads from data.

func (*BitReader) CanReadBitFast 

func (r *BitReader) CanReadBitFast() bool

CanReadBitFast returns true if calling ReadBitFast() is allowed. Fast bit reads are allowed when at least 2 values are in the buffer. This is because it is not required to refilled the buffer and the caller can inline the calls.

func (*BitReader) ReadBit 

func (r *BitReader) ReadBit() (bool, error)

ReadBit returns the next bit from the underlying data.

func (*BitReader) ReadBitFast 

func (r *BitReader) ReadBitFast() bool

ReadBitFast is an optimized bit read. IMPORTANT: Only allowed if CanReadFastBit() is true!

func (*BitReader) ReadBits 

func (r *BitReader) ReadBits(nbits uint) (uint64, error)

ReadBits reads nbits from the underlying data into a uint64. nbits must be from 1 to 64, inclusive.

func (*BitReader) Reset 

func (r *BitReader) Reset(data []byte)

Reset sets the underlying reader on b and reinitializes.

type BlockIterator 

type BlockIterator struct {
	// contains filtered or unexported fields
}

BlockIterator allows iterating over each block in a TSM file in order. It provides raw access to the block bytes without decoding them.

func (*BlockIterator) Err 

func (b *BlockIterator) Err() error

Err returns any errors encounter during iteration.

func (*BlockIterator) Next 

func (b *BlockIterator) Next() bool

Next returns true if there are more blocks to iterate through.

func (*BlockIterator) PeekNext 

func (b *BlockIterator) PeekNext() []byte

PeekNext returns the next key to be iterated or an empty string.

func (*BlockIterator) Read 

func (b *BlockIterator) Read() (key []byte, minTime int64, maxTime int64, typ byte, checksum uint32, buf []byte, err error)

Read reads information about the next block to be iterated.

type BooleanDecoder 

type BooleanDecoder struct {
	// contains filtered or unexported fields
}

BooleanDecoder decodes a series of booleans from an in-memory buffer.

func (*BooleanDecoder) Error 

func (e *BooleanDecoder) Error() error

Error returns the error encountered during decoding, if one occurred.

func (*BooleanDecoder) Next 

func (e *BooleanDecoder) Next() bool

Next returns whether there are any bits remaining in the decoder. It returns false if there was an error decoding. The error is available on the Error method.

func (*BooleanDecoder) Read 

func (e *BooleanDecoder) Read() bool

Read returns the next bit from the decoder.

func (*BooleanDecoder) SetBytes 

func (e *BooleanDecoder) SetBytes(b []byte)

SetBytes initializes the decoder with a new set of bytes to read from. This must be called before calling any other methods.

type BooleanEncoder 

type BooleanEncoder struct {
	// contains filtered or unexported fields
}

BooleanEncoder encodes a series of booleans to an in-memory buffer.

func NewBooleanEncoder 

func NewBooleanEncoder(sz int) BooleanEncoder

NewBooleanEncoder returns a new instance of BooleanEncoder.

func (*BooleanEncoder) Bytes 

func (e *BooleanEncoder) Bytes() ([]byte, error)

Bytes returns a new byte slice containing the encoded booleans from previous calls to Write.

func (*BooleanEncoder) Flush 

func (e *BooleanEncoder) Flush()

Flush is no-op

func (*BooleanEncoder) Reset 

func (e *BooleanEncoder) Reset()

Reset sets the encoder to its initial state.

func (*BooleanEncoder) Write 

func (e *BooleanEncoder) Write(b bool)

Write encodes b to the underlying buffer.

type BooleanValue 

type BooleanValue struct {
	// contains filtered or unexported fields
}

BooleanValue represents a boolean value.

func DecodeBooleanBlock 

func DecodeBooleanBlock(block []byte, a *[]BooleanValue) ([]BooleanValue, error)

DecodeBooleanBlock decodes the boolean block from the byte slice and appends the boolean values to a.

func (BooleanValue) Size 

func (v BooleanValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (BooleanValue) String 

func (v BooleanValue) String() string

String returns the string representation of the value and its timestamp.

func (BooleanValue) UnixNano 

func (v BooleanValue) UnixNano() int64

UnixNano returns the timestamp of the value in nanoseconds since unix epoch.

func (BooleanValue) Value 

func (v BooleanValue) Value() interface{}

Value returns the underlying boolean value.

type BooleanValues 

type BooleanValues []BooleanValue

BooleanValues represents a slice of Boolean values.

func (BooleanValues) Deduplicate 

func (a BooleanValues) Deduplicate() BooleanValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (BooleanValues) Encode 

func (a BooleanValues) Encode(buf []byte) ([]byte, error)

func (BooleanValues) Exclude 

func (a BooleanValues) Exclude(min, max int64) BooleanValues

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (BooleanValues) FindRange 

func (a BooleanValues) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (BooleanValues) Include 

func (a BooleanValues) Include(min, max int64) BooleanValues

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (BooleanValues) Len 

func (a BooleanValues) Len() int

Sort methods

func (BooleanValues) Less 

func (a BooleanValues) Less(i, j int) bool

func (BooleanValues) MaxTime 

func (a BooleanValues) MaxTime() int64

func (BooleanValues) Merge 

func (a BooleanValues) Merge(b BooleanValues) BooleanValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (BooleanValues) MinTime 

func (a BooleanValues) MinTime() int64

func (BooleanValues) Size 

func (a BooleanValues) Size() int

func (BooleanValues) Swap 

func (a BooleanValues) Swap(i, j int)

type Cache 

type Cache struct {
	// contains filtered or unexported fields
}

Cache maintains an in-memory store of Values for a set of keys.

func NewCache 

func NewCache(maxSize uint64) *Cache

NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory. Only used for engine caches, never for snapshots.

func (*Cache) ApplyEntryFn 

func (c *Cache) ApplyEntryFn(f func(key []byte, entry *entry) error) error

ApplyEntryFn applies the function f to each entry in the Cache. ApplyEntryFn calls f on each entry in turn, within the same goroutine. It is safe for use by multiple goroutines.

func (*Cache) ClearSnapshot 

func (c *Cache) ClearSnapshot(success bool)

ClearSnapshot removes the snapshot cache from the list of flushing caches and adjusts the size.

func (*Cache) Count 

func (c *Cache) Count() int

func (*Cache) Deduplicate 

func (c *Cache) Deduplicate()

Deduplicate sorts the snapshot before returning it. The compactor and any queries coming in while it writes will need the values sorted.

func (*Cache) Delete 

func (c *Cache) Delete(keys [][]byte)

Delete removes all values for the given keys from the cache.

func (*Cache) DeleteRange 

func (c *Cache) DeleteRange(keys [][]byte, min, max int64)

DeleteRange removes the values for all keys containing points with timestamps between between min and max from the cache.

TODO(edd): Lock usage could possibly be optimised if necessary.

func (*Cache) Free 

func (c *Cache) Free()

Free releases the underlying store and memory held by the Cache.

func (*Cache) Keys 

func (c *Cache) Keys() [][]byte

Keys returns a sorted slice of all keys under management by the cache.

func (*Cache) LastWriteTime 

func (c *Cache) LastWriteTime() time.Time

func (*Cache) MaxSize 

func (c *Cache) MaxSize() uint64

MaxSize returns the maximum number of bytes the cache may consume.

func (*Cache) SetMaxSize 

func (c *Cache) SetMaxSize(size uint64)

SetMaxSize updates the memory limit of the cache.

func (*Cache) Size 

func (c *Cache) Size() uint64

Size returns the number of point-calcuated bytes the cache currently uses.

func (*Cache) Snapshot 

func (c *Cache) Snapshot() (*Cache, error)

Snapshot takes a snapshot of the current cache, adds it to the slice of caches that are being flushed, and resets the current cache with new values.

func (*Cache) Split 

func (c *Cache) Split(n int) []*Cache

func (*Cache) Type 

func (c *Cache) Type(key []byte) (models.FieldType, error)

Type returns the series type for a key.

func (*Cache) UpdateAge 

func (c *Cache) UpdateAge()

UpdateAge updates the age statistic based on the current time.

func (*Cache) Values 

func (c *Cache) Values(key []byte) Values

Values returns a copy of all values, deduped and sorted, for the given key.

func (*Cache) Write 

func (c *Cache) Write(key []byte, values []Value) error

Write writes the set of values for the key to the cache. This function is goroutine-safe. It returns an error if the cache will exceed its max size by adding the new values.

func (*Cache) WriteMulti 

func (c *Cache) WriteMulti(values map[string][]Value) error

WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe. It returns an error if the cache will exceeded its max size by adding the new values. The write attempts to write as many values as possible. If one key fails, the others can still succeed and an error will be returned.

type CacheConfig 

type CacheConfig struct {
	// MaxMemorySize is the maximum size a shard's cache can reach before it starts
	// rejecting writes.
	MaxMemorySize toml.Size `toml:"max-memory-size"`

	// SnapshotMemorySize is the size at which the engine will snapshot the cache and
	// write it to a TSM file, freeing up memory
	SnapshotMemorySize toml.Size `toml:"snapshot-memory-size"`

	// SnapshotWriteColdDuration is the length of time at which the engine will snapshot
	// the cache and write it to a new TSM file if the shard hasn't received writes or
	// deletes
	SnapshotWriteColdDuration toml.Duration `toml:"snapshot-write-cold-duration"`
}

CacheConfig holds all of the configuration for the in memory cache of values that are waiting to be snapshot.

type CacheLoader 

type CacheLoader struct {
	Logger *zap.Logger
	// contains filtered or unexported fields
}

CacheLoader processes a set of WAL segment files, and loads a cache with the data contained within those files. Processing of the supplied files take place in the order they exist in the files slice.

func NewCacheLoader 

func NewCacheLoader(files []string) *CacheLoader

NewCacheLoader returns a new instance of a CacheLoader.

func (*CacheLoader) Load 

func (cl *CacheLoader) Load(cache *Cache) error

Load returns a cache loaded with the data contained within the segment files. If, during reading of a segment file, corruption is encountered, that segment file is truncated up to and including the last valid byte, and processing continues with the next segment file.

func (*CacheLoader) WithLogger 

func (cl *CacheLoader) WithLogger(log *zap.Logger)

WithLogger sets the logger on the CacheLoader.

type CompactionConfig 

type CompactionConfig struct {
	// FullWriteColdDuration is the duration at which the engine will compact all TSM
	// files in a shard if it hasn't received a write or delete
	FullWriteColdDuration toml.Duration `toml:"full-write-cold-duration"`

	// Throughput is the rate limit in bytes per second that we will allow TSM compactions
	// to write to disk. Not that short bursts are allowed to happen at a possibly larger
	// value, set by CompactThroughputBurst. A value of 0 here will disable compaction rate
	// limiting
	Throughput toml.Size `toml:"throughput"`

	// ThroughputBurst is the rate limit in bytes per second that we will allow TSM compactions
	// to write to disk. If this is not set, the burst value will be set to equal the normal
	// throughput
	ThroughputBurst toml.Size `toml:"throughput-burst"`

	// MaxConcurrent is the maximum number of concurrent full and level compactions that can
	// run at one time.  A value of 0 results in 50% of runtime.GOMAXPROCS(0) used at runtime.
	MaxConcurrent int `toml:"max-concurrent"`
}

CompactionConfing holds all of the configuration for compactions. Eventually we want to move this out of tsm1 so that it can be scheduled more intelligently.

type CompactionGroup 

type CompactionGroup []string

CompactionGroup represents a list of files eligible to be compacted together.

type CompactionPlanner 

type CompactionPlanner interface {
	Plan(lastWrite time.Time) []CompactionGroup
	PlanLevel(level int) []CompactionGroup
	PlanOptimize() []CompactionGroup
	Release(group []CompactionGroup)
	FullyCompacted() bool

	// ForceFull causes the planner to return a full compaction plan the next
	// time Plan() is called if there are files that could be compacted.
	ForceFull()

	SetFileStore(fs *FileStore)
}

CompactionPlanner determines what TSM files and WAL segments to include in a given compaction run.

type Compactor 

type Compactor struct {
	Dir  string
	Size int

	FileStore interface {
		NextGeneration() int
		TSMReader(path string) *TSMReader
	}

	// RateLimit is the limit for disk writes for all concurrent compactions.
	RateLimit limiter.Rate
	// contains filtered or unexported fields
}

Compactor merges multiple TSM files into new files or writes a Cache into 1 or more TSM files.

func NewCompactor 

func NewCompactor() *Compactor

NewCompactor returns a new instance of Compactor.

func (*Compactor) Close 

func (c *Compactor) Close()

Close disables the Compactor.

func (*Compactor) CompactFast 

func (c *Compactor) CompactFast(tsmFiles []string) ([]string, error)

CompactFast writes multiple smaller TSM files into 1 or more larger files.

func (*Compactor) CompactFull 

func (c *Compactor) CompactFull(tsmFiles []string) ([]string, error)

CompactFull writes multiple smaller TSM files into 1 or more larger files.

func (*Compactor) DisableCompactions 

func (c *Compactor) DisableCompactions()

DisableSnapshots disables the compactor from performing compactions.

func (*Compactor) DisableSnapshots 

func (c *Compactor) DisableSnapshots()

DisableSnapshots disables the compactor from performing snapshots.

func (*Compactor) EnableCompactions 

func (c *Compactor) EnableCompactions()

EnableCompactions allows the compactor to perform compactions.

func (*Compactor) EnableSnapshots 

func (c *Compactor) EnableSnapshots()

EnableSnapshots allows the compactor to perform snapshots.

func (*Compactor) Open 

func (c *Compactor) Open()

Open initializes the Compactor.

func (*Compactor) WithFormatFileNameFunc 

func (c *Compactor) WithFormatFileNameFunc(formatFileNameFunc FormatFileNameFunc)

func (*Compactor) WithParseFileNameFunc 

func (c *Compactor) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)

func (*Compactor) WriteSnapshot 

func (c *Compactor) WriteSnapshot(cache *Cache) ([]string, error)

WriteSnapshot writes a Cache snapshot to one or more new TSM files.

type Config 

type Config struct {
	// MacConcurrentOpens controls the concurrency of opening tsm files during
	// engine opening.
	MaxConcurrentOpens int `toml:"max-concurrent-opens"`

	// MADVWillNeed controls whether we hint to the kernel that we intend to page
	// in mmap'd sections of TSM files. This setting defaults to off, as it has
	// been found to be problematic in some cases. It may help users who have
	// slow disks.
	MADVWillNeed bool `toml:"use-madv-willneed"`

	Compaction CompactionConfig `toml:"compaction"`
	Cache      CacheConfig      `toml:"cache"`
}

Config contains all of the configuration necessary to run a tsm1 engine.

func NewConfig 

func NewConfig() Config

NewConfig constructs a Config with the default values.

type DefaultPlanner 

type DefaultPlanner struct {
	FileStore fileStore
	// contains filtered or unexported fields
}

DefaultPlanner implements CompactionPlanner using a strategy to roll up multiple generations of TSM files into larger files in stages. It attempts to minimize the number of TSM files on disk while rolling up a bounder number of files.

func NewDefaultPlanner 

func NewDefaultPlanner(fs fileStore, writeColdDuration time.Duration) *DefaultPlanner

func (*DefaultPlanner) ForceFull 

func (c *DefaultPlanner) ForceFull()

ForceFull causes the planner to return a full compaction plan the next time a plan is requested. When ForceFull is called, level and optimize plans will not return plans until a full plan is requested and released.

func (*DefaultPlanner) FullyCompacted 

func (c *DefaultPlanner) FullyCompacted() bool

FullyCompacted returns true if the shard is fully compacted.

func (*DefaultPlanner) ParseFileName 

func (c *DefaultPlanner) ParseFileName(path string) (int, int, error)

func (*DefaultPlanner) Plan 

func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup

Plan returns a set of TSM files to rewrite for level 4 or higher. The planning returns multiple groups if possible to allow compactions to run concurrently.

func (*DefaultPlanner) PlanLevel 

func (c *DefaultPlanner) PlanLevel(level int) []CompactionGroup

PlanLevel returns a set of TSM files to rewrite for a specific level.

func (*DefaultPlanner) PlanOptimize 

func (c *DefaultPlanner) PlanOptimize() []CompactionGroup

PlanOptimize returns all TSM files if they are in different generations in order to optimize the index across TSM files. Each returned compaction group can be compacted concurrently.

func (*DefaultPlanner) Release 

func (c *DefaultPlanner) Release(groups []CompactionGroup)

Release removes the files reference in each compaction group allowing new plans to be able to use them.

func (*DefaultPlanner) SetFileStore 

func (c *DefaultPlanner) SetFileStore(fs *FileStore)

type DeleteRangeWALEntry 

type DeleteRangeWALEntry struct {
	Keys     [][]byte
	Min, Max int64
	// contains filtered or unexported fields
}

DeleteRangeWALEntry represents the deletion of multiple series.

func (*DeleteRangeWALEntry) Encode 

func (w *DeleteRangeWALEntry) Encode(b []byte) ([]byte, error)

Encode converts the DeleteRangeWALEntry into a byte slice, appending to b.

func (*DeleteRangeWALEntry) MarshalBinary 

func (w *DeleteRangeWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*DeleteRangeWALEntry) MarshalSize 

func (w *DeleteRangeWALEntry) MarshalSize() int

func (*DeleteRangeWALEntry) Type 

func (w *DeleteRangeWALEntry) Type() WalEntryType

Type returns DeleteRangeWALEntryType.

func (*DeleteRangeWALEntry) UnmarshalBinary 

func (w *DeleteRangeWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

type DeleteWALEntry 

type DeleteWALEntry struct {
	Keys [][]byte
	// contains filtered or unexported fields
}

DeleteWALEntry represents the deletion of multiple series.

func (*DeleteWALEntry) Encode 

func (w *DeleteWALEntry) Encode(dst []byte) ([]byte, error)

Encode converts the DeleteWALEntry into a byte slice, appending to dst.

func (*DeleteWALEntry) MarshalBinary 

func (w *DeleteWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*DeleteWALEntry) MarshalSize 

func (w *DeleteWALEntry) MarshalSize() int

func (*DeleteWALEntry) Type 

func (w *DeleteWALEntry) Type() WalEntryType

Type returns DeleteWALEntryType.

func (*DeleteWALEntry) UnmarshalBinary 

func (w *DeleteWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

type EmptyValue 

type EmptyValue struct{}

EmptyValue is used when there is no appropriate other value.

func (EmptyValue) Size 

func (e EmptyValue) Size() int

Size returns 0.

func (EmptyValue) String 

func (e EmptyValue) String() string

String returns the empty string.

func (EmptyValue) UnixNano 

func (e EmptyValue) UnixNano() int64

UnixNano returns tsdb.EOF.

func (EmptyValue) Value 

func (e EmptyValue) Value() interface{}

Value returns nil.

type Engine 

type Engine struct {
	WAL            Log
	Cache          *Cache
	Compactor      *Compactor
	CompactionPlan CompactionPlanner
	FileStore      *FileStore

	MaxPointsPerBlock int

	// CacheFlushMemorySizeThreshold specifies the minimum size threshold for
	// the cache when the engine should write a snapshot to a TSM file
	CacheFlushMemorySizeThreshold uint64

	// CacheFlushWriteColdDuration specifies the length of time after which if
	// no writes have been committed to the WAL, the engine will write
	// a snapshot of the cache to a TSM file
	CacheFlushWriteColdDuration time.Duration
	// contains filtered or unexported fields
}

Engine represents a storage engine with compressed blocks.

func NewEngine 

func NewEngine(path string, idx *tsi1.Index, config Config, options ...EngineOption) *Engine

NewEngine returns a new instance of Engine.

func (*Engine) Close 

func (e *Engine) Close() error

Close closes the engine. Subsequent calls to Close are a nop.

func (*Engine) CreateCursorIterator 

func (e *Engine) CreateCursorIterator(ctx context.Context) (tsdb.CursorIterator, error)

func (*Engine) CreateSeriesListIfNotExists 

func (e *Engine) CreateSeriesListIfNotExists(collection *tsdb.SeriesCollection) error

func (*Engine) DeleteMeasurement 

func (e *Engine) DeleteMeasurement(name []byte) error

DeleteMeasurement deletes a measurement and all related series.

func (*Engine) DeletePrefix 

func (e *Engine) DeletePrefix(prefix []byte, min, max int64) error

func (*Engine) DeleteSeriesRange 

func (e *Engine) DeleteSeriesRange(itr tsdb.SeriesIterator, min, max int64) error

DeleteSeriesRange removes the values between min and max (inclusive) from all series

func (*Engine) DeleteSeriesRangeWithPredicate 

func (e *Engine) DeleteSeriesRangeWithPredicate(itr tsdb.SeriesIterator, predicate func(name []byte, tags models.Tags) (int64, int64, bool)) error

DeleteSeriesRangeWithPredicate removes the values between min and max (inclusive) from all series for which predicate() returns true. If predicate() is nil, then all values in range are removed.

func (*Engine) DiskSize 

func (e *Engine) DiskSize() int64

DiskSize returns the total size in bytes of all TSM and WAL segments on disk.

func (*Engine) ForEachMeasurementName 

func (e *Engine) ForEachMeasurementName(fn func(name []byte) error) error

ForEachMeasurementName iterates over each measurement name in the engine.

func (*Engine) Free 

func (e *Engine) Free() error

Free releases any resources held by the engine to free up memory or CPU.

func (*Engine) HasTagKey 

func (e *Engine) HasTagKey(name, key []byte) (bool, error)

func (*Engine) IsIdle 

func (e *Engine) IsIdle() bool

IsIdle returns true if the cache is empty, there are no running compactions and the shard is fully compacted.

func (*Engine) IteratorCost 

func (e *Engine) IteratorCost(measurement string, opt query.IteratorOptions) (query.IteratorCost, error)

IteratorCost produces the cost of an iterator.

func (*Engine) KeyCursor 

func (e *Engine) KeyCursor(ctx context.Context, key []byte, t int64, ascending bool) *KeyCursor

KeyCursor returns a KeyCursor for the given key starting at time t.

func (*Engine) LastModified 

func (e *Engine) LastModified() time.Time

LastModified returns the time when this shard was last modified.

func (*Engine) MeasurementExists 

func (e *Engine) MeasurementExists(name []byte) (bool, error)

func (*Engine) MeasurementNamesByRegex 

func (e *Engine) MeasurementNamesByRegex(re *regexp.Regexp) ([][]byte, error)

func (*Engine) MeasurementStats 

func (e *Engine) MeasurementStats() (MeasurementStats, error)

MeasurementStats returns the current measurement stats for the engine.

func (*Engine) MeasurementTagKeysByExpr 

func (e *Engine) MeasurementTagKeysByExpr(name []byte, expr influxql.Expr) (map[string]struct{}, error)

func (*Engine) Open 

func (e *Engine) Open() error

Open opens and initializes the engine.

func (*Engine) Path 

func (e *Engine) Path() string

Path returns the path the engine was opened with.

func (*Engine) ScheduleFullCompaction 

func (e *Engine) ScheduleFullCompaction() error

ScheduleFullCompaction will force the engine to fully compact all data stored. This will cancel and running compactions and snapshot any data in the cache to TSM files. This is an expensive operation.

func (*Engine) SeriesN 

func (e *Engine) SeriesN() int64

SeriesN returns the unique number of series in the index.

func (*Engine) SetCompactionsEnabled 

func (e *Engine) SetCompactionsEnabled(enabled bool)

SetCompactionsEnabled enables compactions on the engine. When disabled all running compactions are aborted and new compactions stop running.

func (*Engine) SetDefaultMetricLabels 

func (e *Engine) SetDefaultMetricLabels(labels prometheus.Labels)

SetDefaultMetricLabels sets the default labels for metrics on the engine. It must be called before the Engine is opened.

func (*Engine) SetEnabled 

func (e *Engine) SetEnabled(enabled bool)

SetEnabled sets whether the engine is enabled.

func (*Engine) SetFieldName 

func (e *Engine) SetFieldName(measurement []byte, name string)

func (*Engine) ShouldCompactCache 

func (e *Engine) ShouldCompactCache(t time.Time) bool

ShouldCompactCache returns true if the Cache is over its flush threshold or if the passed in lastWriteTime is older than the write cold threshold.

func (*Engine) TagKeyCardinality 

func (e *Engine) TagKeyCardinality(name, key []byte) int

func (*Engine) WithCompactionPlanner 

func (e *Engine) WithCompactionPlanner(planner CompactionPlanner)

func (*Engine) WithFileStoreObserver 

func (e *Engine) WithFileStoreObserver(obs FileStoreObserver)

func (*Engine) WithFormatFileNameFunc 

func (e *Engine) WithFormatFileNameFunc(formatFileNameFunc FormatFileNameFunc)

func (*Engine) WithLogger 

func (e *Engine) WithLogger(log *zap.Logger)

WithLogger sets the logger for the engine.

func (*Engine) WithParseFileNameFunc 

func (e *Engine) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)

func (*Engine) WritePoints 

func (e *Engine) WritePoints(points []models.Point) error

WritePoints writes metadata and point data into the engine. It returns an error if new points are added to an existing key.

func (*Engine) WriteSnapshot 

func (e *Engine) WriteSnapshot() error

WriteSnapshot will snapshot the cache and write a new TSM file with its contents, releasing the snapshot when done.

func (*Engine) WriteTo 

func (e *Engine) WriteTo(w io.Writer) (n int64, err error)

WriteTo is not implemented.

type EngineOption 

type EngineOption func(i *Engine)

An EngineOption is a functional option for changing the configuration of an Engine.

type FileStat 

type FileStat struct {
	Path             string
	HasTombstone     bool
	Size             uint32
	LastModified     int64
	MinTime, MaxTime int64
	MinKey, MaxKey   []byte
}

FileStat holds information about a TSM file on disk.

func (FileStat) MaybeContainsKey 

func (f FileStat) MaybeContainsKey(key []byte) bool

ContainsKey returns true if the min and max keys of the file overlap the arguments min and max.

func (FileStat) OverlapsKeyRange 

func (f FileStat) OverlapsKeyRange(min, max []byte) bool

OverlapsKeyRange returns true if the min and max keys of the file overlap the arguments min and max.

func (FileStat) OverlapsTimeRange 

func (f FileStat) OverlapsTimeRange(min, max int64) bool

OverlapsTimeRange returns true if the time range of the file intersect min and max.

type FileStore 

type FileStore struct {
	// contains filtered or unexported fields
}

FileStore is an abstraction around multiple TSM files.

func NewFileStore 

func NewFileStore(dir string) *FileStore

NewFileStore returns a new instance of FileStore based on the given directory.

func (*FileStore) Apply 

func (f *FileStore) Apply(fn func(r TSMFile) error) error

func (*FileStore) BlockCount 

func (f *FileStore) BlockCount(path string, idx int) int

BlockCount returns number of values stored in the block at location idx in the file at path. If path does not match any file in the store, 0 is returned. If idx is out of range for the number of blocks in the file, 0 is returned.

func (*FileStore) Close 

func (f *FileStore) Close() error

Close closes the file store.

func (*FileStore) Cost 

func (f *FileStore) Cost(key []byte, min, max int64) query.IteratorCost

func (*FileStore) Count 

func (f *FileStore) Count() int

Count returns the number of TSM files currently loaded.

func (*FileStore) CreateSnapshot 

func (f *FileStore) CreateSnapshot() (string, error)

CreateSnapshot creates hardlinks for all tsm and tombstone files in the path provided.

func (*FileStore) CurrentGeneration 

func (f *FileStore) CurrentGeneration() int

CurrentGeneration returns the current generation of the TSM files.

func (*FileStore) Delete 

func (f *FileStore) Delete(keys [][]byte) error

Delete removes the keys from the set of keys available in this file.

func (*FileStore) DeleteRange 

func (f *FileStore) DeleteRange(keys [][]byte, min, max int64) error

DeleteRange removes the values for keys between timestamps min and max. This should only be used with smaller batches of series keys.

func (*FileStore) DiskSizeBytes 

func (f *FileStore) DiskSizeBytes() int64

DiskSizeBytes returns the total number of bytes consumed by the files in the FileStore.

func (*FileStore) Files 

func (f *FileStore) Files() []TSMFile

Files returns the slice of TSM files currently loaded. This is only used for tests, and the files aren't guaranteed to stay valid in the presense of compactions.

func (*FileStore) Free 

func (f *FileStore) Free() error

Free releases any resources held by the FileStore. The resources will be re-acquired if necessary if they are needed after freeing them.

func (*FileStore) KeyCursor 

func (f *FileStore) KeyCursor(ctx context.Context, key []byte, t int64, ascending bool) *KeyCursor

KeyCursor returns a KeyCursor for key and t across the files in the FileStore.

func (*FileStore) Keys 

func (f *FileStore) Keys() map[string]byte

Keys returns all keys and types for all files in the file store.

func (*FileStore) LastModified 

func (f *FileStore) LastModified() time.Time

LastModified returns the last time the file store was updated with new TSM files or a delete.

func (*FileStore) MeasurementStats 

func (f *FileStore) MeasurementStats() (MeasurementStats, error)

MeasurementStats returns the sum of all measurement stats within the store.

func (*FileStore) NextGeneration 

func (f *FileStore) NextGeneration() int

NextGeneration increments the max file ID and returns the new value.

func (*FileStore) Open 

func (f *FileStore) Open() error

Open loads all the TSM files in the configured directory.

func (*FileStore) ParseFileName 

func (f *FileStore) ParseFileName(path string) (int, int, error)

func (*FileStore) Read 

func (f *FileStore) Read(key []byte, t int64) ([]Value, error)

Read returns the slice of values for the given key and the given timestamp, if any file matches those constraints.

func (*FileStore) Replace 

func (f *FileStore) Replace(oldFiles, newFiles []string) error

Replace replaces oldFiles with newFiles.

func (*FileStore) ReplaceWithCallback 

func (f *FileStore) ReplaceWithCallback(oldFiles, newFiles []string, updatedFn func(r []TSMFile)) error

ReplaceWithCallback replaces oldFiles with newFiles and calls updatedFn with the files to be added the FileStore.

func (*FileStore) Stats 

func (f *FileStore) Stats() []FileStat

Stats returns the stats of the underlying files, preferring the cached version if it is still valid.

func (*FileStore) TSMReader 

func (f *FileStore) TSMReader(path string) *TSMReader

Reader returns a TSMReader for path if one is currently managed by the FileStore. Otherwise it returns nil. If it returns a file, you must call Unref on it when you are done, and never use it after that.

func (*FileStore) Type 

func (f *FileStore) Type(key []byte) (byte, error)

Type returns the type of values store at the block for key.

func (*FileStore) WalkKeys 

func (f *FileStore) WalkKeys(seek []byte, fn func(key []byte, typ byte) error) error

WalkKeys calls fn for every key in every TSM file known to the FileStore. If the key exists in multiple files, it will be invoked for each file.

func (*FileStore) WithLogger 

func (f *FileStore) WithLogger(log *zap.Logger)

WithLogger sets the logger on the file store.

func (*FileStore) WithObserver 

func (f *FileStore) WithObserver(obs FileStoreObserver)

WithObserver sets the observer for the file store.

func (*FileStore) WithParseFileNameFunc 

func (f *FileStore) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)

type FileStoreObserver 

type FileStoreObserver interface {
	// FileFinishing is called before a file is renamed to it's final name.
	FileFinishing(path string) error

	// FileUnlinking is called before a file is unlinked.
	FileUnlinking(path string) error
}

FileStoreObserver is passed notifications before the file store adds or deletes files. In this way, it can be sure to observe every file that is added or removed even in the presence of process death.

type FileStoreStatistics 

type FileStoreStatistics struct {
	SDiskBytes int64
	SFileCount int64
}

FileStoreStatistics keeps statistics about the file store.

type FloatDecoder 

type FloatDecoder struct {
	// contains filtered or unexported fields
}

FloatDecoder decodes a byte slice into multiple float64 values.

func (*FloatDecoder) Error 

func (it *FloatDecoder) Error() error

Error returns the current decoding error.

func (*FloatDecoder) Next 

func (it *FloatDecoder) Next() bool

Next returns true if there are remaining values to read.

func (*FloatDecoder) SetBytes 

func (it *FloatDecoder) SetBytes(b []byte) error

SetBytes initializes the decoder with b. Must call before calling Next().

func (*FloatDecoder) Values 

func (it *FloatDecoder) Values() float64

Values returns the current float64 value.

type FloatEncoder 

type FloatEncoder struct {
	// contains filtered or unexported fields
}

FloatEncoder encodes multiple float64s into a byte slice.

func NewFloatEncoder 

func NewFloatEncoder() *FloatEncoder

NewFloatEncoder returns a new FloatEncoder.

func (*FloatEncoder) Bytes 

func (s *FloatEncoder) Bytes() ([]byte, error)

Bytes returns a copy of the underlying byte buffer used in the encoder.

func (*FloatEncoder) Flush 

func (s *FloatEncoder) Flush()

Flush indicates there are no more values to encode.

func (*FloatEncoder) Reset 

func (s *FloatEncoder) Reset()

Reset sets the encoder back to its initial state.

func (*FloatEncoder) Write 

func (s *FloatEncoder) Write(v float64)

Write encodes v to the underlying buffer.

type FloatValue 

type FloatValue struct {
	// contains filtered or unexported fields
}

FloatValue represents a float64 value.

func DecodeFloatBlock 

func DecodeFloatBlock(block []byte, a *[]FloatValue) ([]FloatValue, error)

DecodeFloatBlock decodes the float block from the byte slice and appends the float values to a.

func (FloatValue) Size 

func (v FloatValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (FloatValue) String 

func (v FloatValue) String() string

String returns the string representation of the value and its timestamp.

func (FloatValue) UnixNano 

func (v FloatValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (FloatValue) Value 

func (v FloatValue) Value() interface{}

Value returns the underlying float64 value.

type FloatValues 

type FloatValues []FloatValue

FloatValues represents a slice of Float values.

func (FloatValues) Deduplicate 

func (a FloatValues) Deduplicate() FloatValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (FloatValues) Encode 

func (a FloatValues) Encode(buf []byte) ([]byte, error)

func (FloatValues) Exclude 

func (a FloatValues) Exclude(min, max int64) FloatValues

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (FloatValues) FindRange 

func (a FloatValues) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (FloatValues) Include 

func (a FloatValues) Include(min, max int64) FloatValues

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (FloatValues) Len 

func (a FloatValues) Len() int

Sort methods

func (FloatValues) Less 

func (a FloatValues) Less(i, j int) bool

func (FloatValues) MaxTime 

func (a FloatValues) MaxTime() int64

func (FloatValues) Merge 

func (a FloatValues) Merge(b FloatValues) FloatValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (FloatValues) MinTime 

func (a FloatValues) MinTime() int64

func (FloatValues) Size 

func (a FloatValues) Size() int

func (FloatValues) Swap 

func (a FloatValues) Swap(i, j int)

type FormatFileNameFunc 

type FormatFileNameFunc func(generation, sequence int) string

FormatFileNameFunc is executed when generating a new TSM filename. Source filenames are provided via src.

type IndexEntry 

type IndexEntry struct {
	// The min and max time of all points stored in the block.
	MinTime, MaxTime int64

	// The absolute position in the file where this block is located.
	Offset int64

	// The size in bytes of the block in the file.
	Size uint32
}

IndexEntry is the index information for a given block in a TSM file.

func (*IndexEntry) AppendTo 

func (e *IndexEntry) AppendTo(b []byte) []byte

AppendTo writes a binary-encoded version of IndexEntry to b, allocating and returning a new slice, if necessary.

func (*IndexEntry) Contains 

func (e *IndexEntry) Contains(t int64) bool

Contains returns true if this IndexEntry may contain values for the given time. The min and max times are inclusive.

func (*IndexEntry) OverlapsTimeRange 

func (e *IndexEntry) OverlapsTimeRange(min, max int64) bool

OverlapsTimeRange returns true if the given time ranges are completely within the entry's time bounds.

func (*IndexEntry) String 

func (e *IndexEntry) String() string

String returns a string representation of the entry.

func (*IndexEntry) UnmarshalBinary 

func (e *IndexEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary decodes an IndexEntry from a byte slice.

type IndexWriter 

type IndexWriter interface {
	// Add records a new block entry for a key in the index.
	Add(key []byte, blockType byte, minTime, maxTime int64, offset int64, size uint32)

	// Entries returns all index entries for a key.
	Entries(key []byte) []IndexEntry

	// KeyCount returns the count of unique keys in the index.
	KeyCount() int

	// Size returns the size of a the current index in bytes.
	Size() uint32

	// MarshalBinary returns a byte slice encoded version of the index.
	MarshalBinary() ([]byte, error)

	// WriteTo writes the index contents to a writer.
	WriteTo(w io.Writer) (int64, error)

	Close() error

	Remove() error
}

IndexWriter writes a TSMIndex.

func NewDiskIndexWriter 

func NewDiskIndexWriter(f *os.File) IndexWriter

NewIndexWriter returns a new IndexWriter.

func NewIndexWriter 

func NewIndexWriter() IndexWriter

NewIndexWriter returns a new IndexWriter.

type IntegerDecoder 

type IntegerDecoder struct {
	// contains filtered or unexported fields
}

IntegerDecoder decodes a byte slice into int64s.

func (*IntegerDecoder) Error 

func (d *IntegerDecoder) Error() error

Error returns the last error encountered by the decoder.

func (*IntegerDecoder) Next 

func (d *IntegerDecoder) Next() bool

Next returns true if there are any values remaining to be decoded.

func (*IntegerDecoder) Read 

func (d *IntegerDecoder) Read() int64

Read returns the next value from the decoder.

func (*IntegerDecoder) SetBytes 

func (d *IntegerDecoder) SetBytes(b []byte)

SetBytes sets the underlying byte slice of the decoder.

type IntegerEncoder 

type IntegerEncoder struct {
	// contains filtered or unexported fields
}

IntegerEncoder encodes int64s into byte slices.

func NewIntegerEncoder 

func NewIntegerEncoder(sz int) IntegerEncoder

NewIntegerEncoder returns a new integer encoder with an initial buffer of values sized at sz.

func (*IntegerEncoder) Bytes 

func (e *IntegerEncoder) Bytes() ([]byte, error)

Bytes returns a copy of the underlying buffer.

func (*IntegerEncoder) Flush 

func (e *IntegerEncoder) Flush()

Flush is no-op

func (*IntegerEncoder) Reset 

func (e *IntegerEncoder) Reset()

Reset sets the encoder back to its initial state.

func (*IntegerEncoder) Write 

func (e *IntegerEncoder) Write(v int64)

Write encodes v to the underlying buffers.

type IntegerValue 

type IntegerValue struct {
	// contains filtered or unexported fields
}

IntegerValue represents an int64 value.

func DecodeIntegerBlock 

func DecodeIntegerBlock(block []byte, a *[]IntegerValue) ([]IntegerValue, error)

DecodeIntegerBlock decodes the integer block from the byte slice and appends the integer values to a.

func (IntegerValue) Size 

func (v IntegerValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (IntegerValue) String 

func (v IntegerValue) String() string

String returns the string representation of the value and its timestamp.

func (IntegerValue) UnixNano 

func (v IntegerValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (IntegerValue) Value 

func (v IntegerValue) Value() interface{}

Value returns the underlying int64 value.

type IntegerValues 

type IntegerValues []IntegerValue

IntegerValues represents a slice of Integer values.

func (IntegerValues) Deduplicate 

func (a IntegerValues) Deduplicate() IntegerValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (IntegerValues) Encode 

func (a IntegerValues) Encode(buf []byte) ([]byte, error)

func (IntegerValues) Exclude 

func (a IntegerValues) Exclude(min, max int64) IntegerValues

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (IntegerValues) FindRange 

func (a IntegerValues) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (IntegerValues) Include 

func (a IntegerValues) Include(min, max int64) IntegerValues

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (IntegerValues) Len 

func (a IntegerValues) Len() int

Sort methods

func (IntegerValues) Less 

func (a IntegerValues) Less(i, j int) bool

func (IntegerValues) MaxTime 

func (a IntegerValues) MaxTime() int64

func (IntegerValues) Merge 

func (a IntegerValues) Merge(b IntegerValues) IntegerValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (IntegerValues) MinTime 

func (a IntegerValues) MinTime() int64

func (IntegerValues) Size 

func (a IntegerValues) Size() int

func (IntegerValues) Swap 

func (a IntegerValues) Swap(i, j int)

type KeyCursor 

type KeyCursor struct {
	// contains filtered or unexported fields
}

KeyCursor allows iteration through keys in a set of files within a FileStore.

func (*KeyCursor) Close 

func (c *KeyCursor) Close()

Close removes all references on the cursor.

func (*KeyCursor) Next 

func (c *KeyCursor) Next()

Next moves the cursor to the next position. Data should be read by the ReadBlock functions.

func (*KeyCursor) ReadBooleanArrayBlock 

func (c *KeyCursor) ReadBooleanArrayBlock(values *tsdb.BooleanArray) (*tsdb.BooleanArray, error)

ReadBooleanArrayBlock reads the next block as a set of boolean values.

func (*KeyCursor) ReadBooleanBlock 

func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error)

ReadBooleanBlock reads the next block as a set of boolean values.

func (*KeyCursor) ReadFloatArrayBlock 

func (c *KeyCursor) ReadFloatArrayBlock(values *tsdb.FloatArray) (*tsdb.FloatArray, error)

ReadFloatArrayBlock reads the next block as a set of float values.

func (*KeyCursor) ReadFloatBlock 

func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error)

ReadFloatBlock reads the next block as a set of float values.

func (*KeyCursor) ReadIntegerArrayBlock 

func (c *KeyCursor) ReadIntegerArrayBlock(values *tsdb.IntegerArray) (*tsdb.IntegerArray, error)

ReadIntegerArrayBlock reads the next block as a set of integer values.

func (*KeyCursor) ReadIntegerBlock 

func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error)

ReadIntegerBlock reads the next block as a set of integer values.

func (*KeyCursor) ReadStringArrayBlock 

func (c *KeyCursor) ReadStringArrayBlock(values *tsdb.StringArray) (*tsdb.StringArray, error)

ReadStringArrayBlock reads the next block as a set of string values.

func (*KeyCursor) ReadStringBlock 

func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error)

ReadStringBlock reads the next block as a set of string values.

func (*KeyCursor) ReadUnsignedArrayBlock 

func (c *KeyCursor) ReadUnsignedArrayBlock(values *tsdb.UnsignedArray) (*tsdb.UnsignedArray, error)

ReadUnsignedArrayBlock reads the next block as a set of unsigned values.

func (*KeyCursor) ReadUnsignedBlock 

func (c *KeyCursor) ReadUnsignedBlock(buf *[]UnsignedValue) ([]UnsignedValue, error)

ReadUnsignedBlock reads the next block as a set of unsigned values.

type KeyIterator 

type KeyIterator interface {
	// Next returns true if there are any values remaining in the iterator.
	Next() bool

	// Read returns the key, time range, and raw data for the next block,
	// or any error that occurred.
	Read() (key []byte, minTime int64, maxTime int64, data []byte, err error)

	// Close closes the iterator.
	Close() error

	// Err returns any errors encountered during iteration.
	Err() error

	// EstimatedIndexSize returns the estimated size of the index that would
	// be required to store all the series and entries in the KeyIterator.
	EstimatedIndexSize() int
}

KeyIterator allows iteration over set of keys and values in sorted order.

func NewCacheKeyIterator 

func NewCacheKeyIterator(cache *Cache, size int, interrupt chan struct{}) KeyIterator

NewCacheKeyIterator returns a new KeyIterator from a Cache.

func NewTSMBatchKeyIterator 

func NewTSMBatchKeyIterator(size int, fast bool, interrupt chan struct{}, readers ...*TSMReader) (KeyIterator, error)

NewTSMBatchKeyIterator returns a new TSM key iterator from readers. size indicates the maximum number of values to encode in a single block.

func NewTSMKeyIterator 

func NewTSMKeyIterator(size int, fast bool, interrupt chan struct{}, readers ...*TSMReader) (KeyIterator, error)

NewTSMKeyIterator returns a new TSM key iterator from readers. size indicates the maximum number of values to encode in a single block.

type Log 

type Log interface {
	Open() error
	Close() error
	Path() string

	LastWriteTime() time.Time
	DiskSizeBytes() int64

	WriteMulti(values map[string][]Value) (int, error)
	DeleteRange(keys [][]byte, min, max int64) (int, error)

	CloseSegment() error
	ClosedSegments() ([]string, error)
	Remove(files []string) error
}

Log describes an interface for a durable disk-based log.

type MeasurementStats 

type MeasurementStats map[string]int

MeasurementStats represents a set of measurement sizes.

func NewMeasurementStats 

func NewMeasurementStats() MeasurementStats

NewStats returns a new instance of Stats.

func (MeasurementStats) Add 

func (s MeasurementStats) Add(other MeasurementStats)

Add adds the values of all measurements in other to s.

func (MeasurementStats) MeasurementNames 

func (s MeasurementStats) MeasurementNames() []string

MeasurementNames returns a list of sorted measurement names.

func (MeasurementStats) ReadFrom 

func (s MeasurementStats) ReadFrom(r io.Reader) (n int64, err error)

ReadFrom reads stats from r in a binary format. Reader must also be an io.ByteReader.

func (MeasurementStats) Sub 

func (s MeasurementStats) Sub(other MeasurementStats)

Sub subtracts the values of all measurements in other from s.

func (MeasurementStats) WriteTo 

func (s MeasurementStats) WriteTo(w io.Writer) (n int64, err error)

WriteTo writes stats to w in a binary format.

type NopWAL 

type NopWAL struct{}

NopWAL implements the Log interface and provides a no-op WAL implementation.

func (NopWAL) Close 

func (w NopWAL) Close() error

func (NopWAL) CloseSegment 

func (w NopWAL) CloseSegment() error

func (NopWAL) ClosedSegments 

func (w NopWAL) ClosedSegments() ([]string, error)

func (NopWAL) DeleteRange 

func (w NopWAL) DeleteRange(keys [][]byte, min, max int64) (int, error)

func (NopWAL) DiskSizeBytes 

func (w NopWAL) DiskSizeBytes() int64

func (NopWAL) LastWriteTime 

func (w NopWAL) LastWriteTime() time.Time

func (NopWAL) Open 

func (w NopWAL) Open() error

func (NopWAL) Path 

func (w NopWAL) Path() string

func (NopWAL) Remove 

func (w NopWAL) Remove(files []string) error

func (NopWAL) WriteMulti 

func (w NopWAL) WriteMulti(values map[string][]Value) (int, error)

type ParseFileNameFunc 

type ParseFileNameFunc func(name string) (generation, sequence int, err error)

ParseFileNameFunc is executed when parsing a TSM filename into generation & sequence.

type StringDecoder 

type StringDecoder struct {
	// contains filtered or unexported fields
}

StringDecoder decodes a byte slice into strings.

func (*StringDecoder) Error 

func (e *StringDecoder) Error() error

Error returns the last error encountered by the decoder.

func (*StringDecoder) Next 

func (e *StringDecoder) Next() bool

Next returns true if there are any values remaining to be decoded.

func (*StringDecoder) Read 

func (e *StringDecoder) Read() string

Read returns the next value from the decoder.

func (*StringDecoder) SetBytes 

func (e *StringDecoder) SetBytes(b []byte) error

SetBytes initializes the decoder with bytes to read from. This must be called before calling any other method.

type StringEncoder 

type StringEncoder struct {
	// contains filtered or unexported fields
}

StringEncoder encodes multiple strings into a byte slice.

func NewStringEncoder 

func NewStringEncoder(sz int) StringEncoder

NewStringEncoder returns a new StringEncoder with an initial buffer ready to hold sz bytes.

func (*StringEncoder) Bytes 

func (e *StringEncoder) Bytes() ([]byte, error)

Bytes returns a copy of the underlying buffer.

func (*StringEncoder) Flush 

func (e *StringEncoder) Flush()

Flush is no-op

func (*StringEncoder) Reset 

func (e *StringEncoder) Reset()

Reset sets the encoder back to its initial state.

func (*StringEncoder) Write 

func (e *StringEncoder) Write(s string)

Write encodes s to the underlying buffer.

type StringValue 

type StringValue struct {
	// contains filtered or unexported fields
}

StringValue represents a string value.

func DecodeStringBlock 

func DecodeStringBlock(block []byte, a *[]StringValue) ([]StringValue, error)

DecodeStringBlock decodes the string block from the byte slice and appends the string values to a.

func (StringValue) Size 

func (v StringValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (StringValue) String 

func (v StringValue) String() string

String returns the string representation of the value and its timestamp.

func (StringValue) UnixNano 

func (v StringValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (StringValue) Value 

func (v StringValue) Value() interface{}

Value returns the underlying string value.

type StringValues 

type StringValues []StringValue

StringValues represents a slice of String values.

func (StringValues) Deduplicate 

func (a StringValues) Deduplicate() StringValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (StringValues) Encode 

func (a StringValues) Encode(buf []byte) ([]byte, error)

func (StringValues) Exclude 

func (a StringValues) Exclude(min, max int64) StringValues

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (StringValues) FindRange 

func (a StringValues) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (StringValues) Include 

func (a StringValues) Include(min, max int64) StringValues

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (StringValues) Len 

func (a StringValues) Len() int

Sort methods

func (StringValues) Less 

func (a StringValues) Less(i, j int) bool

func (StringValues) MaxTime 

func (a StringValues) MaxTime() int64

func (StringValues) Merge 

func (a StringValues) Merge(b StringValues) StringValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (StringValues) MinTime 

func (a StringValues) MinTime() int64

func (StringValues) Size 

func (a StringValues) Size() int

func (StringValues) Swap 

func (a StringValues) Swap(i, j int)

type TSMFile 

type TSMFile interface {
	// Path returns the underlying file path for the TSMFile.  If the file
	// has not be written or loaded from disk, the zero value is returned.
	Path() string

	// Read returns all the values in the block where time t resides.
	Read(key []byte, t int64) ([]Value, error)

	// ReadAt returns all the values in the block identified by entry.
	ReadAt(entry *IndexEntry, values []Value) ([]Value, error)
	ReadFloatBlockAt(entry *IndexEntry, values *[]FloatValue) ([]FloatValue, error)
	ReadFloatArrayBlockAt(entry *IndexEntry, values *tsdb.FloatArray) error
	ReadIntegerBlockAt(entry *IndexEntry, values *[]IntegerValue) ([]IntegerValue, error)
	ReadIntegerArrayBlockAt(entry *IndexEntry, values *tsdb.IntegerArray) error
	ReadUnsignedBlockAt(entry *IndexEntry, values *[]UnsignedValue) ([]UnsignedValue, error)
	ReadUnsignedArrayBlockAt(entry *IndexEntry, values *tsdb.UnsignedArray) error
	ReadStringBlockAt(entry *IndexEntry, values *[]StringValue) ([]StringValue, error)
	ReadStringArrayBlockAt(entry *IndexEntry, values *tsdb.StringArray) error
	ReadBooleanBlockAt(entry *IndexEntry, values *[]BooleanValue) ([]BooleanValue, error)
	ReadBooleanArrayBlockAt(entry *IndexEntry, values *tsdb.BooleanArray) error

	// Entries returns the index entries for all blocks for the given key.
	ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error)

	// Contains returns true if the file contains any values for the given
	// key.
	Contains(key []byte) bool

	// OverlapsTimeRange returns true if the time range of the file intersect min and max.
	OverlapsTimeRange(min, max int64) bool

	// OverlapsKeyRange returns true if the key range of the file intersects min and max.
	OverlapsKeyRange(min, max []byte) bool

	// TimeRange returns the min and max time across all keys in the file.
	TimeRange() (int64, int64)

	// TombstoneRange returns ranges of time that are deleted for the given key.
	TombstoneRange(key []byte, buf []TimeRange) []TimeRange

	// KeyRange returns the min and max keys in the file.
	KeyRange() ([]byte, []byte)

	// KeyCount returns the number of distinct keys in the file.
	KeyCount() int

	// Iterator returns an iterator over the keys starting at the provided key. You must
	// call Next before calling any of the accessors.
	Iterator([]byte) TSMIterator

	// Type returns the block type of the values stored for the key.  Returns one of
	// BlockFloat64, BlockInt64, BlockBoolean, BlockString.  If key does not exist,
	// an error is returned.
	Type(key []byte) (byte, error)

	// BatchDelete return a BatchDeleter that allows for multiple deletes in batches
	// and group commit or rollback.
	BatchDelete() BatchDeleter

	// Delete removes the keys from the set of keys available in this file.
	Delete(keys [][]byte) error

	// DeleteRange removes the values for keys between timestamps min and max.
	DeleteRange(keys [][]byte, min, max int64) error

	// DeletePrefix removes the values for keys beginning with prefix. It calls dead with
	// any keys that became dead as a result of this call.
	DeletePrefix(prefix []byte, min, max int64, dead func([]byte)) error

	// HasTombstones returns true if file contains values that have been deleted.
	HasTombstones() bool

	// TombstoneFiles returns the tombstone filestats if there are any tombstones
	// written for this file.
	TombstoneFiles() []FileStat

	// Close closes the underlying file resources.
	Close() error

	// Size returns the size of the file on disk in bytes.
	Size() uint32

	// Rename renames the existing TSM file to a new name and replaces the mmap backing slice using the new
	// file name.  Index and Reader state are not re-initialized.
	Rename(path string) error

	// Remove deletes the file from the filesystem.
	Remove() error

	// InUse returns true if the file is currently in use by queries.
	InUse() bool

	// Ref records that this file is actively in use.
	Ref()

	// Unref records that this file is no longer in use.
	Unref()

	// Stats returns summary information about the TSM file.
	Stats() FileStat

	// BlockIterator returns an iterator pointing to the first block in the file and
	// allows sequential iteration to each and every block.
	BlockIterator() *BlockIterator

	// Free releases any resources held by the FileStore to free up system resources.
	Free() error

	// Stats returns the statistics for the file.
	MeasurementStats() (MeasurementStats, error)
}

TSMFile represents an on-disk TSM file.

type TSMIndex 

type TSMIndex interface {
	// Delete removes the given keys from the index. Returns true if there were any changes.
	Delete(keys [][]byte) bool

	// DeleteRange removes the given keys with data between minTime and maxTime from the index.
	// Returns true if there were any changes.
	DeleteRange(keys [][]byte, minTime, maxTime int64) bool

	// DeletePrefix removes keys that begin with the given prefix with data between minTime and
	// maxTime from the index. Returns true if there were any changes. It calls dead with any
	// keys that became dead as a result of this call.
	DeletePrefix(prefix []byte, minTime, maxTime int64, dead func([]byte)) bool

	// MaybeContainsKey returns true if the given key may exist in the index. This is faster than
	// Contains but, may return false positives.
	MaybeContainsKey(key []byte) bool

	// Contains return true if the given key exists in the index.
	Contains(key []byte) bool

	// MaybeContainsValue returns true if key and time might exist in this file. This function
	// could return true even though the actual point does not exists. For example, the key may
	// exist in this file, but not have a point exactly at time t.
	MaybeContainsValue(key []byte, timestamp int64) bool

	// ReadEntries reads the index entries for key into entries.
	ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error)

	// Entry returns the index entry for the specified key and timestamp.  If no entry
	// matches the key and timestamp, nil is returned.
	Entry(key []byte, timestamp int64) *IndexEntry

	// KeyCount returns the count of unique keys in the index.
	KeyCount() int

	// Iterator returns an iterator over the keys starting at the provided key. You must
	// call Next before calling any of the accessors.
	Iterator([]byte) *TSMIndexIterator

	// OverlapsTimeRange returns true if the time range of the file intersect min and max.
	OverlapsTimeRange(min, max int64) bool

	// OverlapsKeyRange returns true if the min and max keys of the file overlap the arguments min and max.
	OverlapsKeyRange(min, max []byte) bool

	// Size returns the size of the current index in bytes.
	Size() uint32

	// TimeRange returns the min and max time across all keys in the file.
	TimeRange() (int64, int64)

	// TombstoneRange returns ranges of time that are deleted for the given key.
	TombstoneRange(key []byte, buf []TimeRange) []TimeRange

	// KeyRange returns the min and max keys in the file.
	KeyRange() ([]byte, []byte)

	// Type returns the block type of the values stored for the key.  Returns one of
	// BlockFloat64, BlockInt64, BlockBool, BlockString.  If key does not exist,
	// an error is returned.
	Type(key []byte) (byte, error)

	// UnmarshalBinary populates an index from an encoded byte slice
	// representation of an index.
	UnmarshalBinary(b []byte) error

	// Close closes the index and releases any resources.
	Close() error
}

TSMIndex represent the index section of a TSM file. The index records all blocks, their locations, sizes, min and max times.

type TSMIndexIterator 

type TSMIndexIterator struct {
	// contains filtered or unexported fields
}

TSMIndexIterator allows one to iterate over the TSM index.

func (*TSMIndexIterator) Entries 

func (t *TSMIndexIterator) Entries() []IndexEntry

Entries reports the current list of entries.

func (*TSMIndexIterator) Err 

func (t *TSMIndexIterator) Err() error

Err reports if an error stopped the iteration.

func (*TSMIndexIterator) Key 

func (t *TSMIndexIterator) Key() []byte

Key reports the current key.

func (*TSMIndexIterator) Next 

func (t *TSMIndexIterator) Next() bool

Next advances the iterator and reports if it is still valid.

func (*TSMIndexIterator) Peek 

func (t *TSMIndexIterator) Peek() []byte

Peek reports the next key or nil if there is not one or an error happened.

func (*TSMIndexIterator) Type 

func (t *TSMIndexIterator) Type() byte

Type reports the current type.

type TSMIterator 

type TSMIterator interface {
	Next() bool
	Peek() []byte
	Key() []byte
	Type() byte
	Entries() []IndexEntry
	Err() error
}

type TSMReader 

type TSMReader struct {
	// contains filtered or unexported fields
}

TSMReader is a reader for a TSM file.

func NewTSMReader 

func NewTSMReader(f *os.File, options ...tsmReaderOption) (*TSMReader, error)

NewTSMReader returns a new TSMReader from the given file.

func (*TSMReader) BatchDelete 

func (r *TSMReader) BatchDelete() BatchDeleter

BatchDelete returns a BatchDeleter. Only a single goroutine may run a BatchDelete at a time. Callers must either Commit or Rollback the operation.

func (*TSMReader) BlockIterator 

func (t *TSMReader) BlockIterator() *BlockIterator

BlockIterator returns a BlockIterator for the underlying TSM file.

func (*TSMReader) Close 

func (t *TSMReader) Close() error

Close closes the TSMReader.

func (*TSMReader) Contains 

func (t *TSMReader) Contains(key []byte) bool

Contains returns whether the given key is present in the index.

func (*TSMReader) Delete 

func (t *TSMReader) Delete(keys [][]byte) error

Delete deletes blocks indicated by keys.

func (*TSMReader) DeletePrefix 

func (t *TSMReader) DeletePrefix(prefix []byte, minTime, maxTime int64, dead func([]byte)) error

DeletePrefix removes the given points for keys beginning with prefix. It calls dead with any keys that became dead as a result of this call.

func (*TSMReader) DeleteRange 

func (t *TSMReader) DeleteRange(keys [][]byte, minTime, maxTime int64) error

DeleteRange removes the given points for keys between minTime and maxTime. The series keys passed in must be sorted.

func (*TSMReader) Free 

func (t *TSMReader) Free() error

func (*TSMReader) HasTombstones 

func (t *TSMReader) HasTombstones() bool

HasTombstones return true if there are any tombstone entries recorded.

func (*TSMReader) InUse 

func (t *TSMReader) InUse() bool

InUse returns whether the TSMReader currently has any active references.

func (*TSMReader) IndexSize 

func (t *TSMReader) IndexSize() uint32

IndexSize returns the size of the index in bytes.

func (*TSMReader) Iterator 

func (t *TSMReader) Iterator(key []byte) TSMIterator

Iterator returns an iterator over the keys starting at the provided key. You must call Next before calling any of the accessors.

func (*TSMReader) KeyCount 

func (t *TSMReader) KeyCount() int

KeyCount returns the count of unique keys in the TSMReader.

func (*TSMReader) KeyRange 

func (t *TSMReader) KeyRange() ([]byte, []byte)

KeyRange returns the min and max key across all keys in the file.

func (*TSMReader) LastModified 

func (t *TSMReader) LastModified() int64

LastModified returns the last time the underlying file was modified.

func (*TSMReader) MaybeContainsValue 

func (t *TSMReader) MaybeContainsValue(key []byte, ts int64) bool

MaybeContainsValue returns true if key and time might exists in this file. This function could return true even though the actual point does not exist. For example, the key may exist in this file, but not have a point exactly at time t.

func (*TSMReader) MeasurementStats 

func (t *TSMReader) MeasurementStats() (MeasurementStats, error)

MeasurementStats returns the on-disk measurement stats for this file, if available.

func (*TSMReader) OverlapsKeyRange 

func (t *TSMReader) OverlapsKeyRange(min, max []byte) bool

OverlapsKeyRange returns true if the key range of the file intersect min and max.

func (*TSMReader) OverlapsTimeRange 

func (t *TSMReader) OverlapsTimeRange(min, max int64) bool

OverlapsTimeRange returns true if the time range of the file intersect min and max.

func (*TSMReader) Path 

func (t *TSMReader) Path() string

Path returns the path of the file the TSMReader was initialized with.

func (*TSMReader) Read 

func (t *TSMReader) Read(key []byte, timestamp int64) ([]Value, error)

Read returns the values corresponding to the block at the given key and timestamp.

func (*TSMReader) ReadAll 

func (t *TSMReader) ReadAll(key []byte) ([]Value, error)

ReadAll returns all values for a key in all blocks.

func (*TSMReader) ReadAt 

func (t *TSMReader) ReadAt(entry *IndexEntry, vals []Value) ([]Value, error)

ReadAt returns the values corresponding to the given index entry.

func (*TSMReader) ReadBooleanArrayBlockAt 

func (t *TSMReader) ReadBooleanArrayBlockAt(entry *IndexEntry, vals *tsdb.BooleanArray) error

ReadBooleanArrayBlockAt fills vals with the boolean values corresponding to the given index entry.

func (*TSMReader) ReadBooleanBlockAt 

func (t *TSMReader) ReadBooleanBlockAt(entry *IndexEntry, vals *[]BooleanValue) ([]BooleanValue, error)

ReadBooleanBlockAt returns the boolean values corresponding to the given index entry.

func (*TSMReader) ReadBytes 

func (t *TSMReader) ReadBytes(e *IndexEntry, b []byte) (uint32, []byte, error)

func (*TSMReader) ReadEntries 

func (t *TSMReader) ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error)

ReadEntries reads the index entries for key into entries.

func (*TSMReader) ReadFloatArrayBlockAt 

func (t *TSMReader) ReadFloatArrayBlockAt(entry *IndexEntry, vals *tsdb.FloatArray) error

ReadFloatArrayBlockAt fills vals with the float values corresponding to the given index entry.

func (*TSMReader) ReadFloatBlockAt 

func (t *TSMReader) ReadFloatBlockAt(entry *IndexEntry, vals *[]FloatValue) ([]FloatValue, error)

ReadFloatBlockAt returns the float values corresponding to the given index entry.

func (*TSMReader) ReadIntegerArrayBlockAt 

func (t *TSMReader) ReadIntegerArrayBlockAt(entry *IndexEntry, vals *tsdb.IntegerArray) error

ReadIntegerArrayBlockAt fills vals with the integer values corresponding to the given index entry.

func (*TSMReader) ReadIntegerBlockAt 

func (t *TSMReader) ReadIntegerBlockAt(entry *IndexEntry, vals *[]IntegerValue) ([]IntegerValue, error)

ReadIntegerBlockAt returns the integer values corresponding to the given index entry.

func (*TSMReader) ReadStringArrayBlockAt 

func (t *TSMReader) ReadStringArrayBlockAt(entry *IndexEntry, vals *tsdb.StringArray) error

ReadStringArrayBlockAt fills vals with the string values corresponding to the given index entry.

func (*TSMReader) ReadStringBlockAt 

func (t *TSMReader) ReadStringBlockAt(entry *IndexEntry, vals *[]StringValue) ([]StringValue, error)

ReadStringBlockAt returns the string values corresponding to the given index entry.

func (*TSMReader) ReadUnsignedArrayBlockAt 

func (t *TSMReader) ReadUnsignedArrayBlockAt(entry *IndexEntry, vals *tsdb.UnsignedArray) error

ReadUnsignedArrayBlockAt fills vals with the unsigned values corresponding to the given index entry.

func (*TSMReader) ReadUnsignedBlockAt 

func (t *TSMReader) ReadUnsignedBlockAt(entry *IndexEntry, vals *[]UnsignedValue) ([]UnsignedValue, error)

ReadUnsignedBlockAt returns the unsigned values corresponding to the given index entry.

func (*TSMReader) Ref 

func (t *TSMReader) Ref()

Ref records a usage of this TSMReader. If there are active references when the reader is closed or removed, the reader will remain open until there are no more references.

func (*TSMReader) Remove 

func (t *TSMReader) Remove() error

Remove removes any underlying files stored on disk for this reader.

func (*TSMReader) Rename 

func (t *TSMReader) Rename(path string) error

Rename renames the underlying file to the new path.

func (*TSMReader) Size 

func (t *TSMReader) Size() uint32

Size returns the size of the underlying file in bytes.

func (*TSMReader) Stats 

func (t *TSMReader) Stats() FileStat

Stats returns the FileStat for the TSMReader's underlying file.

func (*TSMReader) TimeRange 

func (t *TSMReader) TimeRange() (int64, int64)

TimeRange returns the min and max time across all keys in the file.

func (*TSMReader) TombstoneFiles 

func (t *TSMReader) TombstoneFiles() []FileStat

TombstoneFiles returns any tombstone files associated with this TSM file.

func (*TSMReader) TombstoneRange 

func (t *TSMReader) TombstoneRange(key []byte, buf []TimeRange) []TimeRange

TombstoneRange returns ranges of time that are deleted for the given key.

func (*TSMReader) Type 

func (t *TSMReader) Type(key []byte) (byte, error)

Type returns the type of values stored at the given key.

func (*TSMReader) Unref 

func (t *TSMReader) Unref()

Unref removes a usage record of this TSMReader. If the Reader was closed by another goroutine while there were active references, the file will be closed and remove

func (*TSMReader) WithObserver 

func (t *TSMReader) WithObserver(obs FileStoreObserver)

WithObserver sets the observer for the TSM reader.

type TSMWriter 

type TSMWriter interface {
	// Write writes a new block for key containing and values.  Writes append
	// blocks in the order that the Write function is called.  The caller is
	// responsible for ensuring keys and blocks are sorted appropriately.
	// Values are encoded as a full block.  The caller is responsible for
	// ensuring a fixed number of values are encoded in each block as well as
	// ensuring the Values are sorted. The first and last timestamp values are
	// used as the minimum and maximum values for the index entry.
	Write(key []byte, values Values) error

	// WriteBlock writes a new block for key containing the bytes in block.  WriteBlock appends
	// blocks in the order that the WriteBlock function is called.  The caller is
	// responsible for ensuring keys and blocks are sorted appropriately, and that the
	// block and index information is correct for the block.  The minTime and maxTime
	// timestamp values are used as the minimum and maximum values for the index entry.
	WriteBlock(key []byte, minTime, maxTime int64, block []byte) error

	// WriteIndex finishes the TSM write streams and writes the index.
	WriteIndex() error

	// Flushes flushes all pending changes to the underlying file resources.
	Flush() error

	// Close closes any underlying file resources.
	Close() error

	// Size returns the current size in bytes of the file.
	Size() uint32

	// Stats returns the statistics generated by the writer.
	MeasurementStats() MeasurementStats

	Remove() error
}

TSMWriter writes TSM formatted key and values.

func NewTSMWriter 

func NewTSMWriter(w io.Writer) (TSMWriter, error)

NewTSMWriter returns a new TSMWriter writing to w.

func NewTSMWriterWithDiskBuffer 

func NewTSMWriterWithDiskBuffer(w io.Writer) (TSMWriter, error)

NewTSMWriterWithDiskBuffer returns a new TSMWriter writing to w and will use a disk based buffer for the TSM index if possible.

type TimeDecoder 

type TimeDecoder struct {
	// contains filtered or unexported fields
}

TimeDecoder decodes a byte slice into timestamps.

func (*TimeDecoder) Error 

func (d *TimeDecoder) Error() error

Error returns the last error encountered by the decoder.

func (*TimeDecoder) Init 

func (d *TimeDecoder) Init(b []byte)

Init initializes the decoder with bytes to read from.

func (*TimeDecoder) Next 

func (d *TimeDecoder) Next() bool

Next returns true if there are any timestamps remaining to be decoded.

func (*TimeDecoder) Read 

func (d *TimeDecoder) Read() int64

Read returns the next timestamp from the decoder.

type TimeEncoder 

type TimeEncoder interface {
	Write(t int64)
	Bytes() ([]byte, error)
	Reset()
}

TimeEncoder encodes time.Time to byte slices.

func NewTimeEncoder 

func NewTimeEncoder(sz int) TimeEncoder

NewTimeEncoder returns a TimeEncoder with an initial buffer ready to hold sz bytes.

type TimeRange 

type TimeRange struct {
	Min, Max int64
}

TimeRange holds a min and max timestamp.

func (TimeRange) Less 

func (t TimeRange) Less(o TimeRange) bool

func (TimeRange) Overlaps 

func (t TimeRange) Overlaps(min, max int64) bool

type Tombstone 

type Tombstone struct {
	// Key is the tombstoned series key.
	Key []byte

	// Prefix indicates if this tombstone entry is a prefix key, meaning all
	// keys with a prefix matching Key should be removed for the [Min, Max] range.
	Prefix bool

	// Min and Max are the min and max unix nanosecond time ranges of Key that are deleted.  If
	// the full range is deleted, both values are -1.
	Min, Max int64
}

Tombstone represents an individual deletion.

func (Tombstone) String 

func (t Tombstone) String() string

type Tombstoner 

type Tombstoner struct {

	// Path is the location of the file to record tombstone. This should be the
	// full path to a TSM file.
	Path string

	FilterFn func(k []byte) bool
	// contains filtered or unexported fields
}

Tombstoner records tombstones when entries are deleted.

func NewTombstoner 

func NewTombstoner(path string, filterFn func(k []byte) bool) *Tombstoner

NewTombstoner constructs a Tombstoner for the given path. FilterFn can be nil.

func (*Tombstoner) Add 

func (t *Tombstoner) Add(keys [][]byte) error

Add adds the all keys, across all timestamps, to the tombstone.

func (*Tombstoner) AddPrefix 

func (t *Tombstoner) AddPrefix(key []byte) error

AddPrefix adds a prefix-based tombstone key.

func (*Tombstoner) AddPrefixRange 

func (t *Tombstoner) AddPrefixRange(key []byte, min, max int64) error

AddPrefixRange adds a prefix-based tombstone key with an explicit range.

func (*Tombstoner) AddRange 

func (t *Tombstoner) AddRange(keys [][]byte, min, max int64) error

AddRange adds all keys to the tombstone specifying only the data between min and max to be removed.

func (*Tombstoner) Delete 

func (t *Tombstoner) Delete() error

Delete removes all the tombstone files from disk.

func (*Tombstoner) Flush 

func (t *Tombstoner) Flush() error

func (*Tombstoner) HasTombstones 

func (t *Tombstoner) HasTombstones() bool

HasTombstones return true if there are any tombstone entries recorded.

func (*Tombstoner) Rollback 

func (t *Tombstoner) Rollback() error

func (*Tombstoner) TombstoneFiles 

func (t *Tombstoner) TombstoneFiles() []FileStat

TombstoneFiles returns any tombstone files associated with Tombstoner's TSM file.

func (*Tombstoner) Walk 

func (t *Tombstoner) Walk(fn func(t Tombstone) error) error

Walk calls fn for every Tombstone under the Tombstoner.

func (*Tombstoner) WithObserver 

func (t *Tombstoner) WithObserver(obs FileStoreObserver)

WithObserver sets a FileStoreObserver for when the tombstone file is written.

type UnsignedValue 

type UnsignedValue struct {
	// contains filtered or unexported fields
}

UnsignedValue represents an int64 value.

func DecodeUnsignedBlock 

func DecodeUnsignedBlock(block []byte, a *[]UnsignedValue) ([]UnsignedValue, error)

DecodeUnsignedBlock decodes the unsigned integer block from the byte slice and appends the unsigned integer values to a.

func (UnsignedValue) Size 

func (v UnsignedValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (UnsignedValue) String 

func (v UnsignedValue) String() string

String returns the string representation of the value and its timestamp.

func (UnsignedValue) UnixNano 

func (v UnsignedValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (UnsignedValue) Value 

func (v UnsignedValue) Value() interface{}

Value returns the underlying int64 value.

type UnsignedValues 

type UnsignedValues []UnsignedValue

UnsignedValues represents a slice of Unsigned values.

func (UnsignedValues) Deduplicate 

func (a UnsignedValues) Deduplicate() UnsignedValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (UnsignedValues) Encode 

func (a UnsignedValues) Encode(buf []byte) ([]byte, error)

func (UnsignedValues) Exclude 

func (a UnsignedValues) Exclude(min, max int64) UnsignedValues

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (UnsignedValues) FindRange 

func (a UnsignedValues) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (UnsignedValues) Include 

func (a UnsignedValues) Include(min, max int64) UnsignedValues

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (UnsignedValues) Len 

func (a UnsignedValues) Len() int

Sort methods

func (UnsignedValues) Less 

func (a UnsignedValues) Less(i, j int) bool

func (UnsignedValues) MaxTime 

func (a UnsignedValues) MaxTime() int64

func (UnsignedValues) Merge 

func (a UnsignedValues) Merge(b UnsignedValues) UnsignedValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (UnsignedValues) MinTime 

func (a UnsignedValues) MinTime() int64

func (UnsignedValues) Size 

func (a UnsignedValues) Size() int

func (UnsignedValues) Swap 

func (a UnsignedValues) Swap(i, j int)

type Value 

type Value interface {
	// UnixNano returns the timestamp of the value in nanoseconds since unix epoch.
	UnixNano() int64

	// Value returns the underlying value.
	Value() interface{}

	// Size returns the number of bytes necessary to represent the value and its timestamp.
	Size() int

	// String returns the string representation of the value and its timestamp.
	String() string
	// contains filtered or unexported methods
}

Value represents a TSM-encoded value.

func DecodeBlock 

func DecodeBlock(block []byte, vals []Value) ([]Value, error)

DecodeBlock takes a byte slice and decodes it into values of the appropriate type based on the block.

func NewBooleanValue 

func NewBooleanValue(t int64, v bool) Value

NewBooleanValue returns a new boolean value.

func NewFloatValue 

func NewFloatValue(t int64, v float64) Value

NewFloatValue returns a new float value.

func NewIntegerValue 

func NewIntegerValue(t int64, v int64) Value

NewIntegerValue returns a new integer value.

func NewStringValue 

func NewStringValue(t int64, v string) Value

NewStringValue returns a new string value.

func NewUnsignedValue 

func NewUnsignedValue(t int64, v uint64) Value

NewUnsignedValue returns a new unsigned integer value.

func NewValue 

func NewValue(t int64, value interface{}) Value

NewValue returns a new Value with the underlying type dependent on value.

type Values 

type Values []Value

Values represents a slice of values.

func (Values) Deduplicate 

func (a Values) Deduplicate() Values

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.

func (Values) Encode 

func (a Values) Encode(buf []byte) ([]byte, error)

Encode converts the values to a byte slice. If there are no values, this function panics.

func (Values) Exclude 

func (a Values) Exclude(min, max int64) Values

Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (Values) FindRange 

func (a Values) FindRange(min, max int64) (int, int)

FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (Values) Include 

func (a Values) Include(min, max int64) Values

Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.

func (Values) InfluxQLType 

func (a Values) InfluxQLType() (influxql.DataType, error)

InfluxQLType returns the influxql.DataType the values map to.

func (Values) Len 

func (a Values) Len() int

Sort methods

func (Values) Less 

func (a Values) Less(i, j int) bool

func (Values) MaxTime 

func (a Values) MaxTime() int64

func (Values) Merge 

func (a Values) Merge(b Values) Values

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (Values) MinTime 

func (a Values) MinTime() int64

func (Values) Size 

func (a Values) Size() int

func (Values) Swap 

func (a Values) Swap(i, j int)

type WAL 

type WAL struct {

	// SegmentSize is the file size at which a segment file will be rotated
	SegmentSize int
	// contains filtered or unexported fields
}

WAL represents the write-ahead log used for writing TSM files.

func NewWAL 

func NewWAL(path string) *WAL

NewWAL initializes a new WAL at the given directory.

func (*WAL) Close 

func (l *WAL) Close() error

Close will finish any flush that is currently in progress and close file handles.

func (*WAL) CloseSegment 

func (l *WAL) CloseSegment() error

CloseSegment closes the current segment if it is non-empty and opens a new one.

func (*WAL) ClosedSegments 

func (l *WAL) ClosedSegments() ([]string, error)

ClosedSegments returns a slice of the names of the closed segment files.

func (*WAL) Delete 

func (l *WAL) Delete(keys [][]byte) (int, error)

Delete deletes the given keys, returning the segment ID for the operation.

func (*WAL) DeleteRange 

func (l *WAL) DeleteRange(keys [][]byte, min, max int64) (int, error)

DeleteRange deletes the given keys within the given time range, returning the segment ID for the operation.

func (*WAL) DiskSizeBytes 

func (l *WAL) DiskSizeBytes() int64

DiskSizeBytes returns the on-disk size of the WAL.

func (*WAL) EnableTraceLogging 

func (l *WAL) EnableTraceLogging(enabled bool)

EnableTraceLogging must be called before the WAL is opened.

func (*WAL) LastWriteTime 

func (l *WAL) LastWriteTime() time.Time

LastWriteTime is the last time anything was written to the WAL.

func (*WAL) Open 

func (l *WAL) Open() error

Open opens and initializes the Log. Open can recover from previous unclosed shutdowns.

func (*WAL) Path 

func (l *WAL) Path() string

Path returns the directory the log was initialized with.

func (*WAL) Remove 

func (l *WAL) Remove(files []string) error

Remove deletes the given segment file paths from disk and cleans up any associated objects.

func (*WAL) WithFsyncDelay 

func (l *WAL) WithFsyncDelay(delay time.Duration)

WithFsyncDelay sets the fsync delay and should be called before the WAL is opened.

func (*WAL) WithLogger 

func (l *WAL) WithLogger(log *zap.Logger)

WithLogger sets the WAL's logger.

func (*WAL) WriteMulti 

func (l *WAL) WriteMulti(values map[string][]Value) (int, error)

WriteMulti writes the given values to the WAL. It returns the WAL segment ID to which the points were written. If an error is returned the segment ID should be ignored.

type WALConfig 

type WALConfig struct {
	// Enabled controls if the WAL is enabled.
	Enabled bool `toml:"enabled"`

	// WALFsyncDelay is the amount of time that a write will wait before fsyncing.  A
	// duration greater than 0 can be used to batch up multiple fsync calls.  This is
	// useful for slower disks or when WAL write contention is seen.  A value of 0 fsyncs
	// every write to the WAL.
	FsyncDelay toml.Duration `toml:"fsync-delay"`
}

WALConfig holds all of the configuration about the WAL.

func NewWALConfig 

func NewWALConfig() WALConfig

type WALEntry 

type WALEntry interface {
	Type() WalEntryType
	Encode(dst []byte) ([]byte, error)
	MarshalBinary() ([]byte, error)
	UnmarshalBinary(b []byte) error
	MarshalSize() int
}

WALEntry is record stored in each WAL segment. Each entry has a type and an opaque, type dependent byte slice data attribute.

type WALSegmentReader 

type WALSegmentReader struct {
	// contains filtered or unexported fields
}

WALSegmentReader reads WAL segments.

func NewWALSegmentReader 

func NewWALSegmentReader(r io.ReadCloser) *WALSegmentReader

NewWALSegmentReader returns a new WALSegmentReader reading from r.

func (*WALSegmentReader) Close 

func (r *WALSegmentReader) Close() error

Close closes the underlying io.Reader.

func (*WALSegmentReader) Count 

func (r *WALSegmentReader) Count() int64

Count returns the total number of bytes read successfully from the segment, as of the last call to Read(). The segment is guaranteed to be valid up to and including this number of bytes.

func (*WALSegmentReader) Error 

func (r *WALSegmentReader) Error() error

Error returns the last error encountered by the reader.

func (*WALSegmentReader) Next 

func (r *WALSegmentReader) Next() bool

Next indicates if there is a value to read.

func (*WALSegmentReader) Read 

func (r *WALSegmentReader) Read() (WALEntry, error)

Read returns the next entry in the reader.

func (*WALSegmentReader) Reset 

func (r *WALSegmentReader) Reset(rc io.ReadCloser)

type WALSegmentWriter 

type WALSegmentWriter struct {
	// contains filtered or unexported fields
}

WALSegmentWriter writes WAL segments.

func NewWALSegmentWriter 

func NewWALSegmentWriter(w io.WriteCloser) *WALSegmentWriter

NewWALSegmentWriter returns a new WALSegmentWriter writing to w.

func (*WALSegmentWriter) Flush 

func (w *WALSegmentWriter) Flush() error

func (*WALSegmentWriter) Write 

func (w *WALSegmentWriter) Write(entryType WalEntryType, compressed []byte) error

Write writes entryType and the buffer containing compressed entry data.

type WalEntryType 

type WalEntryType byte

WalEntryType is a byte written to a wal segment file that indicates what the following compressed block contains. 

const (
	// WriteWALEntryType indicates a write entry.
	WriteWALEntryType WalEntryType = 0x01

	// DeleteWALEntryType indicates a delete entry.
	DeleteWALEntryType WalEntryType = 0x02

	// DeleteRangeWALEntryType indicates a delete range entry.
	DeleteRangeWALEntryType WalEntryType = 0x03
)

type WriteWALEntry 

type WriteWALEntry struct {
	Values map[string][]Value
	// contains filtered or unexported fields
}

WriteWALEntry represents a write of points.

func (*WriteWALEntry) Encode 

func (w *WriteWALEntry) Encode(dst []byte) ([]byte, error)

Encode converts the WriteWALEntry into a byte stream using dst if it is large enough. If dst is too small, the slice will be grown to fit the encoded entry.

func (*WriteWALEntry) MarshalBinary 

func (w *WriteWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*WriteWALEntry) MarshalSize 

func (w *WriteWALEntry) MarshalSize() int

func (*WriteWALEntry) Type 

func (w *WriteWALEntry) Type() WalEntryType

Type returns WriteWALEntryType.

func (*WriteWALEntry) UnmarshalBinary 

func (w *WriteWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.