Back to godoc.org

Package tsdb

v2.0.0 (beta.5)
Latest Go to latest
Published: Feb 13, 2020 | License: MIT | Module: github.com/mildebrandt/influxdb/v2

Index

Package Files

Constants

const (
	SeriesIndexVersion = 1
	SeriesIndexMagic   = "SIDX"
)
const (
	// SeriesIDSize is the size in bytes of a series key ID.
	SeriesIDSize        = 8
	SeriesOffsetSize    = 8
	SeriesIndexElemSize = SeriesOffsetSize + SeriesIDSize

	SeriesIndexLoadFactor = 90 // rhh load factor

	SeriesIndexHeaderSize = 0 +
		4 + 1 +
		8 + 8 +
		8 + 8 +
		8 + 8 +
		8 + 8 +
		0
)
const (
	SeriesSegmentVersion = 1
	SeriesSegmentMagic   = "SSEG"

	SeriesSegmentHeaderSize = 4 + 1 // magic + version
)
const (
	SeriesEntryFlagSize   = 1
	SeriesEntryHeaderSize = 1 + 8 // flag + id

	SeriesEntryInsertFlag    = 0x01
	SeriesEntryTombstoneFlag = 0x02
)

Series entry constants.

const (
	// DefaultLargeSeriesWriteThreshold is the number of series per write
	// that requires the series index be pregrown before insert.
	DefaultLargeSeriesWriteThreshold = 10000
)
const DefaultSeriesPartitionCompactThreshold = 1 << 17 // 128K

DefaultSeriesPartitionCompactThreshold is the number of series IDs to hold in the in-memory series map before compacting and rebuilding the on-disk representation.

const EOF = query.ZeroTime

EOF represents a "not found" key returned by a Cursor.

const (
	// SeriesFilePartitionN is the number of partitions a series file is split into.
	SeriesFilePartitionN = 8
)

Variables

var (
	// ErrFieldTypeConflict is returned when a new field already exists with a different type.
	ErrFieldTypeConflict = errors.New("field type conflict")

	// ErrUnknownFieldType is returned when the type of a field cannot be determined.
	ErrUnknownFieldType = errors.New("unknown field type")
)
var (
	ErrSeriesFileClosed         = errors.New("tsdb: series file closed")
	ErrInvalidSeriesPartitionID = errors.New("tsdb: invalid series partition id")
)
var (
	ErrSeriesPartitionClosed              = errors.New("tsdb: series partition closed")
	ErrSeriesPartitionCompactionCancelled = errors.New("tsdb: series partition compaction cancelled")
)
var (
	ErrInvalidSeriesSegment        = errors.New("invalid series segment")
	ErrInvalidSeriesSegmentVersion = errors.New("invalid series segment version")
	ErrSeriesSegmentNotWritable    = errors.New("series segment not writable")
)
var ErrInvalidSeriesIndex = errors.New("invalid series index")

func AppendSeriesEntry

func AppendSeriesEntry(dst []byte, flag uint8, id SeriesIDTyped, key []byte) []byte

func AppendSeriesKey

func AppendSeriesKey(dst []byte, name []byte, tags models.Tags) []byte

AppendSeriesKey serializes name and tags to a byte slice. The total length is prepended as a uvarint.

func CompareSeriesKeys

func CompareSeriesKeys(a, b []byte) int

func DecodeName

func DecodeName(name [16]byte) (org, bucket platform.ID)

DecodeName converts tsdb internal serialization back to organization and bucket IDs.

func DecodeNameSlice

func DecodeNameSlice(name []byte) (org, bucket platform.ID)

DecodeNameSlice converts tsdb internal serialization back to organization and bucket IDs.

func EncodeName

func EncodeName(org, bucket platform.ID) [16]byte

EncodeName converts org/bucket pairs to the tsdb internal serialization

func EncodeNameSlice

func EncodeNameSlice(org, bucket platform.ID) []byte

EncodeNameSlice converts org/bucket pairs to the tsdb internal serialization but returns a byte slice.

func EncodeNameString

func EncodeNameString(org, bucket platform.ID) string

EncodeNameString converts org/bucket pairs to the tsdb internal serialization

func EncodeOrgName

func EncodeOrgName(org platform.ID) [8]byte

EncodeOrgName converts org to the tsdb internal serialization that may be used as a prefix when searching for keys matching a specific organization.

func ExplodePoints

func ExplodePoints(org, bucket platform.ID, points []models.Point) ([]models.Point, error)

ExplodePoints creates a list of points that only contains one field per point. It also moves the measurement to a tag, and changes the measurement to be the provided argument.

func GenerateSeriesKeys

func GenerateSeriesKeys(names [][]byte, tagsSlice []models.Tags) [][]byte

GenerateSeriesKeys generates series keys for a list of names & tags using a single large memory block.

func IsValidSeriesEntryFlag

func IsValidSeriesEntryFlag(flag byte) bool

IsValidSeriesEntryFlag returns true if flag is valid.

func IsValidSeriesSegmentFilename

func IsValidSeriesSegmentFilename(filename string) bool

IsValidSeriesSegmentFilename returns true if filename is a 4-character lowercase hexadecimal number.

func JoinSeriesOffset

func JoinSeriesOffset(segmentID uint16, pos uint32) int64

JoinSeriesOffset returns an offset that combines the 2-byte segmentID and 4-byte pos.

func MakeTagsKey

func MakeTagsKey(keys []string, tags models.Tags) []byte

MakeTagsKey converts a tag set to bytes for use as a lookup key.

func MarshalTags

func MarshalTags(tags map[string]string) []byte

MarshalTags converts a tag set to bytes for use as a lookup key.

func NewMeasurementSliceIterator

func NewMeasurementSliceIterator(names [][]byte) *measurementSliceIterator

NewMeasurementSliceIterator returns an iterator over a slice of in-memory measurement names.

func NewTagKeySliceIterator

func NewTagKeySliceIterator(keys [][]byte) *tagKeySliceIterator

NewTagKeySliceIterator returns a TagKeyIterator that iterates over a slice.

func NewTagValueSliceIterator

func NewTagValueSliceIterator(values [][]byte) *tagValueSliceIterator

NewTagValueSliceIterator returns a TagValueIterator that iterates over a slice.

func ParseSeriesKey

func ParseSeriesKey(data []byte) (name []byte, tags models.Tags)

ParseSeriesKey extracts the name & tags from a series key.

func ParseSeriesKeyInto

func ParseSeriesKeyInto(data []byte, dstTags models.Tags) ([]byte, models.Tags)

ParseSeriesKeyInto extracts the name and tags for data, parsing the tags into dstTags, which is then returened.

The returned dstTags may have a different length and capacity.

func ParseSeriesSegmentFilename

func ParseSeriesSegmentFilename(filename string) (uint16, error)

ParseSeriesSegmentFilename returns the id represented by the hexadecimal filename.

func PrometheusCollectors

func PrometheusCollectors() []prometheus.Collector

PrometheusCollectors returns all the metrics associated with the tsdb package.

func ReadSeriesKey

func ReadSeriesKey(data []byte) (key, remainder []byte)

ReadSeriesKey returns the series key from the beginning of the buffer.

func ReadSeriesKeyFromSegments

func ReadSeriesKeyFromSegments(a []*SeriesSegment, offset int64) []byte

ReadSeriesKeyFromSegments returns a series key from an offset within a set of segments.

func ReadSeriesKeyLen

func ReadSeriesKeyLen(data []byte) (sz int, remainder []byte)

func ReadSeriesKeyMeasurement

func ReadSeriesKeyMeasurement(data []byte) (name, remainder []byte)

func ReadSeriesKeyTag

func ReadSeriesKeyTag(data []byte) (key, value, remainder []byte)

func ReadSeriesKeyTagN

func ReadSeriesKeyTagN(data []byte) (n int, remainder []byte)

func SeriesKeySize

func SeriesKeySize(name []byte, tags models.Tags) int

SeriesKeySize returns the number of bytes required to encode a series key.

func SeriesKeysSize

func SeriesKeysSize(names [][]byte, tagsSlice []models.Tags) int

SeriesKeysSize returns the number of bytes required to encode a list of name/tags.

func SeriesSegmentSize

func SeriesSegmentSize(id uint16) uint32

SeriesSegmentSize returns the maximum size of the segment. The size goes up by powers of 2 starting from 4MB and reaching 256MB.

func SplitSeriesOffset

func SplitSeriesOffset(offset int64) (segmentID uint16, pos uint32)

SplitSeriesOffset splits a offset into its 2-byte segmentID and 4-byte pos parts.

type BooleanArray

type BooleanArray = cursors.BooleanArray

func NewBooleanArrayLen

func NewBooleanArrayLen(sz int) *BooleanArray

type BooleanArrayCursor

type BooleanArrayCursor = cursors.BooleanArrayCursor

type ByTagKey

type ByTagKey []*query.TagSet

func (ByTagKey) Len

func (t ByTagKey) Len() int

func (ByTagKey) Less

func (t ByTagKey) Less(i, j int) bool

func (ByTagKey) Swap

func (t ByTagKey) Swap(i, j int)

type Config

type Config struct {
	// LargeSeriesWriteThreshold is the threshold before a write requires
	// preallocation to improve throughput. Currently used in the series file.
	LargeSeriesWriteThreshold int `toml:"large-series-write-threshold"`
}

Config contains all of the configuration related to tsdb.

func NewConfig

func NewConfig() Config

NewConfig return a new instance of config with default settings.

type Cursor

type Cursor = cursors.Cursor

type CursorIterator

type CursorIterator = cursors.CursorIterator

type CursorIterators

type CursorIterators = cursors.CursorIterators

type CursorRequest

type CursorRequest = cursors.CursorRequest

type FloatArray

type FloatArray = cursors.FloatArray

func NewFloatArrayLen

func NewFloatArrayLen(sz int) *FloatArray

type FloatArrayCursor

type FloatArrayCursor = cursors.FloatArrayCursor

type IDData

type IDData struct {
	Offset  int64
	Key     []byte
	Deleted bool
}

IDData keeps track of data about a series ID.

type IntegerArray

type IntegerArray = cursors.IntegerArray

func NewIntegerArrayLen

func NewIntegerArrayLen(sz int) *IntegerArray

type IntegerArrayCursor

type IntegerArrayCursor = cursors.IntegerArrayCursor

type MeasurementIterator

type MeasurementIterator interface {
	Close() error
	Next() ([]byte, error)
}

MeasurementIterator represents a iterator over a list of measurements.

func MergeMeasurementIterators

func MergeMeasurementIterators(itrs ...MeasurementIterator) MeasurementIterator

MergeMeasurementIterators returns an iterator that merges a set of iterators. Iterators that are first in the list take precedence and a deletion by those early iterators will invalidate elements by later iterators.

type MeasurementIterators

type MeasurementIterators []MeasurementIterator

func (MeasurementIterators) Close

func (a MeasurementIterators) Close() (err error)

type PartialWriteError

type PartialWriteError struct {
	Reason  string
	Dropped int

	// A sorted slice of series keys that were dropped.
	DroppedKeys [][]byte
}

PartialWriteError indicates a write request could only write a portion of the requested values.

func (PartialWriteError) Error

func (e PartialWriteError) Error() string

type SeriesCollection

type SeriesCollection struct {
	Points     []models.Point
	Keys       [][]byte
	SeriesKeys [][]byte
	Names      [][]byte
	Tags       []models.Tags
	Types      []models.FieldType
	SeriesIDs  []SeriesID

	// Keeps track of invalid entries.
	Dropped     uint64
	DroppedKeys [][]byte
	Reason      string
	// contains filtered or unexported fields
}

SeriesCollection is a struct of arrays representation of a collection of series that allows for efficient filtering.

func NewSeriesCollection

func NewSeriesCollection(points []models.Point) *SeriesCollection

NewSeriesCollection builds a SeriesCollection from a slice of points. It does some filtering of invalid points.

func (*SeriesCollection) Advance

func (s *SeriesCollection) Advance(length int)

Advance will advance all of the slices that can length elements: x = x[length:].

func (*SeriesCollection) ApplyConcurrentDrops

func (s *SeriesCollection) ApplyConcurrentDrops()

ApplyConcurrentDrops will remove all of the dropped values during concurrent iteration. It should not be called concurrently with any calls to Invalid.

func (*SeriesCollection) Copy

func (s *SeriesCollection) Copy(dst, src int)

Copy will copy the element at src into dst in all slices that can: x[dst] = x[src].

func (SeriesCollection) Duplicate

func (s SeriesCollection) Duplicate() *SeriesCollection

Duplicate returns a copy of the SeriesCollection. The slices are shared. Appending to any of them may or may not be reflected.

func (*SeriesCollection) InvalidateAll

func (s *SeriesCollection) InvalidateAll(reason string)

InvalidateAll causes all of the entries to become invalid.

func (*SeriesCollection) Iterator

func (s *SeriesCollection) Iterator() SeriesCollectionIterator

Iterator returns a new iterator over the entries in the collection. Multiple iterators can exist at the same time. Marking entries as invalid/skipped is more expensive, but thread safe. You must call ApplyConcurrentDrops after all of the iterators are finished.

func (*SeriesCollection) Length

func (s *SeriesCollection) Length() int

Length returns the length of the first non-nil slice in the collection, or 0 if there is no non-nil slice.

func (*SeriesCollection) PartialWriteError

func (s *SeriesCollection) PartialWriteError() error

PartialWriteError returns a PartialWriteError if any entries have been marked as invalid. It returns an error to avoid `return collection.PartialWriteError()` always being non-nil.

func (*SeriesCollection) Swap

func (s *SeriesCollection) Swap(i, j int)

Swap will swap the elements at i and j in all slices that can: x[i], x[j] = x[j], x[i].

func (*SeriesCollection) Truncate

func (s *SeriesCollection) Truncate(length int)

Truncate will truncate all of the slices that can down to length: x = x[:length].

type SeriesCollectionIterator

type SeriesCollectionIterator struct {
	// contains filtered or unexported fields
}

SeriesCollectionIterator is an iterator over the collection of series.

func (SeriesCollectionIterator) Index

func (i SeriesCollectionIterator) Index() int

func (*SeriesCollectionIterator) Invalid

func (i *SeriesCollectionIterator) Invalid(reason string)

Invalid flags the current entry as invalid, including it in the set of dropped keys and recording a reason. Only the first reason is kept. This is safe for concurrent callers, but ApplyConcurrentDrops must be called after all iterators are finished.

func (SeriesCollectionIterator) Key

func (i SeriesCollectionIterator) Key() []byte

func (SeriesCollectionIterator) Length

func (i SeriesCollectionIterator) Length() int

func (SeriesCollectionIterator) Name

func (i SeriesCollectionIterator) Name() []byte

func (*SeriesCollectionIterator) Next

func (i *SeriesCollectionIterator) Next() bool

Next advances the iterator and returns false if it's done.

func (SeriesCollectionIterator) Point

func (i SeriesCollectionIterator) Point() models.Point

func (SeriesCollectionIterator) SeriesID

func (i SeriesCollectionIterator) SeriesID() SeriesID

func (SeriesCollectionIterator) SeriesKey

func (i SeriesCollectionIterator) SeriesKey() []byte

func (SeriesCollectionIterator) Tags

func (i SeriesCollectionIterator) Tags() models.Tags

func (SeriesCollectionIterator) Type

func (i SeriesCollectionIterator) Type() models.FieldType

type SeriesFile

type SeriesFile struct {
	LargeWriteThreshold int

	Logger *zap.Logger
	// contains filtered or unexported fields
}

SeriesFile represents the section of the index that holds series data.

func NewSeriesFile

func NewSeriesFile(path string) *SeriesFile

NewSeriesFile returns a new instance of SeriesFile.

func (*SeriesFile) Acquire

func (f *SeriesFile) Acquire() (*lifecycle.Reference, error)

Acquire ensures that the series file won't be closed until after the reference has been released.

func (*SeriesFile) Close

func (f *SeriesFile) Close() error

Close unmaps the data file.

func (*SeriesFile) CreateSeriesListIfNotExists

func (f *SeriesFile) CreateSeriesListIfNotExists(collection *SeriesCollection) error

CreateSeriesListIfNotExists creates a list of series in bulk if they don't exist. It overwrites the collection's Keys and SeriesIDs fields. The collection's SeriesIDs slice will have IDs for every name+tags, creating new series IDs as needed. If any SeriesID is zero, then a type conflict has occurred for that series.

func (*SeriesFile) DeleteSeriesID

func (f *SeriesFile) DeleteSeriesID(id SeriesID) error

DeleteSeriesID flags a series as permanently deleted. If the series is reintroduced later then it must create a new id.

func (*SeriesFile) DisableCompactions

func (f *SeriesFile) DisableCompactions()

DisableCompactions prevents new compactions from running.

func (*SeriesFile) DisableMetrics

func (f *SeriesFile) DisableMetrics()

DisableMetrics ensures that activity is not collected via the prometheus metrics. DisableMetrics must be called before Open.

func (*SeriesFile) EnableCompactions

func (f *SeriesFile) EnableCompactions()

EnableCompactions allows compactions to run.

func (*SeriesFile) HasSeries

func (f *SeriesFile) HasSeries(name []byte, tags models.Tags, buf []byte) bool

HasSeries return true if the series exists.

func (*SeriesFile) IsDeleted

func (f *SeriesFile) IsDeleted(id SeriesID) bool

IsDeleted returns true if the ID has been deleted before.

func (*SeriesFile) Open

func (f *SeriesFile) Open(ctx context.Context) error

Open memory maps the data file at the file's path.

func (*SeriesFile) Partitions

func (f *SeriesFile) Partitions() []*SeriesPartition

Partitions returns all partitions.

func (*SeriesFile) Path

func (f *SeriesFile) Path() string

Path returns the path to the file.

func (*SeriesFile) Series

func (f *SeriesFile) Series(id SeriesID) ([]byte, models.Tags)

Series returns the parsed series name and tags for an offset.

func (*SeriesFile) SeriesCount

func (f *SeriesFile) SeriesCount() uint64

SeriesCount returns the number of series.

func (*SeriesFile) SeriesID

func (f *SeriesFile) SeriesID(name []byte, tags models.Tags, buf []byte) SeriesID

SeriesID returns the series id for the series.

func (*SeriesFile) SeriesIDIterator

func (f *SeriesFile) SeriesIDIterator() SeriesIDIterator

SeriesIterator returns an iterator over all the series.

func (*SeriesFile) SeriesIDPartition

func (f *SeriesFile) SeriesIDPartition(id SeriesID) *SeriesPartition

func (*SeriesFile) SeriesIDPartitionID

func (f *SeriesFile) SeriesIDPartitionID(id SeriesID) int

func (*SeriesFile) SeriesIDTyped

func (f *SeriesFile) SeriesIDTyped(name []byte, tags models.Tags, buf []byte) SeriesIDTyped

SeriesIDTyped returns the typed series id for the series.

func (*SeriesFile) SeriesIDTypedBySeriesKey

func (f *SeriesFile) SeriesIDTypedBySeriesKey(key []byte) SeriesIDTyped

SeriesIDTypedBySeriesKey returns the typed series id for the series.

func (*SeriesFile) SeriesKey

func (f *SeriesFile) SeriesKey(id SeriesID) []byte

SeriesKey returns the series key for a given id.

func (*SeriesFile) SeriesKeyName

func (f *SeriesFile) SeriesKeyName(id SeriesID) []byte

SeriesKeyName returns the measurement name for a series id.

func (*SeriesFile) SeriesKeyPartition

func (f *SeriesFile) SeriesKeyPartition(key []byte) *SeriesPartition

func (*SeriesFile) SeriesKeyPartitionID

func (f *SeriesFile) SeriesKeyPartitionID(key []byte) int

func (*SeriesFile) SeriesKeys

func (f *SeriesFile) SeriesKeys(ids []SeriesID) [][]byte

SeriesKeys returns a list of series keys from a list of ids.

func (*SeriesFile) SeriesKeysPartitionIDs

func (f *SeriesFile) SeriesKeysPartitionIDs(keys [][]byte) []int

func (*SeriesFile) SeriesPartitionPath

func (f *SeriesFile) SeriesPartitionPath(i int) string

SeriesPartitionPath returns the path to a given partition.

func (*SeriesFile) SetDefaultMetricLabels

func (f *SeriesFile) SetDefaultMetricLabels(labels prometheus.Labels)

SetDefaultMetricLabels sets the default labels for metrics on the Series File. It must be called before the SeriesFile is opened.

func (*SeriesFile) WithLogger

func (f *SeriesFile) WithLogger(log *zap.Logger)

WithLogger sets the logger on the SeriesFile and all underlying partitions. It must be called before Open.

type SeriesID

type SeriesID struct{ ID uint64 }

SeriesID is the type of a series id. It is logically a uint64, but encoded as a struct so that we gain more type checking when changing operations on it. The field is exported only so that tests that use reflection based comparisons still work; no one should use the field directly.

func NewSeriesID

func NewSeriesID(id uint64) SeriesID

NewSeriesID constructs a series id from the raw value. It discards any type information.

func ReadAllSeriesIDIterator

func ReadAllSeriesIDIterator(itr SeriesIDIterator) ([]SeriesID, error)

ReadAllSeriesIDIterator returns all ids from the iterator.

func (SeriesID) Greater

func (s SeriesID) Greater(o SeriesID) bool

Greater returns if the SeriesID is greater than the passed in value.

func (SeriesID) IsZero

func (s SeriesID) IsZero() bool

IsZero returns if the SeriesID is zero.

func (SeriesID) Less

func (s SeriesID) Less(o SeriesID) bool

Less returns if the SeriesID is less than the passed in value.

func (SeriesID) RawID

func (s SeriesID) RawID() uint64

ID returns the raw id for the SeriesID.

func (SeriesID) WithType

func (s SeriesID) WithType(typ models.FieldType) SeriesIDTyped

WithType constructs a SeriesIDTyped with the given type.

type SeriesIDElem

type SeriesIDElem struct {
	SeriesID SeriesID
	Expr     influxql.Expr
}

SeriesIDElem represents a single series and optional expression.

type SeriesIDElems

type SeriesIDElems []SeriesIDElem

SeriesIDElems represents a list of series id elements.

func (SeriesIDElems) Len

func (a SeriesIDElems) Len() int

func (SeriesIDElems) Less

func (a SeriesIDElems) Less(i, j int) bool

func (SeriesIDElems) Swap

func (a SeriesIDElems) Swap(i, j int)

type SeriesIDExprIterator

type SeriesIDExprIterator struct {
	// contains filtered or unexported fields
}

seriesIDExprIterator is an iterator that attaches an associated expression.

func (*SeriesIDExprIterator) Close

func (itr *SeriesIDExprIterator) Close() error

func (*SeriesIDExprIterator) Next

func (itr *SeriesIDExprIterator) Next() (SeriesIDElem, error)

Next returns the next element in the iterator.

type SeriesIDIterator

type SeriesIDIterator interface {
	Next() (SeriesIDElem, error)
	Close() error
}

SeriesIDIterator represents a iterator over a list of series ids.

func DifferenceSeriesIDIterators

func DifferenceSeriesIDIterators(itr0, itr1 SeriesIDIterator) SeriesIDIterator

DifferenceSeriesIDIterators returns an iterator that only returns series which occur the first iterator but not the second iterator.

func FilterUndeletedSeriesIDIterator

func FilterUndeletedSeriesIDIterator(sfile *SeriesFile, itr SeriesIDIterator) (SeriesIDIterator, error)

FilterUndeletedSeriesIDIterator returns an iterator which filters all deleted series.

func IntersectSeriesIDIterators

func IntersectSeriesIDIterators(itr0, itr1 SeriesIDIterator) SeriesIDIterator

IntersectSeriesIDIterators returns an iterator that only returns series which occur in both iterators. If both series have associated expressions then they are combined together.

func MergeSeriesIDIterators

func MergeSeriesIDIterators(itrs ...SeriesIDIterator) SeriesIDIterator

MergeSeriesIDIterators returns an iterator that merges a set of iterators. Iterators that are first in the list take precedence and a deletion by those early iterators will invalidate elements by later iterators.

func NewSeriesIDExprIterator

func NewSeriesIDExprIterator(itr SeriesIDIterator, expr influxql.Expr) SeriesIDIterator

newSeriesIDExprIterator returns a new instance of seriesIDExprIterator.

func UnionSeriesIDIterators

func UnionSeriesIDIterators(itr0, itr1 SeriesIDIterator) SeriesIDIterator

UnionSeriesIDIterators returns an iterator that returns series from both both iterators. If both series have associated expressions then they are combined together.

type SeriesIDIterators

type SeriesIDIterators []SeriesIDIterator

func (SeriesIDIterators) Close

func (a SeriesIDIterators) Close() (err error)

type SeriesIDSet

type SeriesIDSet struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

SeriesIDSet represents a lockable bitmap of series ids.

func NewSeriesIDSet

func NewSeriesIDSet(a ...SeriesID) *SeriesIDSet

NewSeriesIDSet returns a new instance of SeriesIDSet.

func NewSeriesIDSetNegate

func NewSeriesIDSetNegate(a, b *SeriesIDSet) *SeriesIDSet

NewSeriesIDSetNegate returns a new SeriesIDSet containing all the elements in a that are not present in b. That is, the set difference between a and b.

func (*SeriesIDSet) Add

func (s *SeriesIDSet) Add(id SeriesID)

Add adds the series id to the set.

func (*SeriesIDSet) AddMany

func (s *SeriesIDSet) AddMany(ids ...SeriesID)

AddMany adds multiple ids to the SeriesIDSet. AddMany takes a lock, so may not be optimal to call many times with few ids.

func (*SeriesIDSet) AddNoLock

func (s *SeriesIDSet) AddNoLock(id SeriesID)

AddNoLock adds the series id to the set. Add is not safe for use from multiple goroutines. Callers must manage synchronization.

func (*SeriesIDSet) And

func (s *SeriesIDSet) And(other *SeriesIDSet) *SeriesIDSet

And returns a new SeriesIDSet containing elements that were present in s and other.

func (*SeriesIDSet) Bytes

func (s *SeriesIDSet) Bytes() int

Bytes estimates the memory footprint of this SeriesIDSet, in bytes.

func (*SeriesIDSet) Cardinality

func (s *SeriesIDSet) Cardinality() uint64

Cardinality returns the cardinality of the SeriesIDSet.

func (*SeriesIDSet) Clear

func (s *SeriesIDSet) Clear()

Clear clears the underlying bitmap for re-use. Clear is safe for use by multiple goroutines.

func (*SeriesIDSet) ClearNoLock

func (s *SeriesIDSet) ClearNoLock()

ClearNoLock clears the underlying bitmap for re-use without taking a lock.

func (*SeriesIDSet) Clone

func (s *SeriesIDSet) Clone() *SeriesIDSet

Clone returns a new SeriesIDSet with a deep copy of the underlying bitmap.

func (*SeriesIDSet) CloneNoLock

func (s *SeriesIDSet) CloneNoLock() *SeriesIDSet

CloneNoLock calls Clone without taking a lock.

func (*SeriesIDSet) Contains

func (s *SeriesIDSet) Contains(id SeriesID) bool

Contains returns true if the id exists in the set.

func (*SeriesIDSet) ContainsNoLock

func (s *SeriesIDSet) ContainsNoLock(id SeriesID) bool

ContainsNoLock returns true if the id exists in the set. ContainsNoLock is not safe for use from multiple goroutines. The caller must manage synchronization.

func (*SeriesIDSet) Diff

func (s *SeriesIDSet) Diff(other *SeriesIDSet)

Diff removes from s any elements also present in other.

func (*SeriesIDSet) Equals

func (s *SeriesIDSet) Equals(other *SeriesIDSet) bool

Equals returns true if other and s are the same set of ids.

func (*SeriesIDSet) ForEach

func (s *SeriesIDSet) ForEach(f func(id SeriesID))

ForEach calls f for each id in the set. The function is applied to the IDs in ascending order.

func (*SeriesIDSet) ForEachNoLock

func (s *SeriesIDSet) ForEachNoLock(f func(id SeriesID))

ForEachNoLock calls f for each id in the set without taking a lock.

func (*SeriesIDSet) Iterator

func (s *SeriesIDSet) Iterator() SeriesIDSetIterable

Iterator returns an iterator to the underlying bitmap. This iterator is not protected by a lock.

func (*SeriesIDSet) Merge

func (s *SeriesIDSet) Merge(others ...*SeriesIDSet)

Merge merged the contents of others into s. The caller does not need to provide s as an argument, and the contents of s will always be present in s after Merge returns.

func (*SeriesIDSet) MergeInPlace

func (s *SeriesIDSet) MergeInPlace(other *SeriesIDSet)

MergeInPlace merges other into s, modifying s in the process.

func (*SeriesIDSet) Remove

func (s *SeriesIDSet) Remove(id SeriesID)

Remove removes the id from the set.

func (*SeriesIDSet) RemoveNoLock

func (s *SeriesIDSet) RemoveNoLock(id SeriesID)

RemoveNoLock removes the id from the set. RemoveNoLock is not safe for use from multiple goroutines. The caller must manage synchronization.

func (*SeriesIDSet) RemoveSet

func (s *SeriesIDSet) RemoveSet(other *SeriesIDSet)

RemoveSet removes all values in other from s, if they exist.

func (*SeriesIDSet) Slice

func (s *SeriesIDSet) Slice() []uint64

Slice returns a slice of series ids.

func (*SeriesIDSet) String

func (s *SeriesIDSet) String() string

func (*SeriesIDSet) UnmarshalBinary

func (s *SeriesIDSet) UnmarshalBinary(data []byte) error

UnmarshalBinary unmarshals data into the set.

func (*SeriesIDSet) UnmarshalBinaryUnsafe

func (s *SeriesIDSet) UnmarshalBinaryUnsafe(data []byte) error

UnmarshalBinaryUnsafe unmarshals data into the set. References to the underlying data are used so data should not be reused by caller.

func (*SeriesIDSet) WriteTo

func (s *SeriesIDSet) WriteTo(w io.Writer) (int64, error)

WriteTo writes the set to w.

type SeriesIDSetIterable

type SeriesIDSetIterable interface {
	HasNext() bool
	Next() uint32
}

type SeriesIDSetIterator

type SeriesIDSetIterator interface {
	SeriesIDIterator
	SeriesIDSet() *SeriesIDSet
}

SeriesIDSetIterator represents an iterator that can produce a SeriesIDSet.

func NewSeriesIDSetIterator

func NewSeriesIDSetIterator(ss *SeriesIDSet) SeriesIDSetIterator

func NewSeriesIDSetIterators

func NewSeriesIDSetIterators(itrs []SeriesIDIterator) []SeriesIDSetIterator

NewSeriesIDSetIterators returns a slice of SeriesIDSetIterator if all itrs can be type casted. Otherwise returns nil.

type SeriesIDSliceIterator

type SeriesIDSliceIterator struct {
	// contains filtered or unexported fields
}

SeriesIDSliceIterator iterates over a slice of series ids.

func NewSeriesIDSliceIterator

func NewSeriesIDSliceIterator(ids []SeriesID) *SeriesIDSliceIterator

NewSeriesIDSliceIterator returns a SeriesIDIterator that iterates over a slice.

func (*SeriesIDSliceIterator) Close

func (itr *SeriesIDSliceIterator) Close() error

func (*SeriesIDSliceIterator) Next

func (itr *SeriesIDSliceIterator) Next() (SeriesIDElem, error)

Next returns the next series id in the slice.

func (*SeriesIDSliceIterator) SeriesIDSet

func (itr *SeriesIDSliceIterator) SeriesIDSet() *SeriesIDSet

SeriesIDSet returns a set of all remaining ids.

type SeriesIDTyped

type SeriesIDTyped struct{ ID uint64 }

SeriesIDType represents a series id with a type. It is logically a uint64, but encoded as a struct so that we gain more type checking when changing operations on it. The field is exported only so that tests that use reflection based comparisons still work; no one should use the field directly.

func NewSeriesIDTyped

func NewSeriesIDTyped(id uint64) SeriesIDTyped

NewSeriesIDTyped constructs a typed series id from the raw values.

func ReadSeriesEntry

func ReadSeriesEntry(data []byte) (flag uint8, id SeriesIDTyped, key []byte, sz int64)

func (SeriesIDTyped) HasType

func (s SeriesIDTyped) HasType() bool

HasType returns if the id actually contains a type.

func (SeriesIDTyped) IsZero

func (s SeriesIDTyped) IsZero() bool

IsZero returns if the SeriesIDTyped is zero. It ignores any type information.

func (SeriesIDTyped) RawID

func (s SeriesIDTyped) RawID() uint64

ID returns the raw id for the SeriesIDTyped.

func (SeriesIDTyped) SeriesID

func (s SeriesIDTyped) SeriesID() SeriesID

SeriesID constructs a SeriesID, discarding any type information.

func (SeriesIDTyped) Type

func (s SeriesIDTyped) Type() models.FieldType

Type returns the associated type.

type SeriesIndex

type SeriesIndex struct {
	// contains filtered or unexported fields
}

SeriesIndex represents an index of key-to-id & id-to-offset mappings.

func NewSeriesIndex

func NewSeriesIndex(path string) *SeriesIndex

func (*SeriesIndex) Clone

func (idx *SeriesIndex) Clone() *SeriesIndex

Clone returns a copy of idx for use during compaction. In-memory maps are not cloned.

func (*SeriesIndex) Close

func (idx *SeriesIndex) Close() (err error)

Close unmaps the index file.

func (*SeriesIndex) Count

func (idx *SeriesIndex) Count() uint64

Count returns the number of series in the index.

func (*SeriesIndex) Delete

func (idx *SeriesIndex) Delete(id SeriesID)

Delete marks the series id as deleted.

func (*SeriesIndex) FindIDByNameTags

func (idx *SeriesIndex) FindIDByNameTags(segments []*SeriesSegment, name []byte, tags models.Tags, buf []byte) SeriesIDTyped

func (*SeriesIndex) FindIDBySeriesKey

func (idx *SeriesIndex) FindIDBySeriesKey(segments []*SeriesSegment, key []byte) SeriesIDTyped

func (*SeriesIndex) FindIDListByNameTags

func (idx *SeriesIndex) FindIDListByNameTags(segments []*SeriesSegment, names [][]byte, tagsSlice []models.Tags, buf []byte) (ids []SeriesIDTyped, ok bool)

func (*SeriesIndex) FindOffsetByID

func (idx *SeriesIndex) FindOffsetByID(id SeriesID) int64

func (*SeriesIndex) GrowBy

func (idx *SeriesIndex) GrowBy(delta int)

GrowBy preallocates the in-memory hashmap to a larger size.

func (*SeriesIndex) InMemCount

func (idx *SeriesIndex) InMemCount() uint64

InMemCount returns the number of series in the in-memory index.

func (*SeriesIndex) InMemSize

func (idx *SeriesIndex) InMemSize() uint64

InMemSize returns the heap size of the index in bytes. The returned value is an estimation and does not include include all allocated memory.

func (*SeriesIndex) Insert

func (idx *SeriesIndex) Insert(key []byte, id SeriesIDTyped, offset int64)

func (*SeriesIndex) IsDeleted

func (idx *SeriesIndex) IsDeleted(id SeriesID) bool

IsDeleted returns true if series id has been deleted.

func (*SeriesIndex) OnDiskCount

func (idx *SeriesIndex) OnDiskCount() uint64

OnDiskCount returns the number of series in the on-disk index.

func (*SeriesIndex) OnDiskSize

func (idx *SeriesIndex) OnDiskSize() uint64

OnDiskSize returns the on-disk size of the index in bytes.

func (*SeriesIndex) Open

func (idx *SeriesIndex) Open() (err error)

Open memory-maps the index file.

func (*SeriesIndex) Recover

func (idx *SeriesIndex) Recover(segments []*SeriesSegment) error

Recover rebuilds the in-memory index for all new entries.

type SeriesIndexHeader

type SeriesIndexHeader struct {
	Version uint8

	MaxSeriesID SeriesID
	MaxOffset   int64

	Count    uint64
	Capacity int64

	KeyIDMap struct {
		Offset int64
		Size   int64
	}

	IDOffsetMap struct {
		Offset int64
		Size   int64
	}
}

SeriesIndexHeader represents the header of a series index.

func NewSeriesIndexHeader

func NewSeriesIndexHeader() SeriesIndexHeader

NewSeriesIndexHeader returns a new instance of SeriesIndexHeader.

func ReadSeriesIndexHeader

func ReadSeriesIndexHeader(data []byte) (hdr SeriesIndexHeader, err error)

ReadSeriesIndexHeader returns the header from data.

func (*SeriesIndexHeader) WriteTo

func (hdr *SeriesIndexHeader) WriteTo(w io.Writer) (n int64, err error)

WriteTo writes the header to w.

type SeriesPartition

type SeriesPartition struct {
	CompactThreshold    int
	LargeWriteThreshold int

	Logger *zap.Logger
	// contains filtered or unexported fields
}

SeriesPartition represents a subset of series file data.

func NewSeriesPartition

func NewSeriesPartition(id int, path string) *SeriesPartition

NewSeriesPartition returns a new instance of SeriesPartition.

func (*SeriesPartition) AppendSeriesIDs

func (p *SeriesPartition) AppendSeriesIDs(a []SeriesID) []SeriesID

AppendSeriesIDs returns a list of all series ids.

func (*SeriesPartition) Close

func (p *SeriesPartition) Close() (err error)

Close unmaps the data files.

func (*SeriesPartition) Compacting

func (p *SeriesPartition) Compacting() bool

Compacting returns if the SeriesPartition is currently compacting.

func (*SeriesPartition) CreateSeriesListIfNotExists

func (p *SeriesPartition) CreateSeriesListIfNotExists(collection *SeriesCollection, keyPartitionIDs []int) error

CreateSeriesListIfNotExists creates a list of series in bulk if they don't exist. The ids parameter is modified to contain series IDs for all keys belonging to this partition. If the type does not match the existing type for the key, a zero id is stored.

func (*SeriesPartition) DeleteSeriesID

func (p *SeriesPartition) DeleteSeriesID(id SeriesID) error

DeleteSeriesID flags a series as permanently deleted. If the series is reintroduced later then it must create a new id.

func (*SeriesPartition) DisableCompactions

func (p *SeriesPartition) DisableCompactions()

func (*SeriesPartition) DiskSize

func (p *SeriesPartition) DiskSize() uint64

DiskSize returns the number of bytes taken up on disk by the partition.

func (*SeriesPartition) EnableCompactions

func (p *SeriesPartition) EnableCompactions()

func (*SeriesPartition) FindIDBySeriesKey

func (p *SeriesPartition) FindIDBySeriesKey(key []byte) SeriesID

FindIDBySeriesKey return the series id for the series key.

func (*SeriesPartition) FindIDTypedBySeriesKey

func (p *SeriesPartition) FindIDTypedBySeriesKey(key []byte) SeriesIDTyped

FindIDTypedBySeriesKey return the typed series id for the series key.

func (*SeriesPartition) ID

func (p *SeriesPartition) ID() int

ID returns the partition id.

func (*SeriesPartition) IndexPath

func (p *SeriesPartition) IndexPath() string

IndexPath returns the path to the series index.

func (*SeriesPartition) IsDeleted

func (p *SeriesPartition) IsDeleted(id SeriesID) bool

IsDeleted returns true if the ID has been deleted before.

func (*SeriesPartition) Open

func (p *SeriesPartition) Open() error

Open memory maps the data file at the partition's path.

func (*SeriesPartition) Path

func (p *SeriesPartition) Path() string

Path returns the path to the partition.

func (*SeriesPartition) Series

func (p *SeriesPartition) Series(id SeriesID) ([]byte, models.Tags)

Series returns the parsed series name and tags for an offset.

func (*SeriesPartition) SeriesCount

func (p *SeriesPartition) SeriesCount() uint64

SeriesCount returns the number of series.

func (*SeriesPartition) SeriesKey

func (p *SeriesPartition) SeriesKey(id SeriesID) []byte

SeriesKey returns the series key for a given id.

type SeriesPartitionCompactor

type SeriesPartitionCompactor struct {
	// contains filtered or unexported fields
}

SeriesPartitionCompactor represents an object reindexes a series partition and optionally compacts segments.

func NewSeriesPartitionCompactor

func NewSeriesPartitionCompactor() *SeriesPartitionCompactor

NewSeriesPartitionCompactor returns a new instance of SeriesPartitionCompactor.

func (*SeriesPartitionCompactor) Compact

func (c *SeriesPartitionCompactor) Compact(p *SeriesPartition) (time.Duration, error)

Compact rebuilds the series partition index.

type SeriesSegment

type SeriesSegment struct {
	// contains filtered or unexported fields
}

SeriesSegment represents a log of series entries.

func CloneSeriesSegments

func CloneSeriesSegments(a []*SeriesSegment) []*SeriesSegment

CloneSeriesSegments returns a copy of a slice of segments.

func CreateSeriesSegment

func CreateSeriesSegment(id uint16, path string) (*SeriesSegment, error)

CreateSeriesSegment generates an empty segment at path.

func FindSegment

func FindSegment(a []*SeriesSegment, id uint16) *SeriesSegment

FindSegment returns a segment by id.

func NewSeriesSegment

func NewSeriesSegment(id uint16, path string) *SeriesSegment

NewSeriesSegment returns a new instance of SeriesSegment.

func (*SeriesSegment) AppendSeriesIDs

func (s *SeriesSegment) AppendSeriesIDs(a []SeriesID) []SeriesID

AppendSeriesIDs appends all the segments ids to a slice. Returns the new slice.

func (*SeriesSegment) CanWrite

func (s *SeriesSegment) CanWrite(data []byte) bool

CanWrite returns true if segment has space to write entry data.

func (*SeriesSegment) Clone

func (s *SeriesSegment) Clone() *SeriesSegment

Clone returns a copy of the segment. Excludes the write handler, if set.

func (*SeriesSegment) Close

func (s *SeriesSegment) Close() (err error)

Close unmaps the segment.

func (*SeriesSegment) CloseForWrite

func (s *SeriesSegment) CloseForWrite() (err error)

func (*SeriesSegment) Data

func (s *SeriesSegment) Data() []byte

Data returns the raw data.

func (*SeriesSegment) Flush

func (s *SeriesSegment) Flush() error

Flush flushes the buffer to disk.

func (*SeriesSegment) ForEachEntry

func (s *SeriesSegment) ForEachEntry(fn func(flag uint8, id SeriesIDTyped, offset int64, key []byte) error) error

ForEachEntry executes fn for every entry in the segment.

func (*SeriesSegment) ID

func (s *SeriesSegment) ID() uint16

ID returns the id the segment was initialized with.

func (*SeriesSegment) InitForWrite

func (s *SeriesSegment) InitForWrite() (err error)

InitForWrite initializes a write handle for the segment. This is only used for the last segment in the series file.

func (*SeriesSegment) MaxSeriesID

func (s *SeriesSegment) MaxSeriesID() SeriesID

MaxSeriesID returns the highest series id in the segment.

func (*SeriesSegment) Open

func (s *SeriesSegment) Open() error

Open memory maps the data file at the file's path.

func (*SeriesSegment) Size

func (s *SeriesSegment) Size() int64

Size returns the size of the data in the segment. This is only populated once InitForWrite() is called.

func (*SeriesSegment) Slice

func (s *SeriesSegment) Slice(pos uint32) []byte

Slice returns a byte slice starting at pos.

func (*SeriesSegment) WriteLogEntry

func (s *SeriesSegment) WriteLogEntry(data []byte) (offset int64, err error)

WriteLogEntry writes entry data into the segment. Returns the offset of the beginning of the entry.

type SeriesSegmentHeader

type SeriesSegmentHeader struct {
	Version uint8
}

SeriesSegmentHeader represents the header of a series segment.

func NewSeriesSegmentHeader

func NewSeriesSegmentHeader() SeriesSegmentHeader

NewSeriesSegmentHeader returns a new instance of SeriesSegmentHeader.

func ReadSeriesSegmentHeader

func ReadSeriesSegmentHeader(data []byte) (hdr SeriesSegmentHeader, err error)

ReadSeriesSegmentHeader returns the header from data.

func (*SeriesSegmentHeader) WriteTo

func (hdr *SeriesSegmentHeader) WriteTo(w io.Writer) (n int64, err error)

WriteTo writes the header to w.

type StringArray

type StringArray = cursors.StringArray

func NewStringArrayLen

func NewStringArrayLen(sz int) *StringArray

type StringArrayCursor

type StringArrayCursor = cursors.StringArrayCursor

type TagKeyIterator

type TagKeyIterator interface {
	Close() error
	Next() ([]byte, error)
}

TagKeyIterator represents a iterator over a list of tag keys.

func MergeTagKeyIterators

func MergeTagKeyIterators(itrs ...TagKeyIterator) TagKeyIterator

MergeTagKeyIterators returns an iterator that merges a set of iterators.

type TagKeyIterators

type TagKeyIterators []TagKeyIterator

func (TagKeyIterators) Close

func (a TagKeyIterators) Close() (err error)

type TagValueIterator

type TagValueIterator interface {
	Close() error
	Next() ([]byte, error)
}

TagValueIterator represents a iterator over a list of tag values.

func MergeTagValueIterators

func MergeTagValueIterators(itrs ...TagValueIterator) TagValueIterator

MergeTagValueIterators returns an iterator that merges a set of iterators.

type TagValueIterators

type TagValueIterators []TagValueIterator

func (TagValueIterators) Close

func (a TagValueIterators) Close() (err error)

type TimestampArray

type TimestampArray = cursors.TimestampArray

type UnsignedArray

type UnsignedArray = cursors.UnsignedArray

func NewUnsignedArrayLen

func NewUnsignedArrayLen(sz int) *UnsignedArray

type UnsignedArrayCursor

type UnsignedArrayCursor = cursors.UnsignedArrayCursor

type Verify

type Verify struct {
	Concurrent int
	Logger     *zap.Logger
	// contains filtered or unexported fields
}

Verify contains configuration for running verification of series files.

func NewVerify

func NewVerify() Verify

NewVerify constructs a Verify with good defaults.

func (Verify) VerifyIndex

func (v Verify) VerifyIndex(indexPath string, segments []*SeriesSegment,
	ids map[uint64]IDData) (valid bool, err error)

VerifyIndex performs verification on an index in a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the partition. The ids map must be built from verifying the passed in segments.

func (Verify) VerifyPartition

func (v Verify) VerifyPartition(partitionPath string) (valid bool, err error)

VerifyPartition performs verifications on a partition of a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the partition.

func (Verify) VerifySegment

func (v Verify) VerifySegment(segmentPath string, ids map[uint64]IDData) (valid bool, err error)

VerifySegment performs verifications on a segment of a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the partition. The ids map is populated with information about the ids stored in the segment.

func (Verify) VerifySeriesFile

func (v Verify) VerifySeriesFile(filePath string) (valid bool, err error)

VerifySeriesFile performs verifications on a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the series file.

Documentation was rendered with GOOS=linux and GOARCH=amd64.

Jump to identifier

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to identifier