wal

package
v0.0.0-...-1b5f0b1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 20, 2017 License: GPL-3.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultSegmentSize of 65MB is the size at which segment files will be rolled over.
	DefaultSegmentSize = 64 * 1024 * 1024

	// WALFileExtension is the file extension we expect for wal segments.
	WALFileExtension = "wal"

	// WALFilePrefix is the prefix on all wal segment files.
	WALFilePrefix = "_"
)
View Source 
const (
	// BlockFloat64 designates a block encodes float64 values.
	BlockFloat64 = byte(0)

	// BlockInteger designates a block encodes int64 values.
	BlockInteger = byte(1)

	// BlockBoolean designates a block encodes boolean values.
	BlockBoolean = byte(2)

	// BlockString designates a block encodes string values.
	BlockString = byte(3)

	// BlockUnsigned designates a block encodes uint64 values.
	BlockUnsigned = byte(4)

	// ZeroTime is the Unix nanosecond timestamp for no time.
	// This time is not used by the query engine or the storage engine as a valid time.
	ZeroTime = int64(math.MinInt64)

	// DefaultMaxPointsPerBlock is the maximum number of points in an encoded
	// block in a TSM file
	DefaultMaxPointsPerBlock = 1000
)

Variables

View Source 
var (
	// ErrSettings is returned when no setting is passed
	ErrSettings = fmt.Errorf("missing settings")
	// ErrWALClosed is returned when attempting to write to a closed WAL file.
	ErrWALClosed = fmt.Errorf("WAL closed")

	// ErrWALCorrupt is returned when reading a corrupt WAL entry.
	ErrWALCorrupt = fmt.Errorf("corrupted WAL entry")
)

Functions

func BlockCount 

func BlockCount(block []byte) int

BlockCount returns the number of timestamps encoded in block.

func BlockType 

func BlockType(block []byte) (byte, error)

BlockType returns the type of value encoded in a block or an error if the block type is unknown.

func SegmentFileNames 

func SegmentFileNames(dir string) ([]string, error)

func ZigZagDecode 

func ZigZagDecode(v uint64) int64

ZigZagDecode converts a previously zigzag encoded uint64 back to a int64.

func ZigZagEncode 

func ZigZagEncode(x int64) uint64

ZigZagEncode converts a int64 to a uint64 by zig zagging negative and positive values across even and odd numbers. Eg. [0,-1,1,-2] becomes [0, 1, 2, 3].

Types

type BooleanValue 

type BooleanValue struct {
	// contains filtered or unexported fields
}

BooleanValue represents a boolean value.

func DecodeBooleanBlock 

func DecodeBooleanBlock(block []byte, a *[]BooleanValue) ([]BooleanValue, error)

DecodeBooleanBlock decodes the boolean block from the byte slice and appends the boolean values to a.

func (BooleanValue) Size 

func (v BooleanValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (BooleanValue) String 

func (v BooleanValue) String() string

String returns the string representation of the value and its timestamp.

func (BooleanValue) UnixNano 

func (v BooleanValue) UnixNano() int64

UnixNano returns the timestamp of the value in nanoseconds since unix epoch.

func (BooleanValue) Value 

func (v BooleanValue) Value() interface{}

Value returns the underlying boolean value.

type Bytes 

type Bytes struct {
	// contains filtered or unexported fields
}

Bytes is a pool of byte slices that can be re-used. Slices in this pool will not be garbage collected when not in use.

func NewBytes 

func NewBytes(max int) *Bytes

NewBytes returns a Bytes pool with capacity for max byte slices to be pool.

func (*Bytes) Get 

func (p *Bytes) Get(sz int) []byte

Get returns a byte slice size with at least sz capacity. Items returned may not be in the zero state and should be reset by the caller.

func (*Bytes) Put 

func (p *Bytes) Put(c []byte)

Put returns a slice back to the pool. If the pool is full, the byte slice is discarded.

type DeleteRangeWALEntry 

type DeleteRangeWALEntry struct {
	Keys     [][]byte
	Min, Max int64
	// contains filtered or unexported fields
}

DeleteRangeWALEntry represents the deletion of multiple series.

func (*DeleteRangeWALEntry) Encode 

func (w *DeleteRangeWALEntry) Encode(b []byte) ([]byte, error)

Encode converts the DeleteRangeWALEntry into a byte slice, appending to b.

func (*DeleteRangeWALEntry) MarshalBinary 

func (w *DeleteRangeWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*DeleteRangeWALEntry) MarshalSize 

func (w *DeleteRangeWALEntry) MarshalSize() int

func (*DeleteRangeWALEntry) Type 

func (w *DeleteRangeWALEntry) Type() WalEntryType

Type returns DeleteRangeWALEntryType.

func (*DeleteRangeWALEntry) UnmarshalBinary 

func (w *DeleteRangeWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

type DeleteWALEntry 

type DeleteWALEntry struct {
	Keys [][]byte
	// contains filtered or unexported fields
}

DeleteWALEntry represents the deletion of multiple series.

func (*DeleteWALEntry) Encode 

func (w *DeleteWALEntry) Encode(dst []byte) ([]byte, error)

Encode converts the DeleteWALEntry into a byte slice, appending to dst.

func (*DeleteWALEntry) MarshalBinary 

func (w *DeleteWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*DeleteWALEntry) MarshalSize 

func (w *DeleteWALEntry) MarshalSize() int

func (*DeleteWALEntry) Type 

func (w *DeleteWALEntry) Type() WalEntryType

Type returns DeleteWALEntryType.

func (*DeleteWALEntry) UnmarshalBinary 

func (w *DeleteWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

type EmptyValue 

type EmptyValue struct{}

EmptyValue is used when there is no appropriate other value.

func (EmptyValue) Size 

func (e EmptyValue) Size() int

Size returns 0.

func (EmptyValue) String 

func (e EmptyValue) String() string

String returns the empty string.

func (EmptyValue) UnixNano 

func (e EmptyValue) UnixNano() int64

UnixNano returns ZeroTime

func (EmptyValue) Value 

func (e EmptyValue) Value() interface{}

Value returns nil.

type FloatValue 

type FloatValue struct {
	// contains filtered or unexported fields
}

FloatValue represents a float64 value.

func DecodeFloatBlock 

func DecodeFloatBlock(block []byte, a *[]FloatValue) ([]FloatValue, error)

DecodeFloatBlock decodes the float block from the byte slice and appends the float values to a.

func (FloatValue) Size 

func (v FloatValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (FloatValue) String 

func (v FloatValue) String() string

String returns the string representation of the value and its timestamp.

func (FloatValue) UnixNano 

func (v FloatValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (FloatValue) Value 

func (v FloatValue) Value() interface{}

Value returns the underlying float64 value.

type Generic 

type Generic struct {
	// contains filtered or unexported fields
}

Generic is a pool of types that can be re-used. Items in this pool will not be garbage collected when not in use.

func NewGeneric 

func NewGeneric(max int, fn func(sz int) interface{}) *Generic

NewGeneric returns a Generic pool with capacity for max items to be pool.

func (*Generic) Get 

func (p *Generic) Get(sz int) interface{}

Get returns a item from the pool or a new instance if the pool is empty. Items returned may not be in the zero state and should be reset by the caller.

func (*Generic) Put 

func (p *Generic) Put(c interface{})

Put returns an item back to the pool. If the pool is full, the item is discarded.

type IntegerValue 

type IntegerValue struct {
	// contains filtered or unexported fields
}

IntegerValue represents an int64 value.

func DecodeIntegerBlock 

func DecodeIntegerBlock(block []byte, a *[]IntegerValue) ([]IntegerValue, error)

DecodeIntegerBlock decodes the integer block from the byte slice and appends the integer values to a.

func (IntegerValue) Size 

func (v IntegerValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (IntegerValue) String 

func (v IntegerValue) String() string

String returns the string representation of the value and its timestamp.

func (IntegerValue) UnixNano 

func (v IntegerValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (IntegerValue) Value 

func (v IntegerValue) Value() interface{}

Value returns the underlying int64 value.

type LimitedBytes 

type LimitedBytes struct {
	// contains filtered or unexported fields
}

LimitedBytes is a pool of byte slices that can be re-used. Slices in this pool will not be garbage collected when not in use. The pool will hold onto a fixed number of byte slices of a maximum size. If the pool is empty and max pool size has not been allocated yet, it will return a new byte slice. Byte slices added to the pool that are over the max size are dropped.

func NewLimitedBytes 

func NewLimitedBytes(capacity int, maxSize int) *LimitedBytes

NewBytes returns a Bytes pool with capacity for max byte slices to be pool.

func (*LimitedBytes) Get 

func (p *LimitedBytes) Get(sz int) []byte

Get returns a byte slice size with at least sz capacity. Items returned may not be in the zero state and should be reset by the caller.

func (*LimitedBytes) Put 

func (p *LimitedBytes) Put(c []byte)

Put returns a slice back to the pool. If the pool is full, the byte slice is discarded. If the byte slice is over the configured max size of any byte slice in the pool, it is discared.

type LoadPoints 

type LoadPoints struct {
	KSTS    string
	Points  []byte
	BlockID int64
}

type Settings 

type Settings struct {
	PathWAL         string
	SyncInterval    string
	CleanupInterval string

	CheckPointInterval string
	CheckPointPath     string

	MaxBufferSize int
	MaxConcWrite  int
}

type Statistic 

type Statistic struct {
	Name   string                 `json:"name"`
	Tags   map[string]string      `json:"tags"`
	Values map[string]interface{} `json:"values"`
}

Statistic is the representation of a statistic used by the monitoring service.

type StringValue 

type StringValue struct {
	// contains filtered or unexported fields
}

StringValue represents a string value.

func DecodeStringBlock 

func DecodeStringBlock(block []byte, a *[]StringValue) ([]StringValue, error)

DecodeStringBlock decodes the string block from the byte slice and appends the string values to a.

func (StringValue) Size 

func (v StringValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (StringValue) String 

func (v StringValue) String() string

String returns the string representation of the value and its timestamp.

func (StringValue) UnixNano 

func (v StringValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (StringValue) Value 

func (v StringValue) Value() interface{}

Value returns the underlying string value.

type UnsignedValue 

type UnsignedValue struct {
	// contains filtered or unexported fields
}

UnsignedValue represents an int64 value.

func DecodeUnsignedBlock 

func DecodeUnsignedBlock(block []byte, a *[]UnsignedValue) ([]UnsignedValue, error)

DecodeUnsignedBlock decodes the unsigned integer block from the byte slice and appends the unsigned integer values to a.

func (UnsignedValue) Size 

func (v UnsignedValue) Size() int

Size returns the number of bytes necessary to represent the value and its timestamp.

func (UnsignedValue) String 

func (v UnsignedValue) String() string

String returns the string representation of the value and its timestamp.

func (UnsignedValue) UnixNano 

func (v UnsignedValue) UnixNano() int64

UnixNano returns the timestamp of the value.

func (UnsignedValue) Value 

func (v UnsignedValue) Value() interface{}

Value returns the underlying int64 value.

type Value 

type Value interface {
	// UnixNano returns the timestamp of the value in nanoseconds since unix epoch.
	UnixNano() int64

	// Value returns the underlying value.
	Value() interface{}

	// Size returns the number of bytes necessary to represent the value and its timestamp.
	Size() int

	// String returns the string representation of the value and its timestamp.
	String() string
	// contains filtered or unexported methods
}

Value represents a TSM-encoded value.

func DecodeBlock 

func DecodeBlock(block []byte, vals []Value) ([]Value, error)

DecodeBlock takes a byte slice and decodes it into values of the appropriate type based on the block.

func NewBooleanValue 

func NewBooleanValue(t int64, v bool) Value

NewBooleanValue returns a new boolean value.

func NewFloatValue 

func NewFloatValue(t int64, v float64) Value

NewFloatValue returns a new float value.

func NewIntegerValue 

func NewIntegerValue(t int64, v int64) Value

NewIntegerValue returns a new integer value.

func NewStringValue 

func NewStringValue(t int64, v string) Value

NewStringValue returns a new string value.

func NewUnsignedValue 

func NewUnsignedValue(t int64, v uint64) Value

NewUnsignedValue returns a new unsigned integer value.

func NewValue 

func NewValue(t int64, value interface{}) Value

NewValue returns a new Value with the underlying type dependent on value.

type Values 

type Values []Value

Values represents a slice of values.

func (Values) Encode 

func (a Values) Encode(buf []byte) ([]byte, error)

Encode converts the values to a byte slice. If there are no values, this function panics.

type WAL 

type WAL struct {

	// SegmentSize is the file size at which a segment file will be rotated
	SegmentSize int
	// contains filtered or unexported fields
}

WAL represents the write-ahead log used for writing TSM files.

func New 

func New(settings *Settings, l *zap.Logger) (*WAL, error)

NewWAL initializes a new WAL at the given directory.

func (*WAL) Add 

func (wal *WAL) Add(p *pb.Point)

Add append point at the end of the file

func (*WAL) Close 

func (l *WAL) Close() error

Close will finish any flush that is currently in progress and close file handles.

func (*WAL) CloseSegment 

func (l *WAL) CloseSegment() error

CloseSegment closes the current segment if it is non-empty and opens a new one.

func (*WAL) ClosedSegments 

func (l *WAL) ClosedSegments() ([]string, error)

ClosedSegments returns a slice of the names of the closed segment files.

func (*WAL) Delete 

func (l *WAL) Delete(keys [][]byte) (int64, error)

Delete deletes the given keys, returning the segment ID for the operation.

func (*WAL) DeleteRange 

func (l *WAL) DeleteRange(keys [][]byte, min, max int64) (int64, error)

DeleteRange deletes the given keys within the given time range, returning the segment ID for the operation.

func (*WAL) DeleteTT 

func (wal *WAL) DeleteTT(ksts string)

func (*WAL) DiskSizeBytes 

func (l *WAL) DiskSizeBytes() int64

func (*WAL) LastWriteTime 

func (l *WAL) LastWriteTime() time.Time

LastWriteTime is the last time anything was written to the WAL.

func (*WAL) Load 

func (wal *WAL) Load() <-chan LoadPoints

func (*WAL) Open 

func (l *WAL) Open() error

Open opens and initializes the Log. Open can recover from previous unclosed shutdowns.

func (*WAL) Path 

func (l *WAL) Path() string

Path returns the directory the log was initialized with.

func (*WAL) Remove 

func (l *WAL) Remove(files []string) error

Remove deletes the given segment file paths from disk and cleans up any associated objects.

func (*WAL) Replay 

func (wal *WAL) Replay(filename string) ([]*pb.Point, error)

func (*WAL) SetStats 

func (wal *WAL) SetStats(sts *tsstats.StatsTS)

Set wal stats

func (*WAL) SetTT 

func (wal *WAL) SetTT(ksts string, date int64)

func (*WAL) Start 

func (wal *WAL) Start()

Start dispatchs a goroutine with a ticker to save and sync points in disk

func (*WAL) Statistics 

func (l *WAL) Statistics(tags map[string]string) []Statistic

Statistics returns statistics for periodic monitoring.

func (*WAL) Stop 

func (wal *WAL) Stop()

func (*WAL) WithLogger 

func (l *WAL) WithLogger(log zap.Logger)

WithLogger sets the WAL's logger.

func (*WAL) WriteMulti 

func (l *WAL) WriteMulti(values map[string][]Value) (int64, error)

WriteMulti writes the given values to the WAL. It returns the WAL segment ID to which the points were written. If an error is returned the segment ID should be ignored.

type WALEntry 

type WALEntry interface {
	Type() WalEntryType
	Encode(dst []byte) ([]byte, error)
	MarshalBinary() ([]byte, error)
	UnmarshalBinary(b []byte) error
	MarshalSize() int
}

WALEntry is record stored in each WAL segment. Each entry has a type and an opaque, type dependent byte slice data attribute.

type WALSegmentReader 

type WALSegmentReader struct {
	// contains filtered or unexported fields
}

WALSegmentReader reads WAL segments.

func NewWALSegmentReader 

func NewWALSegmentReader(r io.ReadCloser) *WALSegmentReader

NewWALSegmentReader returns a new WALSegmentReader reading from r.

func (*WALSegmentReader) Close 

func (r *WALSegmentReader) Close() error

Close closes the underlying io.Reader.

func (*WALSegmentReader) Count 

func (r *WALSegmentReader) Count() int64

Count returns the total number of bytes read successfully from the segment, as of the last call to Read(). The segment is guaranteed to be valid up to and including this number of bytes.

func (*WALSegmentReader) Error 

func (r *WALSegmentReader) Error() error

Error returns the last error encountered by the reader.

func (*WALSegmentReader) Next 

func (r *WALSegmentReader) Next() bool

Next indicates if there is a value to read.

func (*WALSegmentReader) Read 

func (r *WALSegmentReader) Read() (WALEntry, error)

Read returns the next entry in the reader.

type WALSegmentWriter 

type WALSegmentWriter struct {
	// contains filtered or unexported fields
}

WALSegmentWriter writes WAL segments.

func NewWALSegmentWriter 

func NewWALSegmentWriter(w io.WriteCloser) *WALSegmentWriter

NewWALSegmentWriter returns a new WALSegmentWriter writing to w.

func (*WALSegmentWriter) Flush 

func (w *WALSegmentWriter) Flush() error

func (*WALSegmentWriter) Write 

func (w *WALSegmentWriter) Write(entryType WalEntryType, compressed []byte) error

Write writes entryType and the buffer containing compressed entry data.

type WALStatistics 

type WALStatistics struct {
	OldBytes     int64
	CurrentBytes int64
	WriteOK      int64
	WriteErr     int64
}

WALStatistics maintains statistics about the WAL.

type WalEntryType 

type WalEntryType byte

WalEntryType is a byte written to a wal segment file that indicates what the following compressed block contains. 

const (
	// WriteWALEntryType indicates a write entry.
	WriteWALEntryType WalEntryType = 0x01

	// DeleteWALEntryType indicates a delete entry.
	DeleteWALEntryType WalEntryType = 0x02

	// DeleteRangeWALEntryType indicates a delete range entry.
	DeleteRangeWALEntryType WalEntryType = 0x03
)

type WriteWALEntry 

type WriteWALEntry struct {
	Values map[string][]Value
	// contains filtered or unexported fields
}

WriteWALEntry represents a write of points.

func (*WriteWALEntry) Encode 

func (w *WriteWALEntry) Encode(dst []byte) ([]byte, error)

Encode converts the WriteWALEntry into a byte stream using dst if it is large enough. If dst is too small, the slice will be grown to fit the encoded entry.

func (*WriteWALEntry) MarshalBinary 

func (w *WriteWALEntry) MarshalBinary() ([]byte, error)

MarshalBinary returns a binary representation of the entry in a new byte slice.

func (*WriteWALEntry) MarshalSize 

func (w *WriteWALEntry) MarshalSize() int

func (*WriteWALEntry) Type 

func (w *WriteWALEntry) Type() WalEntryType

Type returns WriteWALEntryType.

func (*WriteWALEntry) UnmarshalBinary 

func (w *WriteWALEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary deserializes the byte slice into w.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.