v2

README

V2 File Format

The V2 file format uses a simple binary format based on efficiency and not on backwards compatibility or future compatibility. If there is a need for changes it should be created a separate version.

The library benc was chosen because it was the lowest level library that offered buffer reuse, unsafe string usage and still had reasonable checks for out of bounds, and issues. This makes it more performant that mus and msgp format for example. The source of narrowing down serialization libaries came from https://alecthomas.github.io/go_serialization_benchmarks/.

Tested with mus,gencode,benc,bepod,mmsgp, and fastape. Benc hit the sweetspot on usage, nice api with reasonable checks.

The underlying format stores a type byte metric or metadata, specific metadata for each time and then the raw byte array of the protobuf bytes. This means it does not need to fully deserialize the data.

Documentation

Overview

Package v2 Copied from prometheus.

Index

• Constants
• func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) prompb.Histogram
• func FromIntHistogram(timestamp int64, h *histogram.Histogram) prompb.Histogram
• func FromMetadataType(t model.MetricType) prompb.MetricMetadata_MetricType
• type Format
  • func NewFormat() *Format
  • func (s *Format) AddPrometheusMetadata(name string, unit string, help string, pType string) error
  • func (s *Format) AddPrometheusMetric(ts int64, value float64, lbls labels.Labels, h *histogram.Histogram, ...) error
  • func (s *Format) Marshal(handle func(map[string]string, []byte) error) error
  • func (s *Format) Unmarshal(meta map[string]string, buf []byte) (items []types.Datum, err error)
• type Metadata
  • func (m Metadata) Bytes() []byte
  • func (m Metadata) FileFormat() types.FileFormat
  • func (m Metadata) Free()
  • func (m Metadata) IsMeta() bool
  • func (metadata *Metadata) Marshal(b []byte)
  • func (metadata *Metadata) MarshalPlain(tn int, b []byte) (n int)
  • func (metadata *Metadata) NestedMarshal(tn int, b []byte, id uint16) (n int)
  • func (metadata *Metadata) NestedSize(id uint16) (s int)
  • func (metadata *Metadata) NestedUnmarshal(tn int, b []byte, r []uint16, id uint16) (n int, err error)
  • func (metadata *Metadata) Size() int
  • func (metadata *Metadata) SizePlain() (s int)
  • func (m Metadata) Type() types.Type
  • func (metadata *Metadata) Unmarshal(b []byte) (err error)
  • func (metadata *Metadata) UnmarshalPlain(tn int, b []byte) (n int, err error)
• type Metric
  • func (m Metric) Bytes() []byte
  • func (m Metric) FileFormat() types.FileFormat
  • func (m *Metric) Free()
  • func (m Metric) Hash() uint64
  • func (m Metric) IsHistogram() bool
  • func (metric *Metric) Marshal(b []byte)
  • func (metric *Metric) MarshalPlain(tn int, b []byte) (n int)
  • func (metric *Metric) NestedMarshal(tn int, b []byte, id uint16) (n int)
  • func (metric *Metric) NestedSize(id uint16) (s int)
  • func (metric *Metric) NestedUnmarshal(tn int, b []byte, r []uint16, id uint16) (n int, err error)
  • func (metric *Metric) Size() int
  • func (metric *Metric) SizePlain() (s int)
  • func (m Metric) TimeStampMS() int64
  • func (m Metric) Type() types.Type
  • func (metric *Metric) Unmarshal(b []byte) (err error)
  • func (metric *Metric) UnmarshalPlain(tn int, b []byte) (n int, err error)

Constants

const PrometheusExemplar = uint8(2)

const PrometheusMetadata = uint8(3)

const PrometheusMetric = uint8(1)

Functions

func FromFloatHistogram

func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) prompb.Histogram

FromFloatHistogram returns remote Histogram from the float Histogram.

func FromIntHistogram

func FromIntHistogram(timestamp int64, h *histogram.Histogram) prompb.Histogram

FromIntHistogram returns remote Histogram from the integer Histogram.

func FromMetadataType

func FromMetadataType(t model.MetricType) prompb.MetricMetadata_MetricType

FromMetadataType transforms a Prometheus metricType into prompb metricType. Since the former is a string we need to transform it to an enum.

Types

type Format

type Format struct {
	// contains filtered or unexported fields
}

Format describe the v2 data format.

func NewFormat

func NewFormat() *Format

func (*Format) AddPrometheusMetadata

func (s *Format) AddPrometheusMetadata(name string, unit string, help string, pType string) error

func (*Format) AddPrometheusMetric

func (s *Format) AddPrometheusMetric(ts int64, value float64, lbls labels.Labels, h *histogram.Histogram, fh *histogram.FloatHistogram, e exemplar.Exemplar, externalLabels map[string]string) error

AddPrometheusMetric marshals a prometheus metric to its prombpb.TimeSeries representation and writes it to the buffer.

func (*Format) Marshal

func (s *Format) Marshal(handle func(map[string]string, []byte) error) error

func (*Format) Unmarshal

func (s *Format) Unmarshal(meta map[string]string, buf []byte) (items []types.Datum, err error)

type Metadata

type Metadata struct {
	Buf []byte
}

Struct - metadata

func (Metadata) Bytes

func (m Metadata) Bytes() []byte

func (Metadata) FileFormat

func (m Metadata) FileFormat() types.FileFormat

func (Metadata) Free

func (m Metadata) Free()

func (Metadata) IsMeta

func (m Metadata) IsMeta() bool

func (*Metadata) Marshal

func (metadata *Metadata) Marshal(b []byte)

Marshal - metadata

func (*Metadata) MarshalPlain

func (metadata *Metadata) MarshalPlain(tn int, b []byte) (n int)

MarshalPlain - metadata

func (*Metadata) NestedMarshal

func (metadata *Metadata) NestedMarshal(tn int, b []byte, id uint16) (n int)

Nested Marshal - metadata

func (*Metadata) NestedSize

func (metadata *Metadata) NestedSize(id uint16) (s int)

Nested Size - metadata

func (*Metadata) NestedUnmarshal

func (metadata *Metadata) NestedUnmarshal(tn int, b []byte, r []uint16, id uint16) (n int, err error)

Nested Unmarshal - metadata

func (*Metadata) Size

func (metadata *Metadata) Size() int

Size - metadata

func (*Metadata) SizePlain

func (metadata *Metadata) SizePlain() (s int)

SizePlain - metadata

func (Metadata) Type

func (m Metadata) Type() types.Type

func (*Metadata) Unmarshal

func (metadata *Metadata) Unmarshal(b []byte) (err error)

Unmarshal - metadata

func (*Metadata) UnmarshalPlain

func (metadata *Metadata) UnmarshalPlain(tn int, b []byte) (n int, err error)

UnmarshalPlain - metadata

type Metric

type Metric struct {
	Timestampmsvalue int64
	Hashvalue        uint64
	IsHistogramvalue bool
	Buf              []byte
}

Struct - metric

func (Metric) Bytes

func (m Metric) Bytes() []byte

Bytes represents the underlying data and should not be handled aside from Build* functions that understand the Type.

func (Metric) FileFormat

func (m Metric) FileFormat() types.FileFormat

func (*Metric) Free

func (m *Metric) Free()

func (Metric) Hash

func (m Metric) Hash() uint64

func (Metric) IsHistogram

func (m Metric) IsHistogram() bool

func (*Metric) Marshal

func (metric *Metric) Marshal(b []byte)

Marshal - metric

func (*Metric) MarshalPlain

func (metric *Metric) MarshalPlain(tn int, b []byte) (n int)

MarshalPlain - metric

func (*Metric) NestedMarshal

func (metric *Metric) NestedMarshal(tn int, b []byte, id uint16) (n int)

Nested Marshal - metric

func (*Metric) NestedSize

func (metric *Metric) NestedSize(id uint16) (s int)

Nested Size - metric

func (*Metric) NestedUnmarshal

func (metric *Metric) NestedUnmarshal(tn int, b []byte, r []uint16, id uint16) (n int, err error)

Nested Unmarshal - metric

func (*Metric) Size

func (metric *Metric) Size() int

Size - metric

func (*Metric) SizePlain

func (metric *Metric) SizePlain() (s int)

SizePlain - metric

func (Metric) TimeStampMS

func (m Metric) TimeStampMS() int64

func (Metric) Type

func (m Metric) Type() types.Type

func (*Metric) Unmarshal

func (metric *Metric) Unmarshal(b []byte) (err error)

Unmarshal - metric

func (*Metric) UnmarshalPlain

func (metric *Metric) UnmarshalPlain(tn int, b []byte) (n int, err error)

UnmarshalPlain - metric