arrow

package
v0.23.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 9, 2024 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Overview

Package arrow contains types and functions used to convert OTLP logs into their Arrow representation.

Index

Constants

This section is empty.

Variables

View Source 
var (
	// LogsSchema is the Arrow schema for the OTLP Arrow Logs record.
	LogsSchema = arrow.NewSchema([]arrow.Field{
		{Name: constants.ID, Type: arrow.PrimitiveTypes.Uint16, Metadata: schema.Metadata(schema.Optional, schema.DeltaEncoding)},
		{Name: constants.Resource, Type: acommon.ResourceDT, Metadata: schema.Metadata(schema.Optional)},
		{Name: constants.Scope, Type: acommon.ScopeDT, Metadata: schema.Metadata(schema.Optional)},

		{Name: constants.SchemaUrl, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Optional, schema.Dictionary8)},
		{Name: constants.TimeUnixNano, Type: arrow.FixedWidthTypes.Timestamp_ns},
		{Name: constants.ObservedTimeUnixNano, Type: arrow.FixedWidthTypes.Timestamp_ns},
		{Name: constants.TraceId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 16}, Metadata: schema.Metadata(schema.Optional, schema.Dictionary8)},
		{Name: constants.SpanId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 8}, Metadata: schema.Metadata(schema.Optional, schema.Dictionary8)},
		{Name: constants.SeverityNumber, Type: arrow.PrimitiveTypes.Int32, Metadata: schema.Metadata(schema.Optional, schema.Dictionary8)},
		{Name: constants.SeverityText, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Optional, schema.Dictionary8)},
		{Name: constants.Body, Type: arrow.StructOf([]arrow.Field{
			{Name: constants.BodyType, Type: arrow.PrimitiveTypes.Uint8},
			{Name: constants.BodyStr, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary16)},
			{Name: constants.BodyInt, Type: arrow.PrimitiveTypes.Int64, Metadata: schema.Metadata(schema.Optional, schema.Dictionary16)},
			{Name: constants.BodyDouble, Type: arrow.PrimitiveTypes.Float64, Metadata: schema.Metadata(schema.Optional)},
			{Name: constants.BodyBool, Type: arrow.FixedWidthTypes.Boolean, Metadata: schema.Metadata(schema.Optional)},
			{Name: constants.BodyBytes, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Optional, schema.Dictionary16)},
			{Name: constants.BodySer, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Optional, schema.Dictionary16)},
		}...)},
		{Name: constants.DroppedAttributesCount, Type: arrow.PrimitiveTypes.Uint32, Metadata: schema.Metadata(schema.Optional)},
		{Name: constants.Flags, Type: arrow.PrimitiveTypes.Uint32, Metadata: schema.Metadata(schema.Optional)},
	}, nil)
)

Functions

This section is empty.

Types

type AttrsBuilders 

type AttrsBuilders struct {
	// contains filtered or unexported fields
}

AttrsBuilders groups together AttrsBuilder instances used to build related data attributes (i.e. resource attributes, scope attributes, and log record attributes.

func (*AttrsBuilders) LogRecord 

func (ab *AttrsBuilders) LogRecord() *carrow.Attrs16Builder

func (*AttrsBuilders) Resource 

func (ab *AttrsBuilders) Resource() *carrow.Attrs16Builder

func (*AttrsBuilders) Scope 

func (ab *AttrsBuilders) Scope() *carrow.Attrs16Builder

type AttrsConfig 

type AttrsConfig struct {
	Resource *arrow.Attrs16Config
	Scope    *arrow.Attrs16Config
	Log      *arrow.Attrs16Config
}

type Config 

type Config struct {
	Global *cfg.Config

	Log   *LogConfig
	Attrs *AttrsConfig
}

func DefaultConfig 

func DefaultConfig() *Config

func NewConfig 

func NewConfig(globalConf *cfg.Config) *Config

func NewNoSortConfig 

func NewNoSortConfig(globalConf *cfg.Config) *Config

type FlattenedLog 

type FlattenedLog struct {
	ResScope *ResScope
	Log      plog.LogRecord
}

type LogConfig 

type LogConfig struct {
	Sorter LogSorter
}

type LogRecordStats 

type LogRecordStats struct {
	TotalCount           int64
	Distribution         *hdrhistogram.Histogram
	TimeUnixNano         *carrow.TimestampStats
	ObservedTimeUnixNano *carrow.TimestampStats
	TraceID              *hyperloglog.Sketch
	SpanID               *hyperloglog.Sketch
	SeverityNumber       *hyperloglog.Sketch
	SeverityText         *carrow.StringStats
	Body                 *carrow.AnyValueStats
	Attributes           *carrow.AttributesStats
}

func NewLogRecordStats 

func NewLogRecordStats() *LogRecordStats

func (*LogRecordStats) ShowStats 

func (s *LogRecordStats) ShowStats(indent string)

type LogSorter 

type LogSorter interface {
	Sort(logs []*FlattenedLog)
}

type LogsAnalyzer 

type LogsAnalyzer struct {
	LogRecordCount    int64
	ResourceLogsStats *ResourceLogsStats
}

func NewLogsAnalyzer 

func NewLogsAnalyzer() *LogsAnalyzer

func (*LogsAnalyzer) Analyze 

func (t *LogsAnalyzer) Analyze(logs *LogsOptimized)

func (*LogsAnalyzer) ShowStats 

func (t *LogsAnalyzer) ShowStats(indent string)

type LogsBuilder 

type LogsBuilder struct {
	// contains filtered or unexported fields
}

LogsBuilder is a helper to build a list of resource logs.

func NewLogsBuilder 

func NewLogsBuilder(
	recordBuilder *builder.RecordBuilderExt,
	cfg *Config,
	stats *stats.ProducerStats,
	observer observer.ProducerObserver,
) (*LogsBuilder, error)

NewLogsBuilder creates a new LogsBuilder.

Important Note: The recordBuilder parameter will not be released by this LogsBuilder as it's shared with other instances of log builders.

func (*LogsBuilder) Append 

func (b *LogsBuilder) Append(logs plog.Logs) (err error)

Append appends a new set of resource logs to the builder.

func (*LogsBuilder) Build 

func (b *LogsBuilder) Build() (record arrow.Record, err error)

Build builds an Arrow Record from the builder.

Once the array is no longer needed, Release() must be called to free the memory allocated by the record.

func (*LogsBuilder) RelatedData 

func (b *LogsBuilder) RelatedData() *RelatedData

func (*LogsBuilder) Release 

func (b *LogsBuilder) Release()

Release releases the memory allocated by the builder.

func (*LogsBuilder) ShowSchema 

func (b *LogsBuilder) ShowSchema()

type LogsByNothing 

type LogsByNothing struct{}

func UnsortedLogs 

func UnsortedLogs() *LogsByNothing

func (*LogsByNothing) Sort 

func (s *LogsByNothing) Sort(_ []*FlattenedLog)

type LogsByResourceLogsIDScopeLogsIDTraceID 

type LogsByResourceLogsIDScopeLogsIDTraceID struct{}

func SortLogsByResourceLogsIDScopeLogsIDTraceID 

func SortLogsByResourceLogsIDScopeLogsIDTraceID() *LogsByResourceLogsIDScopeLogsIDTraceID

Sort logs by resource logs ID, scope logs ID, and trace ID.

func (*LogsByResourceLogsIDScopeLogsIDTraceID) Sort 

func (s *LogsByResourceLogsIDScopeLogsIDTraceID) Sort(logs []*FlattenedLog)

type LogsOptimized 

type LogsOptimized struct {
	Logs []*FlattenedLog
}

type LogsOptimizer 

type LogsOptimizer struct {
	// contains filtered or unexported fields
}

func NewLogsOptimizer 

func NewLogsOptimizer(sorter LogSorter) *LogsOptimizer

func (*LogsOptimizer) Optimize 

func (t *LogsOptimizer) Optimize(logs plog.Logs) *LogsOptimized

type RelatedData 

type RelatedData struct {
	// contains filtered or unexported fields
}

RelatedData is a collection of related/dependent data to log record entities.

func NewRelatedData 

func NewRelatedData(cfg *Config, stats *stats.ProducerStats, observer observer.ProducerObserver) (*RelatedData, error)

func (*RelatedData) AttrsBuilders 

func (r *RelatedData) AttrsBuilders() *AttrsBuilders

func (*RelatedData) BuildRecordMessages 

func (r *RelatedData) BuildRecordMessages() ([]*record_message.RecordMessage, error)

func (*RelatedData) RecordBuilderExt 

func (r *RelatedData) RecordBuilderExt(payloadType *carrow.PayloadType) *builder.RecordBuilderExt

func (*RelatedData) Release 

func (r *RelatedData) Release()

func (*RelatedData) Reset 

func (r *RelatedData) Reset()

func (*RelatedData) Schemas 

func (r *RelatedData) Schemas() []carrow.SchemaWithPayload

type ResScope 

type ResScope struct {
	// Resource log section.
	ResourceLogsID    int
	Resource          pcommon.Resource
	ResourceSchemaUrl string

	// Scope log section.
	ScopeLogsID    int
	Scope          pcommon.InstrumentationScope
	ScopeSchemaUrl string
}

type ResourceLogsStats 

type ResourceLogsStats struct {
	TotalCount         int64
	Distribution       *hdrhistogram.Histogram
	ResLogsIDsDistinct *hyperloglog.Sketch
	ResourceStats      *carrow.ResourceStats
	ScopeLogsStats     *ScopeLogsStats
	SchemaUrlStats     *carrow.SchemaUrlStats
}

func (*ResourceLogsStats) ShowStats 

func (r *ResourceLogsStats) ShowStats(indent string)

func (*ResourceLogsStats) UpdateWith 

func (r *ResourceLogsStats) UpdateWith(logs *LogsOptimized)

type ScopeLogsStats 

type ScopeLogsStats struct {
	Distribution         *hdrhistogram.Histogram
	ScopeLogsIDsDistinct *hyperloglog.Sketch
	ScopeStats           *carrow.ScopeStats
	SchemaUrlStats       *carrow.SchemaUrlStats
	LogRecordStats       *LogRecordStats
}

func (*ScopeLogsStats) ShowStats 

func (s *ScopeLogsStats) ShowStats(indent string)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.