logz

README

logz

This library provides in-process aggregated collector of messages and web page to report them. They are useful for last mile observability of logs.

logz is inspired by OpenCensus zPages.

Features

• High performance and low resource consumption.
• Adapter for go.uber.org/zap.
• Adapter for github.com/bool64/ctxd.
• HTTP handler to serve aggregated messages.
• Best effort filtering of dynamic parts of messages.

Example for go.uber.org/zap

zc := zap.NewDevelopmentConfig()
zz, lo := zzap.NewOption(logz.Config{
    MaxCardinality: 5,
    MaxSamples:     10,
	FilterMessage:  true,
})

l, err := zc.Build(zz)
if err != nil {
    panic(err)
}

l.Debug("starting example")
l.Sugar().Infow("sample info", "one", 1, "two", 2)
l.Error("unexpected end of the world")

l.Warn("please be careful with asf872dfse732")
l.Warn("please be careful with sdf890sdf0w9d") // These messages will be grouped together with help of filtering.

l.Info("starting server at http://localhost:6060/")
err = http.ListenAndServe("0.0.0.0:6060", logzpage.Handler(lo...))
if err != nil {
    l.Fatal(err.Error())
}

Example for github.com/bool64/ctxd

var logger ctxd.Logger

lz := ctxz.NewObserver(logger, logz.Config{
    MaxCardinality:      100,
    MaxSamples:          50,
    DistRetentionPeriod: 72 * time.Hour,
})
logger = lz

ctx := context.TODO()

logger.Debug(ctx, "starting example")
logger.Info(ctx, "sample info", "one", 1, "two", 2)
logger.Error(ctx, "unexpected end of the world")

logger.Important(ctx, "starting server at http://localhost:6060/")

err := http.ListenAndServe("0.0.0.0:6060", logzpage.Handler(lz.LevelObservers()...))
if err != nil {
    logger.Error(ctx, err.Error())
    os.Exit(1)
}

Documentation

Overview

Package logz provides events observer.

Index

• type Bucket
• type Config
• type Entry
• type Observer
  • func (l *Observer) ObserveMessage(msg string, data interface{})
• type PreparedObserver
  • func NewObserver(cfg Config) *PreparedObserver
  • func (l *PreparedObserver) Find(msg string) Entry
  • func (l *PreparedObserver) GetEntries() []Entry
  • func (l *PreparedObserver) GetEntriesWithSamples() []Entry
  • func (l *PreparedObserver) ObserveMessage(msg string, data interface{})
  • func (l *PreparedObserver) Other(withSamples bool) Entry
• type Sample

Types

type Bucket

type Bucket struct {
	From  time.Time
	To    time.Time
	Count uint64
}

Bucket contains count of events in time interval.

type Config

type Config struct {
	// Name can be used to identify observer instance in a group, for example a group of log levels.
	Name string

	// MaxCardinality limits number of distinct message families being tracked.
	// All messages that exceed cardinality are grouped together as "other".
	// Default 100.
	MaxCardinality uint32

	// MaxSamples limits a number of latest samples kept for a message family.
	// Default 10.
	MaxSamples uint32

	// SamplingInterval is the minimum amount of time needed to pass from a last Sample collection in particular message family.
	// Messages that are observed too quickly after last sampling are counted, but not sampled.
	// Default 1ms.
	SamplingInterval time.Duration

	// DistResolution is the maximum number of time interval buckets to track distribution in time.
	// Default 100.
	DistResolution int

	// DistRetentionPeriod is maximum age of bucket. Use -1 for unlimited.
	// Default one week (168 hours).
	DistRetentionPeriod time.Duration

	// FilterMessage can reduce cardinality by filtering dynamic parts of messages.
	// It uses github.com/vearutop/lograte/filter.Dynamic
	// See https://pkg.go.dev/github.com/vearutop/lograte/filter#Dynamic.
	// This option is not needed if you already have messages without dynamic interpolated values.
	// This option worsen performance, so use it only if you need it.
	FilterMessage bool
}

Config defines observer configuration.

type Entry

type Entry struct {
	Message string
	Count   uint64
	Samples []Sample
	First   time.Time
	Last    time.Time

	MaxBucketCount int
	Buckets        []Bucket
}

Entry contains aggregated information about message.

type Observer

type Observer struct {
	Config
	PreparedObserver
	// contains filtered or unexported fields
}

Observer keeps track of messages. Use NewObserver() to create new instance.

func (*Observer) ObserveMessage

func (l *Observer) ObserveMessage(msg string, data interface{})

ObserveMessage updates aggregated information about message.

type PreparedObserver

type PreparedObserver struct {
	// contains filtered or unexported fields
}

PreparedObserver keeps track of messages.

func NewObserver

func NewObserver(cfg Config) *PreparedObserver

NewObserver creates PreparedObserver.

func (*PreparedObserver) Find

func (l *PreparedObserver) Find(msg string) Entry

Find lookups entry by message.

func (*PreparedObserver) GetEntries

func (l *PreparedObserver) GetEntries() []Entry

GetEntries returns a list of observed event entries without data samples.

func (*PreparedObserver) GetEntriesWithSamples

func (l *PreparedObserver) GetEntriesWithSamples() []Entry

GetEntriesWithSamples returns a list of observed event entries with data samples.

func (*PreparedObserver) ObserveMessage

func (l *PreparedObserver) ObserveMessage(msg string, data interface{})

ObserveMessage updates aggregated information about message.

func (*PreparedObserver) Other

func (l *PreparedObserver) Other(withSamples bool) Entry

Other returns entry for other events.

type Sample

type Sample struct {
	Msg  string      `json:"msg"`
	Data interface{} `json:"data"`
	Time time.Time   `json:"time"`
}

Sample is a single sample of a message.