Back to godoc.org

Package cloudtail

v0.0.0-...-bd98d2d
Latest Go to latest

The highest tagged major version is .

Published: Sep 19, 2020 | License: BSD-3-Clause | Module: chromium.googlesource.com/infra/infra.git

Index

Constants

const (
	DefaultFlushThreshold  = 5000
	DefaultFlushTimeout    = 5 * time.Second
	DefaultMaxPushAttempts = 5
	DefaultPushRetryDelay  = 500 * time.Millisecond
)

Default config for PushBuffer if none is provided.

const (
	DefaultRotationCheckPeriod = 5 * time.Second
	DefaultPollingPeriod       = 500 * time.Millisecond
	DefaultReadBufferLen       = 1024 * 256
)

See corresponding fields of TailerOptions.

const DefaultResourceType = "machine"

DefaultResourceType is used by NewClient if ClientOptions doesn't specify ResourceType.

type Client

type Client interface {
	// PushEntries sends entries to Cloud Logging. No retries.
	//
	// May return fatal or transient errors. Check with errors.IsTransient.
	//
	// Respects context deadline.
	PushEntries(ctx context.Context, entries []*Entry) error
}

Client knows how to send entries to Cloud Logging log.

func NewClient

func NewClient(opts ClientOptions) (Client, error)

NewClient returns new object that knows how to push log entries to a single log in Cloud Logging.

type ClientID

type ClientID struct {
	// ResourceType identifies a kind of entity that produces this log (e.g.
	// 'machine', 'master'). Default is DefaultResourceType.
	ResourceType string

	// ResourceID identifies exact instance of provided resource type (e.g
	// 'vm12-m4', 'master.chromium.fyi'). Default is machine hostname.
	ResourceID string

	// LogID identifies what sort of log this is. Must be set.
	LogID string
}

ClientID uniquely identifies the log entries sent by this process.

Its values are used to identify the log in Cloud Logging and also included in monitoring metrics.

type ClientOptions

type ClientOptions struct {
	// ClientID uniquely identifies the log entries sent by this process.
	ClientID

	// Client is http.Client to use (that must implement proper authentication).
	Client *http.Client

	// UserAgent is an optional string appended to User-Agent HTTP header.
	UserAgent string

	// ProjectID is Cloud project to sends logs to. Must be set.
	ProjectID string

	// Debug is true to print log entries to stdout instead of sending them.
	Debug bool
}

ClientOptions is passed to NewClient.

type Entry

type Entry struct {
	// InsertId can be used to deduplicate log entries.
	InsertID string

	// Timestamp is an optional timestamp.
	Timestamp time.Time

	// Severity is the severity of the log entry.
	Severity Severity

	// TextPayload is the log entry payload, represented as a text string.
	TextPayload string

	// JSONPayload is the log entry payload, represented as a JSONish structure.
	JSONPayload interface{}

	// ParsedBy is the parser that parsed this line, or nil if it fell through to
	// the default parser.
	ParsedBy LogParser

	// Labels is a set of user-defined (key, value) data for additional
	// information about the log entry.
	Labels map[string]string
}

Entry is a single log entry. It can be a text message, or a JSONish struct.

type LogParser

type LogParser interface {
	// ParseLogLine returns log entry with all recognized info filled in or nil
	// if the line format is not recognized.
	ParseLogLine(line string) *Entry

	// MergeLogLine appends the line of text to the existing Entry.  The Entry was
	// created by this LogParser.  Returns true if the merge succeeded, false if
	// the line should be added as a separate log entry.
	MergeLogLine(line string, e *Entry) bool
}

LogParser takes a line of text and extracts log related info from it.

func NullParser

func NullParser() LogParser

NullParser returns a parser that converts log line into a raw text Entry.

func StdParser

func StdParser() LogParser

StdParser returns a parser that recognizes common types of logs.

type LogParserChain

type LogParserChain []LogParser

LogParserChain is a list of log parsers applied one after another until first hit.

func (LogParserChain) MergeLogLine

func (c LogParserChain) MergeLogLine(line string, e *Entry) bool

MergeLogLine does nothing. Concrete parsers should set the ParsedBy Entry member for their own MergeLogLine methods to be called.

func (LogParserChain) ParseLogLine

func (c LogParserChain) ParseLogLine(line string) *Entry

ParseLogLine invokes all parsers in a chain until a first hit. If no parser recognizes a line, it returns Entry with unparsed text message as payload and default fields.

type PipeReader

type PipeReader struct {
	// ClientID identifies the log stream for monitoring.
	ClientID ClientID

	// Source is a reader to read logs from.
	Source io.Reader

	// PushBuffer knows how to forward log entries to the client.
	PushBuffer PushBuffer

	// Parser converts text lines into log entries, default is StdParser().
	Parser LogParser

	// LineBufferSize defines how many log lines to accumulate (if the flush is
	// blocked) before starting to drop them.
	//
	// Default is 0, which means to never drop lines (stop reading from the
	// source instead).
	LineBufferSize int

	// OnEOF is called immediately when EOF (or reading error) is encountered.
	//
	// Note that this happens before 'Run' returns, because 'Run' waits for data
	// to be pushed to the PushBuffer.
	OnEOF func()

	// OnLineDropped is called whenever a line gets dropped due to full buffer.
	OnLineDropped func()
}

PipeReader reads lines from io.Reader, parses and pushes them to the buffer.

func (*PipeReader) Run

func (r *PipeReader) Run(ctx context.Context) error

Run reads from the reader until EOF or until the context is closed.

Returns error only if reading from io.Reader fails. On EOF or on context cancellation returns nil. Always returns same error as was sent to OnEOF.

Waits for all read data to be pushed to PushBuffer.

type PushBuffer

type PushBuffer interface {
	// Start starts an internal goroutine that periodically flushes entries.
	//
	// The goroutine uses the given context for creating timers and for flushes.
	// If this context is canceled, all incoming entries are dropped. Instead use
	// Stop to shutdown gracefully.
	Start(ctx context.Context)

	// Send appends the entry to the buffer of pending entries and flushes them if
	// the buffer becomes full.
	//
	// It can block, waiting for pending data to be sent. Unblocks (and drops
	// the entry) if the context is canceled.
	//
	// Since flushes may happen asynchronously, doesn't return an error. Instead
	// a failed flush attempt will be logged and the error will eventually be
	// returned by Stop().
	Send(ctx context.Context, e Entry)

	// Stop waits for all entries to be sent and stops the flush timer.
	//
	// Aborts ASAP when the passed context is canceled. Returns an error if any of
	// pending data wasn't flushed successfully.
	Stop(ctx context.Context) error
}

PushBuffer batches log entries together before pushing them to the client.

It accumulates entries in a buffer and flushes them when the buffer is full (see FlushThreshold option) or when the first pushed entry is sufficiently old (see FlushTimeout option).

It uses no more than 1 connection to the Cloud Logging.

func NewPushBuffer

func NewPushBuffer(opts PushBufferOptions) PushBuffer

NewPushBuffer returns PushBuffer that's ready to accept log entries.

type PushBufferOptions

type PushBufferOptions struct {
	// Client is configured client to use to push messages.
	//
	// Required.
	Client Client

	// FlushThreshold defines how many pending messages trigger a flush.
	FlushThreshold int

	// FlushTimeout is maximum time an entry is kept in buffer before it is sent.
	FlushTimeout time.Duration

	// MaxPushAttempts is how many times to push entries when retrying errors.
	MaxPushAttempts int

	// PushRetryDelay is how long to wait before retrying a failed push.
	//
	// Will be doubled on each failed retry attempt.
	PushRetryDelay time.Duration
}

PushBufferOptions defines configuration for a new PushBuffer instance.

type Severity

type Severity string

Severity defines how important a message is.

const (
	Default   Severity = "DEFAULT"
	Debug     Severity = "DEBUG"
	Info      Severity = "INFO"
	Notice    Severity = "NOTICE"
	Warning   Severity = "WARNING"
	Error     Severity = "ERROR"
	Critical  Severity = "CRITICAL"
	Alert     Severity = "ALERT"
	Emergency Severity = "EMERGENCY"
)

All severity types understood by Cloud Logging.

func (*Severity) Set

func (sev *Severity) Set(value string) error

Set is called by 'flag' package when parsing command line options.

func (Severity) String

func (sev Severity) String() string

String is used by 'flag' package.

func (Severity) Validate

func (sev Severity) Validate() error

Validate returns a error if the severity is not recognized.

type Tailer

type Tailer struct {
	// contains filtered or unexported fields
}

Tailer watches a file for changes and pushes new lines to a the buffer.

func NewTailer

func NewTailer(opts TailerOptions) (*Tailer, error)

NewTailer prepares a Tailer.

Use its 'Run' method to start tailing a file.

func (*Tailer) Run

func (tailer *Tailer) Run(ctx context.Context)

Run watches a file for changes and pushes new lines to the buffer.

Use Stop() (from another goroutine) to gracefully terminate the tailer, or cancel the context to abort it ASAP.

func (*Tailer) Stop

func (tailer *Tailer) Stop()

Stop asynchronously notifies tailer to stop (i.e. 'Run' to unblock and return). Panics if called twice.

type TailerOptions

type TailerOptions struct {
	// Path identifies a file to watch.
	Path string

	// PushBuffer knows how to forward log entries to the client.
	PushBuffer PushBuffer

	// TeeOutput (if not nil) receives raw text lines before they are sent to
	// the parser. Write errors are logged, but otherwise ignored.
	TeeOutput io.Writer

	// Parser converts text lines into log entries, default is StdParser().
	Parser LogParser

	// SeekToEnd is true to seek to file's end before tailing.
	SeekToEnd bool

	// UsePolling is true to disable fsnotify watchers and use polling.
	UsePolling bool

	// RotationCheckPeriod defines how often to call os.Stat to see whether
	// the file has been moved or truncated.
	RotationCheckPeriod time.Duration

	// PollingPeriod defines how often to poll file for changes if fsnotify system
	// is not working.
	PollingPeriod time.Duration

	// ReadBufferLen is maximum number of bytes read from file in one operation.
	ReadBufferLen int
	// contains filtered or unexported fields
}

TailerOptions is passed to NewTailer.

Package Files

Documentation was rendered with GOOS=linux and GOARCH=amd64.

Jump to identifier

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to identifier