file

Documentation

Index

• func NewLineEndSplitFunc(re *regexp.Regexp) bufio.SplitFunc
• func NewLineStartSplitFunc(re *regexp.Regexp) bufio.SplitFunc
• func NewNewlineSplitFunc(encoding encoding.Encoding) (bufio.SplitFunc, error)
• type Fingerprint
  • func (f Fingerprint) Copy() *Fingerprint
  • func (f Fingerprint) StartsWith(old *Fingerprint) bool
• type FingerprintUpdatingReader
  • func NewFingerprintUpdatingReader(r io.Reader, offset int64, f *Fingerprint, fingerprintSize int) *FingerprintUpdatingReader
  • func (f *FingerprintUpdatingReader) Read(dst []byte) (int, error)
• type InputConfig
  • func NewInputConfig(operatorID string) *InputConfig
  • func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, error)
• type InputOperator
  • func (f *InputOperator) NewFingerprint(file *os.File) (*Fingerprint, error)
  • func (f *InputOperator) NewReader(path string, file *os.File, fp *Fingerprint) (*Reader, error)
  • func (f *InputOperator) Start() error
  • func (f *InputOperator) Stop() error
• type MultilineConfig
• type PositionalScanner
  • func NewPositionalScanner(r io.Reader, maxLogSize int, startOffset int64, splitFunc bufio.SplitFunc) *PositionalScanner
  • func (ps *PositionalScanner) Pos() int64
• type Reader
  • func (f *Reader) Copy(file *os.File) (*Reader, error)
  • func (f *Reader) InitializeOffset(startAtBeginning bool) error
  • func (f *Reader) ReadToEnd(ctx context.Context)

Functions

func NewLineEndSplitFunc

func NewLineEndSplitFunc(re *regexp.Regexp) bufio.SplitFunc

NewLineEndSplitFunc creates a bufio.SplitFunc that splits an incoming stream into tokens that end with a match to the regex pattern provided

func NewLineStartSplitFunc

func NewLineStartSplitFunc(re *regexp.Regexp) bufio.SplitFunc

NewLineStartSplitFunc creates a bufio.SplitFunc that splits an incoming stream into tokens that start with a match to the regex pattern provided

func NewNewlineSplitFunc

func NewNewlineSplitFunc(encoding encoding.Encoding) (bufio.SplitFunc, error)

NewNewlineSplitFunc splits log lines by newline, just as bufio.ScanLines, but never returning an token using EOF as a terminator

Types

type Fingerprint

type Fingerprint struct {
	FirstBytes []byte
}

Fingerprint is used to identify a file

func (Fingerprint) Copy

func (f Fingerprint) Copy() *Fingerprint

Copy creates a new copy of the fingerprint

func (Fingerprint) StartsWith

func (f Fingerprint) StartsWith(old *Fingerprint) bool

StartsWith returns true if the fingerprints are the same or if the new fingerprint starts with the old one

type FingerprintUpdatingReader

type FingerprintUpdatingReader struct {
	// contains filtered or unexported fields
}

FingerprintUpdatingReader wraps another reader, and updates the fingerprint with each read in the first fingerPrintSize bytes

func NewFingerprintUpdatingReader

func NewFingerprintUpdatingReader(r io.Reader, offset int64, f *Fingerprint, fingerprintSize int) *FingerprintUpdatingReader

NewFingerprintUpdatingReader creates a new FingerprintUpdatingReader starting starting at the given offset

func (*FingerprintUpdatingReader) Read

func (f *FingerprintUpdatingReader) Read(dst []byte) (int, error)

Read reads from the wrapped reader, saving the read bytes to the fingerprint

type InputConfig

type InputConfig struct {
	helper.InputConfig `yaml:",inline"`

	Include []string `json:"include,omitempty" yaml:"include,omitempty"`
	Exclude []string `json:"exclude,omitempty" yaml:"exclude,omitempty"`

	PollInterval       helper.Duration  `json:"poll_interval,omitempty"        yaml:"poll_interval,omitempty"`
	Multiline          *MultilineConfig `json:"multiline,omitempty"            yaml:"multiline,omitempty"`
	IncludeFileName    bool             `json:"include_file_name,omitempty"    yaml:"include_file_name,omitempty"`
	IncludeFilePath    bool             `json:"include_file_path,omitempty"    yaml:"include_file_path,omitempty"`
	StartAt            string           `json:"start_at,omitempty"             yaml:"start_at,omitempty"`
	FingerprintSize    helper.ByteSize  `json:"fingerprint_size,omitempty"     yaml:"fingerprint_size,omitempty"`
	MaxLogSize         helper.ByteSize  `json:"max_log_size,omitempty"         yaml:"max_log_size,omitempty"`
	MaxConcurrentFiles int              `json:"max_concurrent_files,omitempty" yaml:"max_concurrent_files,omitempty"`
	Encoding           string           `json:"encoding,omitempty"             yaml:"encoding,omitempty"`
}

InputConfig is the configuration of a file input operator

func NewInputConfig

func NewInputConfig(operatorID string) *InputConfig

NewInputConfig creates a new input config with default values

func (InputConfig) Build

func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, error)

Build will build a file input operator from the supplied configuration

type InputOperator

type InputOperator struct {
	helper.InputOperator

	Include            []string
	Exclude            []string
	FilePathField      entry.Field
	FileNameField      entry.Field
	PollInterval       time.Duration
	SplitFunc          bufio.SplitFunc
	MaxLogSize         int
	MaxConcurrentFiles int
	SeenPaths          map[string]struct{}
	// contains filtered or unexported fields
}

InputOperator is an operator that monitors files for entries

func (*InputOperator) NewFingerprint

func (f *InputOperator) NewFingerprint(file *os.File) (*Fingerprint, error)

NewFingerprint creates a new fingerprint from an open file

func (*InputOperator) NewReader

func (f *InputOperator) NewReader(path string, file *os.File, fp *Fingerprint) (*Reader, error)

NewReader creates a new file reader

func (*InputOperator) Start

func (f *InputOperator) Start() error

Start will start the file monitoring process

func (*InputOperator) Stop

func (f *InputOperator) Stop() error

Stop will stop the file monitoring process

type MultilineConfig

type MultilineConfig struct {
	LineStartPattern string `json:"line_start_pattern" yaml:"line_start_pattern"`
	LineEndPattern   string `json:"line_end_pattern"   yaml:"line_end_pattern"`
}

MultilineConfig is the configuration a multiline operation

type PositionalScanner

type PositionalScanner struct {
	*bufio.Scanner
	// contains filtered or unexported fields
}

PositionalScanner is a scanner that maintains position

func NewPositionalScanner

func NewPositionalScanner(r io.Reader, maxLogSize int, startOffset int64, splitFunc bufio.SplitFunc) *PositionalScanner

NewPositionalScanner creates a new positional scanner

func (*PositionalScanner) Pos

func (ps *PositionalScanner) Pos() int64

Pos returns the current position of the scanner

type Reader

type Reader struct {
	Fingerprint *Fingerprint
	Offset      int64
	Path        string

	*zap.SugaredLogger `json:"-"`
	// contains filtered or unexported fields
}

Reader manages a single file

func (*Reader) Copy

func (f *Reader) Copy(file *os.File) (*Reader, error)

Copy creates a deep copy of a Reader

func (*Reader) InitializeOffset

func (f *Reader) InitializeOffset(startAtBeginning bool) error

InitializeOffset sets the starting offset

func (*Reader) ReadToEnd

func (f *Reader) ReadToEnd(ctx context.Context)

ReadToEnd will read until the end of the file