scanner

Documentation

Index

• type CollectorStats
• type Config
  • func NewDefaultConfig() *Config
  • func (c *Config) Apply(c1 *Config)
  • func (c *Config) String() string
• type FileFormat
  • func (c *FileFormat) String() string
• type Parser
• type ParserStats
• type Scanner
  • func NewScanner(cfg *Config, ss StateStorage) (*Scanner, error)
  • func (s *Scanner) Events() <-chan *model.Event
  • func (s *Scanner) GetStats() *CollectorStats
  • func (s *Scanner) Start() error
  • func (s *Scanner) Stop() error
• type StateStorage
  • func NewFileStateStorage(fn string) StateStorage
  • func NewInMemStateStorage() StateStorage
• type WorkerStats

Types

type CollectorStats

type CollectorStats struct {
	Config *Config
	// contains list of files that were excluded from processing
	Excludes []string
	Workers  []*WorkerStats
}

type Config

type Config struct {
	// ScanPaths contains a list of path that should be scanned and check
	// for potential sources. Scan Paths are defined like Globs, for example:
	// /var/log/*.log etc.
	ScanPaths            []string `json:"scanPaths"`
	ScanPathsIntervalSec int      `json:"scanPathsIntervalSec"`

	// Exclude contains slice of regular expressions (the slice could be empty)
	// which will be applied to files found by ScanPaths. The Exclude slice
	// of reg-exps intended to exclude some files which could be scanned.
	Exclude []string `json:"exclude"`

	// StateFlushIntervalSec defines the time interval to store the state
	StateFlushIntervalSec int `json:"stateFlushIntervalSec"`

	// FileFormats allows to specify which date-time formats can be used
	// when parsing files by the specific matcher
	FileFormats []*FileFormat `json:"fileFormats"`

	// RecordMaxSizeBytes defines a maximum size of one record
	RecordMaxSizeBytes int `json:"recordMaxSizeBytes"`

	// EventMaxRecords defines how many records can be packed in one event
	EventMaxRecords     int `json:"eventMaxRecords"`
	EventSendIntervalMs int `json:"eventSendIntervalMs"`
}

Config is a structure which contains scanner configuration and settings

func NewDefaultConfig

func NewDefaultConfig() *Config

func (*Config) Apply

func (c *Config) Apply(c1 *Config)

func (*Config) String

func (c *Config) String() string

type FileFormat

type FileFormat struct {
	// PathMatcher defines regExp which should be applied to a file name
	// to identify whether the time format below will be applied to it or not
	// example is '.*'
	PathMatcher string `json:"pathMatcher"`

	// DataFormat defines how data is formatted in the file: text, json etc.
	// By the field value a type of parser for the worker will be created then
	DataFormat string `json:"format"`

	// TimeFormats contains a list of supposed date-time formats that should
	// be tried for parsing the file lines (see dtparser) example is
	// ["MMM D, YYYY h:mm:ss P"]
	TimeFormats []string `json:"timeFormats"`
}

FileFormat defines a pattern of file matching and time formats, if any, that should be applied to the files when parsing

func (*FileFormat) String

func (c *FileFormat) String() string

=== fileFormat

type Parser

type Parser interface {
	io.Closer

	// NextRecord parses next record. It returns error if it could not parse
	// a record from the stream. io.EOF is returned if no new records found, but
	// end is reached.
	NextRecord() (*model.Record, error)

	// SetStreamPos specifies the stream position for the next record read
	SetStreamPos(pos int64) error

	// GetStreamPos returns position of the last successfully (error was nil)
	// returned record by nextRecord(). If nextRecord() returned non-nil
	// error the getStreamPos() returned value is not relevant and should
	// not be used as a valid stream position.
	GetStreamPos() int64

	// GetStat returns the parser statistic
	GetStats() *ParserStats
}

Parser provides an interface for retrieving records from a data-stream. Implementations of the interface are supposed to be initialized by the stream (io.Reader)

type ParserStats

type ParserStats struct {
	DataType string
	Size     int64
	Pos      int64
	// for text parsers provides information about found date-time formats
	DateFormats map[string]int64
}

ParserStat struct contains information about the parser statistics

type Scanner

type Scanner struct {
	// contains filtered or unexported fields
}

func NewScanner

func NewScanner(cfg *Config, ss StateStorage) (*Scanner, error)

=== scanner

func (*Scanner) Events

func (s *Scanner) Events() <-chan *model.Event

func (*Scanner) GetStats

func (s *Scanner) GetStats() *CollectorStats

func (*Scanner) Start

func (s *Scanner) Start() error

func (*Scanner) Stop

func (s *Scanner) Stop() error

type StateStorage

type StateStorage interface {
	// ReadData will return data buffer. If the status is not
	// found os.IsNotExist(err) will be true.
	ReadData() ([]byte, error)
	WriteData(buf []byte) error
}

StateStorage interface allows to read and write serialized data of scanner status

func NewFileStateStorage

func NewFileStateStorage(fn string) StateStorage

func NewInMemStateStorage

func NewInMemStateStorage() StateStorage

type WorkerStats

type WorkerStats struct {
	Filename    string
	Id          string
	ParserStats *ParserStats
}