crawler

Documentation

Index

• type Crawler
  • func (crawler *Crawler) Start(files []config.ProspectorConfig, eventChan chan *input.FileEvent)
  • func (crawler *Crawler) Stop()
• type Prospector
  • func (p *Prospector) Init() error
  • func (p *Prospector) Run(spoolChan chan *input.FileEvent)
  • func (p *Prospector) Stop()
• type ProspectorFileStat
• type Registrar
  • func NewRegistrar(registryFile string) (*Registrar, error)
  • func (r *Registrar) GetFileState(path string) (*FileState, bool)
  • func (r *Registrar) Init() error
  • func (r *Registrar) LoadState()
  • func (r *Registrar) Run()
  • func (r *Registrar) Stop()

Types

type Crawler

type Crawler struct {
	// Registrar object to persist the state
	Registrar *Registrar
	// contains filtered or unexported fields
}

func (*Crawler) Start

func (crawler *Crawler) Start(files []config.ProspectorConfig, eventChan chan *input.FileEvent)

func (*Crawler) Stop

func (crawler *Crawler) Stop()

type Prospector

type Prospector struct {
	ProspectorConfig cfg.ProspectorConfig
	// contains filtered or unexported fields
}

func (*Prospector) Init

func (p *Prospector) Init() error

Init sets up default config for prospector

func (*Prospector) Run

func (p *Prospector) Run(spoolChan chan *input.FileEvent)

Starts scanning through all the file paths and fetch the related files. Start a harvester for each file

func (*Prospector) Stop

func (p *Prospector) Stop()

type ProspectorFileStat

type ProspectorFileStat struct {
	Fileinfo      os.FileInfo /* the file info */
	Harvester     chan int64  /* the harvester will send an event with its offset when it closes */
	LastIteration uint32      /* int number of the last iterations in which we saw this file */
}

Contains statistic about file when it was last seend by the prospector

type Registrar

type Registrar struct {

	// Map with all file paths inside and the corresponding state
	State map[string]*FileState
	// Channel used by the prospector and crawler to send FileStates to be persisted
	Persist chan *input.FileState

	Channel chan []*FileEvent
	// contains filtered or unexported fields
}

func NewRegistrar

func NewRegistrar(registryFile string) (*Registrar, error)

func (*Registrar) GetFileState

func (r *Registrar) GetFileState(path string) (*FileState, bool)

func (*Registrar) Init

func (r *Registrar) Init() error

func (*Registrar) LoadState

func (r *Registrar) LoadState()

loadState fetches the previous reading state from the configure RegistryFile file The default file is .filebeat file which is stored in the same path as the binary is running

func (*Registrar) Run

func (r *Registrar) Run()

func (*Registrar) Stop

func (r *Registrar) Stop()