Documentation ¶
Overview ¶
Package processors implements preprocessors for ingesters. The intended usage is to create a ProcessorSet and call ProcessorSet.Process(). Calls to ProcessorSet.Process() are thread-safe while Process() calls on specific processors is not.
Index ¶
- Constants
- Variables
- func CheckProcessor(id string) error
- func PopSet(cnt int) []*entry.Entry
- func ProcessorLoadConfig(vc *config.VariableConfig) (cfg interface{}, err error)
- type CSVRouteConfig
- type CSVRouter
- type CiscoISE
- type CiscoISEConfig
- type Corelight
- type CorelightConfig
- type Drop
- type DropConfig
- type EntryEncoder
- type Forwarder
- type ForwarderConfig
- type GravwellForwarder
- type GravwellForwarderConfig
- type GzipDecompressor
- type GzipDecompressorConfig
- type JsonArraySplitConfig
- type JsonArraySplitter
- type JsonExtractConfig
- type JsonExtractor
- type JsonFilter
- type JsonFilterConfig
- type JsonTimestamp
- type JsonTimestampConfig
- type PersistentBuffer
- type PersistentBufferConfig
- type PersistentBufferConsumer
- type Plugin
- type PluginConfig
- type PluginData
- type Processor
- type ProcessorConfig
- func (pc ProcessorConfig) CheckConfig(name string) (err error)
- func (pc ProcessorConfig) CheckProcessors(set []string) (err error)
- func (pc ProcessorConfig) MarshalJSON() ([]byte, error)
- func (pc ProcessorConfig) ProcessorSet(t tagWriter, names []string) (pr *ProcessorSet, err error)
- func (pc ProcessorConfig) Validate() (err error)
- type ProcessorSet
- func (pr *ProcessorSet) AddProcessor(p Processor)
- func (pr *ProcessorSet) Close() (err error)
- func (pr *ProcessorSet) Enabled() bool
- func (pr *ProcessorSet) Process(ent *entry.Entry) (err error)
- func (pr *ProcessorSet) ProcessBatch(ents []*entry.Entry) (err error)
- func (pr *ProcessorSet) ProcessBatchContext(ents []*entry.Entry, ctx context.Context) (err error)
- func (pr *ProcessorSet) ProcessContext(ent *entry.Entry, ctx context.Context) (err error)
- type RegexExtractConfig
- type RegexExtractor
- type RegexRouteConfig
- type RegexRouter
- type RegexTimestamp
- type RegexTimestampConfig
- type SetAllocator
- type SrcRouteConfig
- type SrcRouter
- type SyslogRouter
- type SyslogRouterConfig
- type Tagger
- type Vpc
- type VpcConfig
Constants ¶
const ( PluginProcessor string = `plugin` PluginEngineScriggo string = `scriggo` )
const (
CSVRouterProcessor = `csvrouter`
)
const (
CiscoISEProcessor string = `cisco_ise`
)
const (
CorelightProcessor = `corelight`
)
const (
DropProcessor string = `drop`
)
const (
ForwarderProcessor string = `forwarder`
)
const (
GravwellForwarderProcessor string = `gravwellforwarder`
)
const (
GzipProcessor string = `gzip`
)
const (
JsonArraySplitProcessor string = `jsonarraysplit`
)
const (
JsonExtractProcessor string = `jsonextract`
)
const (
JsonFilterProcessor string = `jsonfilter`
)
const (
JsonTimestampProcessor string = `jsontimeextract`
)
const PersistentBufferProcessor = `persistent-buffer`
const (
RegexExtractProcessor = `regexextract`
)
const (
RegexRouterProcessor = `regexrouter`
)
const (
RegexTimestampProcessor string = `regextimestamp`
)
const (
SrcRouterProcessor = `srcrouter`
)
const (
SyslogRouterProcessor = `syslogrouter`
)
const (
VpcProcessor string = `vpc`
)
Variables ¶
var ( ErrInvalidRemoteISEHeader = errors.New("Failed to match remote ISE header") ErrInvalidISEHeader = errors.New("Failed to match ISE header") ErrInvalidRemoteISESeq = errors.New("Invalid multipart message sequence") ErrInvalidISESeq = errors.New("Invalid ISE message sequence") )
var ( ErrUnknownType = errors.New("Unknown entry encoder type") ErrInvalidWriter = errors.New("Writer is nil") )
var ( ErrNoUnixOnWindows = errors.New("Unix transport not available on Windows") ErrMissingTarget = errors.New("Target IP:Port or Unix path required") ErrUnknownProtocol = errors.New("Unknown protocol") ErrUnknownFormat = errors.New("Unknown format") ErrClosed = errors.New("Closed") ErrNilTagger = errors.New("invalid parameter, missing tagger") )
var ( ErrNilGF = errors.New("GravwellForwarder object is nil") ErrFailedTagLookup = errors.New("GravwellForwarder failed to lookup tag") )
var ( ErrMissStrictConflict = errors.New("Strict-Extraction requires Drop-Misses=true") ErrMissingExtractions = errors.New("Extractions specifications missing") ErrNoAdditionalFields = errors.New("Additional-Fields cannot be set if Extractions parameter is unset") ErrInvalidExtractions = errors.New("Invalid Extractions") ErrInvalidKeyname = errors.New("Invalid keyname") ErrDuplicateKey = errors.New("Duplicate extraction key") ErrDuplicateKeyname = errors.New("Duplicate keys") ErrSingleArraySplitOnly = errors.New("jsonarraysplit only supports a single extraction") )
var ( ErrMatchAction = errors.New("Match-Action must be either 'pass' or 'drop' (default pass)") ErrMatchLogic = errors.New("Match-Logic must be either 'and' or 'or' (default and)") )
var ( ErrNoPlugins = errors.New("No plugins provided in Plugin-Path") ErrDuplicateFile = errors.New("dupclicate plugin file") )
var ( ErrUnknownProcessor = errors.New("Unknown preprocessor") ErrNilConfig = errors.New("Nil configuration") ErrNotFound = errors.New("Processor not found") ErrNotReady = errors.New("ProcessorSet not ready") ErrInvalidEntry = errors.New("ErrInvalidEntry") )
var ( ErrMissingRegex = errors.New("Missing regular expression") ErrMissingRouteExtraction = errors.New("Missing route extraction name") ErrMissingRoutes = errors.New("Missing route specifications") ErrMissingExtractNames = errors.New("Regular expression does not extract any names") )
var ( ErrEmptyRegex = errors.New("Empty regular expression") ErrEmptyMatch = errors.New("Empty TS-Match-Name") ErrNoSubexps = errors.New("Must specify at least one subexpression") )
var (
ErrBufferEmpty = errors.New("Buffer is empty")
)
var (
ErrInvalidColumnIndex = errors.New("Invalid column index")
)
var (
ErrNotGzipped = errors.New("Input is not a gzipped stream")
)
Functions ¶
func CheckProcessor ¶
func ProcessorLoadConfig ¶
func ProcessorLoadConfig(vc *config.VariableConfig) (cfg interface{}, err error)
Types ¶
type CSVRouteConfig ¶ added in v3.8.10
func CSVRouteLoadConfig ¶ added in v3.8.10
func CSVRouteLoadConfig(vc *config.VariableConfig) (c CSVRouteConfig, err error)
type CSVRouter ¶ added in v3.8.10
type CSVRouter struct { CSVRouteConfig // contains filtered or unexported fields }
func NewCSVRouter ¶ added in v3.8.10
func NewCSVRouter(cfg CSVRouteConfig, tagger Tagger) (*CSVRouter, error)
type CiscoISE ¶ added in v3.6.0
type CiscoISE struct { CiscoISEConfig // contains filtered or unexported fields }
func NewCiscoISEProcessor ¶ added in v3.6.0
func NewCiscoISEProcessor(cfg CiscoISEConfig) (ise *CiscoISE, err error)
type CiscoISEConfig ¶ added in v3.6.0
type CiscoISEConfig struct { Passthrough_Misses bool //deprecated DO NOT USE Drop_Misses bool Enable_Multipart_Reassembly bool Max_Multipart_Buffer uint64 Max_Multipart_Latency string Output_Format string Attribute_Drop_Filter []string Attribute_Strip_Header bool // contains filtered or unexported fields }
func CiscoISELoadConfig ¶ added in v3.6.0
func CiscoISELoadConfig(vc *config.VariableConfig) (c CiscoISEConfig, err error)
type Corelight ¶ added in v3.8.12
type Corelight struct { CorelightConfig // contains filtered or unexported fields }
A Corelight processor takes JSON-formatted Corelight logs and reformats them as TSV, matching the standard Zeek log types.
func NewCorelight ¶ added in v3.8.12
func NewCorelight(cfg CorelightConfig, tagger Tagger) (*Corelight, error)
type CorelightConfig ¶ added in v3.8.12
type CorelightConfig struct { // Prefix specifies the prefix for corelight logs. Each log type name will // be appended to the prefix to create a tag; thus if Prefix="zeek", // conn logs will be ingested to the 'zeekconn' tag, dhcp logs to 'zeekdhcp', // and so on. Prefix string // Custom_Format specifies a custom override for a path value and headers, there can be many Custom_Format []string }
func CorelightLoadConfig ¶ added in v3.8.12
func CorelightLoadConfig(vc *config.VariableConfig) (c CorelightConfig, err error)
func (*CorelightConfig) Validate ¶ added in v3.8.12
func (cl *CorelightConfig) Validate() (err error)
type Drop ¶ added in v3.4.3
type Drop struct { DropConfig // contains filtered or unexported fields }
Drop does not have any state, and doesn't do much
func NewDrop ¶ added in v3.4.3
func NewDrop(cfg DropConfig) (*Drop, error)
type DropConfig ¶ added in v3.4.3
type DropConfig struct { }
func DropLoadConfig ¶ added in v3.4.3
func DropLoadConfig(vc *config.VariableConfig) (c DropConfig, err error)
type Forwarder ¶
type Forwarder struct { ForwarderConfig sync.Mutex // contains filtered or unexported fields }
func NewForwarder ¶
func NewForwarder(cfg ForwarderConfig, tgr Tagger) (nf *Forwarder, err error)
type ForwarderConfig ¶
type ForwarderConfig struct { Target string Protocol string Delimiter string Format string Tag []string Regex []string Source []string Timeout uint //timeout in seconds for a write Buffer uint //number of entries in flight (basically channel buffer size) Non_Blocking bool Insecure_Skip_TLS_Verify bool }
func ForwarderLoadConfig ¶
func ForwarderLoadConfig(vc *config.VariableConfig) (c ForwarderConfig, err error)
func (*ForwarderConfig) Validate ¶
func (nfc *ForwarderConfig) Validate() (err error)
type GravwellForwarder ¶ added in v3.4.0
type GravwellForwarder struct { GravwellForwarderConfig ingest.UniformMuxerConfig // contains filtered or unexported fields }
func NewGravwellForwarder ¶ added in v3.4.0
func NewGravwellForwarder(cfg GravwellForwarderConfig, tgr Tagger) (*GravwellForwarder, error)
func (*GravwellForwarder) Close ¶ added in v3.4.0
func (gf *GravwellForwarder) Close() error
func (*GravwellForwarder) Flush ¶ added in v3.6.0
func (gf *GravwellForwarder) Flush() []*entry.Entry
type GravwellForwarderConfig ¶ added in v3.4.0
type GravwellForwarderConfig struct {
config.IngestConfig
}
func GravwellForwarderLoadConfig ¶ added in v3.4.0
func GravwellForwarderLoadConfig(vc *config.VariableConfig) (c GravwellForwarderConfig, err error)
func (GravwellForwarderConfig) MarshalJSON ¶ added in v3.6.1
func (gfc GravwellForwarderConfig) MarshalJSON() ([]byte, error)
we DO NOT want to ship the ingest secret here, so we mask it off
type GzipDecompressor ¶
type GzipDecompressor struct { GzipDecompressorConfig // contains filtered or unexported fields }
GzipDecompressor does not have any state
func NewGzipDecompressor ¶
func NewGzipDecompressor(cfg GzipDecompressorConfig) (*GzipDecompressor, error)
func (*GzipDecompressor) Config ¶
func (gd *GzipDecompressor) Config(v interface{}) (err error)
type GzipDecompressorConfig ¶
func GzipLoadConfig ¶
func GzipLoadConfig(vc *config.VariableConfig) (c GzipDecompressorConfig, err error)
func (GzipDecompressorConfig) BufferSizes ¶
func (gdc GzipDecompressorConfig) BufferSizes() (base, max int)
type JsonArraySplitConfig ¶
type JsonArraySplitConfig struct { Passthrough_Misses bool //deprecated DO NOT USE Drop_Misses bool Extraction string Force_JSON_Object bool Additional_Fields string }
func JsonArraySplitLoadConfig ¶
func JsonArraySplitLoadConfig(vc *config.VariableConfig) (c JsonArraySplitConfig, err error)
type JsonArraySplitter ¶
type JsonArraySplitter struct { JsonArraySplitConfig // contains filtered or unexported fields }
func NewJsonArraySplitter ¶
func NewJsonArraySplitter(cfg JsonArraySplitConfig) (*JsonArraySplitter, error)
func (*JsonArraySplitter) Config ¶
func (j *JsonArraySplitter) Config(v interface{}) (err error)
type JsonExtractConfig ¶
type JsonExtractConfig struct { Passthrough_Misses bool //deprecated DO NOT USE Drop_Misses bool Strict_Extraction bool Force_JSON_Object bool Extractions string }
func JsonExtractLoadConfig ¶
func JsonExtractLoadConfig(vc *config.VariableConfig) (c JsonExtractConfig, err error)
type JsonExtractor ¶
type JsonExtractor struct { JsonExtractConfig // contains filtered or unexported fields }
JsonExtractor
func NewJsonExtractor ¶
func NewJsonExtractor(cfg JsonExtractConfig) (*JsonExtractor, error)
func (*JsonExtractor) Config ¶
func (j *JsonExtractor) Config(v interface{}) (err error)
type JsonFilter ¶
type JsonFilter struct { JsonFilterConfig // contains filtered or unexported fields }
func NewJsonFilter ¶
func NewJsonFilter(cfg JsonFilterConfig) (*JsonFilter, error)
NewJsonFilter instantiates a JsonFilter preprocessor. It will attempt to open and read the files specified in the configuration; nonexistent files or permissions problems will return an error.
func (*JsonFilter) Config ¶
func (j *JsonFilter) Config(v interface{}) (err error)
type JsonFilterConfig ¶
type JsonFilterConfig struct { // what to do when an entry matches: "pass" or "drop" Match_Action string // "and" or "or", specifying that either *all* fields must match or that *any* field will be sufficient Match_Logic string // each Field-Filter consists of the field to match, a comma, and the path to the file containing possible values, e.g. "foo.bar,/tmp/values" Field_Filter []string }
func JsonFilterLoadConfig ¶
func JsonFilterLoadConfig(vc *config.VariableConfig) (c JsonFilterConfig, err error)
type JsonTimestamp ¶ added in v3.8.15
type JsonTimestamp struct { JsonTimestampConfig // contains filtered or unexported fields }
func NewJsonTimestamp ¶ added in v3.8.15
func NewJsonTimestamp(cfg JsonTimestampConfig) (*JsonTimestamp, error)
NewJsonTimestamp instantiates a JsonTimestamp preprocessor. It will attempt to open and read the files specified in the configuration; nonexistent files or permissions problems will return an error.
func (*JsonTimestamp) Config ¶ added in v3.8.15
func (j *JsonTimestamp) Config(v interface{}) (err error)
type JsonTimestampConfig ¶ added in v3.8.15
type JsonTimestampConfig struct { // Optional timestamp override Timestamp_Override string // Optional setting of assume local timezone Assume_Local_Timezone bool // Required path used to go find the timesatmp in the JSON blob Path string }
func JsonTimestampLoadConfig ¶ added in v3.8.15
func JsonTimestampLoadConfig(vc *config.VariableConfig) (c JsonTimestampConfig, err error)
type PersistentBuffer ¶ added in v3.7.5
type PersistentBuffer struct { PersistentBufferConfig // contains filtered or unexported fields }
PersistentBuffer does not have any state, and doesn't do much
func NewPersistentBuffer ¶ added in v3.7.5
func NewPersistentBuffer(cfg PersistentBufferConfig, tagger Tagger) (*PersistentBuffer, error)
func (*PersistentBuffer) Close ¶ added in v3.7.5
func (gd *PersistentBuffer) Close() (err error)
func (*PersistentBuffer) Config ¶ added in v3.7.5
func (gd *PersistentBuffer) Config(v interface{}) (err error)
func (*PersistentBuffer) Flush ¶ added in v3.7.5
func (gd *PersistentBuffer) Flush() []*entry.Entry
type PersistentBufferConfig ¶ added in v3.7.5
func PersistentBufferLoadConfig ¶ added in v3.7.5
func PersistentBufferLoadConfig(vc *config.VariableConfig) (c PersistentBufferConfig, err error)
type PersistentBufferConsumer ¶ added in v3.7.5
type PersistentBufferConsumer struct {
// contains filtered or unexported fields
}
func OpenPersistentBuffer ¶ added in v3.7.5
func OpenPersistentBuffer(pth string) (pbc *PersistentBufferConsumer, err error)
func (*PersistentBufferConsumer) Close ¶ added in v3.7.5
func (pbc *PersistentBufferConsumer) Close() (err error)
func (*PersistentBufferConsumer) Pop ¶ added in v3.7.5
func (pbc *PersistentBufferConsumer) Pop() ([]types.StringTagEntry, error)
type Plugin ¶ added in v3.8.3
type Plugin struct { PluginConfig // contains filtered or unexported fields }
func NewPluginProcessor ¶ added in v3.8.3
func NewPluginProcessor(cfg PluginConfig, tg Tagger) (p *Plugin, err error)
type PluginConfig ¶ added in v3.8.3
type PluginConfig struct { Plugin_Path []string //path to the plugin files (this may support multifile plugins later Plugin_Engine string // defaults to scriggo Debug bool // defaults to false // contains filtered or unexported fields }
func PluginLoadConfig ¶ added in v3.8.3
func PluginLoadConfig(vc *config.VariableConfig) (pc PluginConfig, err error)
type PluginData ¶ added in v3.8.3
PluginData implements the fs.FS interface
type Processor ¶
type Processor interface { Process([]*entry.Entry) ([]*entry.Entry, error) //process an data item potentially setting a tag Flush() []*entry.Entry Close() error //give the processor a chance to tidy up }
Processor is an interface that takes an entry and processes it, returning a new block
type ProcessorConfig ¶
type ProcessorConfig map[string]*config.VariableConfig
func (ProcessorConfig) CheckConfig ¶
func (pc ProcessorConfig) CheckConfig(name string) (err error)
func (ProcessorConfig) CheckProcessors ¶
func (pc ProcessorConfig) CheckProcessors(set []string) (err error)
func (ProcessorConfig) MarshalJSON ¶ added in v3.6.1
func (pc ProcessorConfig) MarshalJSON() ([]byte, error)
func (ProcessorConfig) ProcessorSet ¶
func (pc ProcessorConfig) ProcessorSet(t tagWriter, names []string) (pr *ProcessorSet, err error)
func (ProcessorConfig) Validate ¶
func (pc ProcessorConfig) Validate() (err error)
type ProcessorSet ¶
func NewProcessorSet ¶
func NewProcessorSet(wtr entWriter) *ProcessorSet
func (*ProcessorSet) AddProcessor ¶
func (pr *ProcessorSet) AddProcessor(p Processor)
func (*ProcessorSet) Close ¶
func (pr *ProcessorSet) Close() (err error)
Close will close the underlying preprocessors within the set. This function DOES NOT close the ingest muxer handle. It is ONLY for shutting down preprocessors
func (*ProcessorSet) Enabled ¶
func (pr *ProcessorSet) Enabled() bool
func (*ProcessorSet) ProcessBatch ¶ added in v3.6.1
func (pr *ProcessorSet) ProcessBatch(ents []*entry.Entry) (err error)
func (*ProcessorSet) ProcessBatchContext ¶ added in v3.6.1
func (*ProcessorSet) ProcessContext ¶
type RegexExtractConfig ¶
type RegexExtractConfig struct { Passthrough_Misses bool //deprecated DO NOT USE Drop_Misses bool Regex string Template string Attach []string // list of regular expression items to attach as intrinsic EVs }
func RegexExtractLoadConfig ¶
func RegexExtractLoadConfig(vc *config.VariableConfig) (c RegexExtractConfig, err error)
type RegexExtractor ¶
type RegexExtractor struct { RegexExtractConfig // contains filtered or unexported fields }
func NewRegexExtractor ¶
func NewRegexExtractor(cfg RegexExtractConfig) (*RegexExtractor, error)
func (*RegexExtractor) Config ¶
func (re *RegexExtractor) Config(v interface{}) (err error)
type RegexRouteConfig ¶
type RegexRouteConfig struct { Regex string Route_Extraction string Route []string Drop_Misses bool }
func RegexRouteLoadConfig ¶
func RegexRouteLoadConfig(vc *config.VariableConfig) (c RegexRouteConfig, err error)
type RegexRouter ¶
type RegexRouter struct { RegexRouteConfig // contains filtered or unexported fields }
func NewRegexRouter ¶
func NewRegexRouter(cfg RegexRouteConfig, tagger Tagger) (*RegexRouter, error)
func (*RegexRouter) Config ¶
func (rr *RegexRouter) Config(v interface{}, tagger Tagger) (err error)
type RegexTimestamp ¶
type RegexTimestamp struct { RegexTimestampConfig // contains filtered or unexported fields }
func NewRegexTimestampProcessor ¶
func NewRegexTimestampProcessor(cfg RegexTimestampConfig) (*RegexTimestamp, error)
func (*RegexTimestamp) Config ¶
func (rt *RegexTimestamp) Config(v interface{}) (err error)
type RegexTimestampConfig ¶
type RegexTimestampConfig struct { Regex string // the regular expression to apply to the data TS_Match_Name string // the submatch which contains the timestamp Timestamp_Format_Override string Timezone_Override string Assume_Local_Timezone bool }
func RegexTimestampLoadConfig ¶
func RegexTimestampLoadConfig(vc *config.VariableConfig) (c RegexTimestampConfig, err error)
type SetAllocator ¶
func NewSetAllocator ¶
func NewSetAllocator(allocSize, reallocSize int) (sa *SetAllocator, err error)
type SrcRouteConfig ¶ added in v3.6.1
func SrcRouteLoadConfig ¶ added in v3.6.1
func SrcRouteLoadConfig(vc *config.VariableConfig) (c SrcRouteConfig, err error)
type SrcRouter ¶ added in v3.6.1
type SrcRouter struct { SrcRouteConfig // contains filtered or unexported fields }
func NewSrcRouter ¶ added in v3.6.1
func NewSrcRouter(cfg SrcRouteConfig, tagger Tagger) (*SrcRouter, error)
type SyslogRouter ¶ added in v3.8.19
type SyslogRouter struct { SyslogRouterConfig // contains filtered or unexported fields }
func NewSyslogRouter ¶ added in v3.8.19
func NewSyslogRouter(cfg SyslogRouterConfig, tagger Tagger) (*SyslogRouter, error)
func (*SyslogRouter) Config ¶ added in v3.8.19
func (sr *SyslogRouter) Config(v interface{}) (err error)
type SyslogRouterConfig ¶ added in v3.8.19
func SyslogRouterLoadConfig ¶ added in v3.8.19
func SyslogRouterLoadConfig(vc *config.VariableConfig) (c SyslogRouterConfig, err error)
type Vpc ¶
type Vpc struct { VpcConfig // contains filtered or unexported fields }
func NewVpcProcessor ¶
type VpcConfig ¶
func VpcLoadConfig ¶
func VpcLoadConfig(vc *config.VariableConfig) (c VpcConfig, err error)
func (VpcConfig) BufferSizes ¶
Source Files ¶
- cisco_ise.go
- corelight.go
- csvrouter.go
- drop.go
- entryencoders.go
- extracttools.go
- forwarder.go
- gravdup.go
- gzip.go
- jsonextract.go
- jsonfilter.go
- jsonsplit.go
- jsontimeextract.go
- persistent_buffer.go
- plugin.go
- processors.go
- processors_linux.go
- regexextract.go
- regexrouter.go
- regextimestamp.go
- srcrouter.go
- syslogrouter.go
- utils.go
- vpc.go