filter

package
Version: v0.1.0-alpha Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 19, 2021 License: MIT Imports: 22 Imported by: 0

Documentation

Overview

Package filter provides filter components.

Index

Constants

This section is empty.

Variables

View Source 
var All = []baker.FilterDesc{
	ClauseFilterDesc,
	ClearFieldsDesc,
	ConcatenateDesc,
	DedupDesc,
	ExpandJSONDesc,
	ExpandListDesc,
	FormatTimeDesc,
	HashDesc,
	MetadataLastModifiedDesc,
	MetadataUrlDesc,
	NotNullDesc,
	PartialCloneDesc,
	RegexMatchDesc,
	ReplaceFieldsDesc,
	SetStringFromURLDesc,
	StringMatchDesc,
	TimestampDesc,
	TimestampRangeDesc,
}

All is the list of all baker filters.

View Source 
var ClauseFilterDesc = baker.FilterDesc{
	Name:   "ClauseFilter",
	New:    NewClauseFilter,
	Config: &ClauseFilterConfig{},
	Help:   clauseFilterHelpMsg,
}

ClauseFilterDesc describes the ClauseFilter filter

View Source 
var ClearFieldsDesc = baker.FilterDesc{
	Name:   "ClearFields",
	New:    NewClearFields,
	Config: &ClearFieldsConfig{},
	Help:   "Reset a set of fields of all records passing through",
}

ClearFieldsDesc describes the ClearFields filter

View Source 
var ConcatenateDesc = baker.FilterDesc{
	Name:   "Concatenate",
	New:    NewConcatenate,
	Config: &ConcatenateConfig{},
	Help:   `Concatenate up to 10 fields' values to a single field`,
}

ConcatenateDesc describes the Concatenate filter

View Source 
var DedupDesc = baker.FilterDesc{
	Name:   "Dedup",
	New:    NewDedup,
	Config: &DedupConfig{},
	Help:   dedupHelp,
}
View Source 
var ExpandJSONDesc = baker.FilterDesc{
	Name:   "ExpandJSON",
	New:    NewExpandJSON,
	Config: &ExpandJSONConfig{},
	Help:   expandJSONhelp,
}
View Source 
var ExpandListDesc = baker.FilterDesc{
	Name:   "ExpandList",
	New:    NewExpandList,
	Config: &ExpandListConfig{},
	Help:   expandListhelp,
}
View Source 
var FormatTimeDesc = baker.FilterDesc{
	Name:   "FormatTime",
	New:    NewFormatTime,
	Config: &FormatTimeConfig{},
	Help:   formatTimeHelp,
}
View Source 
var HashDesc = baker.FilterDesc{
	Name:   "Hash",
	New:    NewHash,
	Config: &HashConfig{},
	Help: `This filter hashes a field using a specified hash function and writes the value 
to another (or the same) field. In order to have control over the set of characters
present, the hashed value can optionally be encoded.
	
	
Supported hash functions:
 - md5
 - sha256

Supported encodings:
- hex (hexadecimal encoding)
`,
}

HashDesc describes the Hash filter

View Source 
var MetadataLastModifiedDesc = baker.FilterDesc{
	Name:   "MetadataLastModified",
	New:    NewMetadataLastModified,
	Config: &MetadataLastModifiedConfig{},
	Help:   `Extract the "last modified" timestamp from the record Metadata and write it to the selected field.`,
}
View Source 
var MetadataUrlDesc = baker.FilterDesc{
	Name:   "MetadataUrl",
	New:    NewMetadataUrl,
	Config: &MetadataUrlConfig{},
	Help:   metadataUrlHelp,
}
View Source 
var NotNullDesc = baker.FilterDesc{
	Name:   "NotNull",
	New:    NewNotNull,
	Config: &NotNullConfig{},
	Help:   "Discard the records having null (i.e empty) fields.\n",
}

NotNullDesc describes the NotNull filter.

View Source 
var PartialCloneDesc = baker.FilterDesc{
	Name:   "PartialClone",
	New:    NewPartialClone,
	Config: &PartialCloneConfig{},
	Help:   "Copy a list of fields to a new record and process this new record, discarding the original one",
}
View Source 
var RegexMatchDesc = baker.FilterDesc{
	Name:   "RegexMatch",
	New:    NewRegexMatch,
	Config: &RegexMatchConfig{},
	Help:   "Discard a record if one or more fields don't match the corresponding regular expressions",
}

RegexMatchDesc describes the RegexMatch filter

View Source 
var ReplaceFieldsDesc = baker.FilterDesc{
	Name:   "ReplaceFields",
	New:    NewReplaceFields,
	Config: &ReplaceFieldsConfig{},
	Help:   `Copy a field value or a fixed value to another field. Can copy multiple fields.`,
}

ReplaceFieldsDesc describes the ReplaceFields filter

View Source 
var SetStringFromURLDesc = baker.FilterDesc{
	Name:   "SetStringFromURL",
	New:    NewSetStringFromURL,
	Config: &SetStringFromURLConfig{},
	Help:   helpSetStringFromURL,
}

SetStringFromURLDesc describes the SetStringFromURL filter.

View Source 
var StringMatchDesc = baker.FilterDesc{
	Name:   "StringMatch",
	New:    NewStringMatch,
	Config: &StringMatchConfig{},
	Help:   "Discard records if a field matches any of the provided strings",
}

StringMatchDesc describes the StringMatch filter

View Source 
var TimestampDesc = baker.FilterDesc{
	Name:   "Timestamp",
	New:    NewTimestamp,
	Config: &TimestampConfig{},
	Help:   "Sets a field to the Unix Epoch timestamp at which the record is processed",
}

TimestampDesc describes the Timestamp filter.

View Source 
var TimestampRangeDesc = baker.FilterDesc{
	Name:   "TimestampRange",
	New:    NewTimestampRange,
	Config: &TimestampRangeConfig{},
	Help:   "Discard records if the value of a field containing a timestamp is out of the given time range (i.e StartDateTime <= value < EndDateTime)",
}

TimestampRangeDesc describes the TimestampRange filter.

Functions

func NewClauseFilter 

func NewClauseFilter(cfg baker.FilterParams) (baker.Filter, error)

func NewClearFields 

func NewClearFields(cfg baker.FilterParams) (baker.Filter, error)

NewClearFields returns a ClearFields filter.

func NewConcatenate 

func NewConcatenate(cfg baker.FilterParams) (baker.Filter, error)

func NewDedup 

func NewDedup(cfg baker.FilterParams) (baker.Filter, error)

func NewExpandJSON 

func NewExpandJSON(cfg baker.FilterParams) (baker.Filter, error)

func NewExpandList 

func NewExpandList(cfg baker.FilterParams) (baker.Filter, error)

func NewFormatTime 

func NewFormatTime(cfg baker.FilterParams) (baker.Filter, error)

func NewHash 

func NewHash(cfg baker.FilterParams) (baker.Filter, error)

func NewMetadataLastModified 

func NewMetadataLastModified(cfg baker.FilterParams) (baker.Filter, error)

func NewMetadataUrl 

func NewMetadataUrl(cfg baker.FilterParams) (baker.Filter, error)

func NewNotNull 

func NewNotNull(cfg baker.FilterParams) (baker.Filter, error)

NewNotNull creates and configures a new NotNull filter.

func NewPartialClone 

func NewPartialClone(cfg baker.FilterParams) (baker.Filter, error)

func NewRegexMatch 

func NewRegexMatch(cfg baker.FilterParams) (baker.Filter, error)

NewRegexMatch returns a RegexMatch filter.

func NewReplaceFields 

func NewReplaceFields(cfg baker.FilterParams) (baker.Filter, error)

func NewSetStringFromURL 

func NewSetStringFromURL(cfg baker.FilterParams) (baker.Filter, error)

func NewStringMatch 

func NewStringMatch(cfg baker.FilterParams) (baker.Filter, error)

NewStringMatch returns a StringMatch filter.

func NewTimestamp 

func NewTimestamp(cfg baker.FilterParams) (baker.Filter, error)

NewTimestamp creates and configures a Timestamp filter.

func NewTimestampRange 

func NewTimestampRange(cfg baker.FilterParams) (baker.Filter, error)

NewTimestampRange creates and configures a TimestampRange filter.

Types

type Clause 

type Clause struct {
	// contains filtered or unexported fields
}

Will real sum types ever make it to go?

Validness of the fields in this structure depend on clauseType

type ClauseFilter 

type ClauseFilter struct {
	// contains filtered or unexported fields
}

func (*ClauseFilter) Process 

func (f *ClauseFilter) Process(l baker.Record, next func(baker.Record))

func (*ClauseFilter) Stats 

func (f *ClauseFilter) Stats() baker.FilterStats

type ClauseFilterConfig 

type ClauseFilterConfig struct {
	Clause string `help:"Boolean formula describing which events to let through. If empty, let everything through."`
}

ClauseFilterConfig describes the ClauseFilter filter config

type ClearFields 

type ClearFields struct {
	// contains filtered or unexported fields
}

ClearFields filter clears (i.e set to the empty string) a set of fields.

func (*ClearFields) Process 

func (f *ClearFields) Process(l baker.Record, next func(baker.Record))

Process is where the actual filtering takes place.

func (*ClearFields) Stats 

func (f *ClearFields) Stats() baker.FilterStats

Stats returns filter statistics.

type ClearFieldsConfig 

type ClearFieldsConfig struct {
	Fields []string `help:"set of fields to clear" required:"true"`
}

ClearFieldsConfig holds config parameters of the ClearFields filter.

type Concatenate 

type Concatenate struct {
	// contains filtered or unexported fields
}

func (*Concatenate) Process 

func (c *Concatenate) Process(l baker.Record, next func(baker.Record))

func (*Concatenate) Stats 

func (c *Concatenate) Stats() baker.FilterStats

type ConcatenateConfig 

type ConcatenateConfig struct {
	Fields    []string `help:"The field names to concatenate, in order"`
	Target    string   `help:"The field name to save the concatenated value to"`
	Separator string   `help:"Separator to concatenate the values. Must either be empty or a single ASCII, non-nil char"`
}

type Dedup 

type Dedup struct {
	// contains filtered or unexported fields
}

func (*Dedup) Process 

func (f *Dedup) Process(l baker.Record, next func(baker.Record))

func (*Dedup) Stats 

func (f *Dedup) Stats() baker.FilterStats

type DedupConfig 

type DedupConfig struct {
	Fields       []string `help:"fields to consider when comparing records" required:"true"`
	KeySeparator string   `help:"character separator used to build a key from the fields" default:"\\x1e"`
}

type ExpandJSON 

type ExpandJSON struct {
	// contains filtered or unexported fields
}

func (*ExpandJSON) Process 

func (f *ExpandJSON) Process(l baker.Record, next func(baker.Record))

func (*ExpandJSON) Stats 

func (f *ExpandJSON) Stats() baker.FilterStats

type ExpandJSONConfig 

type ExpandJSONConfig struct {
	Source          string            `help:"record field that contains the json" required:"true"`
	Fields          map[string]string `help:"<JMESPath -> record field> map, the rest will be ignored" required:"true"`
	TrueFalseValues []string          `help:"bind the json boolean values to correstponding strings" default:"[\"true\", \"false\"]"`
}

type ExpandList 

type ExpandList struct {
	// contains filtered or unexported fields
}

func (*ExpandList) Process 

func (f *ExpandList) Process(l baker.Record, next func(baker.Record))

func (*ExpandList) Stats 

func (f *ExpandList) Stats() baker.FilterStats

type ExpandListConfig 

type ExpandListConfig struct {
	Source    string            `help:"record field that contains the list" required:"true"`
	Fields    map[string]string `help:"<list index -> record field> map, the rest will be ignored" required:"true"`
	Separator string            `help:"character separator of the list" default:";"`
}

type FormatTime 

type FormatTime struct {
	// contains filtered or unexported fields
}

func (*FormatTime) Process 

func (f *FormatTime) Process(l baker.Record, next func(baker.Record))

func (*FormatTime) Stats 

func (f *FormatTime) Stats() baker.FilterStats

type FormatTimeConfig 

type FormatTimeConfig struct {
	SrcField  string `help:"Field name of the input time" required:"true"`
	DstField  string `help:"Field name of the output time" required:"true"`
	SrcFormat string `help:"Format of the input time" required:"false" default:"UnixDate"`
	DstFormat string `help:"Format of the output time" required:"false" default:"unixms"`
}

type Hash 

type Hash struct {
	// contains filtered or unexported fields
}

func (*Hash) Process 

func (h *Hash) Process(r baker.Record, next func(baker.Record))

func (*Hash) Stats 

func (h *Hash) Stats() baker.FilterStats

type HashConfig 

type HashConfig struct {
	SrcField string `help:"Name of the field to hash" required:"true"`
	DstField string `help:"Name of the field to write the result to" required:"true"`
	Function string `help:"Name of the hash function to use" required:"true"`
	Encoding string `help:"Name of the encoding function to use" required:"false"`
}

type MetadataLastModified 

type MetadataLastModified struct {
	// contains filtered or unexported fields
}

func (*MetadataLastModified) Process 

func (f *MetadataLastModified) Process(l baker.Record, next func(baker.Record))

func (*MetadataLastModified) Stats 

func (f *MetadataLastModified) Stats() baker.FilterStats

type MetadataLastModifiedConfig 

type MetadataLastModifiedConfig struct {
	DstField string `help:"Name of the field into which write the timestamp to" required:"true"`
}

type MetadataUrl 

type MetadataUrl struct {
	metadata.Cache // type: map[*url.URL]string
	// contains filtered or unexported fields
}

func (*MetadataUrl) Process 

func (f *MetadataUrl) Process(l baker.Record, next func(baker.Record))

func (*MetadataUrl) Stats 

func (f *MetadataUrl) Stats() baker.FilterStats

type MetadataUrlConfig 

type MetadataUrlConfig struct {
	DstField string `help:"Name of the field into to write the url to (or to clear if there's no url)" required:"true"`
}

type NotNull 

type NotNull struct {
	// contains filtered or unexported fields
}

NotNull is a baker filter that discards records having null fields.

func (*NotNull) Process 

func (v *NotNull) Process(l baker.Record, next func(baker.Record))

Process implements baker.Filter.

func (*NotNull) Stats 

func (v *NotNull) Stats() baker.FilterStats

Stats implements baker.Filter.

type NotNullConfig 

type NotNullConfig struct {
	Fields []string `help:"Fields is the list of fields to check for null/empty values" required:"true"`
}

NotNullConfig holds configuration parameters for the NotNull filter.

type PartialClone 

type PartialClone struct {
	// contains filtered or unexported fields
}

func (*PartialClone) Process 

func (s *PartialClone) Process(r baker.Record, next func(baker.Record))

func (*PartialClone) Stats 

func (s *PartialClone) Stats() baker.FilterStats

type PartialCloneConfig 

type PartialCloneConfig struct {
	Fields []string `help:"Fields that must be copied to the new line" required:"true"`
}

type RegexMatch 

type RegexMatch struct {
	// contains filtered or unexported fields
}

RegexMatch filter clears (i.e set to the empty string) a set of fields.

func (*RegexMatch) Process 

func (f *RegexMatch) Process(l baker.Record, next func(baker.Record))

Process is where the actual filtering takes place.

func (*RegexMatch) Stats 

func (f *RegexMatch) Stats() baker.FilterStats

Stats returns filter statistics.

type RegexMatchConfig 

type RegexMatchConfig struct {
	Fields []string `help:"list of fields to match with the corresponding regular expression in Regexs" default:"[]"`
	Regexs []string `help:"list of regular expression to match. Fields[0] must match Regexs[0], Fields[1] Regexs[1] and so on" default:"[]"`
}

RegexMatchConfig holds config parameters of the RegexMatch filter.

type ReplaceFields 

type ReplaceFields struct {
	// contains filtered or unexported fields
}

func (*ReplaceFields) Process 

func (f *ReplaceFields) Process(l baker.Record, next func(baker.Record))

func (*ReplaceFields) Stats 

func (c *ReplaceFields) Stats() baker.FilterStats

type ReplaceFieldsConfig 

type ReplaceFieldsConfig struct {
	CopyFields    []string `help:"List of src, dst field pairs, for example [\"srcField1\", \"dstField1\", \"srcField2\", \"dstField2\"]"`
	ReplaceFields []string `help:"List of field, value pairs, for example: [\"Foo\", \"dstField1\", \"Bar\", \"dstField2\"]"`
}

type SetStringFromURL 

type SetStringFromURL struct {
	// contains filtered or unexported fields
}

func (*SetStringFromURL) Process 

func (f *SetStringFromURL) Process(l baker.Record, next func(baker.Record))

func (*SetStringFromURL) Stats 

func (f *SetStringFromURL) Stats() baker.FilterStats

type SetStringFromURLConfig 

type SetStringFromURLConfig struct {
	Field   string   `help:"Name of the field to set to" required:"true"`
	Strings []string `help:"Strings to look for in the URL. Discard records not containing any of them." required:"true"`
}

type StringMatch 

type StringMatch struct {
	// contains filtered or unexported fields
}

StringMatch filter clears (i.e set to the empty string) a set of fields.

func (*StringMatch) Process 

func (f *StringMatch) Process(l baker.Record, next func(baker.Record))

Process is where the actual filtering takes place.

func (*StringMatch) Stats 

func (f *StringMatch) Stats() baker.FilterStats

Stats returns filter statistics.

type StringMatchConfig 

type StringMatchConfig struct {
	Field       string   `help:"name of the field which value is used for string comparison" required:"true"`
	Strings     []string `help:"list of strings to match." required:"true"`
	InvertMatch bool     `help:"Invert the match outcome, so that records are discarded if they don't match any of the strings" default:"false"`
}

StringMatchConfig holds config parameters of the StringMatch filter.

type Timestamp 

type Timestamp struct {
	// contains filtered or unexported fields
}

Timestamp is a baker filter that sets the value of a certain field to the Unix Epoch timestamp (in seconds, UTC) for all the records it processes.

func (*Timestamp) Process 

func (f *Timestamp) Process(l baker.Record, next func(baker.Record))

Process implements baker.Filter.

func (*Timestamp) Stats 

func (f *Timestamp) Stats() baker.FilterStats

Stats implements baker.Filter.

type TimestampConfig 

type TimestampConfig struct {
	Field string `help:"field to set to the unix Epoch timestamp" required:"true"`
}

TimestampConfig holds configuration paramters for the Timestamp filter.

type TimestampRange 

type TimestampRange struct {
	// contains filtered or unexported fields
}

TimestampRange is a baker filter that discards records depending on the value of a field representing a Unix timestamp.

func (*TimestampRange) Process 

func (f *TimestampRange) Process(l baker.Record, next func(baker.Record))

Process implements baker.Filter.

func (*TimestampRange) Stats 

func (f *TimestampRange) Stats() baker.FilterStats

Stats implements baker.Filter.

type TimestampRangeConfig 

type TimestampRangeConfig struct {
	StartDatetime string `` /* 149-byte string literal not displayed */
	EndDatetime   string `` /* 149-byte string literal not displayed */
	Field         string `help:"Name of the field containing the Unix EPOCH timestamp" required:"true"`
}

TimestampRangeConfig holds configuration paramters for the TimestampRange filter.

Source Files

View all

Directories

Path Synopsis
Package metadata provides a metadata-specific (concurrent and size-limited) cache.
 Package metadata provides a metadata-specific (concurrent and size-limited) cache.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL