Documentation

Overview

    Package filter provides filter components.

    Index

    Constants

    This section is empty.

    Variables

      All is the list of all baker filters.

      View Source
      var ClauseFilterDesc = baker.FilterDesc{
      	Name:   "ClauseFilter",
      	New:    NewClauseFilter,
      	Config: &ClauseFilterConfig{},
      	Help:   clauseFilterHelpMsg,
      }

        ClauseFilterDesc describes the ClauseFilter filter

        View Source
        var ClearFieldsDesc = baker.FilterDesc{
        	Name:   "ClearFields",
        	New:    NewClearFields,
        	Config: &ClearFieldsConfig{},
        	Help:   "Reset a set of fields of all records passing through",
        }

          ClearFieldsDesc describes the ClearFields filter

          View Source
          var ConcatenateDesc = baker.FilterDesc{
          	Name:   "Concatenate",
          	New:    NewConcatenate,
          	Config: &ConcatenateConfig{},
          	Help:   `Concatenate up to 10 fields' values to a single field`,
          }

            ConcatenateDesc describes the Concatenate filter

            View Source
            var DedupDesc = baker.FilterDesc{
            	Name:   "Dedup",
            	New:    NewDedup,
            	Config: &DedupConfig{},
            	Help:   dedupHelp,
            }
            View Source
            var ExpandJSONDesc = baker.FilterDesc{
            	Name:   "ExpandJSON",
            	New:    NewExpandJSON,
            	Config: &ExpandJSONConfig{},
            	Help:   expandJSONhelp,
            }
            View Source
            var HashDesc = baker.FilterDesc{
            	Name:   "Hash",
            	New:    NewHash,
            	Config: &HashConfig{},
            	Help: `This filter hashes a field using a specified hash function and writes the value 
            to another (or the same) field. In order to have control over the set of characters
            present, the hashed value can optionally be encoded.
            	
            	
            Supported hash functions:
             - md5
             - sha256
            
            Supported encodings:
            - hex (hexadecimal encoding)
            `,
            }

              HashDesc describes the Hash filter

              View Source
              var MetadataLastModifiedDesc = baker.FilterDesc{
              	Name:   "MetadataLastModified",
              	New:    NewMetadataLastModified,
              	Config: &MetadataLastModifiedConfig{},
              	Help:   `Extract the "last modified" timestamp from the record Metadata and write it to the selected field.`,
              }
              View Source
              var NotNullDesc = baker.FilterDesc{
              	Name:   "NotNull",
              	New:    NewNotNull,
              	Config: &NotNullConfig{},
              	Help:   "Discard the records having null (i.e empty) fields.\n",
              }

                NotNullDesc describes the NotNull filter.

                View Source
                var PartialCloneDesc = baker.FilterDesc{
                	Name:   "PartialClone",
                	New:    NewPartialClone,
                	Config: &PartialCloneConfig{},
                	Help:   "Copy a list of fields to a new record and process this new record, discarding the original one",
                }
                View Source
                var RegexMatchDesc = baker.FilterDesc{
                	Name:   "RegexMatch",
                	New:    NewRegexMatch,
                	Config: &RegexMatchConfig{},
                	Help:   "Discard a record if one or more fields don't match the corresponding regular expressions",
                }

                  RegexMatchDesc describes the RegexMatch filter

                  View Source
                  var ReplaceFieldsDesc = baker.FilterDesc{
                  	Name:   "ReplaceFields",
                  	New:    NewReplaceFields,
                  	Config: &ReplaceFieldsConfig{},
                  	Help:   `Copy a field value or a fixed value to another field. Can copy multiple fields.`,
                  }

                    ReplaceFieldsDesc describes the ReplaceFields filter

                    View Source
                    var SetStringFromURLDesc = baker.FilterDesc{
                    	Name:   "SetStringFromURL",
                    	New:    NewSetStringFromURL,
                    	Config: &SetStringFromURLConfig{},
                    	Help:   `Extract some strings from metadata url and sets a field with it.`,
                    }

                      SetStringFromURLDesc describes the SetStringFromURL filter.

                      View Source
                      var StringMatchDesc = baker.FilterDesc{
                      	Name:   "StringMatch",
                      	New:    NewStringMatch,
                      	Config: &StringMatchConfig{},
                      	Help:   "Discard records if a field matches any of the provided strings",
                      }

                        StringMatchDesc describes the StringMatch filter

                        View Source
                        var TimestampDesc = baker.FilterDesc{
                        	Name:   "Timestamp",
                        	New:    NewTimestamp,
                        	Config: &TimestampConfig{},
                        	Help:   "Sets a field to the Unix Epoch timestamp at which the record is processed",
                        }

                          TimestampDesc describes the Timestamp filter.

                          View Source
                          var TimestampRangeDesc = baker.FilterDesc{
                          	Name:   "TimestampRange",
                          	New:    NewTimestampRange,
                          	Config: &TimestampRangeConfig{},
                          	Help:   "Discard records if the value of a field containing a timestamp is out of the given time range (i.e StartDateTime <= value < EndDateTime)",
                          }

                            TimestampRangeDesc describes the TimestampRange filter.

                            Functions

                            func NewClauseFilter

                            func NewClauseFilter(cfg baker.FilterParams) (baker.Filter, error)

                            func NewClearFields

                            func NewClearFields(cfg baker.FilterParams) (baker.Filter, error)

                              NewClearFields returns a ClearFields filter.

                              func NewConcatenate

                              func NewConcatenate(cfg baker.FilterParams) (baker.Filter, error)

                              func NewDedup

                              func NewDedup(cfg baker.FilterParams) (baker.Filter, error)

                              func NewExpandJSON

                              func NewExpandJSON(cfg baker.FilterParams) (baker.Filter, error)

                              func NewHash

                              func NewHash(cfg baker.FilterParams) (baker.Filter, error)

                              func NewMetadataLastModified

                              func NewMetadataLastModified(cfg baker.FilterParams) (baker.Filter, error)

                              func NewNotNull

                              func NewNotNull(cfg baker.FilterParams) (baker.Filter, error)

                                NewNotNull creates and configures a new NotNull filter.

                                func NewPartialClone

                                func NewPartialClone(cfg baker.FilterParams) (baker.Filter, error)

                                func NewRegexMatch

                                func NewRegexMatch(cfg baker.FilterParams) (baker.Filter, error)

                                  NewRegexMatch returns a RegexMatch filter.

                                  func NewReplaceFields

                                  func NewReplaceFields(cfg baker.FilterParams) (baker.Filter, error)

                                  func NewSetStringFromURL

                                  func NewSetStringFromURL(cfg baker.FilterParams) (baker.Filter, error)

                                  func NewStringMatch

                                  func NewStringMatch(cfg baker.FilterParams) (baker.Filter, error)

                                    NewStringMatch returns a StringMatch filter.

                                    func NewTimestamp

                                    func NewTimestamp(cfg baker.FilterParams) (baker.Filter, error)

                                      NewTimestamp creates and configures a Timestamp filter.

                                      func NewTimestampRange

                                      func NewTimestampRange(cfg baker.FilterParams) (baker.Filter, error)

                                        NewTimestampRange creates and configures a TimestampRange filter.

                                        Types

                                        type Clause

                                        type Clause struct {
                                        	// contains filtered or unexported fields
                                        }

                                          Will real sum types ever make it to go?

                                          Validness of the fields in this structure depend on clauseType

                                          type ClauseFilter

                                          type ClauseFilter struct {
                                          	// contains filtered or unexported fields
                                          }

                                          func (*ClauseFilter) Process

                                          func (f *ClauseFilter) Process(l baker.Record, next func(baker.Record))

                                          func (*ClauseFilter) Stats

                                          func (f *ClauseFilter) Stats() baker.FilterStats

                                          type ClauseFilterConfig

                                          type ClauseFilterConfig struct {
                                          	Clause string `help:"Boolean formula describing which events to let through. If empty, let everything through."`
                                          }

                                            ClauseFilterConfig describes the ClauseFilter filter config

                                            type ClearFields

                                            type ClearFields struct {
                                            	// contains filtered or unexported fields
                                            }

                                              ClearFields filter clears (i.e set to the empty string) a set of fields.

                                              func (*ClearFields) Process

                                              func (f *ClearFields) Process(l baker.Record, next func(baker.Record))

                                                Process is where the actual filtering takes place.

                                                func (*ClearFields) Stats

                                                func (f *ClearFields) Stats() baker.FilterStats

                                                  Stats returns filter statistics.

                                                  type ClearFieldsConfig

                                                  type ClearFieldsConfig struct {
                                                  	Fields []string `help:"set of fields to clear" required:"true"`
                                                  }

                                                    ClearFieldsConfig holds config parameters of the ClearFields filter.

                                                    type Concatenate

                                                    type Concatenate struct {
                                                    	// contains filtered or unexported fields
                                                    }

                                                    func (*Concatenate) Process

                                                    func (c *Concatenate) Process(l baker.Record, next func(baker.Record))

                                                    func (*Concatenate) Stats

                                                    func (c *Concatenate) Stats() baker.FilterStats

                                                    type ConcatenateConfig

                                                    type ConcatenateConfig struct {
                                                    	Fields    []string `help:"The field names to concatenate, in order"`
                                                    	Target    string   `help:"The field name to save the concatenated value to"`
                                                    	Separator string   `help:"Separator to concatenate the values. Must either be empty or a single ASCII, non-nil char"`
                                                    }

                                                    type Dedup

                                                    type Dedup struct {
                                                    	// contains filtered or unexported fields
                                                    }

                                                    func (*Dedup) Process

                                                    func (f *Dedup) Process(l baker.Record, next func(baker.Record))

                                                    func (*Dedup) Stats

                                                    func (f *Dedup) Stats() baker.FilterStats

                                                    type DedupConfig

                                                    type DedupConfig struct {
                                                    	Fields       []string `help:"fields to consider when comparing records" required:"true"`
                                                    	KeySeparator string   `help:"character separator used to build a key from the fields" default:"\\x1e"`
                                                    }

                                                    type ExpandJSON

                                                    type ExpandJSON struct {
                                                    	// contains filtered or unexported fields
                                                    }

                                                    func (*ExpandJSON) Process

                                                    func (f *ExpandJSON) Process(l baker.Record, next func(baker.Record))

                                                    func (*ExpandJSON) Stats

                                                    func (f *ExpandJSON) Stats() baker.FilterStats

                                                    type ExpandJSONConfig

                                                    type ExpandJSONConfig struct {
                                                    	Source          string            `help:"record field that contains the json" required:"true"`
                                                    	Fields          map[string]string `help:"<JMESPath -> record field> map, the rest will be ignored" required:"true"`
                                                    	TrueFalseValues []string          `help:"bind the json boolean values to correstponding strings" default:"[\"true\", \"false\"]"`
                                                    }

                                                    type Hash

                                                    type Hash struct {
                                                    	// contains filtered or unexported fields
                                                    }

                                                    func (*Hash) Process

                                                    func (h *Hash) Process(r baker.Record, next func(baker.Record))

                                                    func (*Hash) Stats

                                                    func (h *Hash) Stats() baker.FilterStats

                                                    type HashConfig

                                                    type HashConfig struct {
                                                    	SrcField string `help:"Name of the field to hash" required:"true"`
                                                    	DstField string `help:"Name of the field to write the result to" required:"true"`
                                                    	Function string `help:"Name of the hash function to use" required:"true"`
                                                    	Encoding string `help:"Name of the encoding function to use" required:"false"`
                                                    }

                                                    type MetadataLastModified

                                                    type MetadataLastModified struct {
                                                    	// contains filtered or unexported fields
                                                    }

                                                    func (*MetadataLastModified) Process

                                                    func (f *MetadataLastModified) Process(l baker.Record, next func(baker.Record))

                                                    func (*MetadataLastModified) Stats

                                                    type MetadataLastModifiedConfig

                                                    type MetadataLastModifiedConfig struct {
                                                    	DstField string `help:"Name of the field into which write the timestamp to" required:"true"`
                                                    }

                                                    type NotNull

                                                    type NotNull struct {
                                                    	// contains filtered or unexported fields
                                                    }

                                                      NotNull is a baker filter that discards records having null fields.

                                                      func (*NotNull) Process

                                                      func (v *NotNull) Process(l baker.Record, next func(baker.Record))

                                                        Process implements baker.Filter.

                                                        func (*NotNull) Stats

                                                        func (v *NotNull) Stats() baker.FilterStats

                                                          Stats implements baker.Filter.

                                                          type NotNullConfig

                                                          type NotNullConfig struct {
                                                          	Fields []string `help:"Fields is the list of fields to check for null/empty values" required:"true"`
                                                          }

                                                            NotNullConfig holds configuration parameters for the NotNull filter.

                                                            type PartialClone

                                                            type PartialClone struct {
                                                            	// contains filtered or unexported fields
                                                            }

                                                            func (*PartialClone) Process

                                                            func (s *PartialClone) Process(r baker.Record, next func(baker.Record))

                                                            func (*PartialClone) Stats

                                                            func (s *PartialClone) Stats() baker.FilterStats

                                                            type PartialCloneConfig

                                                            type PartialCloneConfig struct {
                                                            	Fields []string `help:"Fields that must be copied to the new line" required:"true"`
                                                            }

                                                            type RegexMatch

                                                            type RegexMatch struct {
                                                            	// contains filtered or unexported fields
                                                            }

                                                              RegexMatch filter clears (i.e set to the empty string) a set of fields.

                                                              func (*RegexMatch) Process

                                                              func (f *RegexMatch) Process(l baker.Record, next func(baker.Record))

                                                                Process is where the actual filtering takes place.

                                                                func (*RegexMatch) Stats

                                                                func (f *RegexMatch) Stats() baker.FilterStats

                                                                  Stats returns filter statistics.

                                                                  type RegexMatchConfig

                                                                  type RegexMatchConfig struct {
                                                                  	Fields []string `help:"list of fields to match with the corresponding regular expression in Regexs" default:"[]"`
                                                                  	Regexs []string `help:"list of regular expression to match. Fields[0] must match Regexs[0], Fields[1] Regexs[1] and so on" default:"[]"`
                                                                  }

                                                                    RegexMatchConfig holds config parameters of the RegexMatch filter.

                                                                    type ReplaceFields

                                                                    type ReplaceFields struct {
                                                                    	// contains filtered or unexported fields
                                                                    }

                                                                    func (*ReplaceFields) Process

                                                                    func (f *ReplaceFields) Process(l baker.Record, next func(baker.Record))

                                                                    func (*ReplaceFields) Stats

                                                                    func (c *ReplaceFields) Stats() baker.FilterStats

                                                                    type ReplaceFieldsConfig

                                                                    type ReplaceFieldsConfig struct {
                                                                    	CopyFields    []string `help:"List of src, dst field pairs, for example [\"srcField1\", \"dstField1\", \"srcField2\", \"dstField2\"]"`
                                                                    	ReplaceFields []string `help:"List of field, value pairs, for example: [\"Foo\", \"dstField1\", \"Bar\", \"dstField2\"]"`
                                                                    }

                                                                    type SetStringFromURL

                                                                    type SetStringFromURL struct {
                                                                    	// contains filtered or unexported fields
                                                                    }

                                                                    func (*SetStringFromURL) Process

                                                                    func (f *SetStringFromURL) Process(l baker.Record, next func(baker.Record))

                                                                    func (*SetStringFromURL) Stats

                                                                    func (f *SetStringFromURL) Stats() baker.FilterStats

                                                                    type SetStringFromURLConfig

                                                                    type SetStringFromURLConfig struct {
                                                                    	Field   string   `help:"Name of the field to set to" required:"true"`
                                                                    	Strings []string `help:"Strings to look for in the URL. Discard records not containing any of them." required:"true"`
                                                                    }

                                                                    type StringMatch

                                                                    type StringMatch struct {
                                                                    	// contains filtered or unexported fields
                                                                    }

                                                                      StringMatch filter clears (i.e set to the empty string) a set of fields.

                                                                      func (*StringMatch) Process

                                                                      func (f *StringMatch) Process(l baker.Record, next func(baker.Record))

                                                                        Process is where the actual filtering takes place.

                                                                        func (*StringMatch) Stats

                                                                        func (f *StringMatch) Stats() baker.FilterStats

                                                                          Stats returns filter statistics.

                                                                          type StringMatchConfig

                                                                          type StringMatchConfig struct {
                                                                          	Field       string   `help:"name of the field which value is used for string comparison" required:"true"`
                                                                          	Strings     []string `help:"list of strings to match." required:"true"`
                                                                          	InvertMatch bool     `help:"Invert the match outcome, so that records are discarded if they don't match any of the strings" default:"false"`
                                                                          }

                                                                            StringMatchConfig holds config parameters of the StringMatch filter.

                                                                            type Timestamp

                                                                            type Timestamp struct {
                                                                            	// contains filtered or unexported fields
                                                                            }

                                                                              Timestamp is a baker filter that sets the value of a certain field to the Unix Epoch timestamp (in seconds, UTC) for all the records it processes.

                                                                              func (*Timestamp) Process

                                                                              func (f *Timestamp) Process(l baker.Record, next func(baker.Record))

                                                                                Process implements baker.Filter.

                                                                                func (*Timestamp) Stats

                                                                                func (f *Timestamp) Stats() baker.FilterStats

                                                                                  Stats implements baker.Filter.

                                                                                  type TimestampConfig

                                                                                  type TimestampConfig struct {
                                                                                  	Field string `help:"field to set to the unix Epoch timestamp" required:"true"`
                                                                                  }

                                                                                    TimestampConfig holds configuration paramters for the Timestamp filter.

                                                                                    type TimestampRange

                                                                                    type TimestampRange struct {
                                                                                    	// contains filtered or unexported fields
                                                                                    }

                                                                                      TimestampRange is a baker filter that discards records depending on the value of a field representing a Unix timestamp.

                                                                                      func (*TimestampRange) Process

                                                                                      func (f *TimestampRange) Process(l baker.Record, next func(baker.Record))

                                                                                        Process implements baker.Filter.

                                                                                        func (*TimestampRange) Stats

                                                                                        func (f *TimestampRange) Stats() baker.FilterStats

                                                                                          Stats implements baker.Filter.

                                                                                          type TimestampRangeConfig

                                                                                          type TimestampRangeConfig struct {
                                                                                          	StartDatetime string `` /* 149-byte string literal not displayed */
                                                                                          	EndDatetime   string `` /* 149-byte string literal not displayed */
                                                                                          	Field         string `help:"Name of the field containing the Unix EPOCH timestamp" required:"true"`
                                                                                          }

                                                                                            TimestampRangeConfig holds configuration paramters for the TimestampRange filter.

                                                                                            Directories

                                                                                            Path Synopsis