custom

Documentation

Index

• func ParseCustom(path string, cc *CompiledConfig, skipFunc func(string) bool, concurrency int) (<-chan *model.OfflineInput, error)
• func PopulateFromJSON(data []byte, out *model.OfflineInput, cc *CompiledConfig)
• func PopulateFromRegex(record []byte, out *model.OfflineInput, cc *CompiledConfig)
• type CompiledConfig
  • func LoadConfig(path string) (*CompiledConfig, error)
• type Config

Functions

func ParseCustom

func ParseCustom(path string, cc *CompiledConfig, skipFunc func(string) bool, concurrency int) (<-chan *model.OfflineInput, error)

ParseCustom handles parsing based on the YAML configuration.

func PopulateFromJSON

func PopulateFromJSON(data []byte, out *model.OfflineInput, cc *CompiledConfig)

func PopulateFromRegex

func PopulateFromRegex(record []byte, out *model.OfflineInput, cc *CompiledConfig)

Types

type CompiledConfig

type CompiledConfig struct {
	Config       *Config
	RecordSep    *regexp.Regexp
	URLRegex     *regexp.Regexp
	DomainRegex  *regexp.Regexp
	HeadersRegex *regexp.Regexp
	BodyRegex    *regexp.Regexp
}

CompiledConfig holds the compiled regexes for performance

func LoadConfig

func LoadConfig(path string) (*CompiledConfig, error)

LoadConfig loads the custom parsing configuration from a YAML file.

type Config

type Config struct {
	// General
	Format string `yaml:"format"` // "json" or "regex"

	// JSON specific (GJSON paths)
	JSON struct {
		URLPath     string `yaml:"url_path"`
		DomainPath  string `yaml:"domain_path"`
		HeadersPath string `yaml:"headers_path"`
		BodyPath    string `yaml:"body_path"`
	} `yaml:"json"`

	// Regex specific
	Regex struct {
		RecordSeparator string `yaml:"record_separator"` // Regex to split file into records
		URLRegex        string `yaml:"url_regex"`
		DomainRegex     string `yaml:"domain_regex"`
		HeadersRegex    string `yaml:"headers_regex"` // Extract a block or JSON string
		BodyRegex       string `yaml:"body_regex"`
	} `yaml:"regex"`
}

Config defines how to extract data from custom input formats.