utils

Documentation

Index

• Constants
• Variables
• func GetImportFormat(override, fp string) (format string, err error)
• func GetQuitChannel() chan os.Signal
• func IngestLineDelimitedStream(cfg LineDelimitedStream) (uint64, uint64, error)
• func MaxProcTune(val int) bool
• func ParseInt(v string) (r uint64, err error)
• func ParseSource(v string) (ret net.IP, err error)
• func StartProfile()
• func StopProfile()
• func WaitForQuit() (r os.Signal)
• type CSVReader
  • func NewCSVReader(rdr io.Reader, th TagHandler) (*CSVReader, error)
  • func (c *CSVReader) DisableEVs()
  • func (c *CSVReader) ReadEntry() (*entry.Entry, error)
• type JSONReader
  • func NewJSONReader(rdr io.Reader, th TagHandler) (*JSONReader, error)
  • func (jr *JSONReader) DisableEVs()
  • func (j *JSONReader) ReadEntry() (ent *entry.Entry, err error)
• type JsonLimitedDecoder
  • func NewJsonLimitedDecoder(rdr io.Reader, max int64) (jld *JsonLimitedDecoder, err error)
  • func (j *JsonLimitedDecoder) Decode(v interface{}) (err error)
  • func (j *JsonLimitedDecoder) TotalRead() int64
• type LineDelimitedStream
• type LineIgnorer
  • func NewIgnorer(prefixes, globs []string) (*LineIgnorer, error)
  • func (l *LineIgnorer) Ignore(b []byte) bool
• type ReadResetCloser
  • func GetReader(fin *os.File, tp types.Type) (r ReadResetCloser, err error)
  • func NewFileReadResetCloser(fin *os.File) ReadResetCloser
  • func OpenBufferedFileReader(p string, buffer int) (r ReadResetCloser, err error)
  • func OpenFileReader(p string) (r ReadResetCloser, err error)
• type ReimportReader
  • func GetImportReader(format string, fin io.ReadCloser, th TagHandler) (ir ReimportReader, err error)
• type State
  • func NewState(pth string, perm os.FileMode) (s *State, err error)
  • func (s *State) Read(f interface{}) (err error)
  • func (s *State) Write(f interface{}) (err error)
• type StatsItem
  • func (si *StatsItem) Add(v uint64)
• type StatsManager
  • func NewStatsManager(interval time.Duration, lgr *log.Logger) (*StatsManager, error)
  • func (sm *StatsManager) RegisterItem(name string) (si *StatsItem, err error)
  • func (sm *StatsManager) Start() error
  • func (sm *StatsManager) Stop()
• type TagHandler
  • func NewIngestTagHandler(igst *ingest.IngestMuxer) TagHandler

Constants

const (
	JsonFormat string = `json`
	CsvFormat  string = `csv`
)

Variables

var (
	ErrOversizedObject = errors.New("JSON object too large")
	ErrInvalidLimit    = errors.New("limit mus be greater than zero")
	ErrInvalidReader   = errors.New("reader is invalid")
)

var (
	ErrInvalidStatePath = errors.New("Invalid state file path")
	ErrNoState          = errors.New("No state available")
)

Functions

func GetImportFormat

func GetImportFormat(override, fp string) (format string, err error)

func GetQuitChannel

func GetQuitChannel() chan os.Signal

GetQuitChannel registers and returns a channel that will be notified upon receipt of the following signals: SIGHUP, SIGINT, SIGQUIT, SIGTERM

func IngestLineDelimitedStream

func IngestLineDelimitedStream(cfg LineDelimitedStream) (uint64, uint64, error)

func MaxProcTune

func MaxProcTune(val int) bool

this will set the GOMAXPROC value ONLY if the environment variable hasn't been set to a valid integer

func ParseInt

func ParseInt(v string) (r uint64, err error)

func ParseSource

func ParseSource(v string) (ret net.IP, err error)

ParseSource attempts to parse a string as a source override The priority logic is:

1. IP address
2. Numeric ID
3. hexadecimal hash

func StartProfile

func StartProfile()

func StopProfile

func StopProfile()

func WaitForQuit

func WaitForQuit() (r os.Signal)

WaitForQuit waits until it receives one of the following signals: SIGHUP, SIGINT, SIGQUIT, SIGTERM It returns the received signal.

Types

type CSVReader

type CSVReader struct {
	TagHandler
	// contains filtered or unexported fields
}

func NewCSVReader

func NewCSVReader(rdr io.Reader, th TagHandler) (*CSVReader, error)

func (*CSVReader) DisableEVs

func (c *CSVReader) DisableEVs()

func (*CSVReader) ReadEntry

func (c *CSVReader) ReadEntry() (*entry.Entry, error)

type JSONReader

type JSONReader struct {
	TagHandler
	// contains filtered or unexported fields
}

func NewJSONReader

func NewJSONReader(rdr io.Reader, th TagHandler) (*JSONReader, error)

func (*JSONReader) DisableEVs

func (jr *JSONReader) DisableEVs()

func (*JSONReader) ReadEntry

func (j *JSONReader) ReadEntry() (ent *entry.Entry, err error)

type JsonLimitedDecoder

type JsonLimitedDecoder struct {
	*json.Decoder
	// contains filtered or unexported fields
}

func NewJsonLimitedDecoder

func NewJsonLimitedDecoder(rdr io.Reader, max int64) (jld *JsonLimitedDecoder, err error)

NewJsonLImitedDecoder will return a new JsonLimitedDecoder ready for use. The json.Decoder object is directly exposed so that buffer methods can be used. This is a drop in replacement for the json.Decoder but we can return additional errors about oversized objects.

func (*JsonLimitedDecoder) Decode

func (j *JsonLimitedDecoder) Decode(v interface{}) (err error)

Decode an object using a JSON decoder

func (*JsonLimitedDecoder) TotalRead

func (j *JsonLimitedDecoder) TotalRead() int64

type LineDelimitedStream

type LineDelimitedStream struct {
	Rdr                    io.Reader
	Proc                   *processors.ProcessorSet
	Tag                    entry.EntryTag
	TG                     *timegrinder.TimeGrinder
	SRC                    net.IP
	IgnorePrefixes         [][]byte
	IgnoreGlobs            []string
	CleanQuotes            bool
	Verbose                bool
	Quotable               bool
	BatchSize              int
	AttachEnumeratedValues []entry.EnumeratedValue // These will be attached to every entry
}

type LineIgnorer

type LineIgnorer struct {
	// contains filtered or unexported fields
}

func NewIgnorer

func NewIgnorer(prefixes, globs []string) (*LineIgnorer, error)

func (*LineIgnorer) Ignore

func (l *LineIgnorer) Ignore(b []byte) bool

Ignore returns true if the given byte slice matches any of the prefixes or globs in the ignorer.

type ReadResetCloser

type ReadResetCloser interface {
	Read([]byte) (int, error)
	Close() error
	Reset() error
}

func GetReader

func GetReader(fin *os.File, tp types.Type) (r ReadResetCloser, err error)

func NewFileReadResetCloser

func NewFileReadResetCloser(fin *os.File) ReadResetCloser

func OpenBufferedFileReader

func OpenBufferedFileReader(p string, buffer int) (r ReadResetCloser, err error)

func OpenFileReader

func OpenFileReader(p string) (r ReadResetCloser, err error)

type ReimportReader

type ReimportReader interface {
	ReadEntry() (*entry.Entry, error)
	OverrideTags(tg entry.EntryTag)
	DisableEVs()
}

func GetImportReader

func GetImportReader(format string, fin io.ReadCloser, th TagHandler) (ir ReimportReader, err error)

type State

type State struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewState

func NewState(pth string, perm os.FileMode) (s *State, err error)

func (*State) Read

func (s *State) Read(f interface{}) (err error)

func (*State) Write

func (s *State) Write(f interface{}) (err error)

type StatsItem

type StatsItem struct {
	// contains filtered or unexported fields
}

func (*StatsItem) Add

func (si *StatsItem) Add(v uint64)

type StatsManager

type StatsManager struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewStatsManager

func NewStatsManager(interval time.Duration, lgr *log.Logger) (*StatsManager, error)

func (*StatsManager) RegisterItem

func (sm *StatsManager) RegisterItem(name string) (si *StatsItem, err error)

func (*StatsManager) Start

func (sm *StatsManager) Start() error

func (*StatsManager) Stop

func (sm *StatsManager) Stop()

type TagHandler

type TagHandler interface {
	OverrideTags(entry.EntryTag)
	GetTag(string) (entry.EntryTag, error)
}

func NewIngestTagHandler

func NewIngestTagHandler(igst *ingest.IngestMuxer) TagHandler