logmunch

README

logmunch

Install with go install github.com/msiebuhr/logmunch/bin/logmunch.

Somewhat ad-hoc tool for parsing logs in logfmt and json-styles from logentries.

Create a file, ~/.logmunch with a list of providers and their default parameters described as URLs. In particular, logentries' key for pull.logentries.com:

logentries://:xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxxx@pull.logentries.com/

After that, running logmunch -source logentries:Test/heroku -filter "H12" -limit 10 will fetch ten entries with the text H12 from hosts/Test/heroku in logentries.

logenries:

• https://logentries.com/doc/api-download/

file:

• Defaults to stdin.
• file:///./local.txt to read local files.

Basic use

(I'm not terribly good at keeping README's up to date, so please run logmunch -h to get a current overview.)

// Parse a local file
logmunch -source=file:/./local.log

// Round timestamps and generate compound key X from A and B
logmunch -source=- \
    -round-time=1h \
    -compound=X,A,B

Developer docs

The internal API used for fetching/parsing/filtering/outputting logs is split out from the main binary, so it should be possible to re-use.

Documentation is at godoc.org.

License

ISC

Documentation

Index

• func DrainCSV(join string) func(<-chan LogLine, io.WriteCloser)
• func DrainCountOverTime(key string) func(<-chan LogLine, io.WriteCloser)
• func DrainGnuplotDistinctKeyCount(key string) func(<-chan LogLine, io.WriteCloser)
• func DrainSqlite3() func(<-chan LogLine, io.WriteCloser)
• func DrainStandard() func(<-chan LogLine, io.WriteCloser)
• func FilterLogChan(filters []Filterer, in <-chan LogLine, out chan<- LogLine)
• func Fuzz(data []byte) int
• func MakeBucketizeKey(key string) func(*LogLine) *LogLine
• func MakeCompondKey(newKey string, sourceKeys []string) func(*LogLine) *LogLine
• func MakeLuaFilter(prog string) func(*LogLine) *LogLine
• func MakeNormaliseUrlPaths(key string, urlTemplates []string) func(*LogLine) *LogLine
• func MakePickFilter(keys []string) func(*LogLine) *LogLine
• func MakeRemoveHerokuDrainId() func(*LogLine) *LogLine
• func MakeRoundTimestampFilter(d time.Duration) func(*LogLine) *LogLine
• func ParseLogEntries(in <-chan string, out chan<- LogLine)
• type Drainer
  • func DrainJson() Drainer
• type Filterer
• type LogLine
  • func NewLogLine(when time.Time, name string, entries map[string]string) LogLine
  • func NoOpFilter(in *LogLine) *LogLine
  • func (l LogLine) Equal(other LogLine) bool
  • func (l *LogLine) GetNumber(key string) float64
  • func (l *LogLine) HandleLogfmt(key, val []byte) error
  • func (l *LogLine) HasKey(key string) bool
  • func (l *LogLine) HasPrefix(prefix string) bool
  • func (l LogLine) MarshalJSON() ([]byte, error)
  • func (l *LogLine) SetNumber(key string, val float64)
  • func (l *LogLine) String() string
• type Query
  • func FileSource(config *url.URL, query Query, out chan<- string) (Query, error)
  • func LogEntriesSource(config *url.URL, query Query, out chan<- string) (Query, error)
• type QueryGroup
  • func NewQueryGroup(name string, keys []string) QueryGroup
  • func (q *QueryGroup) Group(l *LogLine)
• type Source
• type SourceLoader
  • func (s SourceLoader) GetConfig(configUrl string) (Source, *url.URL, error)
  • func (s SourceLoader) GetData(configUrl string, query Query, out chan<- string) (Query, error)
  • func (s *SourceLoader) TryLoadConfigs(filenames []string) error

Functions

func DrainCSV

func DrainCSV(join string) func(<-chan LogLine, io.WriteCloser)

func DrainCountOverTime

func DrainCountOverTime(key string) func(<-chan LogLine, io.WriteCloser)

func DrainGnuplotDistinctKeyCount

func DrainGnuplotDistinctKeyCount(key string) func(<-chan LogLine, io.WriteCloser)

func DrainSqlite3

func DrainSqlite3() func(<-chan LogLine, io.WriteCloser)

func DrainStandard

func DrainStandard() func(<-chan LogLine, io.WriteCloser)

func FilterLogChan

func FilterLogChan(filters []Filterer, in <-chan LogLine, out chan<- LogLine)

Run the LogLines on `in` through the given filters and put them on `out`. Note: Closes `out` when `in` does so.

func Fuzz

func Fuzz(data []byte) int

func MakeBucketizeKey

func MakeBucketizeKey(key string) func(*LogLine) *LogLine

Creates a filter that will bucketize the named key.

Numbers are basically rounded to their nearest value per their order or magnitude; i.e. numbers in the hundredes, will be rounded down to nearest hundred, numbers in the tousands will be rounded down towards nearest thousand.

Note: Round down here actually meands "towards zero"

func MakeCompondKey

func MakeCompondKey(newKey string, sourceKeys []string) func(*LogLine) *LogLine

Make a new key, `newKey`, based on a concatenation of `sourceKeys`.

func MakeLuaFilter

func MakeLuaFilter(prog string) func(*LogLine) *LogLine

func MakeNormaliseUrlPaths

func MakeNormaliseUrlPaths(key string, urlTemplates []string) func(*LogLine) *LogLine

Normalise paths from a given URL after given express-style templates.

Ex `path=/users/msiebuhr/add` w. template `/users/:uid/add` will normalize to `path=/users/:uid/add uid=msiebuhr`.

func MakePickFilter

func MakePickFilter(keys []string) func(*LogLine) *LogLine

Creates a filter that keeps only the given keys for each log-line

func MakeRemoveHerokuDrainId

func MakeRemoveHerokuDrainId() func(*LogLine) *LogLine

Make a new key, `newKey`, based on a concatenation of `sourceKeys`.

func MakeRoundTimestampFilter

func MakeRoundTimestampFilter(d time.Duration) func(*LogLine) *LogLine

Creates a filter that rounds the LogEntries time to the given duration

func ParseLogEntries

func ParseLogEntries(in <-chan string, out chan<- LogLine)

Types

type Drainer

type Drainer func(<-chan LogLine, io.WriteCloser)

func DrainJson

func DrainJson() Drainer

type Filterer

type Filterer func(*LogLine) *LogLine

Given a log line, transforms and returns the logline (or perhaps a new one/none)

type LogLine

type LogLine struct {
	Time    time.Time
	Name    string
	Entries map[string]string
}

func NewLogLine

func NewLogLine(when time.Time, name string, entries map[string]string) LogLine

func NoOpFilter

func NoOpFilter(in *LogLine) *LogLine

Creates a No-Op filter

func (LogLine) Equal

func (l LogLine) Equal(other LogLine) bool

func (*LogLine) GetNumber

func (l *LogLine) GetNumber(key string) float64

func (*LogLine) HandleLogfmt

func (l *LogLine) HandleLogfmt(key, val []byte) error

Implements interface from "github.com/kr/logfmt"

func (*LogLine) HasKey

func (l *LogLine) HasKey(key string) bool

func (*LogLine) HasPrefix

func (l *LogLine) HasPrefix(prefix string) bool

func (LogLine) MarshalJSON

func (l LogLine) MarshalJSON() ([]byte, error)

func (*LogLine) SetNumber

func (l *LogLine) SetNumber(key string, val float64)

func (*LogLine) String

func (l *LogLine) String() string

type Query

type Query struct {
	// A prefix to group things by
	Filter string

	// When the log start and end
	Start time.Time
	End   time.Time

	// How many lines to fetch
	Limit int

	GroupBy []QueryGroup
}

func FileSource

func FileSource(config *url.URL, query Query, out chan<- string) (Query, error)

Get data from a file

func LogEntriesSource

func LogEntriesSource(config *url.URL, query Query, out chan<- string) (Query, error)

Get data from logentries

type QueryGroup

type QueryGroup struct {
	Name string
	Keys []string
}

QueryGroup

func NewQueryGroup

func NewQueryGroup(name string, keys []string) QueryGroup

func (*QueryGroup) Group

func (q *QueryGroup) Group(l *LogLine)

Create a new key in l by name and composed of the given keys.

type Source

type Source func(config *url.URL, query Query, out chan<- string) (Query, error)

type SourceLoader

type SourceLoader struct {
	Config map[string]url.URL
}

Keep a map of protocols -> default settings, all parsed from URLs

func (SourceLoader) GetConfig

func (s SourceLoader) GetConfig(configUrl string) (Source, *url.URL, error)

func (SourceLoader) GetData

func (s SourceLoader) GetData(configUrl string, query Query, out chan<- string) (Query, error)

func (*SourceLoader) TryLoadConfigs

func (s *SourceLoader) TryLoadConfigs(filenames []string) error