logparser

README

Logparser

Logparser is a log parsing library used by node-agent and aws-agent to extract metrics from unstructured logs. Also in this repository, you can find a CLI tool that calculates a log summary from stdin.

Run

cat some.log | docker run -i --rm ghcr.io/coroot/logparser

Sample output

▇                      12 ( 0%) - ERROR [LearnerHandler-/10.10.34.11:52225:LearnerHandler@562] - Unexpected exception causing shutdown while sock still open
▇                       1 ( 0%) - ERROR [CommitProcessor:1:NIOServerCnxn@180] - Unexpected Exception:
▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 314 (23%) - WARN [SendWorker:188978561024:QuorumCnxManager$SendWorker@679] - Interrupted while waiting for message on queue
▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇   291 (21%) - WARN [RecvWorker:188978561024:QuorumCnxManager$RecvWorker@762] - Connection broken for id 188978561024, my id = 1, error =
▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇     266 (19%) - WARN [RecvWorker:188978561024:QuorumCnxManager$RecvWorker@765] - Interrupting SendWorker
▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇     262 (19%) - WARN [SendWorker:188978561024:QuorumCnxManager$SendWorker@688] - Send worker leaving thread
▇▇▇▇▇▇                 86 ( 6%) - WARN [WorkerSender[myid=1]:QuorumCnxManager@368] - Cannot open channel to 2 at election address /10.10.34.12:3888
▇▇▇                    39 ( 2%) - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:ZooKeeperServer@793] - Connection request from old client /10.10.34.19:33442; will be dropped if server is in r-o mode
▇▇▇                    37 ( 2%) - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxn@349] - caught end of stream exception
▇▇                     19 ( 1%) - WARN [LearnerHandler-/10.10.34.12:35276:LearnerHandler@575] - ******* GOODBYE /10.10.34.12:35276 ********
▇                       3 ( 0%) - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxn@354] - Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
▇                       1 ( 0%) - WARN [LearnerHandler-/10.10.34.13:42241:Leader@576] - First is 0x0

1998 messages processed in 0.137 seconds:
  error: 13
  warning: 1318
  info: 667

License

Logparser is licensed under the Apache License, Version 2.0.

Documentation

Index

• type CriDecoder
  • func (d CriDecoder) Decode(src string) (string, error)
• type Decoder
• type DockerJsonDecoder
  • func (d DockerJsonDecoder) Decode(src string) (string, error)
• type DockerLogJson
• type Level
  • func GuessLevel(line string) Level
  • func LevelByPriority(priority string) Level
  • func (l Level) String() string
• type LogCounter
• type LogEntry
• type Message
• type MultilineCollector
  • func NewMultilineCollector(ctx context.Context, timeout time.Duration, limit int) *MultilineCollector
  • func (m *MultilineCollector) Add(entry LogEntry)
• type OnMsgCallbackF
• type Parser
  • func NewParser(ch <-chan LogEntry, decoder Decoder, onMsgCallback OnMsgCallbackF, ...) *Parser
  • func (p *Parser) GetCounters() []LogCounter
  • func (p *Parser) GetSensitiveCounters() []SensitiveLogCounter
  • func (p *Parser) Stop()
• type Pattern
  • func NewPattern(input string) *Pattern
  • func NewPatternFromWords(input string) *Pattern
  • func (p *Pattern) Hash() string
  • func (p *Pattern) String() string
  • func (p *Pattern) WeakEqual(other *Pattern) bool
• type PrecompiledPattern
  • func LoadPatterns() ([]PrecompiledPattern, error)
• type SensitiveLogCounter
• type SensitivePattern
• type SensitivePatternMatch
  • func DetectSensitiveData(line string, hash string, precompiledPatterns []PrecompiledPattern) []SensitivePatternMatch

Types

type CriDecoder

type CriDecoder struct{}

func (CriDecoder) Decode

func (d CriDecoder) Decode(src string) (string, error)

type Decoder

type Decoder interface {
	Decode(string) (string, error)
}

type DockerJsonDecoder

type DockerJsonDecoder struct{}

func (DockerJsonDecoder) Decode

func (d DockerJsonDecoder) Decode(src string) (string, error)

type DockerLogJson

type DockerLogJson struct {
	Log string
}

type Level

type Level int

const (
	LevelUnknown Level = iota
	LevelCritical
	LevelError
	LevelWarning
	LevelInfo
	LevelDebug
)

func GuessLevel

func GuessLevel(line string) Level

func LevelByPriority

func LevelByPriority(priority string) Level

func (Level) String

func (l Level) String() string

type LogCounter

type LogCounter struct {
	Level    Level
	Hash     string
	Sample   string
	Messages int
}

type LogEntry

type LogEntry struct {
	Timestamp time.Time
	Content   string
	Level     Level
}

type Message

type Message struct {
	Timestamp time.Time
	Content   string
	Level     Level
}

type MultilineCollector

type MultilineCollector struct {
	Messages chan Message
	// contains filtered or unexported fields
}

func NewMultilineCollector

func NewMultilineCollector(ctx context.Context, timeout time.Duration, limit int) *MultilineCollector

func (*MultilineCollector) Add

func (m *MultilineCollector) Add(entry LogEntry)

type OnMsgCallbackF

type OnMsgCallbackF func(ts time.Time, level Level, patternHash string, msg string)

type Parser

type Parser struct {
	// contains filtered or unexported fields
}

func NewParser

func NewParser(ch <-chan LogEntry, decoder Decoder, onMsgCallback OnMsgCallbackF, multilineCollectorTimeout time.Duration, disableSensitiveDataDetection bool) *Parser

func (*Parser) GetCounters

func (p *Parser) GetCounters() []LogCounter

func (*Parser) GetSensitiveCounters

func (p *Parser) GetSensitiveCounters() []SensitiveLogCounter

func (*Parser) Stop

func (p *Parser) Stop()

type Pattern

type Pattern struct {
	// contains filtered or unexported fields
}

func NewPattern

func NewPattern(input string) *Pattern

func NewPatternFromWords

func NewPatternFromWords(input string) *Pattern

func (*Pattern) Hash

func (p *Pattern) Hash() string

func (*Pattern) String

func (p *Pattern) String() string

func (*Pattern) WeakEqual

func (p *Pattern) WeakEqual(other *Pattern) bool

type PrecompiledPattern

type PrecompiledPattern struct {
	Name    string
	Pattern *regexp.Regexp
}

func LoadPatterns

func LoadPatterns() ([]PrecompiledPattern, error)

type SensitiveLogCounter

type SensitiveLogCounter struct {
	Sample   string
	Messages int
	Pattern  string
	Regex    string
	Name     string
	Hash     string
}

type SensitivePattern

type SensitivePattern struct {
	Name    string `json:"name"`
	Pattern string `json:"pattern"`
}

type SensitivePatternMatch

type SensitivePatternMatch struct {
	// contains filtered or unexported fields
}

func DetectSensitiveData

func DetectSensitiveData(line string, hash string, precompiledPatterns []PrecompiledPattern) []SensitivePatternMatch