logkeeper

README

To set up and run (be sure to have a mongod instance running on localhost on the default port):

git clone git@github.com:evergreen-ci/logkeeper
cd logkeeper
. ./set_gopath.sh
go run main/logkeeper.go

Example of running resmoke with logkeeper

# Run this from the root directory where mongodb is cloned to:
python buildscripts/resmoke.py --suites=core --log=buildlogger  --buildloggerUrl="http://localhost:8080"

To create the necessary indexes, run mongo buildlogs setup.js

Documentation

Index

• Constants
• Variables
• func CleanupOldLogsByBuild(id interface{}) (int, error)
• func GetCtxRequestId(r *http.Request) int
• func GetHandlerPprof() http.Handler
• func GetSender(queue amboy.Queue, fn string) (send.Sender, error)
• func IsLeader() bool
• func MergeLog(logger1 chan *LogLineItem, logger2 chan *LogLineItem) chan *LogLineItem
• func New(opts Options) *logKeeper
• func SetCtxRequestId(reqID int, r *http.Request) *http.Request
• func StartBackgroundLogging(ctx context.Context)
• func StreamingGetOldTests(ctx context.Context, timeout time.Duration) (<-chan Test, <-chan error)
• func UpdateFailedTestsByBuildID(id interface{}) (int, error)
• type ColorDef
• type ColorSet
  • func NewColorSet() *ColorSet
  • func (self *ColorSet) GetAllColors() []ColorDef
  • func (self *ColorSet) GetColor(inputKey interface{}) (string, error)
• type LimitedReader
  • func (l *LimitedReader) Read(p []byte) (n int, err error)
• type Log
• type LogKeeperBuild
• type LogLine
  • func NewLogLine(data []interface{}) *LogLine
  • func (s LogLine) Msg() string
  • func (s LogLine) Time() time.Time
• type LogLineItem
  • func (self *LogLineItem) Color() string
  • func (lli LogLineItem) Global() bool
  • func (self *LogLineItem) OlderThanThreshold(previousItem interface{}) bool
• type Logger
  • func NewLogger() *Logger
  • func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc)
• type MutableVar
  • func (self *MutableVar) Get() interface{}
  • func (self *MutableVar) Set(v interface{}) interface{}
• type Options
• type Test
  • func GetOldTests(limit int) ([]Test, error)

Constants

const (
	CleanupFactor    = 5
	CleanupBatchSize = 30000 * CleanupFactor

	AmboyInterval      = time.Minute * CleanupFactor
	AmboyWorkers       = 8
	AmboyTargetNumJobs = CleanupBatchSize

	AmboyDBName             = "amboy"
	AmboyMigrationQueueName = "logkeeper.etl"

	AmboyLeaderFile = "/srv/logkeeper/amboy.leader"
)

Variables

var BuildRevision = "-dev"

Should be specified with -ldflags at build time

var Colors = []string{"#333", "seagreen", "steelblue",
	"mediumpurple", "crimson", "darkkhaki",
	"darkgreen", "rosybrown", "chocolate",
	"orangered", "darkseagreen", "royalblue",
	"slategray",
}

var ErrReadSizeLimitExceeded = errors.New("read size limit exceeded")

Functions

func CleanupOldLogsByBuild

func CleanupOldLogsByBuild(id interface{}) (int, error)

func GetCtxRequestId

func GetCtxRequestId(r *http.Request) int

func GetHandlerPprof

func GetHandlerPprof() http.Handler

GetHandlerPprof returns a handler for pprof endpoints.

func GetSender

func GetSender(queue amboy.Queue, fn string) (send.Sender, error)

func IsLeader

func IsLeader() bool

func MergeLog

func MergeLog(logger1 chan *LogLineItem, logger2 chan *LogLineItem) chan *LogLineItem

MergeLog takes two channels of LogLineItem and returns a single channel that feeds the result of merging the two input channels sorted by timestamp.

func New

func New(opts Options) *logKeeper

func SetCtxRequestId

func SetCtxRequestId(reqID int, r *http.Request) *http.Request

func StartBackgroundLogging

func StartBackgroundLogging(ctx context.Context)

func StreamingGetOldTests

func StreamingGetOldTests(ctx context.Context, timeout time.Duration) (<-chan Test, <-chan error)

func UpdateFailedTestsByBuildID

func UpdateFailedTestsByBuildID(id interface{}) (int, error)

Types

type ColorDef

type ColorDef struct {
	Name  string
	Color string
}

type ColorSet

type ColorSet struct {
	// contains filtered or unexported fields
}

ColorSet is a structure to track unique logger names and assign a color number to each one.

func NewColorSet

func NewColorSet() *ColorSet

func (*ColorSet) GetAllColors

func (self *ColorSet) GetAllColors() []ColorDef

GetAllColors returns a list of all the ColorDef entries stored internally, where each ColorDef is composed of a name and actual HTML color value.

func (*ColorSet) GetColor

func (self *ColorSet) GetColor(inputKey interface{}) (string, error)

GetColor returns a unique color name for the given key, creating a new one in its internal map if it does not already exist.

type LimitedReader

type LimitedReader struct {
	R io.Reader // underlying reader
	N int       // max bytes remaining
}

A LimitedReader reads from R but limits the amount of data returned to just N bytes. Each call to Read updates N to reflect the new amount remaining. Note: this is identical to io.LimitedReader, but throws ErrReadSizeLimitExceeded so it can be distinguished from a normal EOF.

func (*LimitedReader) Read

func (l *LimitedReader) Read(p []byte) (n int, err error)

Read returns an error if the bytes in the reader exceed the maximum size threshold for the reader, but fail to

type Log

type Log struct {
	BuildId interface{}    `bson:"build_id"`
	TestId  *bson.ObjectId `bson:"test_id"`
	Seq     int            `bson:"seq"`
	Started *time.Time     `bson:"started"`
	Lines   []LogLine      `bson:"lines"`
}

type LogKeeperBuild

type LogKeeperBuild struct {
	Id       interface{}            `bson:"_id"`
	Builder  string                 `bson:"builder"`
	BuildNum int                    `bson:"buildnum"`
	Started  time.Time              `bson:"started"`
	Name     string                 `bson:"name"`
	Info     map[string]interface{} `bson:"info"`
	Phases   []string               `bson:"phases"`
	Seq      int                    `bson:"seq"`
}

type LogLine

type LogLine []interface{}

func NewLogLine

func NewLogLine(data []interface{}) *LogLine

func (LogLine) Msg

func (s LogLine) Msg() string

func (LogLine) Time

func (s LogLine) Time() time.Time

type LogLineItem

type LogLineItem struct {
	LineNum   int
	Timestamp time.Time
	Data      string
	TestId    *bson.ObjectId
}

func (*LogLineItem) Color

func (self *LogLineItem) Color() string

func (LogLineItem) Global

func (lli LogLineItem) Global() bool

Global returns true if this log line comes from a global log, otherwise false (from a test log).

func (*LogLineItem) OlderThanThreshold

func (self *LogLineItem) OlderThanThreshold(previousItem interface{}) bool

type Logger

type Logger struct {
	// contains filtered or unexported fields
}

is a middleware handler that logs the request as it goes in and the response as it goes out.

func NewLogger

func NewLogger() *Logger

NewLogger returns a new Logger instance

func (*Logger) ServeHTTP

func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc)

type MutableVar

type MutableVar struct {
	Value interface{}
}

func (*MutableVar) Get

func (self *MutableVar) Get() interface{}

func (*MutableVar) Set

func (self *MutableVar) Set(v interface{}) interface{}

type Options

type Options struct {
	// Name of DB in mongod to use for reading/writing log data
	DB string

	//Base URL to append to relative paths
	URL string

	// Maximum Request Size
	MaxRequestSize int
}

type Test

type Test struct {
	Id        bson.ObjectId          `bson:"_id"`
	BuildId   interface{}            `bson:"build_id"`
	BuildName string                 `bson:"build_name"`
	Name      string                 `bson:"name"`
	Command   string                 `bson:"command"`
	Started   time.Time              `bson:"started"`
	Ended     *time.Time             `bson:"ended"`
	Info      map[string]interface{} `bson:"info"`
	Failed    bool                   `bson:"failed"`
	Phase     string                 `bson:"phase"`
	Seq       int                    `bson:"seq"`
}

func GetOldTests

func GetOldTests(limit int) ([]Test, error)