logsauce

README

logsauce

Simple but powerful self-hosted logging system built in Go. Based on the principle of recording raw data first, and analyzing it later. The opposite of how logstash works. The system consists of a server responsible for sourcing all data and a remote agent responsible for monitoring data containers (text files, databases, etc.).

Depending on the format of the sourced data, grok can be used to tokenize the data to make mapreduce operations possible. This makes it possible to easily monitor logfiles with a specific pattern (apache, iptables, auth). If the data is pure JSON, tokenization will not be necessary.

A client API will give frontend developers a way to do simple mapreduce operations on the tokenized data to display it in different ways.

Documentation

Index

• func InitializeDB(config Configuration)
• func RegisterRoutes(listenPort int, relayMode, serverMode bool)
• func SendLogLine(destinationHost, line, filepath, filename, token string)
• func WatchFile(out chan<- string, fileToWatch string)
• func WatchFiles(filesToWatch []string, loadExistingData bool, ...)
• func WriteExampleServerConfig()
• type ComputeRequest
• type ComputeResponse
• type Configuration
  • func ReadConfig(path string) (Configuration, error)
• type Host
• type LogLine
• type LogType

Functions

func InitializeDB

func InitializeDB(config Configuration)

func RegisterRoutes

func RegisterRoutes(listenPort int, relayMode, serverMode bool)

func SendLogLine

func SendLogLine(destinationHost, line, filepath, filename, token string)

func WatchFile

func WatchFile(out chan<- string, fileToWatch string)

func WatchFiles

func WatchFiles(filesToWatch []string, loadExistingData bool, serverAddress, clientToken string)

func WriteExampleServerConfig

func WriteExampleServerConfig()

Types

type ComputeRequest

type ComputeRequest struct {
	Operation   string `json:"op,omitempty"`
	Parameter1  string `json:"p1,omitempty"`
	Parameter2  string `json:"p2,omitempty"`
	Parameter3  string `json:"p3,omitempty"`
	TimeStart   int64  `json:"t0,omitempty"`
	TimeEnd     int64  `json:"t1,omitempty"`
	Filename    string `json:"f,omitempty"`
	Host        string `json:"h,omitempty"`
	LogtypeName string `json:"l,omitempty"`
}

type ComputeResponse

type ComputeResponse struct {
	Values       map[string]int `json:"v,omitempty"`
	TimeStart    int64          `json:"t,omitempty"`
	IntervalSize int64          `json:"is,omitempty"`
	FieldName    string         `json:"fn,omitempty"`
	Filename     string         `json:"f,omitempty"`
	HostId       bson.ObjectId  `bson:",omitempty" json:"h,omitempty"`
	LogtypeName  string         `json:"l,omitempty"`
}

type Configuration

type Configuration struct {
	ListenPort int  `json:"listen_port"`
	ClientMode bool `json:"client_mode"`
	ServerMode bool `json:"server_mode"`
	Relaymode  bool `json:"relay_mode"`

	ServerConfiguration struct {
		DbAddress            string `json:"db_address"`
		DbName               string `json:"db_name"`
		DbUsername           string `json:"db_username"`
		DbPassword           string `json:"db_password"`
		ServerCertificate    string `json:"server_cert"`
		ServerCertificateKey string `json:"server_key"`
		Hosts                []Host `json:"hosts"`
	} `json:"server_config"`

	ClientConfiguration struct {
		ServerAddress     string   `json:"server_address"`
		FilesToWatch      []string `json:"files_to_watch"`
		ServerCertificate string   `json:"server_cert"`
		ClientToken       string   `json:"token"`
	} `json:"client_config"`
}

func ReadConfig

func ReadConfig(path string) (Configuration, error)

type Host

type Host struct {
	Id    bson.ObjectId `bson:"_id,omitempty"`
	Token string
	Name  string
}

type LogLine

type LogLine struct {
	Id              bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
	Line            string
	Timestamp       int64
	DataTimestamp   int64
	HostId          bson.ObjectId `bson:",omitempty" json:",omitempty"`
	TypeId          bson.ObjectId `bson:",omitempty" json:",omitempty"`
	Filename        string
	Filepath        string
	Alias           string            `bson:",omitempty" json:",omitempty"`
	TokenizedObject map[string]string `bson:",omitempty" json:",omitempty"`
}

type LogType

type LogType struct {
	Id            bson.ObjectId `bson:"_id,omitempty"`
	Name          string        `bson:",omitempty" json:",omitempty"`
	Description   string        `bson:",omitempty" json:",omitempty"`
	Pattern       string        `bson:",omitempty" json:",omitempty"`
	DateFormat    string        `bson:",omitempty" json:",omitempty"`
	DateFieldname string        `bson:",omitempty" json:",omitempty"`
}