logpeck

package module
v0.0.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 29, 2017 License: Apache-2.0 Imports: 19 Imported by: 0

README

Logpeck - A Simple, RESTful Log Collector

Build Status Documentation Status

Objectives

Logpeck aims to be an easy-to-use module that parsing and collecting contents from log file and posting into ElasticSearch. We want to control collection tasks remotely with HTTP API (NONE configuration file).

We highly recommend to install logpeck-kibana-plugin into Kibana. With this plugin, we can control all machines and collection tasks conveniently. At the same time, we can take advantage of powerful searching and visualization features of Kibana.

Build & Launch

go build cmd/logpeckd/logpeckd.go

./logpeckd -config logpeckd.conf

We can also use supervisor or other service management software to manage logpeck process.

Try RESTful API

  1. Add a new task first. (Want more task config, filter, json, long, etc.? see here.)
curl -XPOST http://127.0.0.1:7117/peck_task/add -d {
  	"Name":"SystemLog",
	"LogPath":"/var/log/syslog",
	"ESConfig":{
	  	"Hosts":["127.0.0.1:9200"],
		"Index":"syslog",
		"Type":"raw"
	}
}

Add Success
  1. Start task.
curl -XPOST http://127.0.0.1:7117/peck_task/start -d {
  	"Name":"SystemLog"
}

Start Success
  1. Stop task
curl -XPOST http://127.0.0.1:7117/peck_task/stop -d {
  	"Name":"SystemLog"
}
  1. Remove task
curl -XPOST http://127.0.0.1:7117/peck_task/remove -d {
  	"Name":"SystemLog"
}
  1. List tasks
curl -XPOST http://127.0.0.1:7117/peck_task/list

Documentation

Dependencies

Saulte to all these excellent projects.

Discussion

Any suggestions or questions, please create an issue to feedback.

Documentation

Index

Constants

View Source 
const VersionString string = "0.2.0"

Variables

This section is empty.

Functions

func FormatJsonValue 

func FormatJsonValue(iValue interface{}) interface{}

func GetHost 

func GetHost() string

func GetString added in v0.0.3 

func GetString(j *sjson.Json, key string, required bool) (string, error)

func GetStringArray added in v0.0.3 

func GetStringArray(j *sjson.Json, key string) ([]string, error)

func HttpCall 

func HttpCall(method, url string, bodyString string)

func InitConfig 

func InitConfig(file *string) bool

func LogExecTime 

func LogExecTime(start time.Time, prefix string)

func NewAddTaskHandler 

func NewAddTaskHandler(pecker *Pecker) http.HandlerFunc

func NewListPathHandler added in v0.0.3 

func NewListPathHandler() http.HandlerFunc

func NewListStatsHandler added in v0.0.3 

func NewListStatsHandler(pecker *Pecker) http.HandlerFunc

func NewListTaskHandler 

func NewListTaskHandler(pecker *Pecker) http.HandlerFunc

func NewRemoveTaskHandler 

func NewRemoveTaskHandler(pecker *Pecker) http.HandlerFunc

func NewStartTaskHandler 

func NewStartTaskHandler(pecker *Pecker) http.HandlerFunc

func NewStopTaskHandler 

func NewStopTaskHandler(pecker *Pecker) http.HandlerFunc

func NewTestTaskHandler added in v0.0.3 

func NewTestTaskHandler() http.HandlerFunc

func NewUpdateTaskHandler 

func NewUpdateTaskHandler(pecker *Pecker) http.HandlerFunc

func NewVersionHandler added in v0.0.3 

func NewVersionHandler() http.HandlerFunc

func OpenDB 

func OpenDB(path string) (err error)

func SelectRandom 

func SelectRandom(candidates []string) (string, error)

func SplitString 

func SplitString(content, delims string) []string

func TestPeckTask added in v0.0.3 

func TestPeckTask(config *PeckTaskConfig) ([]map[string]interface{}, error)

Types

type DB 

type DB struct {
	// contains filtered or unexported fields
}

func GetDBHandler 

func GetDBHandler() *DB

func (*DB) Close 

func (p *DB) Close() error

func (*DB) GetAllConfigs 

func (p *DB) GetAllConfigs() (configs []PeckTaskConfig, err error)

func (*DB) GetAllStats 

func (p *DB) GetAllStats() (stats []PeckTaskStat, err error)

func (*DB) GetConfig 

func (p *DB) GetConfig(logPath, name string) (*PeckTaskConfig, error)

func (*DB) GetStat 

func (p *DB) GetStat(logPath, name string) (*PeckTaskStat, error)

func (*DB) RemoveConfig 

func (p *DB) RemoveConfig(logPath, name string) error

func (*DB) RemoveStat 

func (p *DB) RemoveStat(logPath, name string) error

func (*DB) SaveConfig 

func (p *DB) SaveConfig(config *PeckTaskConfig) error

func (*DB) SaveStat 

func (p *DB) SaveStat(stat *PeckTaskStat) error

type ElasticSearchConfig 

type ElasticSearchConfig struct {
	Hosts   []string
	Index   string
	Type    string
	Mapping map[string]interface{}
}

func ParseESConfig added in v0.0.3 

func ParseESConfig(j *sjson.Json) (config ElasticSearchConfig, e error)

type ElasticSearchSender 

type ElasticSearchSender struct {
	// contains filtered or unexported fields
}

func NewElasticSearchSender 

func NewElasticSearchSender(config *ElasticSearchConfig, fields []PeckField) *ElasticSearchSender

func (*ElasticSearchSender) GetIndexName added in v0.0.3 

func (p *ElasticSearchSender) GetIndexName() (indexName string)

func (*ElasticSearchSender) InitMapping added in v0.0.3 

func (p *ElasticSearchSender) InitMapping() error

func (*ElasticSearchSender) Send 

func (p *ElasticSearchSender) Send(fields map[string]interface{})

type LogPeckConfig 

type LogPeckConfig struct {
	Port          int32         `toml:"port"`
	LogLevel      string        `toml:"log_level"`
	MaxTaskNum    int32         `toml:"max_task_num"`
	DatabaseFile  string        `toml:"database_file"`
	PeckTaskLimit PeckTaskLimit `toml:"peck_task_limit"`
}
var Config LogPeckConfig

type LogStat 

type LogStat struct {
	LogPath         string
	PeckTaskConfigs []PeckTaskConfig
	PeckTaskStats   []PeckTaskStat
}

type LogTask 

type LogTask struct {
	LogPath string
	// contains filtered or unexported fields
}

func NewLogTask 

func NewLogTask(path string) *LogTask

func (*LogTask) AddPeckTask 

func (p *LogTask) AddPeckTask(task *PeckTask) error

func (*LogTask) Close 

func (p *LogTask) Close() error

func (*LogTask) Empty 

func (p *LogTask) Empty() bool

func (*LogTask) Exist 

func (p *LogTask) Exist(config *PeckTaskConfig) bool

func (*LogTask) GetStat 

func (p *LogTask) GetStat() *LogStat

func (*LogTask) IsStop 

func (p *LogTask) IsStop() bool

func (*LogTask) RemovePeckTask 

func (p *LogTask) RemovePeckTask(config *PeckTaskConfig) error

func (*LogTask) Start 

func (p *LogTask) Start() error

func (*LogTask) StartPeckTask 

func (p *LogTask) StartPeckTask(config *PeckTaskConfig) error

func (*LogTask) Stop 

func (p *LogTask) Stop() error

func (*LogTask) StopPeckTask 

func (p *LogTask) StopPeckTask(config *PeckTaskConfig) error

func (*LogTask) UpdatePeckTask 

func (p *LogTask) UpdatePeckTask(task *PeckTask) error

type MockLog 

type MockLog struct {
	Path      string
	IsRunning bool
	// contains filtered or unexported fields
}

func NewMockLog 

func NewMockLog(path string) (*MockLog, error)

func (*MockLog) Close 

func (p *MockLog) Close()

func (*MockLog) Run 

func (p *MockLog) Run() error

func (*MockLog) Stop 

func (p *MockLog) Stop()

type PeckField 

type PeckField struct {
	Name  string
	Value string
}

func (*PeckField) Check 

func (p *PeckField) Check() error

Field format: 

"Value": "$num"

type PeckFilter 

type PeckFilter struct {
	// contains filtered or unexported fields
}

func NewPeckFilter 

func NewPeckFilter(filterExpr string) *PeckFilter

func (*PeckFilter) Drop 

func (p *PeckFilter) Drop(str string) bool

type PeckTask 

type PeckTask struct {
	Config PeckTaskConfig
	Stat   PeckTaskStat
	// contains filtered or unexported fields
}

func NewPeckTask 

func NewPeckTask(c *PeckTaskConfig, s *PeckTaskStat) (*PeckTask, error)

func (*PeckTask) ExtractFields 

func (p *PeckTask) ExtractFields(content string) map[string]interface{}

func (*PeckTask) ExtractFieldsFromJson 

func (p *PeckTask) ExtractFieldsFromJson(content string) map[string]interface{}

func (*PeckTask) ExtractFieldsFromPlain 

func (p *PeckTask) ExtractFieldsFromPlain(content string) map[string]interface{}

func (*PeckTask) IsStop 

func (p *PeckTask) IsStop() bool

func (*PeckTask) Process 

func (p *PeckTask) Process(content string)

func (*PeckTask) ProcessTest added in v0.0.3 

func (p *PeckTask) ProcessTest(content string) (map[string]interface{}, error)

func (*PeckTask) Start 

func (p *PeckTask) Start()

func (*PeckTask) Stop 

func (p *PeckTask) Stop()

type PeckTaskConfig 

type PeckTaskConfig struct {
	Name     string
	LogPath  string
	ESConfig ElasticSearchConfig

	LogFormat  string
	FilterExpr string
	Fields     []PeckField
	Delimiters string
	Test       TestModule
}

func (*PeckTaskConfig) Check 

func (p *PeckTaskConfig) Check() error

func (*PeckTaskConfig) Unmarshal added in v0.0.3 

func (p *PeckTaskConfig) Unmarshal(jsonStr []byte) (e error)

type PeckTaskLimit 

type PeckTaskLimit struct {
	MaxLinesPerSec int64 `toml:"max_lines_per_sec"`
	MaxBytesPerSec int64 `toml:"max_bytes_per_sec"`
}

type PeckTaskStat 

type PeckTaskStat struct {
	Name        string
	LogPath     string
	LinesPerSec int64
	BytesPerSec int64
	LinesTotal  int64
	BytesTotal  int64
	Stop        bool
}

type Pecker 

type Pecker struct {
	// contains filtered or unexported fields
}

func NewPecker 

func NewPecker(db *DB) (*Pecker, error)

func (*Pecker) AddPeckTask 

func (p *Pecker) AddPeckTask(config *PeckTaskConfig, stat *PeckTaskStat) error

func (*Pecker) GetStat 

func (p *Pecker) GetStat() *PeckerStat

func (*Pecker) ListPeckTask 

func (p *Pecker) ListPeckTask() ([]PeckTaskConfig, error)

func (*Pecker) ListTaskStats added in v0.0.3 

func (p *Pecker) ListTaskStats() ([]PeckTaskStat, error)

func (*Pecker) RemovePeckTask 

func (p *Pecker) RemovePeckTask(config *PeckTaskConfig) error

func (*Pecker) Start 

func (p *Pecker) Start() error

func (*Pecker) StartPeckTask 

func (p *Pecker) StartPeckTask(config *PeckTaskConfig) error

func (*Pecker) StopPeckTask 

func (p *Pecker) StopPeckTask(config *PeckTaskConfig) error

func (*Pecker) UpdatePeckTask 

func (p *Pecker) UpdatePeckTask(config *PeckTaskConfig) error

type PeckerStat 

type PeckerStat struct {
	Name     string
	Stat     Stat
	LogStats []LogStat
}

type Stat 

type Stat struct {
	Name        string
	LinesPerSec int64
	BytesPerSec int64
	LinesTotal  int64
	BytesTotal  int64
}

type TestModule added in v0.0.3 

type TestModule struct {
	TestNum int
	Timeout int
}

Source Files

View all Source files

Directories

Path Synopsis
cmd
logpeckd
util

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.