parser

package
v1.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 27, 2017 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	KeyCSVSchema   = "csv_schema"   // csv 每个列的列名和类型 long/string/float
	KeyCSVSplitter = "csv_splitter" // csv 的分隔符
	KeyCSVLabels   = "csv_labels"   // csv 额外增加的标签信息，比如机器信息等
)
View Source 
const (
	KeyGrokMode               = "grok_mode"     //是否替换\n以匹配多行
	KeyGrokPatterns           = "grok_patterns" // grok 模式串名
	KeyGrokCustomPatternFiles = "grok_custom_pattern_files"
	KeyGrokCustomPatterns     = "grok_custom_patterns"

	KeyTimeZoneOffset = "timezone_offset"
)
View Source 
const (
	LONG   = "long"
	FLOAT  = "float"
	STRING = "string"
	DATE   = "date"
	DROP   = "drop"
)
View Source 
const (
	KEY_SRC_IP   = "source_ip"
	KEY_METHOD   = "method"
	KEY_TOPIC    = "topic"
	KEY_CODE     = "code"
	KEY_RESP_LEN = "resp_len"
	KEY_DURATION = "duration"
	KEY_LOG_TIME = "log_time"
	KEY_ERROR    = "error"
	KEY_WARN     = "warn"
	EMPTY_STRING = ""
)
View Source 
const (
	KeyParserName = utils.GlobalKeyName
	KeyParserType = "type"
	KeyRunnerName = "runner_name"
	KeyLabels     = "labels" // 额外增加的标签信息，比如机器信息等
)

conf 字段

View Source 
const (
	TypeCSV        = "csv"
	TypeLogv1      = "qiniulog"
	TypeKafkaRest  = "kafkarest"
	TypeRaw        = "raw"
	TypeEmpty      = "empty"
	TypeGrok       = "grok"
	TypeInnerSQL   = "_sql"
	TypeInnerMysql = "_mysql"
	TypeJson       = "json"
)

parser 的类型

View Source 
const (
	LogHeadPrefix string = "prefix"
	LogHeadDate   string = "date"
	LogHeadTime   string = "time"
	LogHeadReqid  string = "reqid"
	LogHeadLevel  string = "level"
	LogHeadFile   string = "file"
	LogHeadLog    string = "log" //默认在最后，不能改变顺序
)
View Source 
const (
	KeyQiniulogPrefix = "qiniulog_prefix"   //qiniulog的日志前缀
	KeyLogMaxLine     = "qiniulog_max_line" //qiniulog最大一条日志行数
	KeyLogHeaders     = "qiniulog_log_headers"
)

conf 字段

View Source 
const (
	KeyRaw       = "raw"       //qiniulog的日志前缀
	KeyTimestamp = "timestamp" //qiniulog最大一条日志行数
)
View Source 
const DEFAULT_PATTERNS = `` /* 2684-byte string literal not displayed */
View Source 
const MaxGrokMultiLineBuffer = 64 * 1024 * 1024 // 64MB
View Source 
const MaxParserSchemaErrOutput = 5
View Source 
const (
	ModeMulti = "multi"
)
View Source 
const SECOND_PER_5MIN = 5 * 60
View Source 
const SECOND_PER_DAY = 24 * 60 * 60

Variables

View Source 
var (
	HeaderPattern = map[string]string{
		LogHeadDate:  "^[1-9]\\d{3}/[0-1]\\d/[0-3]\\d$",
		LogHeadTime:  "^[0-2]\\d:[0-6]\\d:[0-6]\\d(\\.\\d{6})?$",
		LogHeadReqid: "^\\[\\w+\\]\\[\\w+\\]$",
		LogHeadLevel: "^\\[[A-Z]+\\]$",
		LogHeadFile:  ":\\d+:$",
	}
	CompliedPatterns map[string]*regexp.Regexp
)

Functions

func Time1Day 

func Time1Day(t int64) string

func Time1DayInt 

func Time1DayInt(t int64) int64

func Time5Min 

func Time5Min(t int64) string

func Time5MinInt 

func Time5MinInt(t int64) int64

Types

type CsvParser 

type CsvParser struct {
	// contains filtered or unexported fields
}

func (*CsvParser) Name 

func (p *CsvParser) Name() string

func (*CsvParser) Parse 

func (p *CsvParser) Parse(lines []string) ([]sender.Data, error)

type CsvType 

type CsvType string

Type 类型常量 

const (
	TypeFloat   CsvType = "float"
	TypeLong    CsvType = "long"
	TypeString  CsvType = "string"
	TypeJsonMap CsvType = "jsonmap"
)

type EmptyParser 

type EmptyParser struct {
	// contains filtered or unexported fields
}

func (*EmptyParser) Name 

func (p *EmptyParser) Name() string

func (*EmptyParser) Parse 

func (p *EmptyParser) Parse(lines []string) (datas []sender.Data, err error)

type GrokParser 

type GrokParser struct {
	Patterns []string // 正式的pattern名称

	CustomPatterns     string
	CustomPatternFiles []string
	// contains filtered or unexported fields
}

func (*GrokParser) Name 

func (gp *GrokParser) Name() string

func (*GrokParser) Parse 

func (gp *GrokParser) Parse(lines []string) ([]sender.Data, error)

type JsonParser 

type JsonParser struct {
	// contains filtered or unexported fields
}

func (*JsonParser) Name 

func (im *JsonParser) Name() string

func (*JsonParser) Parse 

func (im *JsonParser) Parse(lines []string) ([]sender.Data, error)

type KafaRestlogParser 

type KafaRestlogParser struct {
	// contains filtered or unexported fields
}

func (*KafaRestlogParser) Name 

func (krp *KafaRestlogParser) Name() string

func (*KafaRestlogParser) Parse 

func (krp *KafaRestlogParser) Parse(lines []string) ([]sender.Data, error)

func (*KafaRestlogParser) ParseCode 

func (krp *KafaRestlogParser) ParseCode(fields []string) int

func (*KafaRestlogParser) ParseDuration 

func (krp *KafaRestlogParser) ParseDuration(fields []string) int

func (*KafaRestlogParser) ParseIp 

func (krp *KafaRestlogParser) ParseIp(fields []string) string

func (*KafaRestlogParser) ParseLogTime 

func (krp *KafaRestlogParser) ParseLogTime(fields []string) int64

func (*KafaRestlogParser) ParseMethod 

func (krp *KafaRestlogParser) ParseMethod(fields []string) string

func (*KafaRestlogParser) ParseRespCL 

func (krp *KafaRestlogParser) ParseRespCL(fields []string) int

func (*KafaRestlogParser) ParseTopic 

func (krp *KafaRestlogParser) ParseTopic(fields []string) string

type LogParser 

type LogParser interface {
	Name() string
	// parse lines into structured datas
	Parse(lines []string) (datas []sender.Data, err error)
}

func NewCsvParser 

func NewCsvParser(c conf.MapConf) (LogParser, error)

func NewEmptyParser 

func NewEmptyParser(c conf.MapConf) (LogParser, error)

func NewGrokParser 

func NewGrokParser(c conf.MapConf) (LogParser, error)

func NewJsonParser 

func NewJsonParser(c conf.MapConf) (LogParser, error)

func NewKafaRestlogParser 

func NewKafaRestlogParser(c conf.MapConf) (LogParser, error)

func NewQiniulogParser 

func NewQiniulogParser(c conf.MapConf) (LogParser, error)

func NewRawlogParser 

func NewRawlogParser(c conf.MapConf) (LogParser, error)

type ParserRegistry 

type ParserRegistry struct {
	// contains filtered or unexported fields
}

func NewParserRegistry 

func NewParserRegistry() *ParserRegistry

func (*ParserRegistry) NewLogParser 

func (ps *ParserRegistry) NewLogParser(conf conf.MapConf) (p LogParser, err error)

func (*ParserRegistry) RegisterParser 

func (ps *ParserRegistry) RegisterParser(parserType string, constructor func(conf.MapConf) (LogParser, error)) error

type QiniulogParser 

type QiniulogParser struct {
	// contains filtered or unexported fields
}

func (*QiniulogParser) GetParser 

func (p *QiniulogParser) GetParser(head string) (func(string) (string, string, error), error)

func (*QiniulogParser) Name 

func (p *QiniulogParser) Name() string

func (*QiniulogParser) Parse 

func (p *QiniulogParser) Parse(lines []string) ([]sender.Data, error)

type RawlogParser 

type RawlogParser struct {
	// contains filtered or unexported fields
}

func (*RawlogParser) Name 

func (p *RawlogParser) Name() string

func (*RawlogParser) Parse 

func (p *RawlogParser) Parse(lines []string) ([]sender.Data, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.