parser

package
v1.3.5 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 31, 2017 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source
const (
	KeyCSVSchema   = "csv_schema"   // csv 每个列的列名和类型 long/string/float/date
	KeyCSVSplitter = "csv_splitter" // csv 的分隔符
	KeyCSVLabels   = "csv_labels"   // csv 额外增加的标签信息,比如机器信息等
)
View Source
const (
	KeyGrokMode               = "grok_mode"     //是否替换\n以匹配多行
	KeyGrokPatterns           = "grok_patterns" // grok 模式串名
	KeyGrokCustomPatternFiles = "grok_custom_pattern_files"
	KeyGrokCustomPatterns     = "grok_custom_patterns"

	KeyTimeZoneOffset = "timezone_offset"
)
View Source
const (
	LONG   = "long"
	FLOAT  = "float"
	STRING = "string"
	DATE   = "date"
	DROP   = "drop"
)
View Source
const (
	KEY_SRC_IP   = "source_ip"
	KEY_METHOD   = "method"
	KEY_TOPIC    = "topic"
	KEY_CODE     = "code"
	KEY_RESP_LEN = "resp_len"
	KEY_DURATION = "duration"
	KEY_LOG_TIME = "log_time"
	KEY_ERROR    = "error"
	KEY_WARN     = "warn"
	EMPTY_STRING = ""
)
View Source
const (
	NginxSchema      string = "nginx_schema"
	NginxConfPath           = "nginx_log_format_path"
	NginxLogFormat          = "nginx_log_format_name"
	NginxFormatRegex        = "nginx_log_format_regex"
)
View Source
const (
	KeyParserName = utils.GlobalKeyName
	KeyParserType = "type"
	KeyRunnerName = "runner_name"
	KeyLabels     = "labels" // 额外增加的标签信息,比如机器信息等
)

conf 字段

View Source
const (
	TypeCSV        = "csv"
	TypeLogv1      = "qiniulog"
	TypeKafkaRest  = "kafkarest"
	TypeRaw        = "raw"
	TypeEmpty      = "empty"
	TypeGrok       = "grok"
	TypeInnerSQL   = "_sql"
	TypeInnerMysql = "_mysql"
	TypeJson       = "json"
	TypeNginx      = "nginx"
)

parser 的类型

View Source
const (
	LogHeadPrefix string = "prefix"
	LogHeadDate   string = "date"
	LogHeadTime   string = "time"
	LogHeadReqid  string = "reqid"
	LogHeadLevel  string = "level"
	LogHeadFile   string = "file"
	LogHeadLog    string = "log" //默认在最后,不能改变顺序
)
View Source
const (
	KeyQiniulogPrefix = "qiniulog_prefix" //qiniulog的日志前缀
	KeyLogHeaders     = "qiniulog_log_headers"
)

conf 字段

View Source
const (
	KeyRaw       = "raw"
	KeyTimestamp = "timestamp"
)
View Source
const DEFAULT_PATTERNS = `` /* 41293-byte string literal not displayed */
View Source
const MaxGrokMultiLineBuffer = 64 * 1024 * 1024 // 64MB
View Source
const MaxParserSchemaErrOutput = 5
View Source
const (
	ModeMulti = "multi"
)
View Source
const SECOND_PER_5MIN = 5 * 60
View Source
const SECOND_PER_DAY = 24 * 60 * 60

Variables

View Source
var (
	HeaderPattern = map[string]string{
		LogHeadDate:  "^[1-9]\\d{3}/[0-1]\\d/[0-3]\\d$",
		LogHeadTime:  "^[0-2]\\d:[0-6]\\d:[0-6]\\d(\\.\\d{6})?$",
		LogHeadReqid: "^\\[\\w+\\]\\[\\w+\\]$",
		LogHeadLevel: "^\\[[A-Z]+\\]$",
		LogHeadFile:  ":\\d+:$",
	}
	CompliedPatterns map[string]*regexp.Regexp
)
View Source
var ModeKeyOptions = map[string][]utils.Option{
	TypeJson: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeNginx: {
		{
			KeyName:      NginxConfPath,
			ChooseOnly:   false,
			Default:      "/opt/nginx/conf/nginx.conf",
			DefaultNoUse: true,
			Description:  "nginx配置路径(nginx_log_format_path)",
		},
		{
			KeyName:      NginxLogFormat,
			ChooseOnly:   false,
			Default:      "main",
			DefaultNoUse: true,
			Description:  "nginx日志格式名称(nginx_log_format_name)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      NginxSchema,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "指定nginx字段类型(nginx_schema)",
		},
		{
			KeyName:      NginxFormatRegex,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "直接通过正则表达式解析(nginx_log_format_regex)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeGrok: {
		{
			KeyName:      KeyGrokPatterns,
			ChooseOnly:   false,
			Default:      "%{COMMON_LOG_FORMAT}",
			DefaultNoUse: true,
			Description:  "匹配日志的grok表达式(grok_patterns)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyGrokMode,
			ChooseOnly:    true,
			ChooseOptions: []string{"oneline", ModeMulti},
			Default:       "oneline",
			DefaultNoUse:  false,
			Description:   "grok单行多行模式(grok_mode)",
		},
		{
			KeyName:      KeyGrokCustomPatternFiles,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern 文件路径(grok_custom_pattern_files)",
		},
		{
			KeyName:      KeyGrokCustomPatterns,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern (grok_custom_patterns)",
		},
		{
			KeyName:    KeyTimeZoneOffset,
			ChooseOnly: true,
			Default:    "0",
			ChooseOptions: []string{"0", "-1", "-2", "-3", "-4",
				"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
				"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
			DefaultNoUse: false,
			Description:  "时区偏移量(timezone_offset)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},

	TypeCSV: {
		{
			KeyName:      KeyCSVSchema,
			ChooseOnly:   false,
			Default:      "abc string,xyz long,data1 string,data2 float",
			DefaultNoUse: true,
			Description:  "csv格式的字段类型(csv_schema)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyCSVSplitter,
			ChooseOnly:   false,
			Default:      ",",
			DefaultNoUse: false,
			Description:  "csv分隔符(csv_splitter)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:    KeyTimeZoneOffset,
			ChooseOnly: true,
			Default:    "0",
			ChooseOptions: []string{"0", "-1", "-2", "-3", "-4",
				"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
				"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
			DefaultNoUse: false,
			Description:  "时区偏移量(timezone_offset)",
		},
	},
	TypeRaw: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyTimestamp,
			ChooseOnly:    true,
			ChooseOptions: []string{"true", "false"},
			Default:       "true",
			DefaultNoUse:  false,
			Description:   "数据附带时间戳(timestamp)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeLogv1: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:      KeyQiniulogPrefix,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "日志前缀(qiniulog_prefix)",
		},
		{
			KeyName:      KeyLogHeaders,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "七牛日志格式顺序(qiniulog_log_headers)",
		},
	},
	TypeKafkaRest: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeEmpty: {},
}
View Source
var ModeUsages = []utils.KeyValue{
	{TypeJson, "json 格式解析"},
	{TypeNginx, "nginx 日志解析"},
	{TypeGrok, "grok 方式解析"},
	{TypeCSV, "csv 格式日志解析"},
	{TypeRaw, "raw 原始日志按行解析"},
	{TypeLogv1, "qiniulog 七牛日志库解析"},
	{TypeKafkaRest, "kafkarest 日志格式解析"},
	{TypeEmpty, "empty 通过解析清空数据"},
}

ModeUsages 用途说明

View Source
var SampleLogs = map[string]string{
	TypeNginx: `110.110.101.101 - - [21/Mar/2017:18:14:17 +0800] "GET /files/yyyysx HTTP/1.1" 206 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com llEAAFgmnoIa3q0U "0.040" 0.040 760 "-" "-" - - QCloud
1.11.1.1 - - [25/Mar/2017:18:14:17 +0800] "GET /files HTTP/1.1" 200 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com sfvfv123axs "0.040" 0.040 760 "-" "-" - - QCloud`,
	TypeGrok: `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
123.45.12.1 user-identifier bob [10/Oct/2013:13:55:36 -0700] "GET /hello.gif HTTP/1.0" 200 2326`,
	TypeJson: `{"a":"b","c":1,"d":1.1}`,
	TypeCSV: `a,123,bcd,1.2
xsxs,456,asv,5.12`,
	TypeRaw: `raw log1[05-May-2017 13:44:39]  [pool log] pid 4109
script_filename = /data/html/
[0x00007fec119d1720] curl_exec() /data/html/xyframework/base.go:123
[05-May-2017 13:45:39]  [pool log] pid 4108`,
	TypeLogv1:     `2016/10/20 17:30:21.433423 [GE2owHck-Y4IWJHS][WARN] github.com/qiniu/http/rpcutil.v1/rpc_util.go:203: E18102: The specified repo does not exist under the provided appid ~`,
	TypeKafkaRest: `[2016-12-05 03:35:20,682] INFO 172.16.16.191 - - [05/Dec/2016:03:35:20 +0000] "POST /topics/VIP_VvBVy0tuMPPspm1A_0000000000 HTTP/1.1" 200 101640  46 (io.confluent.rest-utils.requests)`,
	TypeEmpty:     "empty 通过解析清空数据",
}

SampleLogs 样例日志,用于前端界面试玩解析器

Functions

func Time1Day

func Time1Day(t int64) string

func Time1DayInt

func Time1DayInt(t int64) int64

func Time5Min

func Time5Min(t int64) string

func Time5MinInt

func Time5MinInt(t int64) int64

Types

type CsvParser

type CsvParser struct {
	// contains filtered or unexported fields
}

func (*CsvParser) Name

func (p *CsvParser) Name() string

func (*CsvParser) Parse

func (p *CsvParser) Parse(lines []string) ([]sender.Data, error)

type CsvType

type CsvType string

Type 类型常量

const (
	TypeFloat   CsvType = "float"
	TypeLong    CsvType = "long"
	TypeString  CsvType = "string"
	TypeDate    CsvType = "date"
	TypeJsonMap CsvType = "jsonmap"
)

type EmptyParser

type EmptyParser struct {
	// contains filtered or unexported fields
}

func (*EmptyParser) Name

func (p *EmptyParser) Name() string

func (*EmptyParser) Parse

func (p *EmptyParser) Parse(lines []string) (datas []sender.Data, err error)

type GrokParser

type GrokParser struct {
	Patterns []string // 正式的pattern名称

	CustomPatterns     string
	CustomPatternFiles []string
	// contains filtered or unexported fields
}

func (*GrokParser) Name

func (gp *GrokParser) Name() string

func (*GrokParser) Parse

func (gp *GrokParser) Parse(lines []string) ([]sender.Data, error)

type JsonParser

type JsonParser struct {
	// contains filtered or unexported fields
}

func (*JsonParser) Name

func (im *JsonParser) Name() string

func (*JsonParser) Parse

func (im *JsonParser) Parse(lines []string) ([]sender.Data, error)

type KafaRestlogParser

type KafaRestlogParser struct {
	// contains filtered or unexported fields
}

func (*KafaRestlogParser) Name

func (krp *KafaRestlogParser) Name() string

func (*KafaRestlogParser) Parse

func (krp *KafaRestlogParser) Parse(lines []string) ([]sender.Data, error)

func (*KafaRestlogParser) ParseCode

func (krp *KafaRestlogParser) ParseCode(fields []string) int

func (*KafaRestlogParser) ParseDuration

func (krp *KafaRestlogParser) ParseDuration(fields []string) int

func (*KafaRestlogParser) ParseIp

func (krp *KafaRestlogParser) ParseIp(fields []string) string

func (*KafaRestlogParser) ParseLogTime

func (krp *KafaRestlogParser) ParseLogTime(fields []string) int64

func (*KafaRestlogParser) ParseMethod

func (krp *KafaRestlogParser) ParseMethod(fields []string) string

func (*KafaRestlogParser) ParseRespCL

func (krp *KafaRestlogParser) ParseRespCL(fields []string) int

func (*KafaRestlogParser) ParseTopic

func (krp *KafaRestlogParser) ParseTopic(fields []string) string

type Label added in v1.2.1

type Label struct {
	Name  string
	Value string
}

func GetLabels added in v1.2.1

func GetLabels(labelList []string, nameMap map[string]struct{}) (labels []Label)

type LogParser

type LogParser interface {
	Name() string
	// parse lines into structured datas
	Parse(lines []string) (datas []sender.Data, err error)
}

func NewCsvParser

func NewCsvParser(c conf.MapConf) (LogParser, error)

func NewEmptyParser

func NewEmptyParser(c conf.MapConf) (LogParser, error)

func NewGrokParser

func NewGrokParser(c conf.MapConf) (LogParser, error)

func NewJsonParser

func NewJsonParser(c conf.MapConf) (LogParser, error)

func NewKafaRestlogParser

func NewKafaRestlogParser(c conf.MapConf) (LogParser, error)

func NewNginxParser added in v1.2.1

func NewNginxParser(c conf.MapConf) (LogParser, error)

func NewQiniulogParser

func NewQiniulogParser(c conf.MapConf) (LogParser, error)

func NewRawlogParser

func NewRawlogParser(c conf.MapConf) (LogParser, error)

type NginxParser added in v1.2.1

type NginxParser struct {
	// contains filtered or unexported fields
}

func NewNginxAccParser added in v1.2.1

func NewNginxAccParser(c conf.MapConf) (p *NginxParser, err error)

func (*NginxParser) Name added in v1.2.1

func (p *NginxParser) Name() string

func (*NginxParser) Parse added in v1.2.1

func (p *NginxParser) Parse(lines []string) ([]sender.Data, error)

type ParserRegistry

type ParserRegistry struct {
	// contains filtered or unexported fields
}

func NewParserRegistry

func NewParserRegistry() *ParserRegistry

func (*ParserRegistry) NewLogParser

func (ps *ParserRegistry) NewLogParser(conf conf.MapConf) (p LogParser, err error)

func (*ParserRegistry) RegisterParser

func (ps *ParserRegistry) RegisterParser(parserType string, constructor func(conf.MapConf) (LogParser, error)) error

type QiniulogParser

type QiniulogParser struct {
	// contains filtered or unexported fields
}

func (*QiniulogParser) GetParser

func (p *QiniulogParser) GetParser(head string) (func(string) (string, string, error), error)

func (*QiniulogParser) Name

func (p *QiniulogParser) Name() string

func (*QiniulogParser) Parse

func (p *QiniulogParser) Parse(lines []string) ([]sender.Data, error)

type RawlogParser

type RawlogParser struct {
	// contains filtered or unexported fields
}

func (*RawlogParser) Name

func (p *RawlogParser) Name() string

func (*RawlogParser) Parse

func (p *RawlogParser) Parse(lines []string) ([]sender.Data, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL