models

package
v1.5.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 18, 2020 License: Apache-2.0 Imports: 32 Imported by: 168

Documentation

Index

Constants

View Source 
const (
	// 一切正常
	ErrNothing = "L200"

	// 单机版 Runner 操作
	ErrConfigName     = "L1001"
	ErrRunnerAdd      = "L1002"
	ErrRunnerDelete   = "L1003"
	ErrRunnerStart    = "L1004"
	ErrRunnerStop     = "L1005"
	ErrRunnerReset    = "L1006"
	ErrRunnerUpdate   = "L1007"
	ErrRunnerErrorGet = "L1008"

	// read 相关
	ErrReadRead = "L1101"
	// parse 相关
	ErrParseParse = "L1201"
	// transform 相关
	ErrTransformTransform = "L1301"
	// send 相关
	ErrSendSend = "L1401"

	// 集群版 master API
	ErrClusterSlaves   = "L2001"
	ErrClusterStatus   = "L2002"
	ErrClusterConfigs  = "L2003"
	ErrClusterRegister = "L2004"
	ErrClusterConfig   = "L2014"

	// 集群版 slave API
	ErrClusterTag = "L2005"

	// 集群版 master 对 slaves 的操作
	ErrClusterRunnerAdd    = "L2006"
	ErrClusterRunnerDelete = "L2007"
	ErrClusterRunnerStart  = "L2008"
	ErrClusterRunnerStop   = "L2009"
	ErrClusterRunnerReset  = "L2010"
	ErrClusterRunnerUpdate = "L2011"
	ErrClusterSlavesDelete = "L2012"
	ErrClusterSlavesTag    = "L2013"
)
View Source 
const (
	B = 1 << (iota * 10)
	KB
	MB
	GB
)
View Source 
const (
	CheckPattern       = "^[a-zA-Z_][a-zA-Z0-9_]{0,127}$"
	CheckPatternKey    = "^[a-zA-Z_.][a-zA-Z0-9_.]{0,127}$"
	DefaultEncodingWay = "UTF-8"

	InternalKeyPrefix = "internal_key_prefix"
	Lst               = "lst"

	KeyType   = "type"
	ProcessAt = "process_at"
	Local     = "local"
	Server    = "server"

	GlobalKeyName = "name"
	ExtraInfo     = "extra_info"

	KeyCore     = "core"
	KeyHostName = "hostname"
	KeyOsInfo   = "osinfo"
	KeyLocalIp  = "localip"

	ContentTypeHeader     = "Content-Type"
	ContentEncodingHeader = "Content-Encoding"

	ApplicationJson = "application/json"
	TextPlain       = "text/plain"
	ApplicationGzip = "application/gzip"

	KeyPandoraStash      = "pandora_stash"       // 当只有一条数据且 sendError 时候，将其转化为 raw 发送到 pandora_stash 这个字段
	KeyPandoraSeparateId = "pandora_separate_id" // 当一条数据大于2M且 sendError 时候，将其切片，切片记录到 pandora_separate_id 这个字段
	TypeIP               = "ip"                  // schema ip

	SchemaFreeTokensPrefix = "schema_free_tokens_"
	LogDBTokensPrefix      = "logdb_tokens_"
	TsDBTokensPrefix       = "tsdb_tokens_"
	KodoTokensPrefix       = "kodo_tokens_"

	KeyRunnerName       = "runner_name"
	KeyRunnerMaxLineLen = "runner_max_line_Len"
	KeyRunnerIsBlock    = "is_block"

	DefaultDirPerm  = 0755
	DefaultFilePerm = 0600

	DefaultMaxBatchSize = 2 * MB

	DefaultSendIntervalSeconds = 60

	DefaultTruncateMaxSize = 1024

	DefaultErrorsListCap = 100

	PipeLineError = "ErrorMessage="

	Text        = "text"
	Checkbox    = "checkbox"
	Radio       = "radio"
	InputNumber = "inputNumber"

	LONG   = "long"
	FLOAT  = "float"
	STRING = "string"
	DATE   = "date"
	DROP   = "drop"

	DefaultSelfRunnerName = DefaultInternalPrefix + "CollectLogRunner"
	DefaultInternalPrefix = "LogkitInternal"
)

Variables

View Source 
var (
	ErrNotSupport  = errors.New("runner does not support")
	ErrNotExist    = errors.New("runner does not exist")
	ErrQueueClosed = errors.New("queue is closed")
)
View Source 
var (
	MaxProcs                     = 1
	NumCPU                       = runtime.NumCPU()
	LogkitAutoCreateDescription  = "由logkit日志收集自动创建"
	MetricAutoCreateDescription  = "由logkit监控收集自动创建"
	SelfLogAutoCreateDescription = "由logkit收集自身日志创建"

	// matches named captures that contain a modifier.
	//   ie,
	//     %{NUMBER:bytes:long}
	//     %{IPORHOST:clientip:date}
	//     %{HTTPDATE:ts1:float}
	ModifierRe = regexp.MustCompile(`%{\w+:(\w+):(long|string|date|float|drop)}`)
	// matches a plain pattern name. ie, %{NUMBER}
	PatternOnlyRe = regexp.MustCompile(`%{(\w+)}`)

	Encoding = []interface{}{"UTF-8", "UTF-16", "US-ASCII", "ISO-8859-1",
		"GBK", "latin1", "GB18030", "EUC-JP", "UTF-16BE", "UTF-16LE", "Big5", "Shift_JIS",
		"ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7",
		"ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13",
		"ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "macos-0_2-10.2", "macos-6_2-10.4",
		"macos-7_3-10.2", "macos-29-10.2", "macos-35-10.2", "windows-1250", "windows-1251",
		"windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256",
		"windows-1257", "windows-1258", "windows-874", "IBM037", "ibm-273_P100-1995",
		"ibm-277_P100-1995", "ibm-278_P100-1995", "ibm-280_P100-1995", "ibm-284_P100-1995",
		"ibm-285_P100-1995", "ibm-290_P100-1995", "ibm-297_P100-1995", "ibm-420_X120-1999",

		"KOI8-R", "KOI8-U", "ebcdic-xml-us"}
)
View Source 
var ErrorCodeHumanize = map[string]string{
	ErrNothing: "操作成功",

	ErrConfigName:   "获取 Config 出现错误",
	ErrRunnerAdd:    "添加 Runner 出现错误",
	ErrRunnerDelete: "删除 Runner 出现错误",
	ErrRunnerStart:  "开启 Runner 出现错误",
	ErrRunnerStop:   "关闭 Runner 出现错误",
	ErrRunnerReset:  "重置 Runner 出现错误",
	ErrRunnerUpdate: "更新 Runner 出现错误",

	ErrParseParse: "解析字符串失败",

	ErrTransformTransform: "转化字段失败",

	ErrClusterSlaves:   "获取 Slaves 列表出现错误",
	ErrClusterStatus:   "获取 Slaves 状态出现错误",
	ErrClusterConfig:   "获取 Slaves Config 出现错误",
	ErrClusterConfigs:  "获取 Slaves Configs 出现错误",
	ErrClusterRegister: "接受 Slaves 注册出现错误",

	ErrClusterTag: "更改 Tag 出现错误",

	ErrClusterRunnerAdd:    "Slaves 添加 Runner 出现错误",
	ErrClusterRunnerDelete: "Slaves 删除 Runner 出现错误",
	ErrClusterRunnerStart:  "Slaves 启动 Runner 出现错误",
	ErrClusterRunnerStop:   "Slaves 关闭 Runner 出现错误",
	ErrClusterRunnerReset:  "Slaves 重置 Runner 出现错误",
	ErrClusterRunnerUpdate: "Slaves 更新 Runner 出现错误",
	ErrClusterSlavesDelete: "Slaves 从列表中移除时出现错误",
	ErrClusterSlavesTag:    "Slaves 更改 Tag 出现错误",
}

Functions

func AddHttpProtocal added in v1.4.6 

func AddHttpProtocal(url string) string

func Bool2String added in v1.4.7 

func Bool2String(i bool) string

func CheckErr added in v1.5.2 

func CheckErr(err error) error

func CheckFileMode added in v1.4.6 

func CheckFileMode(path string, fileMode os.FileMode) error

func CheckLogPath added in v1.5.5 

func CheckLogPath(logpath string) (dir string, path string)

func CheckPandoraKey added in v1.5.2 

func CheckPandoraKey(key string) bool

为了提升性能做的一个预先检查，避免CPU浪费

func CheckPath added in v1.5.4 

func CheckPath(path string) (string, error)

func ConvertDate added in v1.4.8 

func ConvertDate(layoutBefore, layoutAfter string, offset int, loc *time.Location, v interface{}) (interface{}, error)

func CreateDirIfNotExist added in v1.4.6 

func CreateDirIfNotExist(dir string) (err error)

CreateDirIfNotExist 检查文件夹，不存在时创建

func DecodeString added in v1.4.7 

func DecodeString(target string) (result string, err error)

func DecompressTarGzip added in v1.5.3 

func DecompressTarGzip(srcPath, dstPath, targetFile string) (targetDir string, _ error)

DecompressTarGzip 将 TAR.GZ 格式的文件解包到指定目录并返回指定文件所在的解包后的目录， 如果存在多个同名指定文件，则返回第一个找到的目录

func DecompressZip added in v1.4.6 

func DecompressZip(srcPath, dstPath, targetFile string) (targetDir string, _ error)

DecompressZip 将 ZIP 格式的文件解包到指定目录并返回指定文件所在的解包后的目录， 如果存在多个同名指定文件，则返回第一个找到的目录

func DeepConvertKey added in v1.5.2 

func DeepConvertKey(data map[string]interface{}) map[string]interface{}

func DeepConvertKeyWithCache added in v1.5.2 

func DeepConvertKeyWithCache(data map[string]interface{}, cache map[string]KeyInfo) map[string]interface{}

注意：cache如果是nil，这个函数就完全没有意义，不如调用 DeepConvertKey

func DeleteMapValue added in v1.4.6 

func DeleteMapValue(m map[string]interface{}, keys ...string) (interface{}, bool)

通过层级key删除key-val,并返回被删除的val,是否删除成功 如果key不存在,则返回 nil,false

func EncodeString added in v1.4.7 

func EncodeString(target string) (result string)

func ExtractField added in v1.4.6 

func ExtractField(slice []string) ([]string, error)

func FormatWithUserOption added in v1.4.8 

func FormatWithUserOption(layoutAfter string, offset int, t time.Time) interface{}

func GetCmd added in v1.5.4 

func GetCmd(keyStr string) []string

根据key字符串,拆分出层级keys数据

func GetKeys added in v1.4.6 

func GetKeys(keyStr string) []string

根据key字符串,拆分出层级keys数据

func GetMapList added in v1.5.0 

func GetMapList(data string) map[string]string

func GetMapValue added in v1.4.6 

func GetMapValue(m map[string]interface{}, keys ...string) (interface{}, error)

通过层级key获取value. 所有层级的map必须为 map[string]interface{} 类型. keys为空切片,返回原m

func GetRealPath added in v1.4.6 

func GetRealPath(path string) (newPath string, fi os.FileInfo, err error)

GetRealPath 处理软链接等，找到文件真实路径

func GetTime added in v1.5.4 

func GetTime(timestamp string) (time.Time, error)

func Hash added in v1.4.6 

func Hash(s string) string

func IsFileModified added in v1.5.4 

func IsFileModified(path string, interval time.Duration, compare time.Time) bool

func IsJsonString added in v1.4.6 

func IsJsonString(s string) bool

func IsNotExist added in v1.5.2 

func IsNotExist(err error) bool

func IsNotSupport added in v1.5.2 

func IsNotSupport(err error) bool

func IsSelfRunner added in v1.5.4 

func IsSelfRunner(runnerName string) bool

func IsSpace added in v1.5.4 

func IsSpace(separator rune) bool

func IsSubMetaExpire added in v1.5.4 

func IsSubMetaExpire(submetaExpire, expire time.Duration) bool

func IsSubmetaExpireValid added in v1.5.4 

func IsSubmetaExpireValid(submetaExpire, expire time.Duration) bool

func KeyExist added in v1.5.4 

func KeyExist(m map[string]interface{}, val interface{}, keys ...string) (bool, error)

func LogDirAndPattern added in v1.4.6 

func LogDirAndPattern(logpath string) (dir, pattern string, err error)

创建目录，并返回日志模式

func MergeEnvTags added in v1.5.4 

func MergeEnvTags(name string, tags map[string]interface{}) map[string]interface{}

MergeEnvTags 获取环境变量里的内容

func ModTimeLater added in v1.5.0 

func ModTimeLater(f1, f2 os.FileInfo) bool

ModTimeLater 按最后修改时间进行比较

func ParseTimeZoneOffset added in v1.5.4 

func ParseTimeZoneOffset(zoneoffset string) (ret int)

func PickMapValue added in v1.5.1 

func PickMapValue(m map[string]interface{}, pick map[string]interface{}, keys ...string)

func ReadDirByTime added in v1.4.6 

func ReadDirByTime(dir string) (files []os.FileInfo, err error)

ReadDirByTime 读取文件目录后按时间排序，时间最新的文件在前

func ReadFileContent added in v1.4.8 

func ReadFileContent(path string) (content []string, err error)

func RemoveHttpProtocal added in v1.4.6 

func RemoveHttpProtocal(url string) (hostport, schema string)

func SetExtractMapValue added in v1.5.5 

func SetExtractMapValue(m map[string]interface{}, val interface{}, coercive bool, newKey string, keys ...string) error

func SetMapValue added in v1.4.6 

func SetMapValue(m map[string]interface{}, val interface{}, coercive bool, keys ...string) error

通过层级key设置value值. 如果key不存在,将会自动创建. 当coercive为true时,会强制将非map[string]interface{}类型替换为map[string]interface{}类型,有可能导致数据丢失

func SetMapValueExistWithPrefix added in v1.5.2 

func SetMapValueExistWithPrefix(m map[string]interface{}, val interface{}, prefix string, keys ...string) error

通过层级key设置value值, 如果keys不存在则不加前缀，否则加前缀

func SortFilesByTime added in v1.4.6 

func SortFilesByTime(files FileInfos) (soredfiles []os.FileInfo)

SortFilesByTime 按照文件更新的unixnano从大到小排，即最新的文件在前,相同时间的则按照文件名字典序，字典序在后面的排在前面

func TrimInvalidSpace added in v1.5.4 

func TrimInvalidSpace(pattern string) string

func TrimeList added in v1.4.6 

func TrimeList(strs []string) (ret []string)

func TruncateStrSize added in v1.5.3 

func TruncateStrSize(err string, size int) string

func TuoDecode added in v1.4.6 

func TuoDecode(value []byte) (values [][]byte, err error)

TuoDecode 按照 TuoEncode的压缩算法解压，还原出[]byte数组。

func TuoEncode added in v1.4.6 

func TuoEncode(values []sql.RawBytes) (ret []byte)

TuoEncode 把[]byte数组按照长度拼接到一起，每个sql.RawBytes之间间隔4个byte用于存储长度。

Types

type AuthTokens added in v1.4.4 

type AuthTokens struct {
	RunnerName   string
	SenderIndex  int
	SenderTokens conf.MapConf
}

type Data 

type Data map[string]interface{}

Data store as use key/value map

func AddTagsToData added in v1.5.4 

func AddTagsToData(tags map[string]interface{}, datas []Data, runnername string) []Data

type Deleteable added in v1.5.4 

type Deleteable interface {
	Delete() error
}

type ErrorResponse added in v1.4.6 

type ErrorResponse struct {
	Error error `json:"error"`
}

func NewErrorResponse added in v1.4.6 

func NewErrorResponse(err error) *ErrorResponse

type ErrorStatistic added in v1.5.4 

type ErrorStatistic struct {
	ErrorSlice []equeue.ErrorInfo `json:"error_slice"`

	//对于runnerstatus的CompatibleErrorResult结构体来说，下面的三个都没用，只是为了保证兼容性
	//服务端用这个结构体也没有用到下面这三个成员
	MaxSize int `json:"max_size"`

	//以下为v1.0.4及以前版本的结构，为了兼容保留
	Front int `json:"front"`
	Rear  int `json:"rear"`
}

func (ErrorStatistic) GetMaxSize added in v1.5.4 

func (e ErrorStatistic) GetMaxSize() int

func (ErrorStatistic) IsNewVersion added in v1.5.4 

func (e ErrorStatistic) IsNewVersion() bool

type File added in v1.4.6 

type File struct {
	Info os.FileInfo
	Path string
}

func GetLogFiles added in v1.4.6 

func GetLogFiles(doneFilePath string) (files []File)

type FileInfos added in v1.5.0 

type FileInfos []os.FileInfo

FileInfos attaches the methods of Interface to []int64, sorting in decreasing order.

func (FileInfos) Len added in v1.5.0 

func (p FileInfos) Len() int

func (FileInfos) Less added in v1.5.0 

func (p FileInfos) Less(i, j int) bool

func (FileInfos) Sort added in v1.5.0 

func (p FileInfos) Sort()

Sort is a convenience method.

func (FileInfos) Swap added in v1.5.0 

func (p FileInfos) Swap(i, j int)

type GrokLabel added in v1.5.4 

type GrokLabel struct {
	Name  string
	Value string
}

func GetGrokLabels added in v1.5.4 

func GetGrokLabels(labelList []string, nameMap map[string]struct{}) (labels []GrokLabel)

func NewGrokLabel added in v1.5.4 

func NewGrokLabel(name, dataValue string) GrokLabel

type HashSet added in v1.4.6 

type HashSet struct {
	// contains filtered or unexported fields
}

func NewHashSet added in v1.4.6 

func NewHashSet() *HashSet

func (*HashSet) Add added in v1.4.6 

func (s *HashSet) Add(ele interface{})

func (*HashSet) AddStringArray added in v1.4.6 

func (s *HashSet) AddStringArray(ele []string)

func (*HashSet) Clear added in v1.4.6 

func (s *HashSet) Clear()

func (*HashSet) Elements added in v1.4.6 

func (s *HashSet) Elements() []interface{}

func (*HashSet) IsEmpty added in v1.4.6 

func (s *HashSet) IsEmpty() bool

func (*HashSet) IsIn added in v1.4.6 

func (s *HashSet) IsIn(ele interface{}) bool

func (*HashSet) Len added in v1.4.6 

func (s *HashSet) Len() int

func (*HashSet) Remove added in v1.4.6 

func (s *HashSet) Remove(ele interface{})

type KeyInfo added in v1.5.2 

type KeyInfo struct {
	Valid  bool
	NewKey string
}

type KeyValue 

type KeyValue struct {
	Key     string `json:"key"`
	Value   string `json:"value"`
	SortKey string `json:"sort_key"`
}

type KeyValueSlice added in v1.5.2 

type KeyValueSlice []KeyValue

func (KeyValueSlice) Len added in v1.5.2 

func (slice KeyValueSlice) Len() int

func (KeyValueSlice) Less added in v1.5.2 

func (slice KeyValueSlice) Less(i, j int) bool

func (KeyValueSlice) Swap added in v1.5.2 

func (slice KeyValueSlice) Swap(i, j int)

type LagInfo added in v1.4.4 

type LagInfo struct {
	Size     int64  `json:"size"`
	SizeUnit string `json:"sizeunit"`
	Ftlags   int64  `json:"ftlags"`
	Total    int64  `json:"total"`
}

type Option 

type Option struct {
	KeyName            string
	ChooseOnly         bool
	Element            string // 前端显示类型
	ChooseOptions      []interface{}
	Default            interface{}
	DefaultNoUse       bool // 是否使用默认值，true为不使用默认值，false为使用默认值
	Description        string
	CheckRegex         string
	Style              string `json:"style"`
	Required           bool   `json:"required"` // 是否必填
	Placeholder        string `json:"placeholder"`
	Type               string `json:"Type,omitempty"`
	Secret             bool
	Advance            bool                   `json:"advance,omitempty"`
	AdvanceDepend      string                 `json:"advance_depend,omitempty"`
	AdvanceDependValue interface{}            `json:"advance_depend_value,omitempty"`
	ToolTip            string                 `json:"tooltip,omitempty"` // 该选项说明
	MutiDefaultSource  bool                   `json:"muti_default_source"`
	MultiDefault       map[string]interface{} `json:"multi_default,omitempty"`
	MultiDefaultDepend string                 `json:"multi_default_depend,omitempty"`
}

type Pandora added in v1.5.4 

type Pandora struct {
	Name     string `json:"pandora_name"`
	Region   string `json:"pandora_region"`
	Pipeline string `json:"pandora_pipeline"`
	LogDB    string `json:"pandora_logdb"`
	AK       string `json:"pandora_ak"`
	SK       string `json:"pandora_sk"`
}

type Resetable added in v1.5.2 

type Resetable interface {
	Reset() error
}

type SchemaErr added in v1.4.6 

type SchemaErr struct {
	Number int64
	Last   time.Time
}

func (*SchemaErr) Output added in v1.4.6 

func (s *SchemaErr) Output(count int64, err error)

type StatsError added in v1.4.6 

type StatsError struct {
	StatsInfo
	SendError           *reqerr.SendError `json:"error"`
	Ft                  bool              `json:"-"`
	FtNotRetry          bool              `json:"-"`
	DatasourceSkipIndex []int
	RemainDatas         []Data
}

func (*StatsError) AddErrors added in v1.4.6 

func (se *StatsError) AddErrors()

func (*StatsError) AddErrorsNum added in v1.5.1 

func (se *StatsError) AddErrorsNum(n int)

func (*StatsError) AddSuccess added in v1.4.6 

func (se *StatsError) AddSuccess()

func (*StatsError) AddSuccessNum added in v1.5.1 

func (se *StatsError) AddSuccessNum(n int)

func (*StatsError) Error added in v1.4.6 

func (se *StatsError) Error() string

func (*StatsError) ErrorIndexIn added in v1.4.6 

func (se *StatsError) ErrorIndexIn(idx int) bool

type StatsInfo added in v1.4.6 

type StatsInfo struct {
	Errors     int64  `json:"errors"`
	Success    int64  `json:"success"`
	Speed      int64  `json:"speed"`
	Trend      string `json:"trend"`
	LastError  string `json:"last_error"`
	FtQueueLag int64  `json:"-"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.