common

package
Version: v0.9.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 19, 2021 License: MIT Imports: 27 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ConfCheckSnapTimeout        = "check_snap_timeout"
	ConfCheckRaftTimeout        = "check_raft_timeout"
	ConfIgnoreStartupNoBackup   = "ignore_startup_nobackup"
	ConfIgnoreRemoteFileSync    = "ignore_remote_file_sync"
	ConfMaxRemoteRecover        = "max_remote_recover"
	ConfSlowLimiterSwitch       = "slow_limiter_switch"
	ConfSlowLimiterRefuseCostMs = "slow_limiter_refuse_cost_ms"
	ConfSlowLimiterHalfOpenSec  = "slow_limiter_half_open_sec"
)
View Source
const (
	//max key size
	MaxKeySize int = 10240

	// subkey length for hash/set/zset
	MaxSubKeyLen int = 10240

	//max value size
	MaxValueSize int = 1024 * 1024 * 8
)
View Source
const (
	LOG_ERR int32 = iota
	LOG_WARN
	LOG_INFO
	LOG_DEBUG
	LOG_DETAIL
)
View Source
const (
	DIR_PERM  = 0755
	FILE_PERM = 0644
)
View Source
const (
	LearnerRoleLogSyncer = "role_log_syncer"
	LearnerRoleSearcher  = "role_searcher"
)
View Source
const (
	KVName   = "KV"
	ListName = "LIST"
	HashName = "HASH"
	SetName  = "SET"
	ZSetName = "ZSET"
)
View Source
const (
	DefaultExpirationPolicy     = "local_deletion"
	WaitCompactExpirationPolicy = "wait_compact"
)
View Source
const (
	ValueHeaderV1Str      = "value_header_v1"
	ValueHeaderDefaultStr = "default"
)
View Source
const (
	RangeClose uint8 = 0x00
	RangeLOpen uint8 = 0x01
	RangeROpen uint8 = 0x10
	RangeOpen  uint8 = 0x11
)
View Source
const (
	MAX_PARTITION_NUM = 1024
	MAX_REPLICATOR    = 5
)
View Source
const (
	// api used by data node
	APIAddNode        = "/cluster/node/add"
	APIAddLearnerNode = "/cluster/node/addlearner"
	APIRemoveNode     = "/cluster/node/remove"
	APIGetMembers     = "/cluster/members"
	APIGetLeader      = "/cluster/leader"
	APICheckBackup    = "/cluster/checkbackup"
	APIGetIndexes     = "/schema/indexes"
	APINodeAllReady   = "/node/allready"
	// check if the namespace raft node is synced and can be elected as leader immediately
	APIIsRaftSynced = "/cluster/israftsynced"
	APITableStats   = "/tablestats"

	// below api for pd
	APIGetSnapshotSyncInfo = "/pd/snapshot_sync_info"
)
View Source
const (
	InternalPrefix = "##"
)
View Source
const (
	KEYSEP = byte(':')
)
View Source
const (
	NamespaceTableSeperator = byte(':')
)
View Source
const (
	SnapWaitTimeout = time.Minute * 20
)

Variables

View Source
var (
	VerBinary = "unset"
	BuildTime = "unset"
	Commit    = "unset"
)
View Source
var (
	SCAN_CURSOR_SEP = []byte(";")
	SCAN_NODE_SEP   = []byte(":")
)
View Source
var (
	ErrInvalidCommand    = errors.New("invalid command")
	ErrStopped           = errors.New("the node stopped")
	ErrQueueTimeout      = errors.New("queue request timeout")
	ErrInvalidArgs       = errors.New("invalid arguments")
	ErrInvalidTTL        = errors.New("invalid expire time")
	ErrInvalidRedisKey   = errors.New("invalid redis key")
	ErrInvalidScanType   = errors.New("invalid scan type")
	ErrEpochMismatch     = errors.New("epoch mismatch")
	ErrInvalidTableName  = errors.New("table name is invalid")
	ErrInvalidScanCursor = errors.New("invalid scan cursor")
	ErrScanCursorNoTable = errors.New("scan cursor must has table")
	ErrUnexpectError     = errors.New("unexpected error")
	ErrInvalidPrefix     = errors.New("invalid prefix")
	ErrNotSupport        = errors.New("not supported")
)
View Source
var (
	DefaultSnapCount   = 600000
	DefaultSnapCatchup = 500000
)
View Source
var (
	MAX_BATCH_NUM         = 5000
	MinScore      float64 = math.Inf(-1)
	MaxScore      float64 = math.Inf(1)
	InvalidScore  int64   = -1 << 63
	MAX_SCAN_JOB          = 10
)
View Source
var (
	ErrDeprecatedAPI = errors.New("deprecated api")
)
View Source
var ErrRsyncFailed = errors.New("transfer snapshot failed due to rsync error")
View Source
var ErrTransferOutofdate = errors.New("waiting transfer snapshot too long, maybe out of date")

Functions

func APIRequest

func APIRequest(method string, endpoint string, body io.Reader, timeout time.Duration, ret interface{}) (int, error)

stores the result in the value pointed to by ret(must be a pointer)

func BuildCommand added in v0.7.1

func BuildCommand(args [][]byte) redcon.Command

func CheckKey added in v0.9.3

func CheckKey(key []byte) error

func CheckKeySubKey added in v0.9.3

func CheckKeySubKey(key []byte, field []byte) error

func CheckSubKey added in v0.9.3

func CheckSubKey(subkey []byte) error
func CopyFileForHardLink(src, dst string) error

This will copy file as hard link, if failed it will failover to do the file content copy

func CutNamesapce added in v0.7.1

func CutNamesapce(rawKey []byte) ([]byte, error)

func Decorate

func Decorate(f APIHandler, ds ...Decorator) httprouter.Handle

func DeepCopyCmd

func DeepCopyCmd(cmd redcon.Command) redcon.Command

func DumpDynamicConf added in v0.6.2

func DumpDynamicConf() []string

func ExtractNamesapce

func ExtractNamesapce(rawKey []byte) (string, []byte, error)

func ExtractTable

func ExtractTable(rawKey []byte) ([]byte, []byte, error)

func FlushZapDefault added in v0.9.3

func FlushZapDefault()

func GetIPv4ForInterfaceName

func GetIPv4ForInterfaceName(ifname string) string

func GetIntDynamicConf added in v0.6.2

func GetIntDynamicConf(k string) int

func GetNamespaceAndPartition

func GetNamespaceAndPartition(fullNamespace string) (string, int)

func GetNsDesp

func GetNsDesp(ns string, part int) string

func GetStrDynamicConf added in v0.6.2

func GetStrDynamicConf(k string) string

func IsConfSetted added in v0.6.2

func IsConfSetted(k string) bool

func IsFullScanCommand

func IsFullScanCommand(cmd string) bool

func IsInternalTableName

func IsInternalTableName(tb string) bool

func IsMergeCommand

func IsMergeCommand(cmd string) bool

func IsMergeIndexSearchCommand

func IsMergeIndexSearchCommand(cmd string) bool

func IsMergeKeysCommand

func IsMergeKeysCommand(cmd string) bool

func IsMergeScanCommand

func IsMergeScanCommand(cmd string) bool

func IsRoleLogSyncer added in v0.6.2

func IsRoleLogSyncer(role string) bool

func IsValidNamespaceName

func IsValidNamespaceName(ns string) bool

func NewDeadlineTransport

func NewDeadlineTransport(timeout time.Duration) *http.Transport

func NewDefaultLogger

func NewDefaultLogger(module string) *defaultLogger

func RegisterConfChangedHandler added in v0.8.2

func RegisterConfChangedHandler(key string, h KeyChangedHandler)

func RespondV1

func RespondV1(w http.ResponseWriter, code int, data interface{})

func RunFileSync

func RunFileSync(remote string, srcPath string, dstPath string, stopCh chan struct{}) error

make sure the file sync will not overwrite hard link file inplace. (Because the hard link file content which may be used in rocksdb should not be changed ) So with hard link sync, we make sure we do unlink on the file before we update it. (rsync just do it)

func SetIntDynamicConf added in v0.6.2

func SetIntDynamicConf(k string, newV int)

func SetRsyncLimit added in v0.4.3

func SetRsyncLimit(limit int64)

func SetStrDynamicConf added in v0.6.2

func SetStrDynamicConf(k string, newV string)

func SetZapRotateOptions added in v0.9.3

func SetZapRotateOptions(alsoLogToStdout bool, alsoLogErrToStdErr bool, logfile string, maxMB int, maxBackup int, maxAgeDay int)

func VerString

func VerString(app string) string

Types

type APIHandler

type APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)

func PlainText

func PlainText(f APIHandler) APIHandler

func V1

func V1(f APIHandler) APIHandler

type CmdRouter

type CmdRouter struct {
	// contains filtered or unexported fields
}

func NewCmdRouter

func NewCmdRouter() *CmdRouter

func (*CmdRouter) GetCmdHandler

func (r *CmdRouter) GetCmdHandler(name string) (CommandFunc, bool)

func (*CmdRouter) GetMergeCmdHandler

func (r *CmdRouter) GetMergeCmdHandler(name string) (MergeCommandFunc, bool, bool)

return handler, iswrite, isexist

func (*CmdRouter) GetWCmdHandler added in v0.8.0

func (r *CmdRouter) GetWCmdHandler(name string) (WriteCommandFunc, bool)

func (*CmdRouter) RegisterMerge

func (r *CmdRouter) RegisterMerge(name string, f MergeCommandFunc) bool

func (*CmdRouter) RegisterRead added in v0.8.0

func (r *CmdRouter) RegisterRead(name string, f CommandFunc) bool

func (*CmdRouter) RegisterWrite added in v0.8.0

func (r *CmdRouter) RegisterWrite(name string, f WriteCommandFunc) bool

func (*CmdRouter) RegisterWriteMerge

func (r *CmdRouter) RegisterWriteMerge(name string, f MergeCommandFunc) bool

type CommandFunc

type CommandFunc func(redcon.Conn, redcon.Command)

type CommandRspFunc

type CommandRspFunc func(redcon.Command, interface{}) (interface{}, error)

type DataType

type DataType byte

for out use

const (
	NONE DataType = iota
	KV
	LIST
	HASH
	SET
	ZSET
	ALL
)

func (DataType) String

func (d DataType) String() string

type DataVersionT added in v0.8.0

type DataVersionT int
const (
	DefaultDataVer DataVersionT = iota

	//ValueHeaderV1 will add header to kv values to store ttl or other header data
	ValueHeaderV1

	UnknownDataType
)

func StringToDataVersionType added in v0.8.0

func StringToDataVersionType(s string) (DataVersionT, error)

type Decorator

type Decorator func(APIHandler) APIHandler

func HttpLog

func HttpLog(log *LevelLogger, level int32) Decorator

type ExpirationPolicy

type ExpirationPolicy byte
const (
	// LocalDeletion indicates the expired data would be deleted by the underlying storage system automatically and the logical layer
	// do not need to care about the data expiration. Every node in the cluster should start the 'TTLChecker' of the storage system
	// with this policy.
	LocalDeletion ExpirationPolicy = iota
	//
	PeriodicalRotation

	// WaitCompact indicates that all ttl will be stored in the values and will be checked while compacting and reading
	WaitCompact

	UnknownPolicy
)

func StringToExpirationPolicy

func StringToExpirationPolicy(s string) (ExpirationPolicy, error)

type ExpiredDataBuffer

type ExpiredDataBuffer interface {
	Write(DataType, []byte) error
}

type FieldPair

type FieldPair struct {
	Field []byte
	Value []byte
}

type FullScanResult

type FullScanResult struct {
	Results    []interface{}
	Type       DataType
	NextCursor []byte
	PartionId  string
	Error      error
}

type HIndexRespWithValues

type HIndexRespWithValues struct {
	PKey       []byte
	IndexV     interface{}
	HsetValues [][]byte
	// contains filtered or unexported fields
}

type HsetIndexSchema

type HsetIndexSchema struct {
	Name       string             `json:"name"`
	IndexField string             `json:"index_field"`
	PrefixLen  int32              `json:"prefix_len"`
	Unique     int32              `json:"unique"`
	ValueType  IndexPropertyDType `json:"value_type"`
	State      IndexState         `json:"state"`
}

func (*HsetIndexSchema) IsValidNewSchema

func (s *HsetIndexSchema) IsValidNewSchema() bool

type HttpErr

type HttpErr struct {
	Code int
	Text string
}

func (HttpErr) Error

func (e HttpErr) Error() string

type IClusterInfo

type IClusterInfo interface {
	GetClusterName() string
	GetSnapshotSyncInfo(fullNS string) ([]SnapshotSyncInfo, error)
	UpdateMeForNamespaceLeader(fullNS string) (bool, error)
}

type IndexPropertyDType

type IndexPropertyDType int32
const (
	Int64V  IndexPropertyDType = 0
	Int32V  IndexPropertyDType = 1
	StringV IndexPropertyDType = 2
	MaxVT   IndexPropertyDType = 3
)

type IndexSchema

type IndexSchema struct {
	HsetIndexes []*HsetIndexSchema `json:"hset_indexes"`
	JSONIndexes []*JSONIndexSchema `json:"json_indexes"`
}

type IndexState

type IndexState int32
const (
	InitIndex      IndexState = 0
	BuildingIndex  IndexState = 1
	BuildDoneIndex IndexState = 2
	ReadyIndex     IndexState = 3
	DeletedIndex   IndexState = 4
	MaxIndexState  IndexState = 5
)

type InternalCommandFunc

type InternalCommandFunc func(redcon.Command, int64) (interface{}, error)

type JSONIndexSchema

type JSONIndexSchema struct {
	State IndexState `json:"state"`
}

type KFVals

type KFVals struct {
	PK   []byte
	Vals []KVRecordRet
}

type KVRecord

type KVRecord struct {
	Key   []byte
	Value []byte
}

type KVRecordRet

type KVRecordRet struct {
	Rec KVRecord
	Err error
}

type KVals

type KVals struct {
	PK   []byte
	Vals [][]byte
}

type KeyChangedHandler added in v0.8.2

type KeyChangedHandler func(newV interface{})

type LevelLogger

type LevelLogger struct {
	Logger Logger
	// contains filtered or unexported fields
}

func NewLevelLogger

func NewLevelLogger(level int32, l Logger) *LevelLogger

func (*LevelLogger) Debug

func (ll *LevelLogger) Debug(args ...interface{})

func (*LevelLogger) DebugDepth

func (ll *LevelLogger) DebugDepth(d int, l string)

func (*LevelLogger) Debugf

func (ll *LevelLogger) Debugf(f string, args ...interface{})

func (*LevelLogger) Error

func (ll *LevelLogger) Error(args ...interface{})

func (*LevelLogger) ErrorDepth

func (ll *LevelLogger) ErrorDepth(d int, l string)

func (*LevelLogger) Errorf

func (ll *LevelLogger) Errorf(f string, args ...interface{})

func (*LevelLogger) Fatal

func (ll *LevelLogger) Fatal(args ...interface{})

func (*LevelLogger) Fatalf

func (ll *LevelLogger) Fatalf(f string, args ...interface{})

func (*LevelLogger) Info

func (ll *LevelLogger) Info(args ...interface{})

func (*LevelLogger) InfoDepth

func (ll *LevelLogger) InfoDepth(d int, l string)

func (*LevelLogger) Infof

func (ll *LevelLogger) Infof(f string, args ...interface{})

func (*LevelLogger) Level

func (ll *LevelLogger) Level() int32

func (*LevelLogger) Panic

func (ll *LevelLogger) Panic(args ...interface{})

func (*LevelLogger) Panicf

func (ll *LevelLogger) Panicf(f string, args ...interface{})

func (*LevelLogger) SetLevel

func (ll *LevelLogger) SetLevel(l int32)

func (*LevelLogger) Warning

func (ll *LevelLogger) Warning(args ...interface{})

func (*LevelLogger) Warningf

func (ll *LevelLogger) Warningf(f string, args ...interface{})

type Logger

type Logger interface {
	Output(maxdepth int, s string) error
	OutputErr(maxdepth int, s string) error
	OutputWarning(maxdepth int, s string) error
}

func NewLogger added in v0.9.1

func NewLogger() Logger

type MemberInfo

type MemberInfo struct {
	// the replica id
	ID uint64 `json:"id"`
	// the node id replica belong
	NodeID    uint64 `json:"node_id"`
	GroupName string `json:"group_name"`
	// group id the replica belong (different from namespace)
	GroupID  uint64   `json:"group_id"`
	RaftURLs []string `json:"peer_urls"`
}

func (*MemberInfo) IsEqual

func (self *MemberInfo) IsEqual(other *MemberInfo) bool

type MergeCommandFunc

type MergeCommandFunc func(redcon.Command) (interface{}, error)

type MergeLogger

type MergeLogger struct {
	*LevelLogger
	// contains filtered or unexported fields
}

MergeLogger supports merge logging, which merges repeated log lines and prints summary log lines instead.

For merge logging, MergeLogger prints out the line when the line appears at the first time. MergeLogger holds the same log line printed within defaultMergePeriod, and prints out summary log line at the end of defaultMergePeriod. It stops merging when the line doesn't appear within the defaultMergePeriod.

func NewMergeLogger

func NewMergeLogger(logger *LevelLogger) *MergeLogger

func (*MergeLogger) MergeError

func (l *MergeLogger) MergeError(entries ...interface{})

func (*MergeLogger) MergeErrorf

func (l *MergeLogger) MergeErrorf(format string, args ...interface{})

func (*MergeLogger) MergeInfo

func (l *MergeLogger) MergeInfo(entries ...interface{})

func (*MergeLogger) MergeInfof

func (l *MergeLogger) MergeInfof(format string, args ...interface{})

func (*MergeLogger) MergeWarning

func (l *MergeLogger) MergeWarning(entries ...interface{})

func (*MergeLogger) MergeWarningf

func (l *MergeLogger) MergeWarningf(format string, args ...interface{})

type MergeWriteCommandFunc

type MergeWriteCommandFunc func(redcon.Command, interface{}) (interface{}, error)

type SMCmdRouter

type SMCmdRouter struct {
	// contains filtered or unexported fields
}

func NewSMCmdRouter

func NewSMCmdRouter() *SMCmdRouter

func (*SMCmdRouter) GetInternalCmdHandler

func (r *SMCmdRouter) GetInternalCmdHandler(name string) (InternalCommandFunc, bool)

func (*SMCmdRouter) RegisterInternal

func (r *SMCmdRouter) RegisterInternal(name string, f InternalCommandFunc) bool

type ScanResult

type ScanResult struct {
	Keys       [][]byte
	NextCursor []byte
	PartionId  string
	Error      error
}

type ScorePair

type ScorePair struct {
	Score  float64
	Member []byte
}

type SearchResultHeap

type SearchResultHeap []*HIndexRespWithValues

func (SearchResultHeap) Len

func (sh SearchResultHeap) Len() int

func (SearchResultHeap) Less

func (sh SearchResultHeap) Less(i, j int) bool

func (*SearchResultHeap) Pop

func (sh *SearchResultHeap) Pop() interface{}

func (*SearchResultHeap) Push

func (sh *SearchResultHeap) Push(x interface{})

func (SearchResultHeap) Swap

func (sh SearchResultHeap) Swap(i, j int)

type SnapshotSyncInfo

type SnapshotSyncInfo struct {
	ReplicaID   uint64
	NodeID      uint64
	RemoteAddr  string
	HttpAPIPort string
	DataRoot    string
	RsyncModule string
}

type StoppableListener

type StoppableListener struct {
	*net.TCPListener
	// contains filtered or unexported fields
}

stoppableListener sets TCP keep-alive timeouts on accepted connections and waits on stopc message

func NewStoppableListener

func NewStoppableListener(addr string, stopc <-chan struct{}) (*StoppableListener, error)

func (StoppableListener) Accept

func (ln StoppableListener) Accept() (c net.Conn, err error)

type StringArray

type StringArray []string

func (*StringArray) Set

func (a *StringArray) Set(s string) error

func (*StringArray) String

func (a *StringArray) String() string

type WriteCmd

type WriteCmd struct {
	Operation string
	Args      [][]byte
}

type WriteCommandFunc added in v0.8.0

type WriteCommandFunc func(redcon.Command) (interface{}, error)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL