common

package
v0.9.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 19, 2021 License: MIT Imports: 27 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ConfCheckSnapTimeout        = "check_snap_timeout"
	ConfCheckRaftTimeout        = "check_raft_timeout"
	ConfIgnoreStartupNoBackup   = "ignore_startup_nobackup"
	ConfIgnoreRemoteFileSync    = "ignore_remote_file_sync"
	ConfMaxRemoteRecover        = "max_remote_recover"
	ConfSlowLimiterSwitch       = "slow_limiter_switch"
	ConfSlowLimiterRefuseCostMs = "slow_limiter_refuse_cost_ms"
	ConfSlowLimiterHalfOpenSec  = "slow_limiter_half_open_sec"
)
View Source 
const (
	//max key size
	MaxKeySize int = 10240

	// subkey length for hash/set/zset
	MaxSubKeyLen int = 10240

	//max value size
	MaxValueSize int = 1024 * 1024 * 8
)
View Source 
const (
	LOG_ERR int32 = iota
	LOG_WARN
	LOG_INFO
	LOG_DEBUG
	LOG_DETAIL
)
View Source 
const (
	DIR_PERM  = 0755
	FILE_PERM = 0644
)
View Source 
const (
	LearnerRoleLogSyncer = "role_log_syncer"
	LearnerRoleSearcher  = "role_searcher"
)
View Source 
const (
	KVName   = "KV"
	ListName = "LIST"
	HashName = "HASH"
	SetName  = "SET"
	ZSetName = "ZSET"
)
View Source 
const (
	DefaultExpirationPolicy     = "local_deletion"
	WaitCompactExpirationPolicy = "wait_compact"
)
View Source 
const (
	ValueHeaderV1Str      = "value_header_v1"
	ValueHeaderDefaultStr = "default"
)
View Source 
const (
	RangeClose uint8 = 0x00
	RangeLOpen uint8 = 0x01
	RangeROpen uint8 = 0x10
	RangeOpen  uint8 = 0x11
)
View Source 
const (
	MAX_PARTITION_NUM = 1024
	MAX_REPLICATOR    = 5
)
View Source 
const (
	// api used by data node
	APIAddNode        = "/cluster/node/add"
	APIAddLearnerNode = "/cluster/node/addlearner"
	APIRemoveNode     = "/cluster/node/remove"
	APIGetMembers     = "/cluster/members"
	APIGetLeader      = "/cluster/leader"
	APICheckBackup    = "/cluster/checkbackup"
	APIGetIndexes     = "/schema/indexes"
	APINodeAllReady   = "/node/allready"
	// check if the namespace raft node is synced and can be elected as leader immediately
	APIIsRaftSynced = "/cluster/israftsynced"
	APITableStats   = "/tablestats"

	// below api for pd
	APIGetSnapshotSyncInfo = "/pd/snapshot_sync_info"
)
View Source 
const (
	InternalPrefix = "##"
)
View Source 
const (
	KEYSEP = byte(':')
)
View Source 
const (
	NamespaceTableSeperator = byte(':')
)
View Source 
const (
	SnapWaitTimeout = time.Minute * 20
)

Variables

View Source 
var (
	VerBinary = "unset"
	BuildTime = "unset"
	Commit    = "unset"
)
View Source 
var (
	SCAN_CURSOR_SEP = []byte(";")
	SCAN_NODE_SEP   = []byte(":")
)
View Source 
var (
	ErrInvalidCommand    = errors.New("invalid command")
	ErrStopped           = errors.New("the node stopped")
	ErrQueueTimeout      = errors.New("queue request timeout")
	ErrInvalidArgs       = errors.New("invalid arguments")
	ErrInvalidTTL        = errors.New("invalid expire time")
	ErrInvalidRedisKey   = errors.New("invalid redis key")
	ErrInvalidScanType   = errors.New("invalid scan type")
	ErrEpochMismatch     = errors.New("epoch mismatch")
	ErrInvalidTableName  = errors.New("table name is invalid")
	ErrInvalidScanCursor = errors.New("invalid scan cursor")
	ErrScanCursorNoTable = errors.New("scan cursor must has table")
	ErrUnexpectError     = errors.New("unexpected error")
	ErrInvalidPrefix     = errors.New("invalid prefix")
	ErrNotSupport        = errors.New("not supported")
)
View Source 
var (
	DefaultSnapCount   = 600000
	DefaultSnapCatchup = 500000
)
View Source 
var (
	MAX_BATCH_NUM         = 5000
	MinScore      float64 = math.Inf(-1)
	MaxScore      float64 = math.Inf(1)
	InvalidScore  int64   = -1 << 63
	MAX_SCAN_JOB          = 10
)
View Source 
var (
	ErrDeprecatedAPI = errors.New("deprecated api")
)
View Source 
var ErrRsyncFailed = errors.New("transfer snapshot failed due to rsync error")
View Source 
var ErrTransferOutofdate = errors.New("waiting transfer snapshot too long, maybe out of date")

Functions

func APIRequest 

func APIRequest(method string, endpoint string, body io.Reader, timeout time.Duration, ret interface{}) (int, error)

stores the result in the value pointed to by ret(must be a pointer)

func BuildCommand added in v0.7.1 

func BuildCommand(args [][]byte) redcon.Command

func CheckKey added in v0.9.3 

func CheckKey(key []byte) error

func CheckKeySubKey added in v0.9.3 

func CheckKeySubKey(key []byte, field []byte) error

func CheckSubKey added in v0.9.3 

func CheckSubKey(subkey []byte) error
func CopyFileForHardLink(src, dst string) error

This will copy file as hard link, if failed it will failover to do the file content copy

func CutNamesapce added in v0.7.1 

func CutNamesapce(rawKey []byte) ([]byte, error)

func Decorate 

func Decorate(f APIHandler, ds ...Decorator) httprouter.Handle

func DeepCopyCmd 

func DeepCopyCmd(cmd redcon.Command) redcon.Command

func DumpDynamicConf added in v0.6.2 

func DumpDynamicConf() []string

func ExtractNamesapce 

func ExtractNamesapce(rawKey []byte) (string, []byte, error)

func ExtractTable 

func ExtractTable(rawKey []byte) ([]byte, []byte, error)

func FlushZapDefault added in v0.9.3 

func FlushZapDefault()

func GetIPv4ForInterfaceName 

func GetIPv4ForInterfaceName(ifname string) string

func GetIntDynamicConf added in v0.6.2 

func GetIntDynamicConf(k string) int

func GetNamespaceAndPartition 

func GetNamespaceAndPartition(fullNamespace string) (string, int)

func GetNsDesp 

func GetNsDesp(ns string, part int) string

func GetStrDynamicConf added in v0.6.2 

func GetStrDynamicConf(k string) string

func IsConfSetted added in v0.6.2 

func IsConfSetted(k string) bool

func IsFullScanCommand 

func IsFullScanCommand(cmd string) bool

func IsInternalTableName 

func IsInternalTableName(tb string) bool

func IsMergeCommand 

func IsMergeCommand(cmd string) bool

func IsMergeIndexSearchCommand 

func IsMergeIndexSearchCommand(cmd string) bool

func IsMergeKeysCommand 

func IsMergeKeysCommand(cmd string) bool

func IsMergeScanCommand 

func IsMergeScanCommand(cmd string) bool

func IsRoleLogSyncer added in v0.6.2 

func IsRoleLogSyncer(role string) bool

func IsValidNamespaceName 

func IsValidNamespaceName(ns string) bool

func NewDeadlineTransport 

func NewDeadlineTransport(timeout time.Duration) *http.Transport

func NewDefaultLogger 

func NewDefaultLogger(module string) *defaultLogger

func RegisterConfChangedHandler added in v0.8.2 

func RegisterConfChangedHandler(key string, h KeyChangedHandler)

func RespondV1 

func RespondV1(w http.ResponseWriter, code int, data interface{})

func RunFileSync 

func RunFileSync(remote string, srcPath string, dstPath string, stopCh chan struct{}) error

make sure the file sync will not overwrite hard link file inplace. (Because the hard link file content which may be used in rocksdb should not be changed ) So with hard link sync, we make sure we do unlink on the file before we update it. (rsync just do it)

func SetIntDynamicConf added in v0.6.2 

func SetIntDynamicConf(k string, newV int)

func SetRsyncLimit added in v0.4.3 

func SetRsyncLimit(limit int64)

func SetStrDynamicConf added in v0.6.2 

func SetStrDynamicConf(k string, newV string)

func SetZapRotateOptions added in v0.9.3 

func SetZapRotateOptions(alsoLogToStdout bool, alsoLogErrToStdErr bool, logfile string, maxMB int, maxBackup int, maxAgeDay int)

func VerString 

func VerString(app string) string

Types

type APIHandler 

type APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)

func PlainText 

func PlainText(f APIHandler) APIHandler

func V1 

func V1(f APIHandler) APIHandler

type CmdRouter 

type CmdRouter struct {
	// contains filtered or unexported fields
}

func NewCmdRouter 

func NewCmdRouter() *CmdRouter

func (*CmdRouter) GetCmdHandler 

func (r *CmdRouter) GetCmdHandler(name string) (CommandFunc, bool)

func (*CmdRouter) GetMergeCmdHandler 

func (r *CmdRouter) GetMergeCmdHandler(name string) (MergeCommandFunc, bool, bool)

return handler, iswrite, isexist

func (*CmdRouter) GetWCmdHandler added in v0.8.0 

func (r *CmdRouter) GetWCmdHandler(name string) (WriteCommandFunc, bool)

func (*CmdRouter) RegisterMerge 

func (r *CmdRouter) RegisterMerge(name string, f MergeCommandFunc) bool

func (*CmdRouter) RegisterRead added in v0.8.0 

func (r *CmdRouter) RegisterRead(name string, f CommandFunc) bool

func (*CmdRouter) RegisterWrite added in v0.8.0 

func (r *CmdRouter) RegisterWrite(name string, f WriteCommandFunc) bool

func (*CmdRouter) RegisterWriteMerge 

func (r *CmdRouter) RegisterWriteMerge(name string, f MergeCommandFunc) bool

type CommandFunc 

type CommandFunc func(redcon.Conn, redcon.Command)

type CommandRspFunc 

type CommandRspFunc func(redcon.Command, interface{}) (interface{}, error)

type DataType 

type DataType byte

for out use 

const (
	NONE DataType = iota
	KV
	LIST
	HASH
	SET
	ZSET
	ALL
)

func (DataType) String 

func (d DataType) String() string

type DataVersionT added in v0.8.0 

type DataVersionT int
const (
	DefaultDataVer DataVersionT = iota

	//ValueHeaderV1 will add header to kv values to store ttl or other header data
	ValueHeaderV1

	UnknownDataType
)

func StringToDataVersionType added in v0.8.0 

func StringToDataVersionType(s string) (DataVersionT, error)

type Decorator 

type Decorator func(APIHandler) APIHandler

func HttpLog 

func HttpLog(log *LevelLogger, level int32) Decorator

type ExpirationPolicy 

type ExpirationPolicy byte
const (
	// LocalDeletion indicates the expired data would be deleted by the underlying storage system automatically and the logical layer
	// do not need to care about the data expiration. Every node in the cluster should start the 'TTLChecker' of the storage system
	// with this policy.
	LocalDeletion ExpirationPolicy = iota
	//
	PeriodicalRotation

	// WaitCompact indicates that all ttl will be stored in the values and will be checked while compacting and reading
	WaitCompact

	UnknownPolicy
)

func StringToExpirationPolicy 

func StringToExpirationPolicy(s string) (ExpirationPolicy, error)

type ExpiredDataBuffer 

type ExpiredDataBuffer interface {
	Write(DataType, []byte) error
}

type FieldPair 

type FieldPair struct {
	Field []byte
	Value []byte
}

type FullScanResult 

type FullScanResult struct {
	Results    []interface{}
	Type       DataType
	NextCursor []byte
	PartionId  string
	Error      error
}

type HIndexRespWithValues 

type HIndexRespWithValues struct {
	PKey       []byte
	IndexV     interface{}
	HsetValues [][]byte
	// contains filtered or unexported fields
}

type HsetIndexSchema 

type HsetIndexSchema struct {
	Name       string             `json:"name"`
	IndexField string             `json:"index_field"`
	PrefixLen  int32              `json:"prefix_len"`
	Unique     int32              `json:"unique"`
	ValueType  IndexPropertyDType `json:"value_type"`
	State      IndexState         `json:"state"`
}

func (*HsetIndexSchema) IsValidNewSchema 

func (s *HsetIndexSchema) IsValidNewSchema() bool

type HttpErr 

type HttpErr struct {
	Code int
	Text string
}

func (HttpErr) Error 

func (e HttpErr) Error() string

type IClusterInfo 

type IClusterInfo interface {
	GetClusterName() string
	GetSnapshotSyncInfo(fullNS string) ([]SnapshotSyncInfo, error)
	UpdateMeForNamespaceLeader(fullNS string) (bool, error)
}

type IndexPropertyDType 

type IndexPropertyDType int32
const (
	Int64V  IndexPropertyDType = 0
	Int32V  IndexPropertyDType = 1
	StringV IndexPropertyDType = 2
	MaxVT   IndexPropertyDType = 3
)

type IndexSchema 

type IndexSchema struct {
	HsetIndexes []*HsetIndexSchema `json:"hset_indexes"`
	JSONIndexes []*JSONIndexSchema `json:"json_indexes"`
}

type IndexState 

type IndexState int32
const (
	InitIndex      IndexState = 0
	BuildingIndex  IndexState = 1
	BuildDoneIndex IndexState = 2
	ReadyIndex     IndexState = 3
	DeletedIndex   IndexState = 4
	MaxIndexState  IndexState = 5
)

type InternalCommandFunc 

type InternalCommandFunc func(redcon.Command, int64) (interface{}, error)

type JSONIndexSchema 

type JSONIndexSchema struct {
	State IndexState `json:"state"`
}

type KFVals 

type KFVals struct {
	PK   []byte
	Vals []KVRecordRet
}

type KVRecord 

type KVRecord struct {
	Key   []byte
	Value []byte
}

type KVRecordRet 

type KVRecordRet struct {
	Rec KVRecord
	Err error
}

type KVals 

type KVals struct {
	PK   []byte
	Vals [][]byte
}

type KeyChangedHandler added in v0.8.2 

type KeyChangedHandler func(newV interface{})

type LevelLogger 

type LevelLogger struct {
	Logger Logger
	// contains filtered or unexported fields
}

func NewLevelLogger 

func NewLevelLogger(level int32, l Logger) *LevelLogger

func (*LevelLogger) Debug 

func (ll *LevelLogger) Debug(args ...interface{})

func (*LevelLogger) DebugDepth 

func (ll *LevelLogger) DebugDepth(d int, l string)

func (*LevelLogger) Debugf 

func (ll *LevelLogger) Debugf(f string, args ...interface{})

func (*LevelLogger) Error 

func (ll *LevelLogger) Error(args ...interface{})

func (*LevelLogger) ErrorDepth 

func (ll *LevelLogger) ErrorDepth(d int, l string)

func (*LevelLogger) Errorf 

func (ll *LevelLogger) Errorf(f string, args ...interface{})

func (*LevelLogger) Fatal 

func (ll *LevelLogger) Fatal(args ...interface{})

func (*LevelLogger) Fatalf 

func (ll *LevelLogger) Fatalf(f string, args ...interface{})

func (*LevelLogger) Info 

func (ll *LevelLogger) Info(args ...interface{})

func (*LevelLogger) InfoDepth 

func (ll *LevelLogger) InfoDepth(d int, l string)

func (*LevelLogger) Infof 

func (ll *LevelLogger) Infof(f string, args ...interface{})

func (*LevelLogger) Level 

func (ll *LevelLogger) Level() int32

func (*LevelLogger) Panic 

func (ll *LevelLogger) Panic(args ...interface{})

func (*LevelLogger) Panicf 

func (ll *LevelLogger) Panicf(f string, args ...interface{})

func (*LevelLogger) SetLevel 

func (ll *LevelLogger) SetLevel(l int32)

func (*LevelLogger) Warning 

func (ll *LevelLogger) Warning(args ...interface{})

func (*LevelLogger) Warningf 

func (ll *LevelLogger) Warningf(f string, args ...interface{})

type Logger 

type Logger interface {
	Output(maxdepth int, s string) error
	OutputErr(maxdepth int, s string) error
	OutputWarning(maxdepth int, s string) error
}

func NewLogger added in v0.9.1 

func NewLogger() Logger

type MemberInfo 

type MemberInfo struct {
	// the replica id
	ID uint64 `json:"id"`
	// the node id replica belong
	NodeID    uint64 `json:"node_id"`
	GroupName string `json:"group_name"`
	// group id the replica belong (different from namespace)
	GroupID  uint64   `json:"group_id"`
	RaftURLs []string `json:"peer_urls"`
}

func (*MemberInfo) IsEqual 

func (self *MemberInfo) IsEqual(other *MemberInfo) bool

type MergeCommandFunc 

type MergeCommandFunc func(redcon.Command) (interface{}, error)

type MergeLogger 

type MergeLogger struct {
	*LevelLogger
	// contains filtered or unexported fields
}

MergeLogger supports merge logging, which merges repeated log lines and prints summary log lines instead.

For merge logging, MergeLogger prints out the line when the line appears at the first time. MergeLogger holds the same log line printed within defaultMergePeriod, and prints out summary log line at the end of defaultMergePeriod. It stops merging when the line doesn't appear within the defaultMergePeriod.

func NewMergeLogger 

func NewMergeLogger(logger *LevelLogger) *MergeLogger

func (*MergeLogger) MergeError 

func (l *MergeLogger) MergeError(entries ...interface{})

func (*MergeLogger) MergeErrorf 

func (l *MergeLogger) MergeErrorf(format string, args ...interface{})

func (*MergeLogger) MergeInfo 

func (l *MergeLogger) MergeInfo(entries ...interface{})

func (*MergeLogger) MergeInfof 

func (l *MergeLogger) MergeInfof(format string, args ...interface{})

func (*MergeLogger) MergeWarning 

func (l *MergeLogger) MergeWarning(entries ...interface{})

func (*MergeLogger) MergeWarningf 

func (l *MergeLogger) MergeWarningf(format string, args ...interface{})

type MergeWriteCommandFunc 

type MergeWriteCommandFunc func(redcon.Command, interface{}) (interface{}, error)

type SMCmdRouter 

type SMCmdRouter struct {
	// contains filtered or unexported fields
}

func NewSMCmdRouter 

func NewSMCmdRouter() *SMCmdRouter

func (*SMCmdRouter) GetInternalCmdHandler 

func (r *SMCmdRouter) GetInternalCmdHandler(name string) (InternalCommandFunc, bool)

func (*SMCmdRouter) RegisterInternal 

func (r *SMCmdRouter) RegisterInternal(name string, f InternalCommandFunc) bool

type ScanResult 

type ScanResult struct {
	Keys       [][]byte
	NextCursor []byte
	PartionId  string
	Error      error
}

type ScorePair 

type ScorePair struct {
	Score  float64
	Member []byte
}

type SearchResultHeap 

type SearchResultHeap []*HIndexRespWithValues

func (SearchResultHeap) Len 

func (sh SearchResultHeap) Len() int

func (SearchResultHeap) Less 

func (sh SearchResultHeap) Less(i, j int) bool

func (*SearchResultHeap) Pop 

func (sh *SearchResultHeap) Pop() interface{}

func (*SearchResultHeap) Push 

func (sh *SearchResultHeap) Push(x interface{})

func (SearchResultHeap) Swap 

func (sh SearchResultHeap) Swap(i, j int)

type SnapshotSyncInfo 

type SnapshotSyncInfo struct {
	ReplicaID   uint64
	NodeID      uint64
	RemoteAddr  string
	HttpAPIPort string
	DataRoot    string
	RsyncModule string
}

type StoppableListener 

type StoppableListener struct {
	*net.TCPListener
	// contains filtered or unexported fields
}

stoppableListener sets TCP keep-alive timeouts on accepted connections and waits on stopc message

func NewStoppableListener 

func NewStoppableListener(addr string, stopc <-chan struct{}) (*StoppableListener, error)

func (StoppableListener) Accept 

func (ln StoppableListener) Accept() (c net.Conn, err error)

type StringArray 

type StringArray []string

func (*StringArray) Set 

func (a *StringArray) Set(s string) error

func (*StringArray) String 

func (a *StringArray) String() string

type WriteCmd 

type WriteCmd struct {
	Operation string
	Args      [][]byte
}

type WriteCommandFunc added in v0.8.0 

type WriteCommandFunc func(redcon.Command) (interface{}, error)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.