kv

package
Version: v0.0.0-...-3633c1a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 11, 2021 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source
const (
	PriorityNormal = iota
	PriorityLow
	PriorityHigh
)

Priority value for transaction priority.

View Source
const (
	ReqTypeSelect   = 101
	ReqTypeIndex    = 102
	ReqTypeDAG      = 103
	ReqTypeAnalyze  = 104
	ReqTypeChecksum = 105

	ReqSubTypeBasic      = 0
	ReqSubTypeDesc       = 10000
	ReqSubTypeGroupBy    = 10001
	ReqSubTypeTopN       = 10002
	ReqSubTypeSignature  = 10003
	ReqSubTypeAnalyzeIdx = 10004
	ReqSubTypeAnalyzeCol = 10005
)

ReqTypes.

View Source
const (
	DefBackoffLockFast = 100
	DefBackOffWeight   = 2
)

Default values

View Source
const TxnRetryableMark = "[try again later]"

TxnRetryableMark is used to uniform the commit error messages which could retry the transaction. *WARNING*: changing this string will affect the backward compatibility.

Variables

View Source
var (
	// DefaultTxnMembufCap is the default transaction membuf capability.
	DefaultTxnMembufCap = 4 * 1024
	// ImportingTxnMembufCap is the capability of tidb importing data situation.
	ImportingTxnMembufCap = 32 * 1024
	// TempTxnMemBufCap is the capability of temporary membuf.
	TempTxnMemBufCap = 64
)
View Source
var (
	// ErrClosed is used when close an already closed txn.
	ErrClosed = terror.ClassKV.New(codeClosed, "Error: Transaction already closed")
	// ErrNotExist is used when try to get an entry with an unexist key from KV store.
	ErrNotExist = terror.ClassKV.New(codeNotExist, "Error: key not exist")
	// ErrTxnRetryable is used when KV store occurs retryable error which SQL layer can safely retry the transaction.
	// When using TiKV as the storage node, the error is returned ONLY when lock not found (txnLockNotFound) in Commit,
	// subject to change it in the future.
	ErrTxnRetryable = terror.ClassKV.New(codeTxnRetryable, "Error: KV error safe to retry %s "+TxnRetryableMark)
	// ErrCannotSetNilValue is the error when sets an empty value.
	ErrCannotSetNilValue = terror.ClassKV.New(codeCantSetNilValue, "can not set nil value")
	// ErrInvalidTxn is the error when commits or rollbacks in an invalid transaction.
	ErrInvalidTxn = terror.ClassKV.New(codeInvalidTxn, "invalid transaction")
	// ErrTxnTooLarge is the error when transaction is too large, lock time reached the maximum value.
	ErrTxnTooLarge = terror.ClassKV.New(codeTxnTooLarge, "transaction is too large")
	// ErrEntryTooLarge is the error when a key value entry is too large.
	ErrEntryTooLarge = terror.ClassKV.New(codeEntryTooLarge, "entry too large, the max entry size is %d, the size of data is %d")
	// ErrKeyExists returns when key is already exist.
	ErrKeyExists = terror.ClassKV.New(codeKeyExists, "key already exist")
	// ErrNotImplemented returns when a function is not implemented yet.
	ErrNotImplemented = terror.ClassKV.New(codeNotImplemented, "not implemented")
	// ErrWriteConflict is the error when the commit meets an write conflict error.
	ErrWriteConflict = terror.ClassKV.New(mysql.ErrWriteConflict,
		mysql.MySQLErrName[mysql.ErrWriteConflict]+" "+TxnRetryableMark)
	// ErrWriteConflictInTiDB is the error when the commit meets an write conflict error when local latch is enabled.
	ErrWriteConflictInTiDB = terror.ClassKV.New(mysql.ErrWriteConflictInTiDB,
		mysql.MySQLErrName[mysql.ErrWriteConflictInTiDB]+" "+TxnRetryableMark)
)
View Source
var (
	// TxnEntrySizeLimit is limit of single entry size (len(key) + len(value)).
	TxnEntrySizeLimit = 6 * 1024 * 1024
	// TxnEntryCountLimit  is limit of number of entries in the MemBuffer.
	TxnEntryCountLimit uint64 = config.DefTxnEntryCountLimit
	// TxnTotalSizeLimit is limit of the sum of all entry size.
	TxnTotalSizeLimit uint64 = config.DefTxnTotalSizeLimit
)

Those limits is enforced to make sure the transaction can be well handled by TiKV.

View Source
var (
	// MaxVersion is the maximum version, notice that it's not a valid version.
	MaxVersion = Version{Ver: math.MaxUint64}
	// MinVersion is the minimum version, it's not a valid version, too.
	MinVersion = Version{Ver: 0}
)
View Source
var DefaultVars = NewVariables()

DefaultVars is the default variables instance.

Functions

func BackOff

func BackOff(attempts uint) int

BackOff Implements exponential backoff with full jitter. Returns real back off time in microsecond. See http://www.awsarchitectureblog.com/2015/03/backoff.html.

func GetInt64

func GetInt64(r Retriever, k Key) (int64, error)

GetInt64 get int64 value which created by IncInt64 method.

func IncInt64

func IncInt64(rm RetrieverMutator, k Key, step int64) (int64, error)

IncInt64 increases the value for key k in kv store by step.

func IsErrNotFound

func IsErrNotFound(err error) bool

IsErrNotFound checks if err is a kind of NotFound error.

func IsMockCommitErrorEnable

func IsMockCommitErrorEnable() bool

IsMockCommitErrorEnable exports for gofail testing.

func IsTxnRetryableError

func IsTxnRetryableError(err error) bool

IsTxnRetryableError checks if the error could safely retry the transaction.

func MockCommitErrorDisable

func MockCommitErrorDisable()

MockCommitErrorDisable exports for gofail testing.

func MockCommitErrorEnable

func MockCommitErrorEnable()

MockCommitErrorEnable exports for gofail testing.

func NextUntil

func NextUntil(it Iterator, fn FnKeyCmp) error

NextUntil applies FnKeyCmp to each entry of the iterator until meets some condition. It will stop when fn returns true, or iterator is invalid or an error occurs.

func RunInNewTxn

func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error

RunInNewTxn will run the f in a new transaction environment.

func WalkMemBuffer

func WalkMemBuffer(memBuf MemBuffer, f func(k Key, v []byte) error) error

WalkMemBuffer iterates all buffered kv pairs in memBuf

Types

type AssertionProto

type AssertionProto interface {
	// SetAssertion sets an assertion for an operation on the key.
	SetAssertion(key Key, assertion AssertionType)
	// Confirm assertions to current position if `succ` is true, reset position otherwise.
	ConfirmAssertions(succ bool)
}

AssertionProto is an interface defined for the assertion protocol.

type AssertionType

type AssertionType int

AssertionType is the type of a assertion.

const (
	None AssertionType = iota
	Exist
	NotExist
)

The AssertionType constants.

type BufferStore

type BufferStore struct {
	MemBuffer
	// contains filtered or unexported fields
}

BufferStore wraps a Retriever for read and a MemBuffer for buffered write. Common usage pattern:

bs := NewBufferStore(r) // use BufferStore to wrap a Retriever
// ...
// read/write on bs
// ...
bs.SaveTo(m)	        // save above operations to a Mutator

func NewBufferStore

func NewBufferStore(r Retriever, cap int) *BufferStore

NewBufferStore creates a BufferStore using r for read.

func (*BufferStore) Get

func (s *BufferStore) Get(k Key) ([]byte, error)

Get implements the Retriever interface.

func (*BufferStore) Iter

func (s *BufferStore) Iter(k Key, upperBound Key) (Iterator, error)

Iter implements the Retriever interface.

func (*BufferStore) IterReverse

func (s *BufferStore) IterReverse(k Key) (Iterator, error)

IterReverse implements the Retriever interface.

func (*BufferStore) Reset

func (s *BufferStore) Reset()

Reset resets s.MemBuffer.

func (*BufferStore) SaveTo

func (s *BufferStore) SaveTo(m Mutator) error

SaveTo saves all buffered kv pairs into a Mutator.

func (*BufferStore) SetCap

func (s *BufferStore) SetCap(cap int)

SetCap sets the MemBuffer capability.

func (*BufferStore) WalkBuffer

func (s *BufferStore) WalkBuffer(f func(k Key, v []byte) error) error

WalkBuffer iterates all buffered kv pairs.

type Client

type Client interface {
	// Send sends request to KV layer, returns a Response.
	Send(ctx context.Context, req *Request, vars *Variables) Response

	// IsRequestTypeSupported checks if reqType and subType is supported.
	IsRequestTypeSupported(reqType, subType int64) bool
}

Client is used to send request to KV layer.

type ContextKey

type ContextKey string

ContextKey is the type of context's key

type Driver

type Driver interface {
	// Open returns a new Storage.
	// The path is the string for storage specific format.
	Open(path string) (Storage, error)
}

Driver is the interface that must be implemented by a KV storage.

type FnKeyCmp

type FnKeyCmp func(key Key) bool

FnKeyCmp is the function for iterator the keys

type InjectedSnapshot

type InjectedSnapshot struct {
	Snapshot
	// contains filtered or unexported fields
}

InjectedSnapshot wraps a Snapshot with injections.

func (*InjectedSnapshot) BatchGet

func (t *InjectedSnapshot) BatchGet(keys []Key) (map[string][]byte, error)

BatchGet returns an error if cfg.getError is set.

func (*InjectedSnapshot) Get

func (t *InjectedSnapshot) Get(k Key) ([]byte, error)

Get returns an error if cfg.getError is set.

type InjectedStore

type InjectedStore struct {
	Storage
	// contains filtered or unexported fields
}

InjectedStore wraps a Storage with injections.

func (*InjectedStore) Begin

func (s *InjectedStore) Begin() (Transaction, error)

Begin creates an injected Transaction.

func (*InjectedStore) BeginWithStartTS

func (s *InjectedStore) BeginWithStartTS(startTS uint64) (Transaction, error)

BeginWithStartTS creates an injected Transaction with startTS.

func (*InjectedStore) GetSnapshot

func (s *InjectedStore) GetSnapshot(ver Version) (Snapshot, error)

GetSnapshot creates an injected Snapshot.

type InjectedTransaction

type InjectedTransaction struct {
	Transaction
	// contains filtered or unexported fields
}

InjectedTransaction wraps a Transaction with injections.

func (*InjectedTransaction) BatchGet

func (t *InjectedTransaction) BatchGet(keys []Key) (map[string][]byte, error)

BatchGet returns an error if cfg.getError is set.

func (*InjectedTransaction) Commit

func (t *InjectedTransaction) Commit(ctx context.Context) error

Commit returns an error if cfg.commitError is set.

func (*InjectedTransaction) Get

func (t *InjectedTransaction) Get(k Key) ([]byte, error)

Get returns an error if cfg.getError is set.

type InjectionConfig

type InjectionConfig struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

InjectionConfig is used for fault injections for KV components.

func (*InjectionConfig) SetCommitError

func (c *InjectionConfig) SetCommitError(err error)

SetCommitError injects an error for all Transaction.Commit() methods.

func (*InjectionConfig) SetGetError

func (c *InjectionConfig) SetGetError(err error)

SetGetError injects an error for all kv.Get() methods.

type IsoLevel

type IsoLevel int

IsoLevel is the transaction's isolation level.

const (
	// SI stands for 'snapshot isolation'.
	SI IsoLevel = iota
	// RC stands for 'read committed'.
	RC
)

type Iterator

type Iterator interface {
	Valid() bool
	Key() Key
	Value() []byte
	Next() error
	Close()
}

Iterator is the interface for a iterator on KV store.

type Key

type Key []byte

Key represents high-level Key type.

func (Key) Clone

func (k Key) Clone() Key

Clone returns a copy of the Key.

func (Key) Cmp

func (k Key) Cmp(another Key) int

Cmp returns the comparison result of two key. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

func (Key) HasPrefix

func (k Key) HasPrefix(prefix Key) bool

HasPrefix tests whether the Key begins with prefix.

func (Key) Next

func (k Key) Next() Key

Next returns the next key in byte-order.

func (Key) PrefixNext

func (k Key) PrefixNext() Key

PrefixNext returns the next prefix key.

Assume there are keys like:

rowkey1
rowkey1_column1
rowkey1_column2
rowKey2

If we seek 'rowkey1' Next, we will get 'rowkey1_column1'. If we seek 'rowkey1' PrefixNext, we will get 'rowkey2'.

type KeyRange

type KeyRange struct {
	StartKey Key
	EndKey   Key
}

KeyRange represents a range where StartKey <= key < EndKey.

func (*KeyRange) IsPoint

func (r *KeyRange) IsPoint() bool

IsPoint checks if the key range represents a point.

type MemBuffer

type MemBuffer interface {
	RetrieverMutator
	// Size returns sum of keys and values length.
	Size() int
	// Len returns the number of entries in the DB.
	Len() int
	// Reset cleanup the MemBuffer
	Reset()
	// SetCap sets the MemBuffer capability, to reduce memory allocations.
	// Please call it before you use the MemBuffer, otherwise it will not works.
	SetCap(cap int)
}

MemBuffer is an in-memory kv collection, can be used to buffer write operations.

func NewMemDbBuffer

func NewMemDbBuffer(cap int) MemBuffer

NewMemDbBuffer creates a new memDbBuffer.

type MockTxn

type MockTxn interface {
	Transaction
	GetOption(opt Option) interface{}
}

MockTxn is used for test cases that need more interfaces than Transaction.

type Mutator

type Mutator interface {
	// Set sets the value for key k as v into kv store.
	// v must NOT be nil or empty, otherwise it returns ErrCannotSetNilValue.
	Set(k Key, v []byte) error
	// Delete removes the entry for key k from kv store.
	Delete(k Key) error
}

Mutator is the interface wraps the basic Set and Delete methods.

type Option

type Option int

Option is used for customizing kv store's behaviors during a transaction.

const (
	// PresumeKeyNotExists indicates that when dealing with a Get operation but failing to read data from cache,
	// we presume that the key does not exist in Store. The actual existence will be checked before the
	// transaction's commit.
	// This option is an optimization for frequent checks during a transaction, e.g. batch inserts.
	PresumeKeyNotExists Option = iota + 1
	// PresumeKeyNotExistsError is the option key for error.
	// When PresumeKeyNotExists is set and condition is not match, should throw the error.
	PresumeKeyNotExistsError
	// BinlogInfo contains the binlog data and client.
	BinlogInfo
	// SchemaChecker is used for checking schema-validity.
	SchemaChecker
	// IsolationLevel sets isolation level for current transaction. The default level is SI.
	IsolationLevel
	// Priority marks the priority of this transaction.
	Priority
	// NotFillCache makes this request do not touch the LRU cache of the underlying storage.
	NotFillCache
	// SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized.
	SyncLog
	// KeyOnly retrieve only keys, it can be used in scan now.
	KeyOnly
	// Pessimistic is defined for pessimistic lock
	Pessimistic
	// SnapshotTS is defined to set snapshot ts.
	SnapshotTS
)

Transaction options

type Options

type Options interface {
	// Get gets an option value.
	Get(opt Option) (v interface{}, ok bool)
}

Options is an interface of a set of options. Each option is associated with a value.

type Request

type Request struct {
	// Tp is the request type.
	Tp        int64
	StartTs   uint64
	Data      []byte
	KeyRanges []KeyRange
	// KeepOrder is true, if the response should be returned in order.
	KeepOrder bool
	// Desc is true, if the request is sent in descending order.
	Desc bool
	// Concurrency is 1, if it only sends the request to a single storage unit when
	// ResponseIterator.Next is called. If concurrency is greater than 1, the request will be
	// sent to multiple storage units concurrently.
	Concurrency int
	// IsolationLevel is the isolation level, default is SI.
	IsolationLevel IsoLevel
	// Priority is the priority of this KV request, its value may be PriorityNormal/PriorityLow/PriorityHigh.
	Priority int
	// NotFillCache makes this request do not touch the LRU cache of the underlying storage.
	NotFillCache bool
	// SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized.
	SyncLog bool
	// Streaming indicates using streaming API for this request, result in that one Next()
	// call would not corresponds to a whole region result.
	Streaming bool
	// MemTracker is used to trace and control memory usage in co-processor layer.
	MemTracker *memory.Tracker
}

Request represents a kv request.

type RequestTypeSupportedChecker

type RequestTypeSupportedChecker struct{}

RequestTypeSupportedChecker is used to check expression can be pushed down.

func (RequestTypeSupportedChecker) IsRequestTypeSupported

func (d RequestTypeSupportedChecker) IsRequestTypeSupported(reqType, subType int64) bool

IsRequestTypeSupported checks whether reqType is supported.

type Response

type Response interface {
	// Next returns a resultSubset from a single storage unit.
	// When full result set is returned, nil is returned.
	Next(ctx context.Context) (resultSubset ResultSubset, err error)
	// Close response.
	Close() error
}

Response represents the response returned from KV layer.

type ResultSubset

type ResultSubset interface {
	// GetData gets the data.
	GetData() []byte
	// GetStartKey gets the start key.
	GetStartKey() Key
	// GetExecDetails gets the detail information.
	GetExecDetails() *execdetails.ExecDetails
	// MemSize returns how many bytes of memory this result use for tracing memory usage.
	MemSize() int64
}

ResultSubset represents a result subset from a single storage unit. TODO: Find a better interface for ResultSubset that can reuse bytes.

type Retriever

type Retriever interface {
	// Get gets the value for key k from kv store.
	// If corresponding kv pair does not exist, it returns nil and ErrNotExist.
	Get(k Key) ([]byte, error)
	// Iter creates an Iterator positioned on the first entry that k <= entry's key.
	// If such entry is not found, it returns an invalid Iterator with no error.
	// It yields only keys that < upperBound. If upperBound is nil, it means the upperBound is unbounded.
	// The Iterator must be Closed after use.
	Iter(k Key, upperBound Key) (Iterator, error)

	// IterReverse creates a reversed Iterator positioned on the first entry which key is less than k.
	// The returned iterator will iterate from greater key to smaller key.
	// If k is nil, the returned iterator will be positioned at the last key.
	// TODO: Add lower bound limit
	IterReverse(k Key) (Iterator, error)
}

Retriever is the interface wraps the basic Get and Seek methods.

type RetrieverMutator

type RetrieverMutator interface {
	Retriever
	Mutator
}

RetrieverMutator is the interface that groups Retriever and Mutator interfaces.

type Snapshot

type Snapshot interface {
	Retriever
	// BatchGet gets a batch of values from snapshot.
	BatchGet(keys []Key) (map[string][]byte, error)
	// SetPriority snapshot set the priority
	SetPriority(priority int)
}

Snapshot defines the interface for the snapshot fetched from KV store.

type Storage

type Storage interface {
	// Begin transaction
	Begin() (Transaction, error)
	// BeginWithStartTS begins transaction with startTS.
	BeginWithStartTS(startTS uint64) (Transaction, error)
	// GetSnapshot gets a snapshot that is able to read any data which data is <= ver.
	// if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
	GetSnapshot(ver Version) (Snapshot, error)
	// GetClient gets a client instance.
	GetClient() Client
	// Close store
	Close() error
	// UUID return a unique ID which represents a Storage.
	UUID() string
	// CurrentVersion returns current max committed version.
	CurrentVersion() (Version, error)
	// GetOracle gets a timestamp oracle client.
	GetOracle() oracle.Oracle
	// SupportDeleteRange gets the storage support delete range or not.
	SupportDeleteRange() (supported bool)
	// Name gets the name of the storage engine
	Name() string
	// Describe returns of brief introduction of the storage
	Describe() string
	// ShowStatus returns the specified status of the storage
	ShowStatus(ctx context.Context, key string) (interface{}, error)
}

Storage defines the interface for storage. Isolation should be at least SI(SNAPSHOT ISOLATION)

func NewInjectedStore

func NewInjectedStore(store Storage, cfg *InjectionConfig) Storage

NewInjectedStore creates a InjectedStore with config.

func NewMockStorage

func NewMockStorage() Storage

NewMockStorage creates a new mockStorage.

type Transaction

type Transaction interface {
	MemBuffer
	AssertionProto
	// Commit commits the transaction operations to KV store.
	Commit(context.Context) error
	// Rollback undoes the transaction operations to KV store.
	Rollback() error
	// String implements fmt.Stringer interface.
	String() string
	// LockKeys tries to lock the entries with the keys in KV store.
	LockKeys(ctx context.Context, forUpdateTS uint64, keys ...Key) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// IsReadOnly checks if the transaction has only performed read operations.
	IsReadOnly() bool
	// StartTS returns the transaction start timestamp.
	StartTS() uint64
	// Valid returns if the transaction is valid.
	// A transaction become invalid after commit or rollback.
	Valid() bool
	// GetMemBuffer return the MemBuffer binding to this transaction.
	GetMemBuffer() MemBuffer
	// SetVars sets variables to the transaction.
	SetVars(vars *Variables)
	// BatchGet gets kv from the memory buffer of statement and transaction, and the kv storage.
	BatchGet(keys []Key) (map[string][]byte, error)
	IsPessimistic() bool
}

Transaction defines the interface for operations inside a Transaction. This is not thread safe.

func NewMockTxn

func NewMockTxn() Transaction

NewMockTxn new a mockTxn.

type UnionIter

type UnionIter struct {
	// contains filtered or unexported fields
}

UnionIter is the iterator on an UnionStore.

func NewUnionIter

func NewUnionIter(dirtyIt Iterator, snapshotIt Iterator, reverse bool) (*UnionIter, error)

NewUnionIter returns a union iterator for BufferStore.

func (*UnionIter) Close

func (iter *UnionIter) Close()

Close implements the Iterator Close interface.

func (*UnionIter) Key

func (iter *UnionIter) Key() Key

Key implements the Iterator Key interface.

func (*UnionIter) Next

func (iter *UnionIter) Next() error

Next implements the Iterator Next interface.

func (*UnionIter) Valid

func (iter *UnionIter) Valid() bool

Valid implements the Iterator Valid interface.

func (*UnionIter) Value

func (iter *UnionIter) Value() []byte

Value implements the Iterator Value interface. Multi columns

type UnionStore

type UnionStore interface {
	MemBuffer
	// Returns related condition pair
	LookupConditionPair(k Key) *conditionPair
	// DeleteConditionPair deletes a condition pair.
	DeleteConditionPair(k Key)
	// WalkBuffer iterates all buffered kv pairs.
	WalkBuffer(f func(k Key, v []byte) error) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// GetOption gets an option.
	GetOption(opt Option) interface{}
	// GetMemBuffer return the MemBuffer binding to this UnionStore.
	GetMemBuffer() MemBuffer
}

UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. Also, it provides some transaction related utilities.

func NewUnionStore

func NewUnionStore(snapshot Snapshot) UnionStore

NewUnionStore builds a new UnionStore.

type Variables

type Variables struct {
	// BackoffLockFast specifies the LockFast backoff base duration in milliseconds.
	BackoffLockFast int

	// BackOffWeight specifies the weight of the max back off time duration.
	BackOffWeight int

	// Hook is used for test to verify the variable take effect.
	Hook func(name string, vars *Variables)
}

Variables defines the variables used by KV storage.

func NewVariables

func NewVariables() *Variables

NewVariables create a new Variables instance with default values.

type Version

type Version struct {
	Ver uint64
}

Version is the wrapper of KV's version.

func NewVersion

func NewVersion(v uint64) Version

NewVersion creates a new Version struct.

func (Version) Cmp

func (v Version) Cmp(another Version) int

Cmp returns the comparison result of two versions. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

type VersionProvider

type VersionProvider interface {
	CurrentVersion() (Version, error)
}

VersionProvider provides increasing IDs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL