kv

package
v2.0.11-binlog+incompa... Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 4, 2019 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (
	PriorityNormal = iota
	PriorityLow
	PriorityHigh
)

Priority value for transaction priority.

View Source
const (
	ReqTypeSelect   = 101
	ReqTypeIndex    = 102
	ReqTypeDAG      = 103
	ReqTypeAnalyze  = 104
	ReqTypeChecksum = 105

	ReqSubTypeBasic      = 0
	ReqSubTypeDesc       = 10000
	ReqSubTypeGroupBy    = 10001
	ReqSubTypeTopN       = 10002
	ReqSubTypeSignature  = 10003
	ReqSubTypeAnalyzeIdx = 10004
	ReqSubTypeAnalyzeCol = 10005
)

ReqTypes.

Variables

View Source
var (
	// DefaultTxnMembufCap is the default transaction membuf capability.
	DefaultTxnMembufCap = 4 * 1024
	// ImportingTxnMembufCap is the capability of tidb importing data situation.
	ImportingTxnMembufCap = 32 * 1024
	// TempTxnMemBufCap is the capability of temporary membuf.
	TempTxnMemBufCap = 64
)
View Source
var (
	// ErrClosed is used when close an already closed txn.
	ErrClosed = terror.ClassKV.New(codeClosed, "Error: Transaction already closed")
	// ErrNotExist is used when try to get an entry with an unexist key from KV store.
	ErrNotExist = terror.ClassKV.New(codeNotExist, "Error: key not exist")
	// ErrConditionNotMatch is used when condition is not met.
	ErrConditionNotMatch = terror.ClassKV.New(codeConditionNotMatch, "Error: Condition not match")
	// ErrLockConflict is used when try to lock an already locked key.
	ErrLockConflict = terror.ClassKV.New(codeLockConflict, "Error: Lock conflict")
	// ErrLazyConditionPairsNotMatch is used when value in store differs from expect pairs.
	ErrLazyConditionPairsNotMatch = terror.ClassKV.New(codeLazyConditionPairsNotMatch, "Error: Lazy condition pairs not match")
	// ErrRetryable is used when KV store occurs RPC error or some other
	// errors which SQL layer can safely retry.
	ErrRetryable = terror.ClassKV.New(codeRetryable, "Error: KV error safe to retry")
	// ErrCannotSetNilValue is the error when sets an empty value.
	ErrCannotSetNilValue = terror.ClassKV.New(codeCantSetNilValue, "can not set nil value")
	// ErrInvalidTxn is the error when commits or rollbacks in an invalid transaction.
	ErrInvalidTxn = terror.ClassKV.New(codeInvalidTxn, "invalid transaction")
	// ErrTxnTooLarge is the error when transaction is too large, lock time reached the maximum value.
	ErrTxnTooLarge = terror.ClassKV.New(codeTxnTooLarge, "transaction is too large")
	// ErrEntryTooLarge is the error when a key value entry is too large.
	ErrEntryTooLarge = terror.ClassKV.New(codeEntryTooLarge, "entry is too large")

	// ErrNotCommitted is the error returned by CommitVersion when this
	// transaction is not committed.
	ErrNotCommitted = terror.ClassKV.New(codeNotCommitted, "this transaction has not committed")

	// ErrKeyExists returns when key is already exist.
	ErrKeyExists = terror.ClassKV.New(codeKeyExists, "key already exist")
	// ErrNotImplemented returns when a function is not implemented yet.
	ErrNotImplemented = terror.ClassKV.New(codeNotImplemented, "not implemented")
)
View Source
var (
	// TxnEntrySizeLimit is limit of single entry size (len(key) + len(value)).
	TxnEntrySizeLimit = 6 * 1024 * 1024
	// TxnEntryCountLimit  is limit of number of entries in the MemBuffer.
	TxnEntryCountLimit uint64 = 300 * 1000
	// TxnTotalSizeLimit is limit of the sum of all entry size.
	TxnTotalSizeLimit = 100 * 1024 * 1024
)

Those limits is enforced to make sure the transaction can be well handled by TiKV.

View Source
var (
	// MaxVersion is the maximum version, notice that it's not a valid version.
	MaxVersion = Version{Ver: math.MaxUint64}
	// MinVersion is the minimum version, it's not a valid version, too.
	MinVersion = Version{Ver: 0}
)

Functions

func BackOff

func BackOff(attempts uint) int

BackOff Implements exponential backoff with full jitter. Returns real back off time in microsecond. See http://www.awsarchitectureblog.com/2015/03/backoff.html.

func BatchGetValues

func BatchGetValues(txn Transaction, keys []Key) (map[string][]byte, error)

BatchGetValues gets values in batch. The values from buffer in transaction and the values from the storage node are merged together.

func GetInt64

func GetInt64(r Retriever, k Key) (int64, error)

GetInt64 get int64 value which created by IncInt64 method.

func IncInt64

func IncInt64(rm RetrieverMutator, k Key, step int64) (int64, error)

IncInt64 increases the value for key k in kv store by step.

func IsErrNotFound

func IsErrNotFound(err error) bool

IsErrNotFound checks if err is a kind of NotFound error.

func IsRetryableError

func IsRetryableError(err error) bool

IsRetryableError checks if the err is a fatal error and the under going operation is worth to retry.

func NextUntil

func NextUntil(it Iterator, fn FnKeyCmp) error

NextUntil applies FnKeyCmp to each entry of the iterator until meets some condition. It will stop when fn returns true, or iterator is invalid or an error occurs.

func RunInNewTxn

func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error

RunInNewTxn will run the f in a new transaction environment.

func WalkMemBuffer

func WalkMemBuffer(memBuf MemBuffer, f func(k Key, v []byte) error) error

WalkMemBuffer iterates all buffered kv pairs in memBuf

Types

type BufferStore

type BufferStore struct {
	MemBuffer
	// contains filtered or unexported fields
}

BufferStore wraps a Retriever for read and a MemBuffer for buffered write. Common usage pattern:

bs := NewBufferStore(r) // use BufferStore to wrap a Retriever
// ...
// read/write on bs
// ...
bs.SaveTo(m)	        // save above operations to a Mutator

func NewBufferStore

func NewBufferStore(r Retriever, cap int) *BufferStore

NewBufferStore creates a BufferStore using r for read.

func (*BufferStore) Get

func (s *BufferStore) Get(k Key) ([]byte, error)

Get implements the Retriever interface.

func (*BufferStore) Iter

func (s *BufferStore) Iter(k Key, upperBound Key) (Iterator, error)

Iter implements the Retriever interface.

func (*BufferStore) IterReverse

func (s *BufferStore) IterReverse(k Key) (Iterator, error)

IterReverse implements the Retriever interface.

func (*BufferStore) Reset

func (s *BufferStore) Reset()

Reset resets s.MemBuffer.

func (*BufferStore) SaveTo

func (s *BufferStore) SaveTo(m Mutator) error

SaveTo saves all buffered kv pairs into a Mutator.

func (*BufferStore) SetCap

func (s *BufferStore) SetCap(cap int)

SetCap sets the MemBuffer capability.

func (*BufferStore) WalkBuffer

func (s *BufferStore) WalkBuffer(f func(k Key, v []byte) error) error

WalkBuffer iterates all buffered kv pairs.

type Client

type Client interface {
	// Send sends request to KV layer, returns a Response.
	Send(ctx context.Context, req *Request) Response

	// IsRequestTypeSupported checks if reqType and subType is supported.
	IsRequestTypeSupported(reqType, subType int64) bool
}

Client is used to send request to KV layer.

type Driver

type Driver interface {
	// Open returns a new Storage.
	// The path is the string for storage specific format.
	Open(path string) (Storage, error)
}

Driver is the interface that must be implemented by a KV storage.

type EncodedKey

type EncodedKey []byte

EncodedKey represents encoded key in low-level storage engine.

func (EncodedKey) Cmp

func (k EncodedKey) Cmp(another EncodedKey) int

Cmp returns the comparison result of two key. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

func (EncodedKey) Next

func (k EncodedKey) Next() EncodedKey

Next returns the next key in byte-order.

type FnKeyCmp

type FnKeyCmp func(key Key) bool

FnKeyCmp is the function for iterator the keys

type InjectedSnapshot

type InjectedSnapshot struct {
	Snapshot
	// contains filtered or unexported fields
}

InjectedSnapshot wraps a Snapshot with injections.

func (*InjectedSnapshot) BatchGet added in v1.0.8

func (t *InjectedSnapshot) BatchGet(keys []Key) (map[string][]byte, error)

BatchGet returns an error if cfg.getError is set.

func (*InjectedSnapshot) Get

func (t *InjectedSnapshot) Get(k Key) ([]byte, error)

Get returns an error if cfg.getError is set.

type InjectedStore

type InjectedStore struct {
	Storage
	// contains filtered or unexported fields
}

InjectedStore wraps a Storage with injections.

func (*InjectedStore) Begin

func (s *InjectedStore) Begin() (Transaction, error)

Begin creates an injected Transaction.

func (*InjectedStore) BeginWithStartTS

func (s *InjectedStore) BeginWithStartTS(startTS uint64) (Transaction, error)

BeginWithStartTS creates an injected Transaction with startTS.

func (*InjectedStore) GetSnapshot

func (s *InjectedStore) GetSnapshot(ver Version) (Snapshot, error)

GetSnapshot creates an injected Snapshot.

type InjectedTransaction

type InjectedTransaction struct {
	Transaction
	// contains filtered or unexported fields
}

InjectedTransaction wraps a Transaction with injections.

func (*InjectedTransaction) Commit

func (t *InjectedTransaction) Commit(ctx context.Context) error

Commit returns an error if cfg.commitError is set.

func (*InjectedTransaction) Get

func (t *InjectedTransaction) Get(k Key) ([]byte, error)

Get returns an error if cfg.getError is set.

func (*InjectedTransaction) GetSnapshot added in v1.0.8

func (t *InjectedTransaction) GetSnapshot() Snapshot

GetSnapshot implements Transaction GetSnapshot method.

type InjectionConfig

type InjectionConfig struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

InjectionConfig is used for fault injections for KV components.

func (*InjectionConfig) SetCommitError

func (c *InjectionConfig) SetCommitError(err error)

SetCommitError injects an error for all Transaction.Commit() methods.

func (*InjectionConfig) SetGetError

func (c *InjectionConfig) SetGetError(err error)

SetGetError injects an error for all kv.Get() methods.

type IsoLevel

type IsoLevel int

IsoLevel is the transaction's isolation level.

const (
	// SI stands for 'snapshot isolation'.
	SI IsoLevel = iota
	// RC stands for 'read committed'.
	RC
)

type Iterator

type Iterator interface {
	Valid() bool
	Key() Key
	Value() []byte
	Next() error
	Close()
}

Iterator is the interface for a iterator on KV store.

type Key

type Key []byte

Key represents high-level Key type.

func (Key) Clone

func (k Key) Clone() Key

Clone returns a copy of the Key.

func (Key) Cmp

func (k Key) Cmp(another Key) int

Cmp returns the comparison result of two key. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

func (Key) HasPrefix

func (k Key) HasPrefix(prefix Key) bool

HasPrefix tests whether the Key begins with prefix.

func (Key) Next

func (k Key) Next() Key

Next returns the next key in byte-order.

func (Key) PrefixNext

func (k Key) PrefixNext() Key

PrefixNext returns the next prefix key.

Assume there are keys like:

rowkey1
rowkey1_column1
rowkey1_column2
rowKey2

If we seek 'rowkey1' Next, we will get 'rowkey1_column1'. If we seek 'rowkey1' PrefixNext, we will get 'rowkey2'.

type KeyRange

type KeyRange struct {
	StartKey Key
	EndKey   Key
}

KeyRange represents a range where StartKey <= key < EndKey.

func (*KeyRange) IsPoint

func (r *KeyRange) IsPoint() bool

IsPoint checks if the key range represents a point.

type MemBuffer

type MemBuffer interface {
	RetrieverMutator
	// Size returns sum of keys and values length.
	Size() int
	// Len returns the number of entries in the DB.
	Len() int
	// Reset cleanup the MemBuffer
	Reset()
	// SetCap sets the MemBuffer capability, to reduce memory allocations.
	// Please call it before you use the MemBuffer, otherwise it will not works.
	SetCap(cap int)
}

MemBuffer is an in-memory kv collection, can be used to buffer write operations.

func NewMemDbBuffer

func NewMemDbBuffer(cap int) MemBuffer

NewMemDbBuffer creates a new memDbBuffer.

type MockTxn

type MockTxn interface {
	Transaction
	GetOption(opt Option) interface{}
}

MockTxn is used for test cases that need more interfaces than Transaction.

type Mutator

type Mutator interface {
	// Set sets the value for key k as v into kv store.
	// v must NOT be nil or empty, otherwise it returns ErrCannotSetNilValue.
	Set(k Key, v []byte) error
	// Delete removes the entry for key k from kv store.
	Delete(k Key) error
}

Mutator is the interface wraps the basic Set and Delete methods.

type Option

type Option int

Option is used for customizing kv store's behaviors during a transaction.

const (
	// PresumeKeyNotExists indicates that when dealing with a Get operation but failing to read data from cache,
	// we presume that the key does not exist in Store. The actual existence will be checked before the
	// transaction's commit.
	// This option is an optimization for frequent checks during a transaction, e.g. batch inserts.
	PresumeKeyNotExists Option = iota + 1
	// PresumeKeyNotExistsError is the option key for error.
	// When PresumeKeyNotExists is set and condition is not match, should throw the error.
	PresumeKeyNotExistsError
	// BinlogInfo contains the binlog data and client.
	BinlogInfo
	// Skip existing check when "prewrite".
	SkipCheckForWrite
	// SchemaLeaseChecker is used for schema lease check.
	SchemaLeaseChecker
	// IsolationLevel sets isolation level for current transaction. The default level is SI.
	IsolationLevel
	// Priority marks the priority of this transaction.
	Priority
	// NotFillCache makes this request do not touch the LRU cache of the underlying storage.
	NotFillCache
	// SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized.
	SyncLog
)

Transaction options

type Options

type Options interface {
	// Get gets an option value.
	Get(opt Option) (v interface{}, ok bool)
}

Options is an interface of a set of options. Each option is associated with a value.

type Request

type Request struct {
	// Tp is the request type.
	Tp        int64
	StartTs   uint64
	Data      []byte
	KeyRanges []KeyRange
	// KeepOrder is true, if the response should be returned in order.
	KeepOrder bool
	// Desc is true, if the request is sent in descending order.
	Desc bool
	// Concurrency is 1, if it only sends the request to a single storage unit when
	// ResponseIterator.Next is called. If concurrency is greater than 1, the request will be
	// sent to multiple storage units concurrently.
	Concurrency int
	// IsolationLevel is the isolation level, default is SI.
	IsolationLevel IsoLevel
	// Priority is the priority of this KV request, its value may be PriorityNormal/PriorityLow/PriorityHigh.
	Priority int
	// NotFillCache makes this request do not touch the LRU cache of the underlying storage.
	NotFillCache bool
	// SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized.
	SyncLog bool
	// Streaming indicates using streaming API for this request, result in that one Next()
	// call would not corresponds to a whole region result.
	Streaming bool
}

Request represents a kv request.

type Response

type Response interface {
	// Next returns a resultSubset from a single storage unit.
	// When full result set is returned, nil is returned.
	Next(ctx context.Context) (resultSubset ResultSubset, err error)
	// Close response.
	Close() error
}

Response represents the response returned from KV layer.

type ResultSubset

type ResultSubset interface {
	// GetData gets the data.
	GetData() []byte
	// GetStartKey gets the start key.
	GetStartKey() Key
	// GetExecDetails gets the detail information.
	GetExecDetails() *execdetails.ExecDetails
}

ResultSubset represents a result subset from a single storage unit. TODO: Find a better interface for ResultSubset that can reuse bytes.

type Retriever

type Retriever interface {
	// Get gets the value for key k from kv store.
	// If corresponding kv pair does not exist, it returns nil and ErrNotExist.
	Get(k Key) ([]byte, error)
	// Iter creates an Iterator positioned on the first entry that k <= entry's key.
	// If such entry is not found, it returns an invalid Iterator with no error.
	// It yields only keys that < upperBound. If upperBound is nil, it means the upperBound is unbounded.
	// The Iterator must be Closed after use.
	Iter(k Key, upperBound Key) (Iterator, error)

	// IterReverse creates a reversed Iterator positioned on the first entry which key is less than k.
	// The returned iterator will iterate from greater key to smaller key.
	// If k is nil, the returned iterator will be positioned at the last key.
	// TODO: Add lower bound limit
	IterReverse(k Key) (Iterator, error)
}

Retriever is the interface wraps the basic Get and Seek methods.

type RetrieverMutator

type RetrieverMutator interface {
	Retriever
	Mutator
}

RetrieverMutator is the interface that groups Retriever and Mutator interfaces.

type Snapshot

type Snapshot interface {
	Retriever
	// BatchGet gets a batch of values from snapshot.
	BatchGet(keys []Key) (map[string][]byte, error)
	// SetPriority snapshot set the priority
	SetPriority(priority int)
}

Snapshot defines the interface for the snapshot fetched from KV store.

type Storage

type Storage interface {
	// Begin transaction
	Begin() (Transaction, error)
	// BeginWithStartTS begins transaction with startTS.
	BeginWithStartTS(startTS uint64) (Transaction, error)
	// GetSnapshot gets a snapshot that is able to read any data which data is <= ver.
	// if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
	GetSnapshot(ver Version) (Snapshot, error)
	// GetClient gets a client instance.
	GetClient() Client
	// Close store
	Close() error
	// UUID return a unique ID which represents a Storage.
	UUID() string
	// CurrentVersion returns current max committed version.
	CurrentVersion() (Version, error)
	// GetOracle gets a timestamp oracle client.
	GetOracle() oracle.Oracle
	// SupportDeleteRange gets the storage support delete range or not.
	SupportDeleteRange() (supported bool)
}

Storage defines the interface for storage. Isolation should be at least SI(SNAPSHOT ISOLATION)

func NewInjectedStore

func NewInjectedStore(store Storage, cfg *InjectionConfig) Storage

NewInjectedStore creates a InjectedStore with config.

func NewMockStorage

func NewMockStorage() Storage

NewMockStorage creates a new mockStorage.

type Transaction

type Transaction interface {
	MemBuffer
	// Commit commits the transaction operations to KV store.
	Commit(context.Context) error
	// Rollback undoes the transaction operations to KV store.
	Rollback() error
	// String implements fmt.Stringer interface.
	String() string
	// LockKeys tries to lock the entries with the keys in KV store.
	LockKeys(keys ...Key) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// IsReadOnly checks if the transaction has only performed read operations.
	IsReadOnly() bool
	// StartTS returns the transaction start timestamp.
	StartTS() uint64
	// Valid returns if the transaction is valid.
	// A transaction become invalid after commit or rollback.
	Valid() bool
	// GetMemBuffer return the MemBuffer binding to this transaction.
	GetMemBuffer() MemBuffer
	// GetSnapshot returns the snapshot of this transaction.
	GetSnapshot() Snapshot
}

Transaction defines the interface for operations inside a Transaction. This is not thread safe.

func NewMockTxn

func NewMockTxn() Transaction

NewMockTxn new a mockTxn.

type UnionIter

type UnionIter struct {
	// contains filtered or unexported fields
}

UnionIter is the iterator on an UnionStore.

func NewUnionIter

func NewUnionIter(dirtyIt Iterator, snapshotIt Iterator, reverse bool) (*UnionIter, error)

NewUnionIter returns a union iterator for BufferStore.

func (*UnionIter) Close

func (iter *UnionIter) Close()

Close implements the Iterator Close interface.

func (*UnionIter) Key

func (iter *UnionIter) Key() Key

Key implements the Iterator Key interface.

func (*UnionIter) Next

func (iter *UnionIter) Next() error

Next implements the Iterator Next interface.

func (*UnionIter) Valid

func (iter *UnionIter) Valid() bool

Valid implements the Iterator Valid interface.

func (*UnionIter) Value

func (iter *UnionIter) Value() []byte

Value implements the Iterator Value interface. Multi columns

type UnionStore

type UnionStore interface {
	MemBuffer
	// CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched.
	// Lazy condition pairs should be checked before transaction commit.
	CheckLazyConditionPairs() error
	// WalkBuffer iterates all buffered kv pairs.
	WalkBuffer(f func(k Key, v []byte) error) error
	// SetOption sets an option with a value, when val is nil, uses the default
	// value of this option.
	SetOption(opt Option, val interface{})
	// DelOption deletes an option.
	DelOption(opt Option)
	// GetOption gets an option.
	GetOption(opt Option) interface{}
	// GetMemBuffer return the MemBuffer binding to this UnionStore.
	GetMemBuffer() MemBuffer
}

UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. Also, it provides some transaction related utilities.

func NewUnionStore

func NewUnionStore(snapshot Snapshot) UnionStore

NewUnionStore builds a new UnionStore.

type Version

type Version struct {
	Ver uint64
}

Version is the wrapper of KV's version.

func NewVersion

func NewVersion(v uint64) Version

NewVersion creates a new Version struct.

func (Version) Cmp

func (v Version) Cmp(another Version) int

Cmp returns the comparison result of two versions. The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

type VersionProvider

type VersionProvider interface {
	CurrentVersion() (Version, error)
}

VersionProvider provides increasing IDs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL