kv

Documentation

Index

• Variables
• func GetActualDatum(encoder Encoder, rowID int64, colIndex int, inputDatum *types.Datum) (types.Datum, error)
• func GetEncoderIncrementalID(encoder Encoder, id int64) int64
• func GetEncoderSe(encoder Encoder) *session
• func GetSession4test(encoder Encoder) sessionctx.Context
• func KvPairsFromRow(row Row) []common.KvPair
• func KvPairsFromRows(rows Rows) []common.KvPair
• func LogKVConvertFailed(logger log.Logger, row []types.Datum, j int, colInfo *model.ColumnInfo, ...) error
• func NewPanickingAllocators(base int64) autoid.Allocators
• func NewSession(options *SessionOptions, logger log.Logger) sessionctx.Context
• type Encoder
  • func NewTableKVEncoder(tbl table.Table, options *SessionOptions, metrics *metric.Metrics, ...) (Encoder, error)
• type KvPairs
  • func (kvs *KvPairs) ClassifyAndAppend(data *Rows, dataChecksum *verification.KVChecksum, indices *Rows, ...)
  • func (kvs *KvPairs) Clear() Rows
  • func (kvs *KvPairs) Size() uint64
  • func (kvs *KvPairs) SplitIntoChunks(splitSize int) []Rows
• type Row
  • func MakeRowFromKvPairs(pairs []common.KvPair) Row
• type RowArrayMarshaler
  • func (row RowArrayMarshaler) MarshalLogArray(encoder zapcore.ArrayEncoder) error
• type Rows
  • func MakeRowsFromKvPairs(pairs []common.KvPair) Rows
• type SessionOptions
• type TableKVDecoder
  • func NewTableKVDecoder(tbl table.Table, tableName string, options *SessionOptions, logger log.Logger) (*TableKVDecoder, error)
  • func (t *TableKVDecoder) DecodeHandleFromIndex(indexInfo *model.IndexInfo, key []byte, value []byte) (kv.Handle, error)
  • func (t *TableKVDecoder) DecodeHandleFromRowKey(key []byte) (kv.Handle, error)
  • func (t *TableKVDecoder) DecodeRawRowData(h kv.Handle, value []byte) ([]types.Datum, map[int64]types.Datum, error)
  • func (t *TableKVDecoder) DecodeRawRowDataAsStr(h kv.Handle, value []byte) (res string)
  • func (t *TableKVDecoder) IterRawIndexKeys(h kv.Handle, rawRow []byte, fn func([]byte) error) error
  • func (t *TableKVDecoder) Name() string

Variables

var ExtraHandleColumnInfo = model.NewExtraHandleColInfo()

Functions

func GetActualDatum

func GetActualDatum(encoder Encoder, rowID int64, colIndex int, inputDatum *types.Datum) (types.Datum, error)

GetActualDatum export getActualDatum function.

func GetEncoderIncrementalID

func GetEncoderIncrementalID(encoder Encoder, id int64) int64

GetEncoderIncrementalID return Auto increment id.

func GetEncoderSe

func GetEncoderSe(encoder Encoder) *session

GetEncoderSe return session.

func GetSession4test

func GetSession4test(encoder Encoder) sessionctx.Context

func KvPairsFromRow

func KvPairsFromRow(row Row) []common.KvPair

KvPairsFromRow converts a Row instance constructed from MakeRowFromKvPairs back into a slice of KvPair. This method panics if the Row is not constructed in such way. nolint:golint // kv.KvPairsFromRow sounds good.

func KvPairsFromRows

func KvPairsFromRows(rows Rows) []common.KvPair

KvPairsFromRows converts a Rows instance constructed from MakeRowsFromKvPairs back into a slice of KvPair. This method panics if the Rows is not constructed in such way. nolint:golint // kv.KvPairsFromRows sounds good.

func LogKVConvertFailed

func LogKVConvertFailed(logger log.Logger, row []types.Datum, j int, colInfo *model.ColumnInfo, err error) error

func NewPanickingAllocators

func NewPanickingAllocators(base int64) autoid.Allocators

NewPanickingAllocators creates a PanickingAllocator shared by all allocation types. we use this to collect the max id(either _tidb_rowid or auto_increment id or auto_random) used during import, and we will use this info to do ALTER TABLE xxx AUTO_RANDOM_BASE or AUTO_INCREMENT on post-process phase.

func NewSession

func NewSession(options *SessionOptions, logger log.Logger) sessionctx.Context

NewSession creates a new trimmed down Session matching the options.

Types

type Encoder

type Encoder interface {
	// Close the encoder.
	Close()

	// Encode encodes a row of SQL values into a backend-friendly format.
	Encode(
		logger log.Logger,
		row []types.Datum,
		rowID int64,
		columnPermutation []int,
		path string,
		offset int64,
	) (Row, error)
}

Encoder encodes a row of SQL values into some opaque type which can be consumed by OpenEngine.WriteEncoded.

func NewTableKVEncoder

func NewTableKVEncoder(
	tbl table.Table,
	options *SessionOptions,
	metrics *metric.Metrics,
	logger log.Logger,
) (Encoder, error)

type KvPairs

type KvPairs struct {
	// contains filtered or unexported fields
}

func (*KvPairs) ClassifyAndAppend

func (kvs *KvPairs) ClassifyAndAppend(
	data *Rows,
	dataChecksum *verification.KVChecksum,
	indices *Rows,
	indexChecksum *verification.KVChecksum,
)

func (*KvPairs) Clear

func (kvs *KvPairs) Clear() Rows

func (*KvPairs) Size

func (kvs *KvPairs) Size() uint64

func (*KvPairs) SplitIntoChunks

func (kvs *KvPairs) SplitIntoChunks(splitSize int) []Rows

type Row

type Row interface {
	// ClassifyAndAppend separates the data-like and index-like parts of the
	// encoded row, and appends these parts into the existing buffers and
	// checksums.
	ClassifyAndAppend(
		data *Rows,
		dataChecksum *verification.KVChecksum,
		indices *Rows,
		indexChecksum *verification.KVChecksum,
	)

	// Size represents the total kv size of this Row.
	Size() uint64
}

Row represents a single encoded row.

func MakeRowFromKvPairs

func MakeRowFromKvPairs(pairs []common.KvPair) Row

MakeRowFromKvPairs converts a KvPair slice into a Row instance. This is mainly used for testing only. The resulting Row instance should only be used for the importer backend.

type RowArrayMarshaler

type RowArrayMarshaler []types.Datum

RowArrayMarshaler wraps a slice of types.Datum for logging the content into zap.

func (RowArrayMarshaler) MarshalLogArray

func (row RowArrayMarshaler) MarshalLogArray(encoder zapcore.ArrayEncoder) error

MarshalLogArray implements the zapcore.ArrayMarshaler interface

type Rows

type Rows interface {
	// SplitIntoChunks splits the rows into multiple consecutive parts, each
	// part having total byte size less than `splitSize`. The meaning of "byte
	// size" should be consistent with the value used in `Row.ClassifyAndAppend`.
	SplitIntoChunks(splitSize int) []Rows

	// Clear returns a new collection with empty content. It may share the
	// capacity with the current instance. The typical usage is `x = x.Clear()`.
	Clear() Rows
}

Rows represents a collection of encoded rows.

func MakeRowsFromKvPairs

func MakeRowsFromKvPairs(pairs []common.KvPair) Rows

MakeRowsFromKvPairs converts a KvPair slice into a Rows instance. This is mainly used for testing only. The resulting Rows instance should only be used for the importer backend.

type SessionOptions

type SessionOptions struct {
	SQLMode   mysql.SQLMode
	Timestamp int64
	SysVars   map[string]string
	// a seed used for tableKvEncoder's auto random bits value
	AutoRandomSeed int64
	// IndexID is used by the DuplicateManager. Only the key range with the specified index ID is scanned.
	IndexID int64
}

SessionOptions is the initial configuration of the session.

type TableKVDecoder

type TableKVDecoder struct {
	// contains filtered or unexported fields
}

func NewTableKVDecoder

func NewTableKVDecoder(
	tbl table.Table,
	tableName string,
	options *SessionOptions,
	logger log.Logger,
) (*TableKVDecoder, error)

func (*TableKVDecoder) DecodeHandleFromIndex

func (t *TableKVDecoder) DecodeHandleFromIndex(indexInfo *model.IndexInfo, key []byte, value []byte) (kv.Handle, error)

func (*TableKVDecoder) DecodeHandleFromRowKey

func (t *TableKVDecoder) DecodeHandleFromRowKey(key []byte) (kv.Handle, error)

func (*TableKVDecoder) DecodeRawRowData

func (t *TableKVDecoder) DecodeRawRowData(h kv.Handle, value []byte) ([]types.Datum, map[int64]types.Datum, error)

DecodeRawRowData decodes raw row data into a datum slice and a (columnID:columnValue) map.

func (*TableKVDecoder) DecodeRawRowDataAsStr

func (t *TableKVDecoder) DecodeRawRowDataAsStr(h kv.Handle, value []byte) (res string)

func (*TableKVDecoder) IterRawIndexKeys

func (t *TableKVDecoder) IterRawIndexKeys(h kv.Handle, rawRow []byte, fn func([]byte) error) error

IterRawIndexKeys generates the raw index keys corresponding to the raw row, and then iterate them using `fn`. The input buffer will be reused.

func (*TableKVDecoder) Name

func (t *TableKVDecoder) Name() string