restore

package
v5.1.2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 13, 2021 License: Apache-2.0 Imports: 70 Imported by: 0

Documentation

Index

Constants

View Source
const (

	// OnlineBytesLimitation/OnlineKeysLimitation is the statistics of
	// Bytes/Keys used per region from pdWriteFlow/pdReadFlow
	// this determines whether the cluster has some region that have other loads
	// and might influence the import task in the future.
	OnlineBytesLimitation = 10 * units.MiB
	OnlineKeysLimitation  = 5000
)
View Source
const (
	FullLevelCompact = -1
	Level1Compact    = 1
)
View Source
const (

	// CreateTableMetadataTable stores the per-table sub jobs information used by TiDB Lightning
	CreateTableMetadataTable = `` /* 647-byte string literal not displayed */

	// CreateTaskMetaTable stores the pre-lightning metadata used by TiDB Lightning
	CreateTaskMetaTable = `` /* 262-byte string literal not displayed */

)

Variables

View Source
var DeliverPauser = common.NewPauser()

DeliverPauser is a shared pauser to pause progress to (*chunkRestore).encodeLoop

Functions

func AlterAutoIncrement

func AlterAutoIncrement(ctx context.Context, g glue.SQLExecutor, tableName string, incr int64) error

AlterAutoIncrement rebase the table auto increment id

NOTE: since tidb can make sure the auto id is always be rebase even if the `incr` value is smaller the the auto incremanet base in tidb side, we needn't fetch currently auto increment value here. See: https://github.com/pingcap/tidb/blob/64698ef9a3358bfd0fdc323996bb7928a56cadca/ddl/ddl_api.go#L2528-L2533

func AlterAutoRandom

func AlterAutoRandom(ctx context.Context, g glue.SQLExecutor, tableName string, randomBase int64) error

func DBFromConfig

func DBFromConfig(dsn config.DBStore) (*sql.DB, error)

func InitSchema

func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema map[string]string) error

func LoadSchemaInfo

func LoadSchemaInfo(
	ctx context.Context,
	schemas []*mydump.MDDatabaseMeta,
	getTables func(context.Context, string) ([]*model.TableInfo, error),
) (map[string]*checkpoints.TidbDBInfo, error)

func ObtainGCLifeTime

func ObtainGCLifeTime(ctx context.Context, db *sql.DB) (string, error)

func ObtainImportantVariables

func ObtainImportantVariables(ctx context.Context, g glue.SQLExecutor) map[string]string

func ObtainNewCollationEnabled

func ObtainNewCollationEnabled(ctx context.Context, g glue.SQLExecutor) bool

func UpdateGCLifeTime

func UpdateGCLifeTime(ctx context.Context, db *sql.DB, gcLifeTime string) error

Types

type CheckType

type CheckType string
const (
	Critical CheckType = "critical"
	Warn     CheckType = "performance"
)

type ChecksumManager

type ChecksumManager interface {
	Checksum(ctx context.Context, tableInfo *checkpoints.TidbTableInfo) (*RemoteChecksum, error)
}

type Controller

type Controller struct {
	// contains filtered or unexported fields
}

func NewRestoreController

func NewRestoreController(
	ctx context.Context,
	dbMetas []*mydump.MDDatabaseMeta,
	cfg *config.Config,
	s storage.ExternalStorage,
	g glue.Glue,
) (*Controller, error)

func NewRestoreControllerWithPauser

func NewRestoreControllerWithPauser(
	ctx context.Context,
	dbMetas []*mydump.MDDatabaseMeta,
	cfg *config.Config,
	s storage.ExternalStorage,
	pauser *common.Pauser,
	g glue.Glue,
) (*Controller, error)

func (*Controller) CheckpointIsValid

func (rc *Controller) CheckpointIsValid(ctx context.Context, tableInfo *md.MDTableMeta) ([]string, bool, error)

CheckpointIsValid checks whether we can start this import with this checkpoint.

func (*Controller) Close

func (rc *Controller) Close()

func (*Controller) ClusterIsAvailable

func (rc *Controller) ClusterIsAvailable(ctx context.Context) error

ClusterIsAvailable check cluster is available to import data. this test can be skipped.

func (*Controller) ClusterResource

func (rc *Controller) ClusterResource(ctx context.Context) error

ClusterResource check cluster has enough resource to import data. this test can by skipped.

func (*Controller) DataCheck

func (rc *Controller) DataCheck(ctx context.Context) error

DataCheck checks the data schema which needs #rc.restoreSchema finished.

func (*Controller) HasLargeCSV

func (rc *Controller) HasLargeCSV(dbMetas []*md.MDDatabaseMeta) error

HasLargeCSV checks whether input csvs is fit for Lightning import. If strictFormat is false, and csv file is large. Lightning will have performance issue. this test cannot be skipped.

func (*Controller) LocalResource

func (rc *Controller) LocalResource(ctx context.Context) error

LocalResource checks the local node has enough resources for this import when local backend enabled;

func (*Controller) Run

func (rc *Controller) Run(ctx context.Context) error

func (*Controller) SchemaIsValid

func (rc *Controller) SchemaIsValid(ctx context.Context, tableInfo *md.MDTableMeta) ([]string, error)

SchemaIsValid checks the import file and cluster schema is match.

func (*Controller) StoragePermission

func (rc *Controller) StoragePermission(ctx context.Context) error

StoragePermission checks whether Lightning has enough permission to storage. this test cannot be skipped.

type RemoteChecksum

type RemoteChecksum struct {
	Schema     string
	Table      string
	Checksum   uint64
	TotalKVs   uint64
	TotalBytes uint64
}

RemoteChecksum represents a checksum result got from tidb.

func DoChecksum

func DoChecksum(ctx context.Context, table *checkpoints.TidbTableInfo) (*RemoteChecksum, error)

DoChecksum do checksum for tables. table should be in <db>.<table>, format. e.g. foo.bar

type SimpleTemplate

type SimpleTemplate struct {
	// contains filtered or unexported fields
}

func (*SimpleTemplate) Collect

func (c *SimpleTemplate) Collect(t CheckType, passed bool, msg string)

func (*SimpleTemplate) FailedCount

func (c *SimpleTemplate) FailedCount(t CheckType) int

func (*SimpleTemplate) Output

func (c *SimpleTemplate) Output() string

func (*SimpleTemplate) Success

func (c *SimpleTemplate) Success() bool

type TableRestore

type TableRestore struct {
	// contains filtered or unexported fields
}

func NewTableRestore

func NewTableRestore(
	tableName string,
	tableMeta *mydump.MDTableMeta,
	dbInfo *checkpoints.TidbDBInfo,
	tableInfo *checkpoints.TidbTableInfo,
	cp *checkpoints.TableCheckpoint,
	ignoreColumns []string,
) (*TableRestore, error)

func (*TableRestore) Close

func (tr *TableRestore) Close()

func (*TableRestore) RebaseChunkRowIDs

func (t *TableRestore) RebaseChunkRowIDs(cp *checkpoints.TableCheckpoint, rowIDBase int64)

type Template

type Template interface {
	// Collect mainly collect performance related checks' results and critical level checks' results.
	// If the performance is not as expect. It will output a warn to user and it won't break the whole import task.
	// if one of critical check not passed. it will stop import task.
	Collect(t CheckType, passed bool, msg string)

	// Success represents the whole check has passed or not.
	Success() bool

	// FailedCount represents (the warn check failed count, the critical check failed count)
	FailedCount(t CheckType) int

	// Output print all checks results.
	Output() string
}

func NewSimpleTemplate

func NewSimpleTemplate() Template

type TiDBManager

type TiDBManager struct {
	// contains filtered or unexported fields
}

func NewTiDBManager

func NewTiDBManager(dsn config.DBStore, tls *common.TLS) (*TiDBManager, error)

func NewTiDBManagerWithDB

func NewTiDBManagerWithDB(db *sql.DB, sqlMode mysql.SQLMode) *TiDBManager

NewTiDBManagerWithDB creates a new TiDB manager with an existing database connection.

func (*TiDBManager) Close

func (timgr *TiDBManager) Close()

func (*TiDBManager) DropTable

func (timgr *TiDBManager) DropTable(ctx context.Context, tableName string) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL