diff

package
Version: v3.0.2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 24, 2019 License: Apache-2.0 Imports: 18 Imported by: 0

README

diff

introduction

diff is a library to provide a function to compare tables:

  • support comapre table's data and struct
  • can generate sql for target table to fix data
  • support compare tables with different name
  • support compare one target table with multipe source tables

To comapre tables, you should construct a TableDiff struct:

// TableDiff saves config for diff table
type TableDiff struct {
	// source tables
	SourceTables []*TableInstance
	// target table
	TargetTable *TableInstance

	// columns be ignored, will not check this column's data, 
	// but may use these columns as split field or order by key.
	IgnoreColumns []string

	// columns be removed, will remove these columns from table info,
	// and will not check these columns' data, will not use these columns as split field or order by key too.
	RemoveColumns []string

	// field should be the primary key, unique key or field with index
	Field string

	// select range, for example: "age > 10 AND age < 20"
	Range string

	// for example, the whole data is [1...100]
	// we can split these data to [1...10], [11...20], ..., [91...100]
	// the [1...10] is a chunk, and it's chunk size is 10
	// size of the split chunk
	ChunkSize int

	// sampling check percent, for example 10 means only check 10% data
	Sample int

	// how many goroutines are created to check data
	CheckThreadCount int

	// set false if want to comapre the data directly
    UseChecksum bool
    
    // collation config in mysql/tidb, should corresponding to charset.
	Collation string

	// ignore check table's struct
	IgnoreStructCheck bool

	// ignore check table's data
	IgnoreDataCheck bool
}

Then call TableDiff's function Equal to get the compare result. The Equal function define as:

func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (structEqual bool, dataEqual bool, err error)

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func CreateDB

func CreateDB(ctx context.Context, dbConfig dbutil.DBConfig, num int) (db *sql.DB, err error)

CreateDB creates sql.DB used for select data

func CreateDBForCP

func CreateDBForCP(ctx context.Context, dbConfig dbutil.DBConfig) (cpDB *sql.DB, err error)

CreateDBForCP creates sql.DB used for write data for checkpoint

Types

type Bound

type Bound struct {
	Column      string `json:"column"`
	Lower       string `json:"lower"`
	LowerSymbol string `json:"lower-symbol"`
	Upper       string `json:"upper"`
	UpperSymbol string `json:"upper-symbol"`
}

Bound represents a bound for a column

type ChunkRange

type ChunkRange struct {
	ID     int      `json:"id"`
	Bounds []*Bound `json:"bounds"`
	Mode   string   `json:"mode"`

	Where string   `json:"where"`
	Args  []string `json:"args"`

	State string `json:"state"`
}

ChunkRange represents chunk range

func NewChunkRange

func NewChunkRange(mode string) *ChunkRange

NewChunkRange return a ChunkRange.

func SplitChunks

func SplitChunks(ctx context.Context, table *TableInstance, splitFields, limits string, chunkSize int, collation string, useTiDBStatsInfo bool, cpDB *sql.DB) (chunks []*ChunkRange, err error)

SplitChunks splits the table to some chunks.

func (*ChunkRange) String

func (c *ChunkRange) String() string

String returns the string of ChunkRange, used for log.

type RowData

type RowData struct {
	Data   map[string]*dbutil.ColumnData
	Source string
}

RowData is the struct of rows selected from mysql/tidb

type RowDatas

type RowDatas struct {
	Rows         []RowData
	OrderKeyCols []*model.ColumnInfo
}

RowDatas is a heap of MergeItems.

func (RowDatas) Len

func (r RowDatas) Len() int

func (RowDatas) Less

func (r RowDatas) Less(i, j int) bool

func (*RowDatas) Pop

func (r *RowDatas) Pop() interface{}

Pop implements heap.Interface's Pop function

func (*RowDatas) Push

func (r *RowDatas) Push(x interface{})

Push implements heap.Interface's Push function

func (RowDatas) Swap

func (r RowDatas) Swap(i, j int)

type TableDiff

type TableDiff struct {
	// source tables
	SourceTables []*TableInstance `json:"source-tables"`
	// target table
	TargetTable *TableInstance `json:"target-table"`

	// columns be ignored
	IgnoreColumns []string `json:"-"`

	// columns be removed
	RemoveColumns []string `json:"-"`

	// field should be the primary key, unique key or field with index
	Fields string `json:"fields"`

	// select range, for example: "age > 10 AND age < 20"
	Range string `json:"range"`

	// for example, the whole data is [1...100]
	// we can split these data to [1...10], [11...20], ..., [91...100]
	// the [1...10] is a chunk, and it's chunk size is 10
	// size of the split chunk
	ChunkSize int `json:"chunk-size"`

	// sampling check percent, for example 10 means only check 10% data
	Sample int `json:"sample"`

	// how many goroutines are created to check data
	CheckThreadCount int `json:"-"`

	// set false if want to comapre the data directly
	UseChecksum bool `json:"-"`

	// set true if just want compare data by checksum, will skip select data when checksum is not equal
	OnlyUseChecksum bool `json:"-"`

	// collation config in mysql/tidb, should corresponding to charset.
	Collation string `json:"collation"`

	// ignore check table's struct
	IgnoreStructCheck bool `json:"-"`

	// ignore check table's data
	IgnoreDataCheck bool `json:"-"`

	// set true will continue check from the latest checkpoint
	UseCheckpoint bool `json:"use-checkpoint"`

	// get tidb statistics information from which table instance. if is nil, will split chunk by random.
	TiDBStatsSource *TableInstance `json:"tidb-stats-source"`

	CpDB *sql.DB
	// contains filtered or unexported fields
}

TableDiff saves config for diff table

func (*TableDiff) CheckTableData

func (t *TableDiff) CheckTableData(ctx context.Context) (equal bool, err error)

CheckTableData checks table's data

func (*TableDiff) CheckTableStruct

func (t *TableDiff) CheckTableStruct(ctx context.Context) (bool, error)

CheckTableStruct checks table's struct

func (*TableDiff) Equal

func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (bool, bool, error)

Equal tests whether two database have same data and schema.

func (*TableDiff) LoadCheckpoint

func (t *TableDiff) LoadCheckpoint(ctx context.Context) ([]*ChunkRange, error)

LoadCheckpoint do some prepare work before check data, like adjust config and create checkpoint table

func (*TableDiff) UpdateSummaryInfo

func (t *TableDiff) UpdateSummaryInfo(ctx context.Context) chan bool

UpdateSummaryInfo updaets summary infomation

func (*TableDiff) WriteSqls

func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error) chan bool

WriteSqls write sqls to file

type TableInstance

type TableInstance struct {
	Conn       *sql.DB `json:"-"`
	Schema     string  `json:"schema"`
	Table      string  `json:"table"`
	InstanceID string  `json:"instance-id"`
	// contains filtered or unexported fields
}

TableInstance record a table instance

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL