config

package
v5.1.2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 13, 2021 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// ImportMode defines mode of import for tikv.
	ImportMode = "import"
	// NormalMode defines mode of normal for tikv.
	NormalMode = "normal"

	// BackendTiDB is a constant for choosing the "TiDB" backend in the configuration.
	BackendTiDB = "tidb"
	// BackendImporter is a constant for choosing the "Importer" backend in the configuration.
	BackendImporter = "importer"
	// BackendLocal is a constant for choosing the "Local" backup in the configuration.
	// In this mode, we write & sort kv pairs with local storage and directly write them to tikv.
	BackendLocal = "local"

	// CheckpointDriverMySQL is a constant for choosing the "MySQL" checkpoint driver in the configuration.
	CheckpointDriverMySQL = "mysql"
	// CheckpointDriverFile is a constant for choosing the "File" checkpoint driver in the configuration.
	CheckpointDriverFile = "file"

	// ReplaceOnDup indicates using REPLACE INTO to insert data
	ReplaceOnDup = "replace"
	// IgnoreOnDup indicates using INSERT IGNORE INTO to insert data
	IgnoreOnDup = "ignore"
	// ErrorOnDup indicates using INSERT INTO to insert data, which would violate PK or UNIQUE constraint
	ErrorOnDup = "error"
)

Variables

View Source
var (
	DefaultFilter = []string{
		"*.*",
		"!mysql.*",
		"!sys.*",
		"!INFORMATION_SCHEMA.*",
		"!PERFORMANCE_SCHEMA.*",
		"!METRICS_SCHEMA.*",
		"!INSPECTION_SCHEMA.*",
	}
)

Functions

This section is empty.

Types

type AllIgnoreColumns

type AllIgnoreColumns []*IgnoreColumns

func (AllIgnoreColumns) GetIgnoreColumns

func (igCols AllIgnoreColumns) GetIgnoreColumns(db string, table string, caseSensitive bool) (*IgnoreColumns, error)

GetIgnoreColumns gets Ignore config by schema name/regex and table name/regex.

type ByteSize

type ByteSize int64

ByteSize is an alias of int64 which accepts human-friendly strings like '10G' when read from TOML.

const (
	// mydumper
	ReadBlockSize   ByteSize = 64 * units.KiB
	MinRegionSize   ByteSize = 256 * units.MiB
	MaxRegionSize   ByteSize = 256 * units.MiB
	SplitRegionSize ByteSize = 96 * units.MiB

	BufferSizeScale = 5
)

func (*ByteSize) UnmarshalJSON

func (size *ByteSize) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler (for testing)

func (*ByteSize) UnmarshalText

func (size *ByteSize) UnmarshalText(b []byte) error

UnmarshalText implements encoding.TextUnmarshaler

type CSVConfig

type CSVConfig struct {
	Separator       string `toml:"separator" json:"separator"`
	Delimiter       string `toml:"delimiter" json:"delimiter"`
	Null            string `toml:"null" json:"null"`
	Header          bool   `toml:"header" json:"header"`
	TrimLastSep     bool   `toml:"trim-last-separator" json:"trim-last-separator"`
	NotNull         bool   `toml:"not-null" json:"not-null"`
	BackslashEscape bool   `toml:"backslash-escape" json:"backslash-escape"`
}

type Checkpoint

type Checkpoint struct {
	Schema           string `toml:"schema" json:"schema"`
	DSN              string `toml:"dsn" json:"-"` // DSN may contain password, don't expose this to JSON.
	Driver           string `toml:"driver" json:"driver"`
	Enable           bool   `toml:"enable" json:"enable"`
	KeepAfterSuccess bool   `toml:"keep-after-success" json:"keep-after-success"`
}

type Config

type Config struct {
	TaskID int64 `toml:"-" json:"id"`

	App  Lightning `toml:"lightning" json:"lightning"`
	TiDB DBStore   `toml:"tidb" json:"tidb"`

	Checkpoint   Checkpoint          `toml:"checkpoint" json:"checkpoint"`
	Mydumper     MydumperRuntime     `toml:"mydumper" json:"mydumper"`
	TikvImporter TikvImporter        `toml:"tikv-importer" json:"tikv-importer"`
	PostRestore  PostRestore         `toml:"post-restore" json:"post-restore"`
	Cron         Cron                `toml:"cron" json:"cron"`
	Routes       []*router.TableRule `toml:"routes" json:"routes"`
	Security     Security            `toml:"security" json:"security"`

	BWList filter.MySQLReplicationRules `toml:"black-white-list" json:"black-white-list"`
}

func NewConfig

func NewConfig() *Config

func (*Config) Adjust

func (cfg *Config) Adjust(ctx context.Context) error

Adjust fixes the invalid or unspecified settings to reasonable valid values.

func (*Config) AdjustCheckPoint

func (cfg *Config) AdjustCheckPoint()

func (*Config) AdjustMydumper

func (cfg *Config) AdjustMydumper()

func (*Config) CheckAndAdjustFilePath

func (cfg *Config) CheckAndAdjustFilePath() error

func (*Config) CheckAndAdjustForLocalBackend

func (cfg *Config) CheckAndAdjustForLocalBackend() error

func (*Config) CheckAndAdjustSecurity

func (cfg *Config) CheckAndAdjustSecurity() error

func (*Config) CheckAndAdjustTiDBPort

func (cfg *Config) CheckAndAdjustTiDBPort(ctx context.Context, mustHaveInternalConnections bool) error

func (*Config) DefaultVarsForImporterAndLocalBackend

func (cfg *Config) DefaultVarsForImporterAndLocalBackend()

func (*Config) DefaultVarsForTiDBBackend

func (cfg *Config) DefaultVarsForTiDBBackend()

func (*Config) HasLegacyBlackWhiteList

func (cfg *Config) HasLegacyBlackWhiteList() bool

HasLegacyBlackWhiteList checks whether the deprecated [black-white-list] section was defined.

func (*Config) LoadFromGlobal

func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error

LoadFromGlobal resets the current configuration to the global settings.

func (*Config) LoadFromTOML

func (cfg *Config) LoadFromTOML(data []byte) error

LoadFromTOML overwrites the current configuration by the TOML data If data contains toml items not in Config and GlobalConfig, return an error If data contains toml items not in Config, thus won't take effect, warn user

func (*Config) String

func (cfg *Config) String() string

func (*Config) ToTLS

func (cfg *Config) ToTLS() (*common.TLS, error)

type Cron

type Cron struct {
	SwitchMode     Duration `toml:"switch-mode" json:"switch-mode"`
	LogProgress    Duration `toml:"log-progress" json:"log-progress"`
	CheckDiskQuota Duration `toml:"check-disk-quota" json:"check-disk-quota"`
}

type DBStore

type DBStore struct {
	Host       string    `toml:"host" json:"host"`
	Port       int       `toml:"port" json:"port"`
	User       string    `toml:"user" json:"user"`
	Psw        string    `toml:"password" json:"-"`
	StatusPort int       `toml:"status-port" json:"status-port"`
	PdAddr     string    `toml:"pd-addr" json:"pd-addr"`
	StrSQLMode string    `toml:"sql-mode" json:"sql-mode"`
	TLS        string    `toml:"tls" json:"tls"`
	Security   *Security `toml:"security" json:"security"`

	SQLMode          mysql.SQLMode `toml:"-" json:"-"`
	MaxAllowedPacket uint64        `toml:"max-allowed-packet" json:"max-allowed-packet"`

	DistSQLScanConcurrency     int `toml:"distsql-scan-concurrency" json:"distsql-scan-concurrency"`
	BuildStatsConcurrency      int `toml:"build-stats-concurrency" json:"build-stats-concurrency"`
	IndexSerialScanConcurrency int `toml:"index-serial-scan-concurrency" json:"index-serial-scan-concurrency"`
	ChecksumTableConcurrency   int `toml:"checksum-table-concurrency" json:"checksum-table-concurrency"`
}

type Duration

type Duration struct {
	time.Duration
}

A duration which can be deserialized from a TOML string. Implemented as https://github.com/BurntSushi/toml#using-the-encodingtextunmarshaler-interface

func (*Duration) MarshalJSON

func (d *Duration) MarshalJSON() ([]byte, error)

func (Duration) MarshalText

func (d Duration) MarshalText() ([]byte, error)

func (*Duration) UnmarshalText

func (d *Duration) UnmarshalText(text []byte) error

type FileRouteRule

type FileRouteRule struct {
	Pattern     string `json:"pattern" toml:"pattern" yaml:"pattern"`
	Path        string `json:"path" toml:"path" yaml:"path"`
	Schema      string `json:"schema" toml:"schema" yaml:"schema"`
	Table       string `json:"table" toml:"table" yaml:"table"`
	Type        string `json:"type" toml:"type" yaml:"type"`
	Key         string `json:"key" toml:"key" yaml:"key"`
	Compression string `json:"compression" toml:"compression" yaml:"compression"`
}

type GlobalCheckpoint

type GlobalCheckpoint struct {
	Enable bool `toml:"enable" json:"enable"`
}

type GlobalConfig

type GlobalConfig struct {
	App          GlobalLightning   `toml:"lightning" json:"lightning"`
	Checkpoint   GlobalCheckpoint  `toml:"checkpoint" json:"checkpoint"`
	TiDB         GlobalTiDB        `toml:"tidb" json:"tidb"`
	Mydumper     GlobalMydumper    `toml:"mydumper" json:"mydumper"`
	TikvImporter GlobalImporter    `toml:"tikv-importer" json:"tikv-importer"`
	PostRestore  GlobalPostRestore `toml:"post-restore" json:"post-restore"`
	Security     Security          `toml:"security" json:"security"`

	ConfigFileContent []byte
}

func LoadGlobalConfig

func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalConfig, error)

LoadGlobalConfig reads the arguments and fills in the GlobalConfig.

func Must

func Must(cfg *GlobalConfig, err error) *GlobalConfig

Must should be called after LoadGlobalConfig(). If LoadGlobalConfig() returns any error, this function will exit the program with an appropriate exit code.

func NewGlobalConfig

func NewGlobalConfig() *GlobalConfig

type GlobalImporter

type GlobalImporter struct {
	Addr        string `toml:"addr" json:"addr"`
	Backend     string `toml:"backend" json:"backend"`
	SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
}

type GlobalLightning

type GlobalLightning struct {
	log.Config
	StatusAddr        string `toml:"status-addr" json:"status-addr"`
	ServerMode        bool   `toml:"server-mode" json:"server-mode"`
	CheckRequirements bool   `toml:"check-requirements" json:"check-requirements"`

	// The legacy alias for setting "status-addr". The value should always the
	// same as StatusAddr, and will not be published in the JSON encoding.
	PProfPort int `toml:"pprof-port" json:"-"`
}

type GlobalMydumper

type GlobalMydumper struct {
	SourceDir string `toml:"data-source-dir" json:"data-source-dir"`
	// Deprecated
	NoSchema      bool             `toml:"no-schema" json:"no-schema"`
	Filter        []string         `toml:"filter" json:"filter"`
	IgnoreColumns []*IgnoreColumns `toml:"ignore-columns" json:"ignore-columns"`
}

type GlobalPostRestore

type GlobalPostRestore struct {
	Checksum PostOpLevel `toml:"checksum" json:"checksum"`
	Analyze  PostOpLevel `toml:"analyze" json:"analyze"`
}

type GlobalTiDB

type GlobalTiDB struct {
	Host       string `toml:"host" json:"host"`
	Port       int    `toml:"port" json:"port"`
	User       string `toml:"user" json:"user"`
	Psw        string `toml:"password" json:"-"`
	StatusPort int    `toml:"status-port" json:"status-port"`
	PdAddr     string `toml:"pd-addr" json:"pd-addr"`
	LogLevel   string `toml:"log-level" json:"log-level"`
}

type IgnoreColumns

type IgnoreColumns struct {
	DB          string   `toml:"db" json:"db"`
	Table       string   `toml:"table" json:"table"`
	TableFilter []string `toml:"table-filter" json:"table-filter"`
	Columns     []string `toml:"columns" json:"columns"`
}

type Lightning

type Lightning struct {
	TableConcurrency  int    `toml:"table-concurrency" json:"table-concurrency"`
	IndexConcurrency  int    `toml:"index-concurrency" json:"index-concurrency"`
	RegionConcurrency int    `toml:"region-concurrency" json:"region-concurrency"`
	IOConcurrency     int    `toml:"io-concurrency" json:"io-concurrency"`
	CheckRequirements bool   `toml:"check-requirements" json:"check-requirements"`
	MetaSchemaName    string `toml:"meta-schema-name" json:"meta-schema-name"`
}

type List

type List struct {
	// contains filtered or unexported fields
}

List is a goroutine-safe FIFO list of *Config, which supports removal from the middle. The list is not expected to be very long.

func NewConfigList

func NewConfigList() *List

NewConfigList creates a new ConfigList instance.

func (*List) AllIDs

func (cl *List) AllIDs() []int64

AllIDs returns a list of all task IDs in the list.

func (*List) Get

func (cl *List) Get(taskID int64) (*Config, bool)

Get obtains a task from the list given its task ID. If the task ID did not exist, the returned bool field will be false.

func (*List) MoveToBack

func (cl *List) MoveToBack(taskID int64) bool

MoveToBack moves a task to the back of the list. Returns true if the task is successfully moved (including no-op), false if the task ID did not exist.

func (*List) MoveToFront

func (cl *List) MoveToFront(taskID int64) bool

MoveToFront moves a task to the front of the list. Returns true if the task is successfully moved (including no-op), false if the task ID did not exist.

func (*List) Pop

func (cl *List) Pop(ctx context.Context) (*Config, error)

Pop removes a configuration from the front of the list. If the list is empty, this method will block until either another goroutines calls Push() or the input context expired.

If the context expired, the error field will contain the error from context.

func (*List) Push

func (cl *List) Push(cfg *Config)

Push adds a configuration to the end of the list. The field `cfg.TaskID` will be modified to include a unique ID to identify this task.

func (*List) Remove

func (cl *List) Remove(taskID int64) bool

Remove removes a task from the list given its task ID. Returns true if a task is successfully removed, false if the task ID did not exist.

type MydumperRuntime

type MydumperRuntime struct {
	ReadBlockSize    ByteSize         `toml:"read-block-size" json:"read-block-size"`
	BatchSize        ByteSize         `toml:"batch-size" json:"batch-size"`
	BatchImportRatio float64          `toml:"batch-import-ratio" json:"batch-import-ratio"`
	SourceDir        string           `toml:"data-source-dir" json:"data-source-dir"`
	CharacterSet     string           `toml:"character-set" json:"character-set"`
	CSV              CSVConfig        `toml:"csv" json:"csv"`
	MaxRegionSize    ByteSize         `toml:"max-region-size" json:"max-region-size"`
	Filter           []string         `toml:"filter" json:"filter"`
	FileRouters      []*FileRouteRule `toml:"files" json:"files"`
	// Deprecated: only used to keep the compatibility.
	NoSchema         bool             `toml:"no-schema" json:"no-schema"`
	CaseSensitive    bool             `toml:"case-sensitive" json:"case-sensitive"`
	StrictFormat     bool             `toml:"strict-format" json:"strict-format"`
	DefaultFileRules bool             `toml:"default-file-rules" json:"default-file-rules"`
	IgnoreColumns    AllIgnoreColumns `toml:"ignore-data-columns" json:"ignore-data-columns"`
}

type PostOpLevel

type PostOpLevel int
const (
	OpLevelOff PostOpLevel = iota
	OpLevelOptional
	OpLevelRequired
)

func (*PostOpLevel) FromStringValue

func (t *PostOpLevel) FromStringValue(s string) error

parser command line parameter

func (*PostOpLevel) MarshalJSON

func (t *PostOpLevel) MarshalJSON() ([]byte, error)

func (PostOpLevel) MarshalText

func (t PostOpLevel) MarshalText() ([]byte, error)

func (PostOpLevel) String

func (t PostOpLevel) String() string

func (*PostOpLevel) UnmarshalJSON

func (t *PostOpLevel) UnmarshalJSON(data []byte) error

func (*PostOpLevel) UnmarshalTOML

func (t *PostOpLevel) UnmarshalTOML(v interface{}) error

type PostRestore

type PostRestore struct {
	Checksum          PostOpLevel `toml:"checksum" json:"checksum"`
	Analyze           PostOpLevel `toml:"analyze" json:"analyze"`
	Level1Compact     bool        `toml:"level-1-compact" json:"level-1-compact"`
	PostProcessAtLast bool        `toml:"post-process-at-last" json:"post-process-at-last"`
	Compact           bool        `toml:"compact" json:"compact"`
}

PostRestore has some options which will be executed after kv restored.

type Security

type Security struct {
	CAPath   string `toml:"ca-path" json:"ca-path"`
	CertPath string `toml:"cert-path" json:"cert-path"`
	KeyPath  string `toml:"key-path" json:"key-path"`
	// RedactInfoLog indicates that whether enabling redact log
	RedactInfoLog bool `toml:"redact-info-log" json:"redact-info-log"`
}

func (*Security) RegisterMySQL

func (sec *Security) RegisterMySQL() error

RegistersMySQL registers (or deregisters) the TLS config with name "cluster" for use in `sql.Open()`. This method is goroutine-safe.

type TikvImporter

type TikvImporter struct {
	Addr             string   `toml:"addr" json:"addr"`
	Backend          string   `toml:"backend" json:"backend"`
	OnDuplicate      string   `toml:"on-duplicate" json:"on-duplicate"`
	MaxKVPairs       int      `toml:"max-kv-pairs" json:"max-kv-pairs"`
	SendKVPairs      int      `toml:"send-kv-pairs" json:"send-kv-pairs"`
	RegionSplitSize  ByteSize `toml:"region-split-size" json:"region-split-size"`
	SortedKVDir      string   `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
	DiskQuota        ByteSize `toml:"disk-quota" json:"disk-quota"`
	RangeConcurrency int      `toml:"range-concurrency" json:"range-concurrency"`

	EngineMemCacheSize      ByteSize `toml:"engine-mem-cache-size" json:"engine-mem-cache-size"`
	LocalWriterMemCacheSize ByteSize `toml:"local-writer-mem-cache-size" json:"local-writer-mem-cache-size"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL