bigquerysql

Documentation

Index

• Variables
• func GenCreateSchema(columns []cloudstorage.TableCol, pkColumns []string, datasetID, tableID string) (string, error)
• func GenDDLViaColumnsDiff(datasetID, tableID string, prevColumns []cloudstorage.TableCol, ...) ([]string, error)
• func GenMergeInto(tableDef cloudstorage.TableDefinition, ...) string
• func GetBigQueryColumnString(column cloudstorage.TableCol, createTable bool) (string, error)
• func GetBigQueryColumnTypeString(column cloudstorage.TableCol) (string, error)
• func GetColumnModifyString(diff *tidbsql.ColumnDiff) (string, error)
• type BigQueryConfig
  • func (cfg *BigQueryConfig) NewClient() (*bigquery.Client, error)
• type BigQueryConnector
  • func NewBigQueryConnector(bqConfig *BigQueryConfig, incrementTableID, datasetID, tableID string, ...) (*BigQueryConnector, error)
  • func (bc *BigQueryConnector) Close()
  • func (bc *BigQueryConnector) CopyTableSchema(sourceDatabase string, sourceTable string, sourceTiDBConn *sql.DB) error
  • func (bc *BigQueryConnector) ExecDDL(tableDef cloudstorage.TableDefinition) error
  • func (bc *BigQueryConnector) InitSchema(columns []cloudstorage.TableCol) error
  • func (bc *BigQueryConnector) LoadIncrement(tableDef cloudstorage.TableDefinition, filePath string) error
  • func (bc *BigQueryConnector) LoadSnapshot(targetTable, filePath string) error

Variables

var TiDB2BigQueryTypeMap map[string]string = map[string]string{
	"bigint":             "INT64",
	"bigint unsigned":    "NUMERIC",
	"binary":             "BYTES",
	"bit":                "BOOL",
	"blob":               "BYTES",
	"char":               "STRING",
	"date":               "DATE",
	"datetime":           "DATETIME",
	"decimal":            "NUMERIC",
	"double":             "FLOAT64",
	"float":              "FLOAT64",
	"int":                "INT64",
	"int unsigned":       "INT64",
	"json":               "STRING",
	"longblob":           "BYTES",
	"longtext":           "STRING",
	"mediumblob":         "BYTES",
	"mediumint":          "INT64",
	"mediumint unsigned": "INT64",
	"mediumtext":         "STRING",
	"set":                "STRING",
	"smallint":           "INT64",
	"smallint unsigned":  "INT64",
	"text":               "STRING",
	"time":               "TIME",
	"timestamp":          "TIMESTAMP",
	"tinyblob":           "BYTES",
	"tinyint":            "INT64",
	"tinyint unsigned":   "INT64",
	"tinytext":           "STRING",
	"varbinary":          "BYTES",
	"varchar":            "STRING",
	"year":               "INT64",
}

TiDB2BigQueryTypeMap is a map from TiDB type to BigQuery type. Reference: https://cloud.google.com/data-fusion/docs/reference/replication-data-types#mysql

Functions

func GenCreateSchema

func GenCreateSchema(columns []cloudstorage.TableCol, pkColumns []string, datasetID, tableID string) (string, error)

func GenDDLViaColumnsDiff

func GenDDLViaColumnsDiff(datasetID, tableID string, prevColumns []cloudstorage.TableCol, curTableDef cloudstorage.TableDefinition) ([]string, error)

func GenMergeInto

func GenMergeInto(tableDef cloudstorage.TableDefinition, datasetID, tableID, externalTableID string) string

func GetBigQueryColumnString

func GetBigQueryColumnString(column cloudstorage.TableCol, createTable bool) (string, error)

GetBigQueryColumnString returns a string describing the column in BigQuery, e.g. "id INT NOT NULL DEFAULT '0'" Refer to: https://dev.mysql.com/doc/refman/8.0/en/data-types.html https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#column_name_and_column_schema

func GetBigQueryColumnTypeString

func GetBigQueryColumnTypeString(column cloudstorage.TableCol) (string, error)

func GetColumnModifyString

func GetColumnModifyString(diff *tidbsql.ColumnDiff) (string, error)

Types

type BigQueryConfig

type BigQueryConfig struct {
	ProjectID           string
	DatasetID           string
	CredentialsFilePath string // path to google credentials file
}

func (*BigQueryConfig) NewClient

func (cfg *BigQueryConfig) NewClient() (*bigquery.Client, error)

type BigQueryConnector

type BigQueryConnector struct {
	// contains filtered or unexported fields
}

func NewBigQueryConnector

func NewBigQueryConnector(bqConfig *BigQueryConfig, incrementTableID, datasetID, tableID string, storageURI *url.URL) (*BigQueryConnector, error)

func (*BigQueryConnector) Close

func (bc *BigQueryConnector) Close()

func (*BigQueryConnector) CopyTableSchema

func (bc *BigQueryConnector) CopyTableSchema(sourceDatabase string, sourceTable string, sourceTiDBConn *sql.DB) error

CopyTableSchema copies table schema from TiDB to BigQuery If table exists, delete it first

func (*BigQueryConnector) ExecDDL

func (bc *BigQueryConnector) ExecDDL(tableDef cloudstorage.TableDefinition) error

func (*BigQueryConnector) InitSchema

func (bc *BigQueryConnector) InitSchema(columns []cloudstorage.TableCol) error

func (*BigQueryConnector) LoadIncrement

func (bc *BigQueryConnector) LoadIncrement(tableDef cloudstorage.TableDefinition, filePath string) error

func (*BigQueryConnector) LoadSnapshot

func (bc *BigQueryConnector) LoadSnapshot(targetTable, filePath string) error