databrickssql

Documentation

Index

• Variables
• func GenCreateExternalTableSQL(tableName string, tableColumns []cloudstorage.TableCol, storageUri string, ...) (string, error)
• func GenCreateTableSQL(tableName string, tableColumns []cloudstorage.TableCol) (string, error)
• func GenDDLViaColumnsDiff(prevColumns []cloudstorage.TableCol, curTableDef cloudstorage.TableDefinition) ([]string, error)
• func GenDropTableSQL(sourceTable string) string
• func GenMergeIntoSQL(tableDef cloudstorage.TableDefinition, tableName, externalTableName string) string
• func GetCredentialNameSet(db *sql.DB) (map[string]interface{}, error)
• func GetDatabricksColumnString(column cloudstorage.TableCol) (string, error)
• func GetDatabricksTypeString(column cloudstorage.TableCol) (string, error)
• func LoadCSVFromS3(db *sql.DB, columns []cloudstorage.TableCol, ...) error
• type DataBricksConfig
  • func (config *DataBricksConfig) OpenDB() (*sql.DB, error)
• type DatabricksConnector
  • func NewDatabricksConnector(dbConfig *DataBricksConfig, credential string, storageURI *url.URL) (*DatabricksConnector, error)
  • func (dc *DatabricksConnector) Close()
  • func (dc *DatabricksConnector) CopyTableSchema(sourceDatabase string, sourceTable string, sourceTiDBConn *sql.DB) error
  • func (dc *DatabricksConnector) ExecDDL(tableDef cloudstorage.TableDefinition) error
  • func (dc *DatabricksConnector) InitSchema(columns []cloudstorage.TableCol) error
  • func (dc *DatabricksConnector) LoadIncrement(tableDef cloudstorage.TableDefinition, filePath string) error
  • func (dc *DatabricksConnector) LoadSnapshot(targetTable, filePath string) error

Variables

var TiDB2DatabricksTypeMap = map[string]string{
	"text":       "STRING",
	"tinytext":   "STRING",
	"mediumtext": "STRING",
	"longtext":   "STRING",
	"blob":       "STRING",
	"tinyblob":   "STRING",
	"mediumblob": "STRING",
	"longblob":   "STRING",
	"varchar":    "STRING",
	"char":       "STRING",
	"int":        "INT",
	"mediumint":  "INT",
	"tinyint":    "TINYINT",
	"smallint":   "SMALLINT",
	"bigint":     "BIGINT",
	"float":      "FLOAT",
	"double":     "DOUBLE",
	"decimal":    "DECIMAL",
	"numeric":    "NUMERIC",
	"bool":       "BOOLEAN",
	"boolean":    "BOOLEAN",
	"date":       "DATE",
	"datetime":   "TIMESTAMP_NTZ",
	"timestamp":  "TIMESTAMP",
	"time":       "TIMESTAMP_NTZ",
}

TiDB2DatabricksTypeMap is a map from TiDB type to Databricks type. Please refer to https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html Databricks don't support the `BINARY` type in the external table with the CSV file which are `tidb2dw` used. So we drop the `binary` and `varbinary` type in the map.

Functions

func GenCreateExternalTableSQL

func GenCreateExternalTableSQL(tableName string, tableColumns []cloudstorage.TableCol, storageUri string, credential string) (string, error)

func GenCreateTableSQL

func GenCreateTableSQL(tableName string, tableColumns []cloudstorage.TableCol) (string, error)

func GenDDLViaColumnsDiff

func GenDDLViaColumnsDiff(prevColumns []cloudstorage.TableCol, curTableDef cloudstorage.TableDefinition) ([]string, error)

func GenDropTableSQL

func GenDropTableSQL(sourceTable string) string

func GenMergeIntoSQL

func GenMergeIntoSQL(tableDef cloudstorage.TableDefinition, tableName, externalTableName string) string

func GetCredentialNameSet

func GetCredentialNameSet(db *sql.DB) (map[string]interface{}, error)

GetCredentialNameSet returns all storage credential names in the database

func GetDatabricksColumnString

func GetDatabricksColumnString(column cloudstorage.TableCol) (string, error)

GetDatabricksColumnString returns a string describing the column in Databricks, e.g. "id INT NOT NULL DEFAULT '0'" Refer to: https://dev.mysql.com/doc/refman/8.0/en/data-types.html https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html

func GetDatabricksTypeString

func GetDatabricksTypeString(column cloudstorage.TableCol) (string, error)

func LoadCSVFromS3

func LoadCSVFromS3(db *sql.DB, columns []cloudstorage.TableCol, targetTable, storageUri, filePath string, credential string) error

Types

type DataBricksConfig

type DataBricksConfig struct {
	Host     string
	Port     int
	Token    string
	Endpoint string
	Catalog  string
	Schema   string
}

func (*DataBricksConfig) OpenDB

func (config *DataBricksConfig) OpenDB() (*sql.DB, error)

type DatabricksConnector

type DatabricksConnector struct {
	// contains filtered or unexported fields
}

func NewDatabricksConnector

func NewDatabricksConnector(dbConfig *DataBricksConfig, credential string, storageURI *url.URL) (*DatabricksConnector, error)

func (*DatabricksConnector) Close

func (dc *DatabricksConnector) Close()

func (*DatabricksConnector) CopyTableSchema

func (dc *DatabricksConnector) CopyTableSchema(sourceDatabase string, sourceTable string, sourceTiDBConn *sql.DB) error

func (*DatabricksConnector) ExecDDL

func (dc *DatabricksConnector) ExecDDL(tableDef cloudstorage.TableDefinition) error

func (*DatabricksConnector) InitSchema

func (dc *DatabricksConnector) InitSchema(columns []cloudstorage.TableCol) error

func (*DatabricksConnector) LoadIncrement

func (dc *DatabricksConnector) LoadIncrement(tableDef cloudstorage.TableDefinition, filePath string) error

func (*DatabricksConnector) LoadSnapshot

func (dc *DatabricksConnector) LoadSnapshot(targetTable, filePath string) error