conversion

Documentation

Overview

TODO:(searce) Organize code in go style format to make this file more readable.

public constants first
key public type definitions next (although often it makes sense to put them next to public functions that use them)
then public functions (and relevant type definitions)
and helper functions and other non-public definitions last (generally in order of importance)

Index

• Variables
• func CheckExistingDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (bool, error)
• func CreateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string, ...) error
• func CreateOrUpdateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, ...) error
• func DataConv(ctx context.Context, sourceProfile profiles.SourceProfile, ...) (*writer.BatchWriter, error)
• func GetInfoSchema(sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)
• func ProcessDump(driver string, conv *internal.Conv, r *internal.Reader) error
• func ReadSessionFile(conv *internal.Conv, sessionJSON string) error
• func Report(driver string, badWrites map[string]int64, BytesRead int64, banner string, ...)
• func SchemaConv(sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ...) (*internal.Conv, error)
• func UpdateDDLForeignKeys(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string, ...) error
• func UpdateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string, ...) error
• func ValidateDDL(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) error
• func ValidateTables(ctx context.Context, client *sp.Client, spDialect string) (string, error)
• func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (dbExists bool, err error)
• func WriteBadData(bw *writer.BatchWriter, conv *internal.Conv, banner, name string, out *os.File)
• func WriteConvGeneratedFiles(conv *internal.Conv, dbName string, driver string, BytesRead int64, ...) (string, error)
• func WriteSchemaFile(conv *internal.Conv, now time.Time, name string, out *os.File, driver string)
• func WriteSessionFile(conv *internal.Conv, name string, out *os.File)

Variables

var (
	// Set the maximum number of concurrent workers during foreign key creation.
	// This number should not be too high so as to not hit the AdminQuota limit.
	// AdminQuota limits are mentioned here: https://cloud.google.com/spanner/quotas#administrative_limits
	// If facing a quota limit error, consider reducing this value.
	MaxWorkers = 50
)

Functions

func CheckExistingDb

func CheckExistingDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (bool, error)

CheckExistingDb checks whether the database with dbURI exists or not. If API call doesn't respond then user is informed after every 5 minutes on command line.

func CreateDatabase

func CreateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string, conv *internal.Conv, out *os.File, driver string, migrationType string) error

CreateDatabase returns a newly create Spanner DB. It automatically determines an appropriate project, selects a Spanner instance to use, generates a new Spanner DB name, and call into the Spanner admin interface to create the new DB.

func CreateOrUpdateDatabase

func CreateOrUpdateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI, driver string, conv *internal.Conv, out *os.File, migrationType string) error

CreatesOrUpdatesDatabase updates an existing Spanner database or creates a new one if one does not exist.

func DataConv

func DataConv(ctx context.Context, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, client *sp.Client, conv *internal.Conv, dataOnly bool, writeLimit int64) (*writer.BatchWriter, error)

DataConv performs the data conversion The SourceProfile param provides the connection details to use the go SQL library.

func GetInfoSchema

func GetInfoSchema(sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)

func ProcessDump

func ProcessDump(driver string, conv *internal.Conv, r *internal.Reader) error

ProcessDump invokes process dump function from a sql package based on driver selected.

func ReadSessionFile

func ReadSessionFile(conv *internal.Conv, sessionJSON string) error

ReadSessionFile reads a session JSON file and unmarshal it's content into *internal.Conv.

func Report

func Report(driver string, badWrites map[string]int64, BytesRead int64, banner string, conv *internal.Conv, reportFileName string, dbName string, out *os.File)

Report generates a report of schema and data conversion.

func SchemaConv

func SchemaConv(sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams) (*internal.Conv, error)

SchemaConv performs the schema conversion The SourceProfile param provides the connection details to use the go SQL library.

func UpdateDDLForeignKeys

func UpdateDDLForeignKeys(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string, conv *internal.Conv, out *os.File, driver string, migrationType string) error

UpdateDDLForeignKeys updates the Spanner database with foreign key constraints using ALTER TABLE statements.

func UpdateDatabase

func UpdateDatabase(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string, conv *internal.Conv, out *os.File, driver string) error

UpdateDatabase updates an existing spanner database.

func ValidateDDL

func ValidateDDL(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) error

ValidateDDL verifies if an existing DB's ddl follows what is supported by harbourbridge. Currently, we only support empty schema when db already exists.

func ValidateTables

func ValidateTables(ctx context.Context, client *sp.Client, spDialect string) (string, error)

ValidateTables validates that all the tables in the database are empty. It returns the name of the first non-empty table if found, and an empty string otherwise.

func VerifyDb

func VerifyDb(ctx context.Context, adminClient *database.DatabaseAdminClient, dbURI string) (dbExists bool, err error)

VerifyDb checks whether the db exists and if it does, verifies if the schema is what we currently support.

func WriteBadData

func WriteBadData(bw *writer.BatchWriter, conv *internal.Conv, banner, name string, out *os.File)

WriteBadData prints summary stats about bad rows and writes detailed info to file 'name'.

func WriteConvGeneratedFiles

func WriteConvGeneratedFiles(conv *internal.Conv, dbName string, driver string, BytesRead int64, out *os.File) (string, error)

WriteConvGeneratedFiles creates a directory labeled downloads with the current timestamp where it writes the sessionfile, report summary and DDLs then returns the directory where it writes.

func WriteSchemaFile

func WriteSchemaFile(conv *internal.Conv, now time.Time, name string, out *os.File, driver string)

WriteSchemaFile writes DDL statements in a file. It includes CREATE TABLE statements and ALTER TABLE statements to add foreign keys. The parameter name should end with a .txt.

func WriteSessionFile

func WriteSessionFile(conv *internal.Conv, name string, out *os.File)

WriteSessionFile writes conv struct to a file in JSON format.