common

Documentation

Overview

Example (ParseCommaFlag)

fmt.Println(parseCommaFlag(""))
fmt.Println(parseCommaFlag(" "))
fmt.Println(parseCommaFlag("col1, col2"))
fmt.Println(parseCommaFlag("col1,col2"))

Output:

[]
[]
[col1 col2]
[col1 col2]

Index

• Constants
• Variables
• func Escape(sql string) string
• func GetXlsxWatermark(filename string) (string, error)
• func InitTestEnv()
• func PanicIfError(err error)
• func PrintConfig(c Config)
• func ResourceLimit(cpu int, mem uint64) error
• func SQLReadLine(data []byte, atEOF bool) (advance int, token []byte, err error)
• func SetXlsxWatermark(filename string, watermark string) error
• func StringUnique(stringSlice []string) []string
• type Config
  • func ParseFlags() (Config, error)
  • func (c Config) DBAvailable(conn *sql.DB) bool
  • func (c Config) DBParseColumnTypes(header []*sql.ColumnType) []HeaderColumn
  • func (c Config) DBParseHeaderColumn(header []HeaderColumn) []string
  • func (c Config) DBParseNullString(header []HeaderColumn, columns []string) []sql.NullString
  • func (c Config) DBParserColumnNames(header []*sql.ColumnType) []string
  • func (c Config) ExecResult() (sql.Result, error)
  • func (c Config) GetColumnTypes() ([]*sql.ColumnType, error)
  • func (c Config) Hex(name string, value interface{}) (string, bool)
  • func (c Config) NewConnection() (*sql.DB, error)
  • func (c Config) ParseArray(values []string) string
  • func (c Config) ParseSchema() (header []HeaderColumn, err error)
  • func (c Config) QueryRows() (*sql.Rows, error)
  • func (c Config) QuoteKey(str string) string
  • func (c Config) QuoteString(str string) string
  • func (c Config) SQLInsertPrefix(header Row) (string, error)
  • func (c Config) SQLInsertValues(header []HeaderColumn, columns []sql.NullString) (string, error)
  • func (c Config) SQLMultiValues(counter int, prefix, values string) string
  • func (c Config) SetForeignKeyChecks(enable bool, conn *sql.DB, args ...string) error
  • func (c Config) TableTemplate() ([]HeaderColumn, error)
• type HeaderColumn
• type Row
• type Rows

Examples

• Package (ParseCommaFlag)
• Escape
• GetXlsxWatermark
• StringUnique

Constants

const (
	// UTF8BOM utf8 file BOM header
	UTF8BOM = "\xEF\xBB\xBF"

	// DefaultExcelMaxFileSize excel file max size 10MB
	DefaultExcelMaxFileSize = 10 * 1024 * 1024
)

const (
	WrongEmptySet    = `Empty set`
	WrongJSONFormat  = `JSON format error, only support string list. e.g., [ ["header", "columns" ], [ "col1", "col2" ] ]`
	WrongSQLFormat   = `SQL format error, only support basic INSERT/REPLACE syntax`
	WrongArgsCount   = `arguments count mismatch`
	WrongMaskFunc    = `wrong mask function`
	WrongArgValue    = `wrong arguments`
	WrongQuotesValue = `ANSI_QUOTES mode values not support double quotes`
	WrongColumnsCnt  = `columns count mismatch`
	WrongLargeThan0  = `n should large than 0`
)

ERROR message

const (
	WatermarkPrefix = `` /* 438-byte string literal not displayed */

	WatermarkSuffix = `` /* 201-byte string literal not displayed */

)

Variables

var TestPath string

Functions

func Escape

func Escape(sql string) string

Example

fmt.Println(Escape("abc"))
fmt.Println(Escape("abc'"))
fmt.Println(Escape(`abc"`))
fmt.Println(Escape(`abc\`))
fmt.Println(Escape(`abc中文def`))

Output:

abc
abc\'
abc\"
abc\\
abc中文def

func GetXlsxWatermark

func GetXlsxWatermark(filename string) (string, error)

Example

fmt.Println(GetXlsxWatermark(TestPath + "/test/actor.xlsx"))

Output:

watermark text <nil>

func InitTestEnv

func InitTestEnv()

func PanicIfError

func PanicIfError(err error)

PanicIfError panic if error

func PrintConfig

func PrintConfig(c Config)

func ResourceLimit

func ResourceLimit(cpu int, mem uint64) error

ResourceLimit ...

func SQLReadLine

func SQLReadLine(data []byte, atEOF bool) (advance int, token []byte, err error)

SQLReadLine bufio.Scan() SplitFunc

func SetXlsxWatermark

func SetXlsxWatermark(filename string, watermark string) error

func StringUnique

func StringUnique(stringSlice []string) []string

StringUnique string list 去重，同时也会去除空格

Example

var dupStrings = []string{"foo", "bar", "bar"}
fmt.Println(StringUnique(dupStrings))

Output:

[foo bar]

Types

type Config

type Config struct {
	// server config in my.cnf file, [client] section
	Server   string // mysql, postgres, oracle
	User     string
	Password string
	Charset  string
	DSN      string // Formated DSN string

	// other server config
	Host     string
	Socket   string
	Port     string
	Database string
	Limit    int // result lines limit

	// other config
	Query                   string // select query
	File                    string // storage file abs path
	Schema                  string // create table sql file, use for import data
	Cipher                  string // cipher config file
	Mask                    string // mask config file, csv format
	Sensitive               string // sensitive data detection config
	Verbose                 []bool // verbose mod
	CheckEmpty              bool   // check empty result, if empty raise error
	BOM                     bool   // add BOM ahead of plain text file, windows unicode chaos
	NULLString              string // NULL value write in file, e.g., NULL, None, nil, ""
	ANSIQuotes              bool   // enable ANSIQuotes
	DisableForeignKeyChecks bool   // disable foreign key checks
	MaxBufferSize           int    // bufio default buffer size
	PrintCipher             bool   // print d18n automatically generated cipher
	PrintConfig             bool   // print d18n config
	IgnoreBlank             bool   // ignore blank lines in import file
	ExtendedInsert          int    // mysqldump extended-insert

	// mask
	RandSeed int64 // rand.Seed()

	// tools
	Preview   int    // preview xlsx file, print first N lines
	Lint      bool   // file format check
	LintLevel string // lint break level
	Import    bool   // import file data into database
	Detect    bool   // detect sensitive info from data

	// digital watermark
	Watermark string // add watermark into html、xlsx

	// csv config
	Comma     rune     // Comma is the cell delimiter.
	Comments  []string // Comments are the line comment flags, e.g., #, --, //
	NoHeader  bool     // CSV file without Header line
	LineBreak string   // LineBreak whether it meets expectations
	SkipLines int      // skip first N lines when import data

	// excel config
	ExcelMaxFileSize int // excel file max size

	// sql config
	Replace        bool     // use replace into, instead of insert
	Update         []string // use update, instead of insert, primary key list, separated by comma
	Table          string   // table name
	CompleteInsert bool     // complete-insert
	HexBLOB        []string // blob column names
	IgnoreColumns  []string // ignore column list
}

Config d18n config

var TestConfig Config

func ParseFlags

func ParseFlags() (Config, error)

func (Config) DBAvailable

func (c Config) DBAvailable(conn *sql.DB) bool

DBAvailable ...

func (Config) DBParseColumnTypes

func (c Config) DBParseColumnTypes(header []*sql.ColumnType) []HeaderColumn

DBParseColumnTypes convert *sql.ColumnType to self define HeaderColumn list

func (Config) DBParseHeaderColumn

func (c Config) DBParseHeaderColumn(header []HeaderColumn) []string

DBParseHeaderColumn convert []HeaderColumn to column name string list

func (Config) DBParseNullString

func (c Config) DBParseNullString(header []HeaderColumn, columns []string) []sql.NullString

DBParseNullString convert []string to []sql.NullString

func (Config) DBParserColumnNames

func (c Config) DBParserColumnNames(header []*sql.ColumnType) []string

DBParserColumnNames convert *sql.ColumnType to column name string list

func (Config) ExecResult

func (c Config) ExecResult() (sql.Result, error)

ExecResult mysql query get result

func (Config) GetColumnTypes

func (c Config) GetColumnTypes() ([]*sql.ColumnType, error)

func (Config) Hex

func (c Config) Hex(name string, value interface{}) (string, bool)

HexBLOB ...

func (Config) NewConnection

func (c Config) NewConnection() (*sql.DB, error)

newConnection init database connection Go 各种数据库连接字符串汇总 | 鸟窝 https://colobu.com/2019/01/10/drivers-connection-string-in-Go/

func (Config) ParseArray

func (c Config) ParseArray(values []string) string

func (Config) ParseSchema

func (c Config) ParseSchema() (header []HeaderColumn, err error)

func (Config) QueryRows

func (c Config) QueryRows() (*sql.Rows, error)

QueryRows mysql query get rows

func (Config) QuoteKey

func (c Config) QuoteKey(str string) string

func (Config) QuoteString

func (c Config) QuoteString(str string) string

func (Config) SQLInsertPrefix

func (c Config) SQLInsertPrefix(header Row) (string, error)

SQLInsertPrefix ...

func (Config) SQLInsertValues

func (c Config) SQLInsertValues(header []HeaderColumn, columns []sql.NullString) (string, error)

SQLInsertValues concat values to string

func (Config) SQLMultiValues

func (c Config) SQLMultiValues(counter int, prefix, values string) string

func (Config) SetForeignKeyChecks

func (c Config) SetForeignKeyChecks(enable bool, conn *sql.DB, args ...string) error

SetForeignKeyChecks

func (Config) TableTemplate

func (c Config) TableTemplate() ([]HeaderColumn, error)

TableTemplate get header []HeaderColumn from table tamplate schema must be strict formatted, only support SHOW CREATE output info

type HeaderColumn

type HeaderColumn struct {
	Name         string
	ScanType     string
	DatabaseType string
}

HeaderColumn ...

type Row

type Row []string

Rows ...

type Rows

type Rows []Row

Rows ...