gxutil

package module
v0.0.0-...-4ff5b01 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 23, 2020 License: MIT Imports: 47 Imported by: 0

README

gxutil

xutil for Go

Sling

Please see https://github.com/flarco/gxutil/tree/master/cmd/sling for README.

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// SMTPServer is email SMTP server host
	SMTPServer = "smtp.gmail.com"

	// SMTPPort is email SMTP server port
	SMTPPort = 465

	// SMTPUser is SMTP user name
	SMTPUser = os.Getenv("SMTP_USER")

	// SMTPPass is user password
	SMTPPass = os.Getenv("SMTP_PASS")

	// AlertEmail is the email address to send errors to
	AlertEmail = os.Getenv("ALERT_EMAIL")
)

Functions

func Check 

func Check(e error, msg string)

Check logs an error

func Compress 

func Compress(reader io.Reader) io.Reader

Compress uses gzip to compress

func Decompress 

func Decompress(reader io.Reader) (gReader io.Reader, err error)

Decompress uses gzip to decompress if it is gzip. Otherwise return same reader

func Error 

func Error(e error, msg string) error

Error returns stacktrace error with message

func F 

func F(format string, args ...interface{}) string

F : fmt.Sprintf

func GetType 

func GetType(myvar interface{}) string

GetType : return the type of an interface

func IsErr 

func IsErr(err error, msg string) bool

IsErr : checks for error

func IsErrExit 

func IsErrExit(err error, msg string)

IsErrExit : check for err and exits if error

func Log 

func Log(text string)

Log : print text

func LogC 

func LogC(text string, col string, w io.Writer)

LogC : print text in specified color

func LogCBlue 

func LogCBlue(text string)

LogCBlue prints in blue

func LogCCyan 

func LogCCyan(text string)

LogCCyan prints in white

func LogCGreen 

func LogCGreen(text string)

LogCGreen prints in green

func LogCMagenta 

func LogCMagenta(text string)

LogCMagenta print in magenta

func LogCRed 

func LogCRed(text string)

LogCRed prints in red

func LogCRedErr 

func LogCRedErr(text string)

LogCRedErr prints in red to Stderr

func LogCWhite 

func LogCWhite(text string)

LogCWhite prints in white

func LogError 

func LogError(E error)

LogError handles logging of an error, useful for reporting

func LogErrorExit 

func LogErrorExit(E error)

LogErrorExit handles logging of an error and exits, useful for reporting

func LogErrorMail 

func LogErrorMail(E error)

LogErrorMail handles logging of an error and mail it to self

func LogIfError 

func LogIfError(E error)

LogIfError handles logging of an error if it i not nil, useful for reporting

func Now 

func Now() int64

Now : Get unix epoch time in milli

func Panic 

func Panic(e error, msg string)

Panic panics on error

func ParseString 

func ParseString(s string) interface{}

ParseString return an interface string: "varchar" integer: "integer" decimal: "decimal" date: "date" datetime: "timestamp" timestamp: "timestamp" text: "text"

func PrintT 

func PrintT(v interface{})

PrintT prints the type of object

func PrintV 

func PrintV(v interface{})

PrintV prints the value of object

func Propagate 

func Propagate(err error, msg string) error

Propagate is a modified version of stacktrace Propagate

func R 

func R(format string, args ...string) string

R : Replacer R("File {file} had error {error}", "file", file, "error", err)

func RandString 

func RandString(charset string, n int) string

RandString returns a random string of len n with the provided char set charset can be `aplha` or `aplhanumeric`

func Rm 

func Rm(format string, m map[string]interface{}) string

Rm is like R, for replacing with a map

func SendMail 

func SendMail(from string, to []string, subject string, textHTML string) error

SendMail sends an email to the specific email address https://godoc.org/gopkg.in/gomail.v2#example-package

func Tee 

func Tee(reader io.Reader, limit int) io.Reader

Tee prints stream of text of reader

Types

type BaseConn 

type BaseConn struct {
	Connection
	URL  string
	Type string // the type of database for sqlx: postgres, mysql, sqlite

	Data Dataset
	// contains filtered or unexported fields
}

BaseConn is a database connection

func (*BaseConn) BulkExportStream 

func (conn *BaseConn) BulkExportStream(sql string) (ds Datastream, err error)

BulkExportStream streams the rows in bulk

func (*BaseConn) BulkImportStream 

func (conn *BaseConn) BulkImportStream(tableFName string, ds Datastream) (count uint64, err error)

BulkImportStream import the stream rows in bulk

func (*BaseConn) Close 

func (conn *BaseConn) Close() error

Close closes the connection

func (*BaseConn) Connect 

func (conn *BaseConn) Connect() error

Connect connects to the database

func (*BaseConn) Context 

func (conn *BaseConn) Context() Context

Context returns the db context

func (*BaseConn) Db 

func (conn *BaseConn) Db() *sqlx.DB

Db returns the sqlx db object

func (*BaseConn) DropTable 

func (conn *BaseConn) DropTable(tableNames ...string) (err error)

DropTable drops given table.

func (*BaseConn) DropView 

func (conn *BaseConn) DropView(viewNames ...string) (err error)

DropView drops given view.

func (*BaseConn) GenerateDDL 

func (conn *BaseConn) GenerateDDL(tableFName string, data Dataset) (string, error)

GenerateDDL genrate a DDL based on a dataset

func (*BaseConn) GenerateInsertStatement 

func (conn *BaseConn) GenerateInsertStatement(tableName string, fields []string) string

GenerateInsertStatement returns the proper INSERT statement

func (*BaseConn) GetColumns 

func (conn *BaseConn) GetColumns(tableFName string) (Dataset, error)

GetColumns returns columns for given table. `tableFName` should include schema and table, example: `schema1.table2` fields should be `column_name|data_type`

func (*BaseConn) GetColumnsFull 

func (conn *BaseConn) GetColumnsFull(tableFName string) (Dataset, error)

GetColumnsFull returns columns for given table. `tableName` should include schema and table, example: `schema1.table2` fields should be `schema_name|table_name|table_type|column_name|data_type|column_id`

func (*BaseConn) GetCount 

func (conn *BaseConn) GetCount(tableFName string) (uint64, error)

GetCount returns count of records

func (*BaseConn) GetDDL 

func (conn *BaseConn) GetDDL(tableFName string) (string, error)

GetDDL returns DDL for given table.

func (*BaseConn) GetGormConn 

func (conn *BaseConn) GetGormConn() (*gorm.DB, error)

GetGormConn returns the gorm db connection

func (*BaseConn) GetIndexes 

func (conn *BaseConn) GetIndexes(tableFName string) (Dataset, error)

GetIndexes returns indexes for given table.

func (*BaseConn) GetObjects 

func (conn *BaseConn) GetObjects(schema string, objectType string) (Dataset, error)

GetObjects returns objects (tables or views) for given schema `objectType` can be either 'table', 'view' or 'all'

func (*BaseConn) GetPrimaryKeys 

func (conn *BaseConn) GetPrimaryKeys(tableFName string) (Dataset, error)

GetPrimaryKeys returns primark keys for given table.

func (*BaseConn) GetProp 

func (conn *BaseConn) GetProp(key string) string

GetProp returns the value of a property

func (*BaseConn) GetSchemas 

func (conn *BaseConn) GetSchemas() (Dataset, error)

GetSchemas returns schemas

func (*BaseConn) GetSchemata 

func (conn *BaseConn) GetSchemata(schemaName string) (Schema, error)

GetSchemata obtain full schemata info

func (*BaseConn) GetTables 

func (conn *BaseConn) GetTables(schema string) (Dataset, error)

GetTables returns tables for given schema

func (*BaseConn) GetTemplateValue 

func (conn *BaseConn) GetTemplateValue(path string) (value string)

GetTemplateValue returns the value of the path

func (*BaseConn) GetType 

func (conn *BaseConn) GetType() string

GetType returns the type db object

func (*BaseConn) GetViews 

func (conn *BaseConn) GetViews(schema string) (Dataset, error)

GetViews returns views for given schema

func (*BaseConn) Import 

func (conn *BaseConn) Import(data Dataset, tableName string) error

Import imports `data` into `tableName`

func (*BaseConn) Init 

func (conn *BaseConn) Init() (err error)

Init initiates the connection object

func (*BaseConn) InsertBatchStream 

func (conn *BaseConn) InsertBatchStream(tableFName string, columns []string, streamRow <-chan []interface{}) error

InsertBatchStream inserts a stream into a table in batch

func (*BaseConn) InsertStream 

func (conn *BaseConn) InsertStream(tableFName string, ds Datastream) (count uint64, err error)

InsertStream inserts a stream into a table

func (*BaseConn) Kill 

func (conn *BaseConn) Kill() error

Kill kill the database connection

func (*BaseConn) LoadYAML 

func (conn *BaseConn) LoadYAML() error

LoadYAML loads the approriate yaml template

func (*BaseConn) Query 

func (conn *BaseConn) Query(sql string) (Dataset, error)

Query runs a sql query, returns `result`, `error`

func (*BaseConn) QueryContext 

func (conn *BaseConn) QueryContext(ctx context.Context, sql string) (Dataset, error)

QueryContext runs a sql query with ctx, returns `result`, `error`

func (*BaseConn) RunAnalysis 

func (conn *BaseConn) RunAnalysis(analysisName string, values map[string]interface{}) (Dataset, error)

RunAnalysis runs an analysis

func (*BaseConn) RunAnalysisField 

func (conn *BaseConn) RunAnalysisField(analysisName string, tableFName string, fields ...string) (Dataset, error)

RunAnalysisField runs a field level analysis

func (*BaseConn) RunAnalysisTable 

func (conn *BaseConn) RunAnalysisTable(analysisName string, tableFNames ...string) (Dataset, error)

RunAnalysisTable runs a table level analysis

func (*BaseConn) Schemata 

func (conn *BaseConn) Schemata() *Schemata

Schemata returns the Schemata object

func (*BaseConn) SetProp 

func (conn *BaseConn) SetProp(key string, val string)

SetProp sets the value of a property

func (*BaseConn) StreamRecords 

func (conn *BaseConn) StreamRecords(sql string) (<-chan map[string]interface{}, error)

StreamRecords the records of a sql query, returns `result`, `error`

func (*BaseConn) StreamRows 

func (conn *BaseConn) StreamRows(sql string) (ds Datastream, err error)

StreamRows the rows of a sql query, returns `result`, `error`

func (*BaseConn) StreamRowsContext 

func (conn *BaseConn) StreamRowsContext(ctx context.Context, sql string) (ds Datastream, err error)

StreamRowsContext streams the rows of a sql query with context, returns `result`, `error`

func (*BaseConn) Template 

func (conn *BaseConn) Template() *Template

Template returns the Template object

type CSV 

type CSV struct {
	Path    string
	Columns []Column
	File    *os.File
	Data    *Dataset
	Reader  io.Reader
}

CSV is a csv object

func (*CSV) InferSchema 

func (c *CSV) InferSchema() error

InferSchema returns a sample of n rows

func (*CSV) NewReader 

func (c *CSV) NewReader() (*io.PipeReader, error)

NewReader creates a Reader

func (*CSV) ReadStream 

func (c *CSV) ReadStream() (ds Datastream, err error)

ReadStream returns the read CSV stream with Line 1 as header

func (*CSV) Sample 

func (c *CSV) Sample(n int) (Dataset, error)

Sample returns a sample of n rows

func (*CSV) WriteStream 

func (c *CSV) WriteStream(ds Datastream) (cnt uint64, err error)

WriteStream to CSV file

type Column 

type Column struct {
	Position int64  `json:"position"`
	Name     string `json:"name"`
	Type     string `json:"type"`
	// contains filtered or unexported fields
}

Column represents a schemata column

type ColumnStats 

type ColumnStats struct {
	// contains filtered or unexported fields
}

ColumnStats holds statistics for a column

type Connection 

type Connection interface {
	Init() error
	Connect() error
	Kill() error
	Close() error
	GetType() string
	GetGormConn() (*gorm.DB, error)
	LoadYAML() error
	StreamRows(sql string) (Datastream, error)
	BulkExportStream(sql string) (Datastream, error)
	BulkImportStream(tableFName string, ds Datastream) (count uint64, err error)
	Query(sql string) (Dataset, error)
	QueryContext(ctx context.Context, sql string) (Dataset, error)
	GenerateDDL(tableFName string, data Dataset) (string, error)
	GenerateInsertStatement(tableName string, fields []string) string
	DropTable(...string) error
	DropView(...string) error
	InsertStream(tableFName string, ds Datastream) (count uint64, err error)
	Db() *sqlx.DB
	Schemata() *Schemata
	Template() *Template
	SetProp(string, string)
	GetProp(string) string
	GetTemplateValue(path string) (value string)
	Context() Context

	StreamRecords(sql string) (<-chan map[string]interface{}, error)
	GetDDL(string) (string, error)
	GetSchemata(string) (Schema, error)
	GetSchemas() (Dataset, error)
	GetTables(string) (Dataset, error)
	GetViews(string) (Dataset, error)
	GetColumns(string) (Dataset, error)
	GetPrimaryKeys(string) (Dataset, error)
	GetIndexes(string) (Dataset, error)
	GetColumnsFull(string) (Dataset, error)
	GetCount(string) (uint64, error)
	RunAnalysis(string, map[string]interface{}) (Dataset, error)
	RunAnalysisTable(string, ...string) (Dataset, error)
	RunAnalysisField(string, string, ...string) (Dataset, error)
	// contains filtered or unexported methods
}

Connection is the Base interface for Connections

func GetConn 

func GetConn(URL string) Connection

GetConn return the most proper connection for a given database

type Context 

type Context struct {
	// contains filtered or unexported fields
}

Context is to manage context

type Dataset 

type Dataset struct {
	Result   *sqlx.Rows
	Columns  []Column
	Rows     [][]interface{}
	SQL      string
	Duration float64
}

Dataset is a query returned dataset

func ReadCsv 

func ReadCsv(path string) (Dataset, error)

ReadCsv reads CSV and returns dataset

func (*Dataset) GetFields 

func (data *Dataset) GetFields() []string

GetFields return the fields of the Data

func (*Dataset) InferColumnTypes 

func (data *Dataset) InferColumnTypes()

InferColumnTypes determines the columns types

func (*Dataset) Records 

func (data *Dataset) Records() []map[string]interface{}

Records return rows of maps

func (*Dataset) WriteCsv 

func (data *Dataset) WriteCsv(path string) error

WriteCsv writes to a csv file

type Datastream 

type Datastream struct {
	Columns []Column
	Rows    chan []interface{}
	Buffer  [][]interface{}
	// contains filtered or unexported fields
}

Datastream is a stream of rows

func ReadCsvStream 

func ReadCsvStream(path string) (Datastream, error)

ReadCsvStream reads CSV and returns datasream

func (*Datastream) Collect 

func (ds *Datastream) Collect() Dataset

Collect reads a stream and return a dataset

func (*Datastream) GetFields 

func (ds *Datastream) GetFields() []string

GetFields return the fields of the Data

func (*Datastream) InferTypes 

func (ds *Datastream) InferTypes()

InferTypes infers types if needed and add to Buffer Experimental....

func (*Datastream) NewCsvReader 

func (ds *Datastream) NewCsvReader(limit int) *io.PipeReader

NewCsvReader creates a Reader with limit. If limit == 0, then read all rows.

type MsSQLServerConn 

type MsSQLServerConn struct {
	BaseConn
	URL string
}

MsSQLServerConn is a Microsoft SQL Server connection

func (*MsSQLServerConn) Init 

func (conn *MsSQLServerConn) Init() error

Init initiates the object

type MySQLConn 

type MySQLConn struct {
	BaseConn
	URL string
}

MySQLConn is a Postgres connection

func (*MySQLConn) BulkExportStream 

func (conn *MySQLConn) BulkExportStream(sql string) (ds Datastream, err error)

BulkExportStream bulk Export

func (*MySQLConn) BulkImportStream 

func (conn *MySQLConn) BulkImportStream(tableFName string, ds Datastream) (count uint64, err error)

BulkImportStream bulk import stream

func (*MySQLConn) Init 

func (conn *MySQLConn) Init() error

Init initiates the object

func (*MySQLConn) LoadDataInFile 

func (conn *MySQLConn) LoadDataInFile(tableFName string, ds Datastream) (count uint64, err error)

LoadDataInFile Bulk Import

func (*MySQLConn) LoadDataOutFile 

func (conn *MySQLConn) LoadDataOutFile(sql string) (stdOutReader io.Reader, err error)

LoadDataOutFile Bulk Export Possible error: ERROR 1227 (42000) at line 1: Access denied; you need (at least one of) the FILE privilege(s) for this operation File priviledge needs to be granted to user also the --secure-file-priv option needs to be set properly for it to work.

type OracleConn 

type OracleConn struct {
	BaseConn
	URL string
}

OracleConn is a Postgres connection

func (*OracleConn) BulkImportStream 

func (conn *OracleConn) BulkImportStream(tableFName string, ds Datastream) (count uint64, err error)

BulkImportStream bulk import stream

func (*OracleConn) Init 

func (conn *OracleConn) Init() error

Init initiates the object

func (*OracleConn) SQLLoad 

func (conn *OracleConn) SQLLoad(tableFName string, ds Datastream) (count uint64, err error)

SQLLoad uses sqlldr to Bulk Import cat test1.csv | sqlldr system/oracle@oracle.host:1521/xe control=sqlldr.ctl log=/dev/stdout bad=/dev/stderr

type Parquet 

type Parquet struct {
	Path    string
	Columns []Column
	File    *os.File
	PFile   source.ParquetFile
	Data    *Dataset
}

Parquet is a parquet object

func (*Parquet) ReadStream 

func (p *Parquet) ReadStream() (Datastream, error)

ReadStream returns the read Parquet stream into a Datastream https://github.com/xitongsys/parquet-go/blob/master/example/read_partial.go

func (*Parquet) WriteStream 

func (p *Parquet) WriteStream(ds Datastream) error

WriteStream to Parquet file from datastream

type PostgresConn 

type PostgresConn struct {
	BaseConn
	URL string
}

PostgresConn is a Postgres connection

func (*PostgresConn) BulkExportStream 

func (conn *PostgresConn) BulkExportStream(sql string) (ds Datastream, err error)

BulkExportStream uses the bulk dumping (COPY)

func (*PostgresConn) BulkImportStream 

func (conn *PostgresConn) BulkImportStream(tableFName string, ds Datastream) (count uint64, err error)

BulkImportStream inserts a stream into a table

func (*PostgresConn) CopyToStdout 

func (conn *PostgresConn) CopyToStdout(sql string) (stdOutReader io.Reader, err error)

CopyToStdout Copy TO STDOUT

func (*PostgresConn) Init 

func (conn *PostgresConn) Init() error

Init initiates the object

type RedshiftConn 

type RedshiftConn struct {
	BaseConn
	URL string
}

RedshiftConn is a Redshift connection

func (*RedshiftConn) BulkExportStream 

func (conn *RedshiftConn) BulkExportStream(sql string) (ds Datastream, err error)

BulkExportStream reads in bulk

func (*RedshiftConn) BulkImportStream 

func (conn *RedshiftConn) BulkImportStream(tableFName string, ds Datastream) (count uint64, err error)

BulkImportStream inserts a stream into a table. For redshift we need to create CSVs in S3 and then use the COPY command.

func (*RedshiftConn) Init 

func (conn *RedshiftConn) Init() error

Init initiates the object

func (*RedshiftConn) Unload 

func (conn *RedshiftConn) Unload(sql string) (s3Path string, err error)

Unload unloads a query to S3

type S3 

type S3 struct {
	Bucket string
	Region string
}

S3 is a AWS s3 object

func (*S3) Delete 

func (s *S3) Delete(key string) (err error)

Delete deletes an s3 object at provided key

func (*S3) GetRegion 

func (s *S3) GetRegion() (region string)

GetRegion determines the region of the bucket

func (*S3) List 

func (s *S3) List(key string) (paths []string, err error)

List S3 objects from a key/prefix

func (*S3) ReadStream 

func (s *S3) ReadStream(key string) (*io.PipeReader, error)

ReadStream read from an S3 bucket (download) Example: S3 file stream into Database or CSV

func (*S3) WriteStream 

func (s *S3) WriteStream(key string, reader io.Reader) error

WriteStream write to an S3 bucket (upload) Example: Database or CSV stream into S3 file

type Schema 

type Schema struct {
	Name   string `json:"name"`
	Tables map[string]Table
}

Schema represents a schemata schema

type Schemata 

type Schemata struct {
	Schemas map[string]Schema
	Tables  map[string]*Table // all tables with full name lower case (schema.table)
}

Schemata contains the full schema for a connection

type Table 

type Table struct {
	Name       string `json:"name"`
	FullName   string `json:"full_name"`
	IsView     bool   `json:"is_view"` // whether is a view
	Columns    []Column
	ColumnsMap map[string]*Column
}

Table represents a schemata table

type Template 

type Template struct {
	Core           map[string]string
	Metadata       map[string]string
	Analysis       map[string]string
	Function       map[string]string
	GeneralTypeMap map[string]string `yaml:"general_type_map"`
	NativeTypeMap  map[string]string `yaml:"native_type_map"`
	Variable       map[string]string
}

Template is a database YAML template

Source Files

View all Source files

Directories

Path Synopsis
cmd
sling
sling-api

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.