clickhouse

Documentation

Index

• Constants
• func Init()
• func WithConfig(h *Clickhouse, config *config.Config)
• type Clickhouse
  • func NewClickhouse() *Clickhouse
  • func (ch *Clickhouse) AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)
  • func (*Clickhouse) AlterColumn(_, _, _ string) (model.AlterTableResponse, error)
  • func (ch *Clickhouse) Cleanup()
  • func (ch *Clickhouse) ColumnsWithDataTypes(tableName string, columns map[string]string, notNullableColumns []string) string
  • func (ch *Clickhouse) Connect(warehouse warehouseutils.Warehouse) (client.Client, error)
  • func (ch *Clickhouse) ConnectToClickhouse(cred Credentials, includeDBInConn bool) (*sql.DB, error)
  • func (*Clickhouse) CrashRecover(_ warehouseutils.Warehouse) (err error)
  • func (ch *Clickhouse) CreateSchema() (err error)
  • func (ch *Clickhouse) CreateTable(tableName string, columns map[string]string) (err error)
  • func (*Clickhouse) DeleteBy([]string, warehouseutils.DeleteByParams) error
  • func (*Clickhouse) DownloadIdentityRules(*misc.GZipWriter) (err error)
  • func (ch *Clickhouse) DownloadLoadFiles(tableName string) ([]string, error)
  • func (ch *Clickhouse) DropTable(tableName string) (err error)
  • func (ch *Clickhouse) ErrorMappings() []model.JobError
  • func (ch *Clickhouse) FetchSchema(warehouse warehouseutils.Warehouse) (schema, unrecognizedSchema warehouseutils.SchemaT, err error)
  • func (ch *Clickhouse) GetLogIdentifier(args ...string) string
  • func (ch *Clickhouse) GetTotalCountInTable(ctx context.Context, tableName string) (int64, error)
  • func (*Clickhouse) IsEmpty(_ warehouseutils.Warehouse) (empty bool, err error)
  • func (*Clickhouse) LoadIdentityMappingsTable() (err error)
  • func (*Clickhouse) LoadIdentityMergeRulesTable() (err error)
  • func (ch *Clickhouse) LoadTable(tableName string) error
  • func (ch *Clickhouse) LoadTestTable(_, tableName string, payloadMap map[string]interface{}, _ string) (err error)
  • func (ch *Clickhouse) LoadUserTables() (errorMap map[string]error)
  • func (ch *Clickhouse) SetConnectionTimeout(timeout time.Duration)
  • func (ch *Clickhouse) Setup(warehouse warehouseutils.Warehouse, uploader warehouseutils.UploaderI) (err error)
  • func (ch *Clickhouse) TestConnection(warehouse warehouseutils.Warehouse) (err error)
  • func (ch *Clickhouse) UseS3CopyEngineForLoading() bool
• type Credentials

Constants

const (
	Cluster = "cluster"
)

Functions

func Init

func Init()

func WithConfig

func WithConfig(h *Clickhouse, config *config.Config)

Types

type Clickhouse

type Clickhouse struct {
	DB            *sql.DB
	Namespace     string
	ObjectStorage string
	Warehouse     warehouseutils.Warehouse
	Uploader      warehouseutils.UploaderI

	ConnectTimeout              time.Duration
	Logger                      logger.Logger
	QueryDebugLogs              string
	BlockSize                   string
	PoolSize                    string
	ReadTimeout                 string
	WriteTimeout                string
	Compress                    bool
	DisableNullable             bool
	ExecTimeOutInSeconds        time.Duration
	CommitTimeOutInSeconds      time.Duration
	LoadTableFailureRetries     int
	NumWorkersDownloadLoadFiles int
	S3EngineEnabledWorkspaceIDs []string
	// contains filtered or unexported fields
}

func NewClickhouse

func NewClickhouse() *Clickhouse

func (*Clickhouse) AddColumns

func (ch *Clickhouse) AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)

func (*Clickhouse) AlterColumn

func (*Clickhouse) AlterColumn(_, _, _ string) (model.AlterTableResponse, error)

func (*Clickhouse) Cleanup

func (ch *Clickhouse) Cleanup()

func (*Clickhouse) ColumnsWithDataTypes

func (ch *Clickhouse) ColumnsWithDataTypes(tableName string, columns map[string]string, notNullableColumns []string) string

ColumnsWithDataTypes creates columns and its datatype into sql format for creating table

func (*Clickhouse) Connect

func (ch *Clickhouse) Connect(warehouse warehouseutils.Warehouse) (client.Client, error)

func (*Clickhouse) ConnectToClickhouse

func (ch *Clickhouse) ConnectToClickhouse(cred Credentials, includeDBInConn bool) (*sql.DB, error)

ConnectToClickhouse connects to clickhouse with provided credentials

func (*Clickhouse) CrashRecover

func (*Clickhouse) CrashRecover(_ warehouseutils.Warehouse) (err error)

func (*Clickhouse) CreateSchema

func (ch *Clickhouse) CreateSchema() (err error)

func (*Clickhouse) CreateTable

func (ch *Clickhouse) CreateTable(tableName string, columns map[string]string) (err error)

CreateTable creates table with engine ReplacingMergeTree(), this is used for dedupe event data and replace it will the latest data if duplicate data found. This logic is handled by clickhouse The engine differs from MergeTree in that it removes duplicate entries with the same sorting key value.

func (*Clickhouse) DeleteBy

func (*Clickhouse) DeleteBy([]string, warehouseutils.DeleteByParams) error

func (*Clickhouse) DownloadIdentityRules

func (*Clickhouse) DownloadIdentityRules(*misc.GZipWriter) (err error)

func (*Clickhouse) DownloadLoadFiles

func (ch *Clickhouse) DownloadLoadFiles(tableName string) ([]string, error)

DownloadLoadFiles downloads load files for the tableName and gives file names

func (*Clickhouse) DropTable

func (ch *Clickhouse) DropTable(tableName string) (err error)

func (*Clickhouse) ErrorMappings

func (ch *Clickhouse) ErrorMappings() []model.JobError

func (*Clickhouse) FetchSchema

func (ch *Clickhouse) FetchSchema(warehouse warehouseutils.Warehouse) (schema, unrecognizedSchema warehouseutils.SchemaT, err error)

FetchSchema queries clickhouse and returns the schema associated with provided namespace

func (*Clickhouse) GetLogIdentifier

func (ch *Clickhouse) GetLogIdentifier(args ...string) string

func (*Clickhouse) GetTotalCountInTable

func (ch *Clickhouse) GetTotalCountInTable(ctx context.Context, tableName string) (int64, error)

func (*Clickhouse) IsEmpty

func (*Clickhouse) IsEmpty(_ warehouseutils.Warehouse) (empty bool, err error)

func (*Clickhouse) LoadIdentityMappingsTable

func (*Clickhouse) LoadIdentityMappingsTable() (err error)

func (*Clickhouse) LoadIdentityMergeRulesTable

func (*Clickhouse) LoadIdentityMergeRulesTable() (err error)

func (*Clickhouse) LoadTable

func (ch *Clickhouse) LoadTable(tableName string) error

func (*Clickhouse) LoadTestTable

func (ch *Clickhouse) LoadTestTable(_, tableName string, payloadMap map[string]interface{}, _ string) (err error)

func (*Clickhouse) LoadUserTables

func (ch *Clickhouse) LoadUserTables() (errorMap map[string]error)

func (*Clickhouse) SetConnectionTimeout

func (ch *Clickhouse) SetConnectionTimeout(timeout time.Duration)

func (*Clickhouse) Setup

func (ch *Clickhouse) Setup(warehouse warehouseutils.Warehouse, uploader warehouseutils.UploaderI) (err error)

func (*Clickhouse) TestConnection

func (ch *Clickhouse) TestConnection(warehouse warehouseutils.Warehouse) (err error)

TestConnection is used destination connection tester to test the clickhouse connection

func (*Clickhouse) UseS3CopyEngineForLoading

func (ch *Clickhouse) UseS3CopyEngineForLoading() bool

type Credentials

type Credentials struct {
	Host          string
	DBName        string
	User          string
	Password      string
	Port          string
	Secure        string
	SkipVerify    string
	TLSConfigName string
	// contains filtered or unexported fields
}