clickhouse

package module
v2.1.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 21, 2024 License: MIT Imports: 24 Imported by: 0

README

ClickHouse Build Status Go Report Card Coverage Status

Yet another Golang SQL database driver for Yandex ClickHouse

Key features

DSN

schema://user:password@host[:port]/database?param1=value1&...&paramN=valueN
parameters
  • timeout - is the maximum amount of time a dial will wait for a connect to complete
  • idle_timeout - is the maximum amount of time an idle (keep-alive) connection will remain idle before closing itself.
  • read_timeout - specifies the amount of time to wait for a server's response
  • location - timezone to parse Date and DateTime
  • debug - enables debug logging
  • kill_query - enables killing query on the server side if we have error from transport
  • kill_query_timeout - timeout to kill query (default value is 1 second)
  • other clickhouse options can be specified as well (except default_format)

example:

http://user:password@host:8123/clicks?read_timeout=10s&write_timeout=20s

Supported data types

Notes:

  • database/sql does not allow to use big uint64 values. It is recommended use type UInt64 which is provided by driver for such kind of values.
  • type []byte are used as raw string (without quoting)
  • for passing value of type []uint8 to driver as array - please use the wrapper clickhouse.Array
  • for passing decimal value please use the wrappers clickhouse.Decimal*
  • for passing IPv4/IPv6 types use clickhouse.IP
  • for passing Tuple types use clickhouse.Tuple or structs
  • for passing Map types use clickhouse.Map

Supported request params

Clickhouse supports setting query_id and quota_key for each query. The database driver provides ability to set these parameters as well.

There are constants QueryID and QuotaKey for correct setting these params.

quota_key could be set as empty string, but query_id - does not. Keep in mind, that setting same query_id could produce exception or replace already running query depending on current Clickhouse settings. See replace_running_query for details.

See Example section for use cases.

Install

go get -u github.com/mailru/go-clickhouse/v2

Example

package main

import (
	"context"
	"database/sql"
	"log"
	"time"

	"github.com/mailru/go-clickhouse/v2"
)

func main() {
	connect, err := sql.Open("chhttp", "http://127.0.0.1:8123/default")
	if err != nil {
		log.Fatal(err)
	}
	if err := connect.Ping(); err != nil {
		log.Fatal(err)
	}

	_, err = connect.Exec(`
		CREATE TABLE IF NOT EXISTS example (
			country_code FixedString(2),
			os_id        UInt8,
			browser_id   UInt8,
			categories   Array(Int16),
			action_day   Date,
			action_time  DateTime
		) engine=Memory
	`)

	if err != nil {
		log.Fatal(err)
	}

	tx, err := connect.Begin()
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := tx.Prepare(`
		INSERT INTO example (
			country_code,
			os_id,
			browser_id,
			categories,
			action_day,
			action_time
		) VALUES (
			?, ?, ?, ?, ?, ?
		)`)

	if err != nil {
		log.Fatal(err)
	}

	for i := 0; i < 100; i++ {
		if _, err := stmt.Exec(
			"RU",
			10+i,
			100+i,
			clickhouse.Array([]int16{1, 2, 3}),
			clickhouse.Date(time.Now()),
			time.Now(),
		); err != nil {
			log.Fatal(err)
		}
	}

	if err := tx.Commit(); err != nil {
		log.Fatal(err)
	}

	rows, err := connect.Query(`
		SELECT
			country_code,
			os_id,
			browser_id,
			categories,
			action_day,
			action_time
		FROM
			example`)

	if err != nil {
		log.Fatal(err)
	}

	for rows.Next() {
		var (
			country               string
			os, browser           uint8
			categories            []int16
			actionDay, actionTime time.Time
		)
		if err := rows.Scan(
			&country,
			&os,
			&browser,
			&categories,
			&actionDay,
			&actionTime,
		); err != nil {
			log.Fatal(err)
		}
		log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_day: %s, action_time: %s",
			country, os, browser, categories, actionDay, actionTime,
		)
	}

	ctx := context.Background()
	rows, err = connect.QueryContext(context.WithValue(ctx, clickhouse.QueryID, "dummy-query-id"), `
		SELECT
			country_code,
			os_id,
			browser_id,
			categories,
			action_day,
			action_time
		FROM
			example`)

	if err != nil {
		log.Fatal(err)
	}

	for rows.Next() {
		var (
			country               string
			os, browser           uint8
			categories            []int16
			actionDay, actionTime time.Time
		)
		if err := rows.Scan(
			&country,
			&os,
			&browser,
			&categories,
			&actionDay,
			&actionTime,
		); err != nil {
			log.Fatal(err)
		}
		log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_day: %s, action_time: %s",
			country, os, browser, categories, actionDay, actionTime,
		)
	}
}

Use dbr

package main

import (
	"log"
	"time"

	_ "github.com/mailru/go-clickhouse/v2"
	"github.com/mailru/dbr"
)

func main() {
	connect, err := dbr.Open("chhttp", "http://127.0.0.1:8123/default", nil)
	if err != nil {
		log.Fatal(err)
	}
	var items []struct {
		CountryCode string    `db:"country_code"`
		OsID        uint8     `db:"os_id"`
		BrowserID   uint8     `db:"browser_id"`
		Categories  []int16   `db:"categories"`
		ActionTime  time.Time `db:"action_time"`
	}
	sess := connect.NewSession(nil)
	query := sess.Select("country_code", "os_id", "browser_id", "categories", "action_time").From("example")
	query.Where(dbr.Eq("country_code", "RU"))
	if _, err := query.Load(&items); err != nil {
		log.Fatal(err)
	}

	for _, item := range items {
		log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_time: %s",
			item.CountryCode, item.OsID, item.BrowserID, item.Categories, item.ActionTime,
		)
	}
}

Go versions

Officially support last 4 golang releases

Additional clickhouse libraries

Development

You can check the effect of changes on CI or run tests locally:

make init # dep ensure and install
make test

Remember that make init will add a few binaries used for testing

Documentation

Index

Constants

View Source 
const (
	// QueryID uses for setting query_id request param for request to Clickhouse
	QueryID key = iota
	// QuotaKey uses for setting quota_key request param for request to Clickhouse
	QuotaKey
	// RequestQueryParams uses for custom setting request params for request to Clickhouse
	// presented as map[string]string -> key1=value1&key2=value2... etc
	RequestQueryParams
)

Variables

View Source 
var (
	ErrPlaceholderCount  = errors.New("clickhouse: wrong placeholder count")
	ErrNameParams        = errors.New("clickhouse: driver does not support the use of Named Parameters")
	ErrMalformed         = errors.New("clickhouse: response is malformed")
	ErrTransportNil      = errors.New("clickhouse: transport must be set")
	ErrIncorrectResponse = errors.New("clickhouse: response must contain 'Ok.'")
	ErrNoLastInsertID    = errors.New("no LastInsertId available")
	ErrNoRowsAffected    = errors.New("no RowsAffected available")
)

Various errors the driver might return. Can change between driver versions.

Functions

func Array 

func Array(v interface{}) driver.Valuer

Array wraps slice or array into driver.Valuer interface to allow pass through it from database/sql

func Date 

func Date(t time.Time) driver.Valuer

Date returns date for t

func Decimal128 

func Decimal128(v interface{}, s int32) driver.Valuer

Decimal128 converts value to Decimal128 of precision S. The value can be a number or a string. The S (scale) parameter specifies the number of decimal places.

func Decimal32 

func Decimal32(v interface{}, s int32) driver.Valuer

Decimal32 converts value to Decimal32 of precision S. The value can be a number or a string. The S (scale) parameter specifies the number of decimal places.

func Decimal64 

func Decimal64(v interface{}, s int32) driver.Valuer

Decimal64 converts value to Decimal64 of precision S. The value can be a number or a string. The S (scale) parameter specifies the number of decimal places.

func DeregisterTLSConfig 

func DeregisterTLSConfig(key string)

DeregisterTLSConfig removes the tls.Config associated with key.

func IP 

func IP(i net.IP) driver.Valuer

IP returns compatible database format for net.IP

func Map 

func Map(v interface{}) driver.Valuer

func RegisterTLSConfig 

func RegisterTLSConfig(key string, config *tls.Config) error

RegisterTLSConfig registers a custom tls.Config to be used with sql.Open.

func Tuple 

func Tuple(v interface{}) driver.Valuer

Tuple converts a struct into a tuple struct{A string, B int}{"a", 1} -> ("a", 1)

func UInt64 

func UInt64(u uint64) driver.Valuer

UInt64 returns uint64

Types

type ChDriver 

type ChDriver struct {
}

ChDriver implements sql.Driver interface

func (*ChDriver) Open 

func (d *ChDriver) Open(dsn string) (driver.Conn, error)

Open returns new db connection

type Config 

type Config struct {
	User             string
	Password         string
	Scheme           string
	Host             string
	Database         string
	Timeout          time.Duration
	IdleTimeout      time.Duration
	ReadTimeout      time.Duration
	WriteTimeout     time.Duration
	Location         *time.Location
	Debug            bool
	UseDBLocation    bool
	GzipCompression  bool
	Params           map[string]string
	TLSConfig        string
	KillQueryOnErr   bool // kill query on the server side if we have error from transport
	KillQueryTimeout time.Duration
}

Config is a configuration parsed from a DSN string

func NewConfig 

func NewConfig() *Config

NewConfig creates a new config with default values

func ParseDSN 

func ParseDSN(dsn string) (*Config, error)

ParseDSN parses the DSN string to a Config

func (*Config) FormatDSN 

func (cfg *Config) FormatDSN() string

FormatDSN formats the given Config into a DSN string which can be passed to the driver.

type DataParser 

type DataParser interface {
	Parse(io.RuneScanner) (driver.Value, error)
	Type() reflect.Type
}

DataParser implements parsing of a driver value and reporting its type.

func NewDataParser 

func NewDataParser(t *TypeDesc, opt *DataParserOptions) (DataParser, error)

NewDataParser creates a new DataParser based on the given TypeDesc.

type DataParserOptions 

type DataParserOptions struct {
	// Location describes default location for DateTime and Date field without Timezone argument.
	Location *time.Location
	// UseDBLocation if false: always use Location, ignore DateTime argument.
	UseDBLocation bool
}

DataParserOptions describes DataParser options. Ex.: Fields Location and UseDBLocation specify timezone options.

type Error 

type Error struct {
	Code    int
	Message string
}

Error contains parsed information about server error

func (*Error) Error 

func (e *Error) Error() string

Error implements the interface error

type TypeDesc 

type TypeDesc struct {
	Name string
	Args []*TypeDesc
}

TypeDesc describes a (possibly nested) data type returned by ClickHouse.

func ParseTypeDesc 

func ParseTypeDesc(s string) (*TypeDesc, error)

ParseTypeDesc parses the type description that ClickHouse provides.

The grammar is quite simple: 

desc
    name
    name()
    name(args)
args
    desc
    desc, args

Examples: 

String
Nullable(Nothing)
Array(Tuple(Tuple(String, String), Tuple(String, UInt64)))

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.