config

Documentation

Index

• Variables
• type Config
  • func LoadConfig(configFile string) (*Config, error)
  • func (c *Config) GetTimeRangeBySplitUnit() time.Duration
• type TimeSplitUnit

Variables

var StringToTimeSplitUnit = map[string]TimeSplitUnit{
	"minute":  Minute,
	"quarter": Quarter,
	"hour":    Hour,
	"day":     Day,
}

var TimeSplitUnitToString = map[TimeSplitUnit]string{
	Minute:  "minute",
	Quarter: "quarter",
	Hour:    "hour",
	Day:     "day",
}

Types

type Config

type Config struct {
	// Source configuration
	SourceHost           string `json:"sourceHost"`
	SourcePort           int    `json:"sourcePort"`
	SourceUser           string `json:"sourceUser"`
	SourcePass           string `json:"sourcePass"`
	SourceDB             string `json:"sourceDB"`
	SourceTable          string `json:"sourceTable"`
	SourceQuery          string `json:"sourceQuery"`          // select * from table where condition
	SourceWhereCondition string `json:"sourceWhereCondition"` //example: where id > 100 and id < 200 and time > '2023-01-01'
	SourceSplitKey       string `json:"sourceSplitKey"`       // primary split key for split table, only for int type
	// the format of time field must be: 2006-01-02 15:04:05
	SourceSplitTimeKey string `json:"SourceSplitTimeKey"`           // time field for split table
	TimeSplitUnit      string `json:"TimeSplitUnit" default:"hour"` // time split unit, default is hour, option is: minute, hour, day

	// Databend configuration
	DatabendDSN      string `json:"databendDSN" default:"localhost:8000"`
	DatabendTable    string `json:"databendTable"`
	BatchSize        int    `json:"batchSize" default:"1000"`
	BatchMaxInterval int    `json:"batchMaxInterval" default:"3"` // for rate limit control

	// related docs: https://docs.databend.com/sql/sql-commands/dml/dml-copy-into-table
	CopyPurge           bool   `json:"copyPurge" default:"false"`
	CopyForce           bool   `json:"copyForce" default:"false"`
	DisableVariantCheck bool   `json:"disableVariantCheck" default:"false"`
	UserStage           string `json:"userStage" default:"~"`
	DeleteAfterSync     bool   `json:"deleteAfterSync" default:"false"`
	MaxThread           int    `json:"maxThread" default:"2"`
}

func LoadConfig

func LoadConfig(configFile string) (*Config, error)

func (*Config) GetTimeRangeBySplitUnit

func (c *Config) GetTimeRangeBySplitUnit() time.Duration

type TimeSplitUnit

type TimeSplitUnit int

const (
	Minute  TimeSplitUnit = iota
	Quarter               // 15 minutes
	Hour
	Day
)