business

Documentation

Index

• Constants
• type ArchiveHDFS
  • func (this *ArchiveHDFS) ClearHDFS() error
  • func (this *ArchiveHDFS) Export(host, table string, slots []time.Time)
  • func (this *ArchiveHDFS) ExportSlot(host, table string, seq int, slotBeg, slotEnd time.Time)
  • func (this *ArchiveHDFS) ExportToHDFS() (err error)
  • func (this *ArchiveHDFS) FillArchiveDefault()
  • func (this *ArchiveHDFS) GetSlots(host, table string) (slots []time.Time, err error)
  • func (this *ArchiveHDFS) GetSortingInfo() (err error)
  • func (this *ArchiveHDFS) InitConns() (err error)
  • func (this *ArchiveHDFS) SelectUint64(host, query string) (res uint64, err error)
• type CKRebalance
  • func (this *CKRebalance) DoRebalance() (err error)
  • func (this *CKRebalance) ExecutePlan(database string, tbl *TblPartitions) (err error)
  • func (this *CKRebalance) GeneratePlan(tablename string, tbls []*TblPartitions)
  • func (this *CKRebalance) GetRepTables() (err error)
  • func (this *CKRebalance) GetState(database string, table string) (tbls []*TblPartitions, err error)
  • func (this *CKRebalance) GetTables() (err error)
  • func (this *CKRebalance) InitCKConns() (err error)
  • func (this *CKRebalance) InitSshConns(database string) (err error)
• type PurgerRange
  • func NewPurgerRange(hosts []string, port int, user string, password string, database string, ...) *PurgerRange
  • func (this *PurgerRange) InitConns() (err error)
  • func (this *PurgerRange) PurgeTable(table string) (err error)
• type TblPartitions

Constants

const (
	HdfsUserDefault    string = "root"
	MaxFileSizeDefault int    = 1e10
	CKDatabaseDefault  string = "default"
	ParallelismDefault int    = 4

	DateLayout     string = "2006-01-02"
	DateTimeLayout string = "2006-01-02 15:04:05"
	SlotTimeFormat string = "20060102150405"
)

Types

type ArchiveHDFS

type ArchiveHDFS struct {
	Hosts       []string
	Port        int
	User        string
	Password    string
	Database    string
	Tables      []string
	Begin       string
	End         string
	MaxFileSize int
	HdfsAddr    string
	HdfsUser    string
	HdfsDir     string
	Parallelism int
	Pool        *common.WorkerPool
}

func (*ArchiveHDFS) ClearHDFS

func (this *ArchiveHDFS) ClearHDFS() error

func (*ArchiveHDFS) Export

func (this *ArchiveHDFS) Export(host, table string, slots []time.Time)

func (*ArchiveHDFS) ExportSlot

func (this *ArchiveHDFS) ExportSlot(host, table string, seq int, slotBeg, slotEnd time.Time)

func (*ArchiveHDFS) ExportToHDFS

func (this *ArchiveHDFS) ExportToHDFS() (err error)

func (*ArchiveHDFS) FillArchiveDefault

func (this *ArchiveHDFS) FillArchiveDefault()

func (*ArchiveHDFS) GetSlots

func (this *ArchiveHDFS) GetSlots(host, table string) (slots []time.Time, err error)

https://www.slideshare.net/databricks/the-parquet-format-and-performance-optimization-opportunities P22 sorted data helps to predicate pushdown P25 avoid many small files P27 avoid few huge files - 1GB?

func (*ArchiveHDFS) GetSortingInfo

func (this *ArchiveHDFS) GetSortingInfo() (err error)

func (*ArchiveHDFS) InitConns

func (this *ArchiveHDFS) InitConns() (err error)

func (*ArchiveHDFS) SelectUint64

func (this *ArchiveHDFS) SelectUint64(host, query string) (res uint64, err error)

type CKRebalance

type CKRebalance struct {
	Hosts      []string
	Port       int
	User       string
	Password   string
	Databases  []string
	DataDir    string
	DBTables   map[string][]string
	RepTables  map[string]map[string]string
	OsUser     string
	OsPassword string
	OsPort     int
	SshConns   map[string]*ssh.Client
	Pool       *common.WorkerPool
}

func (*CKRebalance) DoRebalance

func (this *CKRebalance) DoRebalance() (err error)

func (*CKRebalance) ExecutePlan

func (this *CKRebalance) ExecutePlan(database string, tbl *TblPartitions) (err error)

func (*CKRebalance) GeneratePlan

func (this *CKRebalance) GeneratePlan(tablename string, tbls []*TblPartitions)

func (*CKRebalance) GetRepTables

func (this *CKRebalance) GetRepTables() (err error)

func (*CKRebalance) GetState

func (this *CKRebalance) GetState(database string, table string) (tbls []*TblPartitions, err error)

func (*CKRebalance) GetTables

func (this *CKRebalance) GetTables() (err error)

func (*CKRebalance) InitCKConns

func (this *CKRebalance) InitCKConns() (err error)

func (*CKRebalance) InitSshConns

func (this *CKRebalance) InitSshConns(database string) (err error)

type PurgerRange

type PurgerRange struct {
	Hosts    []string
	Port     int
	User     string
	Password string
	Database string
	Tables   []string
	Begin    string
	End      string
}

func NewPurgerRange

func NewPurgerRange(hosts []string, port int, user string, password string, database string, begin string, end string) *PurgerRange

func (*PurgerRange) InitConns

func (this *PurgerRange) InitConns() (err error)

func (*PurgerRange) PurgeTable

func (this *PurgerRange) PurgeTable(table string) (err error)

purgeTable purges specified time range

type TblPartitions

type TblPartitions struct {
	Table      string
	Host       string
	ZooPath    string // zoo-path with macros substituted
	Partitions map[string]int64
	TotalSize  int64             // total size of partitions
	ToMoveOut  map[string]string // plan to move some partitions out to other hosts
	ToMoveIn   bool              // plan to move some partitions in
}

TblPartitions is partitions status of a host. A host never move out and move in at the same iteration.