common

Documentation

Index

• Constants
• Variables
• type Column
  • func (c *Column) IsEnumColumn() bool
  • func (c *Column) IsOverwriteOnlyDataType() bool
• type ColumnConfig
• type HLLConfig
• type MetaStore
• type ShardOwnership
• type Table
• type TableConfig
• type TableSchemaMutator
• type TableSchemaReader
• type TableSchemaWatchable

Constants

const (
	Bool      = "Bool"
	Int8      = "Int8"
	Uint8     = "Uint8"
	Int16     = "Int16"
	Uint16    = "Uint16"
	Int32     = "Int32"
	Uint32    = "Uint32"
	Float32   = "Float32"
	SmallEnum = "SmallEnum"
	BigEnum   = "BigEnum"
	UUID      = "UUID"
	GeoPoint  = "GeoPoint"
	GeoShape  = "GeoShape"
	Int64     = "Int64"

	// array types
	ArrayBool      = "Bool[]"
	ArrayInt8      = "Int8[]"
	ArrayUint8     = "Uint8[]"
	ArrayInt16     = "Int16[]"
	ArrayUint16    = "Uint16[]"
	ArrayInt32     = "Int32[]"
	ArrayUint32    = "Uint32[]"
	ArrayFloat32   = "Float32[]"
	ArraySmallEnum = "SmallEnum[]"
	ArrayBigEnum   = "BigEnum[]"
	ArrayUUID      = "UUID[]"
	ArrayGeoPoint  = "GeoPoint[]"
	ArrayInt64     = "Int64[]"
)

string representations of data types

const (
	// EnumDelimiter
	EnumDelimiter = "\u0000\n"
)

Variables

var (
	// ErrTableDoesNotExist indicates Table does not exist
	ErrTableDoesNotExist = errors.New("Table does not exist")
	// ErrTableAlreadyExist indicates Table already exists
	ErrTableAlreadyExist = errors.New("Table already exists")
	// ErrColumnDoesNotExist indicates Column does not exist error
	ErrColumnDoesNotExist = errors.New("Column does not exist")
	// ErrColumnAlreadyExist indicates Column already exists
	ErrColumnAlreadyExist = errors.New("Column already exists")
	// ErrColumnAlreadyDeleted indicates Column already deleted
	ErrColumnAlreadyDeleted = errors.New("Column already deleted")
	// ErrNotEnumColumn indicates Column is not enum type
	ErrNotEnumColumn = errors.New("Column is not enum type")
	// ErrShardDoesNotExist indicates Shard does not exist
	ErrShardDoesNotExist = errors.New("Shard does not exist")
	// ErrNotFactTable indicates table not a fact table
	ErrNotFactTable = errors.New("Table is not fact table")
	// ErrNotDimensionTable indicates table is not a dimension table
	ErrNotDimensionTable = errors.New("Table is not dimension table")
	// ErrWatcherAlreadyExist indicates table is not a dimension table
	ErrWatcherAlreadyExist = errors.New("Watcher already registered")
	// ErrDeleteTimeColumn indicates column is time column and cannot be deleted
	ErrDeleteTimeColumn = errors.New("Time column cannot be deleted")
	// ErrDeletePrimaryKeyColumn indicates column belongs to primary key cannot be deleted
	ErrDeletePrimaryKeyColumn = errors.New("Primary key column cannot be deleted")
	// ErrChangePrimaryKeyColumn indicates primary key columns cannot be changed
	ErrChangePrimaryKeyColumn = errors.New("Primary key column cannot be changed")
	// ErrAllColumnsInvalid indicates all columns are invalid
	ErrAllColumnsInvalid = errors.New("All columns are invalid")
	// ErrMissingPrimaryKey indicates a schema does not have primary key
	ErrMissingPrimaryKey = errors.New("Primary key columns not specified")
	// ErrColumnNonExist indicates a column used does not exist
	ErrColumnNonExist = errors.New("Column does not exist")
	// ErrColumnDeleted indicates a column used was deleted
	ErrColumnDeleted = errors.New("Column already deleted")
	// ErrInvalidDataType indicates invalid data type
	ErrInvalidDataType = errors.New("Invalid data type")
	// ErrIllegalSchemaVersion indicates new schema is not greater than old one
	ErrIllegalSchemaVersion = errors.New("New schema version not greater than old")
	// ErrSchemaUpdateNotAllowed indicates changes attemped on immutable fields
	ErrSchemaUpdateNotAllowed = errors.New("Illegal schame update on immutable field")
	// ErrInsufficientColumnCount indicates no column in a schame
	ErrInsufficientColumnCount = errors.New("Insufficient column count")
	// ErrReusingColumnIDNotAllowed indicates attempt to reuse id of deleted column
	ErrReusingColumnIDNotAllowed = errors.New("Reusing column id not allowed")
	// ErrIllegalChangeSortColumn indicates illegal changes on sort columns
	ErrIllegalChangeSortColumn = errors.New("Illegal changes on sort columns")
	// ErrDuplicatedColumn indicates a column is used more than onces in sort or pk columns
	ErrDuplicatedColumn = errors.New("Illegal deplicated use of column")
	// ErrDuplicatedColumnName indicates duplicated column name in same table
	ErrDuplicatedColumnName          = errors.New("Duplicated column name found")
	ErrMissingTimeColumn             = errors.New("Fact table has to have time column as first column")
	ErrTimeColumnDoesNotAllowDefault = errors.New("Time column does not allow default value")
	ErrDisallowMissingEventTime      = errors.New("Can not disallow missing event time")
	// ErrTimeColumnDoesNotAllowHLLConfig indicates hll configured for time column
	ErrTimeColumnDoesNotAllowHLLConfig   = errors.New("HLLConfig not allowed for time column")
	ErrHLLColumnDoesNotAllowDefaultValue = errors.New("hll column does not allow default value")
	ErrInvalidTableBatchSize             = errors.New("Table batch size should be larger than zero")
	ErrInvalidPrimaryKeyBucketSize       = errors.New("Table primary key bucket size should be larger than zero")
)

Types

type Column

type Column struct {
	// Immutable, columns cannot be renamed.
	Name string `json:"name"`
	// Immutable, columns cannot have their types changed.
	Type string `json:"type"`
	// Deleted columns are kept as placeholders in Table.Columns.
	// read only: true
	Deleted bool `json:"deleted,omitempty"`
	// We store the default value as string here since it's from user input.
	// Nil means the default value is NULL. Actual default value of column data type
	// should be stored in memstore.
	DefaultValue *string `json:"defaultValue,omitempty"`

	// Whether to compare characters case insensitively for enum columns. It only matters
	// for ingestion client as it's the place to concert enum strings to enum values.
	CaseInsensitive bool `json:"caseInsensitive,omitempty"`

	// Whether disable enum cases auto expansion.
	DisableAutoExpand bool `json:"disableAutoExpand,omitempty"`

	// Mutable column configs.
	Config ColumnConfig `json:"config,omitempty"`

	// HLLEnabled determines whether a column is enabled for hll cardinality estimation
	// HLLConfig is immutable
	HLLConfig HLLConfig `json:"hllConfig,omitempty"`
}

Column defines the schema of a column from MetaStore. swagger:model column

func (*Column) IsEnumColumn

func (c *Column) IsEnumColumn() bool

IsEnumColumn checks whether a column is enum column

func (*Column) IsOverwriteOnlyDataType

func (c *Column) IsOverwriteOnlyDataType() bool

IsOverwriteOnlyDataType checks whether a column is overwrite only

type ColumnConfig

type ColumnConfig struct {
	// ColumnEvictionConfig : For column level in-memory eviction, it’s the best
	// effort TTL for in-memory data.
	// Column level eviction has nothing to do with data availability, but based
	// on how much data we pre-loaded, the major impact will be there for query
	// performance. Here we bring in two priorities configs: Preloading days and
	// Priority.
	//   - Preloading days is defined at each column level to indicate how many
	//     recent days data we want to preload to host memory. This is best effort
	//     operation.
	//   - Priority is defined at each column level to indicate the priority of
	//     each column. When data eviction happens, we will rely on column priority
	//     to decide which column will be evicted first.
	//     High number implies high priority.
	PreloadingDays int   `json:"preloadingDays,omitempty"`
	Priority       int64 `json:"priority,omitempty"`
}

ColumnConfig defines the schema of a column config that can be mutated by UpdateColumn API call. swagger:model columnConfig

type HLLConfig

type HLLConfig struct {
	IsHLLColumn bool `json:"isHLLColumn,omitempty"`
}

HLLConfig defines hll configuration swagger:model hllConfig

type MetaStore

type MetaStore interface {
	GetEnumDict(table, column string) ([]string, error)

	// Sets the watcher for the specified enum column.
	// Should only be called once for each enum column.
	// Returns a events channel that emits enum cases starting from startCase,
	// and a done channel for consumer to ack once the event is processed.
	WatchEnumDictEvents(table, column string, startCase int) (events <-chan string, done chan<- struct{}, err error)

	// Returns the latest archiving/live cutoff for the specified shard.
	GetArchivingCutoff(table string, shard int) (uint32, error)

	// PurgeArchiveBatches deletes the metadata related to the archive batch
	PurgeArchiveBatches(table string, shard, batchIDStart, batchIDEnd int) error
	// Returns the version to use for the specified archive batch and size of the batch with the
	// specified archiving/live cutoff.
	GetArchiveBatchVersion(table string, shard, batchID int, cutoff uint32) (uint32, uint32, int, error)
	// Returns the latest snapshot version for the specified shard.
	// the return value is: redoLogFile, offset, lastReadBatchID, lastReadBatchOffset
	GetSnapshotProgress(table string, shard int) (int64, uint32, int32, uint32, error)

	// Set the watcher for table shard ownership change events.
	// Should only be called once.
	// Returns an event channel that emits desired ownership states,
	// and a done channel for consumer to ack once the event is processed.
	WatchShardOwnershipEvents() (events <-chan ShardOwnership, done chan<- struct{}, err error)

	// A subset of newly added columns can be appended to the end of
	// ArchivingSortColumns by adding their index in columns to archivingSortColumns
	// Update column config.
	// Returns the assigned case IDs for each case string.
	ExtendEnumDict(table, column string, enumCases []string) ([]int, error)

	// List available archive batches
	GetArchiveBatches(table string, shard int, start, end int32) ([]int, error)

	// Adds a version and size for the specified archive batch.
	AddArchiveBatchVersion(table string, shard, batchID int, version uint32, seqNum uint32, batchSize int) error

	// WriteArchiveBatchVersion
	OverwriteArchiveBatchVersion(table string, shard, batchID int, version uint32, seqNum uint32, batchSize int) error

	// Updates the archiving/live cutoff time for the specified shard. This is used
	// by the archiving job after each successful run.
	UpdateArchivingCutoff(table string, shard int, cutoff uint32) error

	// Updates the latest snapshot version for the specified shard.
	UpdateSnapshotProgress(table string, shard int, redoLogFile int64, upsertBatchOffset uint32, lastReadBatchID int32, lastReadBatchOffset uint32) error

	// Updates the latest redolog/offset that have been backfilled for the specified shard.
	UpdateBackfillProgress(table string, shard int, redoLogFile int64, offset uint32) error

	// Retrieve the latest redolog/offset that have been backfilled for the specified shard.
	GetBackfillProgressInfo(table string, shard int) (int64, uint32, error)

	// Update ingestion commit offset, used for kafka like streaming ingestion
	UpdateRedoLogCommitOffset(table string, shard int, offset int64) error

	// Get ingestion commit offset, used for kafka like streaming ingestion
	GetRedoLogCommitOffset(table string, shard int) (int64, error)

	// Update ingestion checkpoint offset, used for kafka like streaming ingestion
	UpdateRedoLogCheckpointOffset(table string, shard int, offset int64) error

	// Get ingestion checkpoint offset, used for kafka like streaming ingestion
	GetRedoLogCheckpointOffset(table string, shard int) (int64, error)

	TableSchemaWatchable
	TableSchemaMutator
}

MetaStore defines interfaces of the external metastore, which can be implemented using file system, SQLite, Zookeeper etc.

type ShardOwnership

type ShardOwnership struct {
	TableName string
	Shard     int
	ShouldOwn bool
}

ShardOwnership defines an instruction on whether the receiving instance should start to own or disown the specified table shard.

type Table

type Table struct {
	// Name of the table, immutable.
	Name string `json:"name"`
	// Index to Columns also serves as column IDs.
	Columns []Column `json:"columns"`
	// IDs of primary key columns. This field is immutable.
	PrimaryKeyColumns []int `json:"primaryKeyColumns"`
	// Whether this is a fact table.
	IsFactTable bool `json:"isFactTable"`

	// table configurations
	Config TableConfig `json:"config"`

	// Fact table only.
	// IDs of columns to sort based upon.
	ArchivingSortColumns []int `json:"archivingSortColumns,omitempty"`

	// Incarnation gets incremented every time an table name is reused
	// only used for controller managed schema in cluster setting
	Incarnation int `json:"incarnation"`
	// Version gets incremented every time when schema is updated
	// only used for controller managed schema in cluster setting
	Version int `json:"version"`
}

Table defines the schema and configurations of a table from MetaStore. swagger:model table

type TableConfig

type TableConfig struct {

	// Initial setting of number of buckets for primary key
	// if equals to 0, default will be used
	InitialPrimaryKeyNumBuckets int `json:"initPrimaryKeyNumBuckets,omitempty"`

	// Size of each live batch, should be sufficiently large.
	BatchSize int `json:"batchSize,omitempty" validate:"min=1"`

	// Specifies how often to create a new redo log file.
	RedoLogRotationInterval int `json:"redoLogRotationInterval,omitempty" validate:"min=1"`

	// Specifies the size limit of a single redo log file.
	MaxRedoLogFileSize int `json:"maxRedoLogFileSize,omitempty" validate:"min=1"`

	// Number of minutes after event time before a record can be archived.
	ArchivingDelayMinutes uint32 `json:"archivingDelayMinutes,omitempty" validate:"min=1"`
	// Specifies how often archiving runs.
	ArchivingIntervalMinutes uint32 `json:"archivingIntervalMinutes,omitempty" validate:"min=1"`

	// Specifies how often backfill runs.
	BackfillIntervalMinutes uint32 `json:"backfillIntervalMinutes,omitempty" validate:"min=1"`

	// Upper limit of current backfill buffer size + backfilling buffer size.
	BackfillMaxBufferSize int64 `json:"backfillMaxBufferSize,omitempty" validate:"min=1"`

	// Backfill buffer size in bytes that will trigger a backfill job.
	BackfillThresholdInBytes int64 `json:"backfillThresholdInBytes,omitempty" validate:"min=1"`

	// Size of each live batch used by backfill job.
	BackfillStoreBatchSize int `json:"backfillStoreBatchSize,omitempty" validate:"min=1"`

	// Records with timestamp older than now - RecordRetentionInDays will be skipped
	// during ingestion and backfill. 0 means unlimited days.
	RecordRetentionInDays int `json:"recordRetentionInDays,omitempty" validate:"min=0"`

	// Number of mutations to accumulate before creating a new snapshot.
	SnapshotThreshold int `json:"snapshotThreshold,omitempty" validate:"min=1"`

	// Specifies how often snapshot runs.
	SnapshotIntervalMinutes int `json:"snapshotIntervalMinutes,omitempty" validate:"min=1"`

	AllowMissingEventTime bool `json:"allowMissingEventTime,omitempty"`
}

TableConfig defines the table configurations that can be changed swagger:model tableConfig

type TableSchemaMutator

type TableSchemaMutator interface {
	TableSchemaReader
	CreateTable(table *Table) error
	DeleteTable(name string) error
	UpdateTableConfig(table string, config TableConfig) error
	UpdateTable(table Table) error

	// A subset of newly added columns can be appended to the end of
	// ArchivingSortColumns by adding their index in columns to archivingSortColumns
	AddColumn(table string, column Column, appendToArchivingSortOrder bool) error
	// Update column config.
	UpdateColumn(table string, column string, config ColumnConfig) error
	DeleteColumn(table string, column string) error
}

TableSchemaMutator mutates table metadata

type TableSchemaReader

type TableSchemaReader interface {
	ListTables() ([]string, error)
	GetTable(name string) (*Table, error)
}

TableSchemaReader reads table schema

type TableSchemaWatchable

type TableSchemaWatchable interface {
	// Sets the watcher for table list change (table deletion) events.
	// Should only be called once.
	// Returns a events channel that emits the entire table list on each table deletion event,
	// and a done channel for consumer to ack once the event is processed.
	WatchTableListEvents() (events <-chan []string, done chan<- struct{}, err error)
	// Sets the watcher for table modification/addition events.
	// Should only be called once.
	// Returns a events channel that emits the table schema on each change event for given table,
	// and a done channel for consumer to ack once the event is processed.
	WatchTableSchemaEvents() (events <-chan *Table, done chan<- struct{}, err error)
}

TableSchemaWatchable watches table schema update events