bigtable

Documentation

Index

• Constants
• func BuildRowList(prefix string, body [][]string) bigtable.RowList
• func NewBtTable(ctx context.Context, projectID, instanceID, tableID string) (*cbt.Table, error)
• func Read(ctx context.Context, btGroup *Group, prefix string, body [][]string, ...) ([][]BtRow, error)
• func ReadWithGroupRowList(ctx context.Context, btGroup *Group, prefix string, accs []*Accessor, ...) ([][]BtRow, error)
• func SetupBigtable(ctx context.Context, data map[string]string) (*bigtable.Table, error)
• func SortTables(tables []*Table)
• type Accessor
• type BtRow
• type Group
  • func GetFrequentGroup(g *Group) *Group
  • func NewGroup(tables []*Table, branchTableName string) *Group
  • func (g *Group) TableNames() []string
  • func (g *Group) Tables() []*cbt.Table
  • func (g *Group) UpdateBranchTable(branchTable *Table)
• type Table
  • func NewTable(name string, table *cbt.Table) *Table
  • func (t *Table) Name() string

Constants

const (
	// BtPlaceStatsVarPrefix for place to statsvar list cache.
	BtPlaceStatsVarPrefix = "d/0/"
	// BtStatVarGroup is the key for statvar group cache.
	BtStatVarGroup = "d/1"
	// BtSVAndSVGExistence is the key for stat var and stat var group existence cache.
	BtSVAndSVGExistence = "d/2/"
	// BtObsTimeSeries is the key for obs time series cache.
	BtObsTimeSeries = "d/3/"
	// BtPlacePagePrefix for place page cache.
	BtPlacePagePrefix = "d/4/"
	// BtBioPagePrefix for biology page cache.
	BtBioPagePrefix = "d/6/"
	// BtObsCollectionDateFrequency for obs collection cache that contains the frequency of each
	// date across places.
	BtObsCollectionDateFrequency = "d/8/"
	// BtArcsPrefix for internal arcs cache.
	BtArcsPrefix = "d/9/"
	// BtStatVarSummary for stat var summary cache.
	BtStatVarSummary = "d/a/"
	// BtPlacesInPrefix for GetPlacesIn cache.
	BtPlacesInPrefix = "d/c/"
	// BtPlacesMetadataPrefix for GetPlaceMetadata cache.
	BtPlacesMetadataPrefix = "d/d/"
	// BtObsCollection for obs collection cache.
	BtObsCollection = "d/e/"
	// BtPagedPropValIn for in-arc paged entities.
	// Key: <dcid^predicate^page>
	BtPagedPropValIn = "d/h/"
	// BtPagedPropValOut for out-arc paged entities.
	// Key: <dcid^predicate^page>
	BtPagedPropValOut = "d/i/"
	// BtRelatedLocationsSameTypePrefix for related places with same type.
	BtRelatedLocationsSameTypePrefix = "d/o/"
	// BtRelatedLocationsSameTypeAndAncestorPrefix for related places with same type and ancestor.
	BtRelatedLocationsSameTypeAndAncestorPrefix = "d/q/"
	// BtRelatedLocationsSameTypePCPrefix for related places with same type, per capita.
	BtRelatedLocationsSameTypePCPrefix = "d/o0/"
	// BtRelatedLocationsSameTypeAndAncestorPCPrefix for related places with same type and ancestor,
	// per capita.
	BtRelatedLocationsSameTypeAndAncestorPCPrefix = "d/q0/"

	// BtReconIDMapPrefix for ID mapping for ID-based recon. The key excludes DCID.
	BtReconIDMapPrefix = "d/5/"
	// BtCoordinateReconPrefix for coordinate recon.
	BtCoordinateReconPrefix = "d/b/"

	// BtFamily is the key for the row.
	BtFamily = "csv"
	// BtCacheLimit is the cache limit. The limit is per predicate and neighbor type.
	BtCacheLimit = 500
	// BtBatchQuerySize is the size of BigTable batch query.
	BtBatchQuerySize = 1000
)

Functions

func BuildRowList

func BuildRowList(prefix string, body [][]string) bigtable.RowList

BuildRowList builds row list from BT prefix and token components.

func NewBtTable

func NewBtTable(ctx context.Context, projectID, instanceID, tableID string) (
	*cbt.Table, error)

NewTable creates a new cbt.Table instance.

func Read

func Read(
	ctx context.Context,
	btGroup *Group,
	prefix string,
	body [][]string,
	action func([]byte) (interface{}, error),
) ([][]BtRow, error)

Read reads BigTable rows from multiple Bigtable in parallel. Note all Bigtable read use the same set of rowList.

func ReadWithGroupRowList

func ReadWithGroupRowList(
	ctx context.Context,
	btGroup *Group,
	prefix string,
	accs []*Accessor,
	unmarshalFunc func([]byte) (interface{}, error),
) ([][]BtRow, error)

ReadWithGroupRowList reads BigTable rows from multiple Bigtable in parallel. Reading is chunked as the size limit for RowSet is 500KB.

Note the read could have different RowList for each import group Bigtable as needed by the pagination APIs.

func SetupBigtable

func SetupBigtable(
	ctx context.Context, data map[string]string) (*bigtable.Table, error)

SetupBigtable creates a testing bigtable instance used for testing

func SortTables

func SortTables(tables []*Table)

SortTables sorts the bigtable by import group preferences

• frequent should always be the highest rank
• infrequent should always be the lowest rank
• if a group is not in ranking list, put it right before "infrequent" and after other groups with ranking.

Types

type Accessor

type Accessor struct {
	// Import group table index.
	ImportGroup int
	// A list of body components, each component contains all the key element
	// for that part. One key would constructed by taking one element from each
	// component and concate them together.
	Body [][]string
}

Accessor represents data used to access bigtable row.

type BtRow

type BtRow struct {
	// The body parts of the BT key, which are used to identify the place, dcid
	// or other properties that related to the data.  This is to be used by the
	// caller to group the result.
	Parts []string
	// Data read from Cloud Bigtable
	Data interface{}
}

BtRow contains the BT read key tokens and the cache data.

type Group

type Group struct {
	// contains filtered or unexported fields
}

Group represents all the cloud bigtables that mixer talks to.

func GetFrequentGroup

func GetFrequentGroup(g *Group) *Group

GetFrequentGroup creates a group that only has frequent import group table.

func NewGroup

func NewGroup(
	tables []*Table,
	branchTableName string,
) *Group

NewGroup creates a BigtableGroup

func (*Group) TableNames

func (g *Group) TableNames() []string

TableNames is the accesser to get all the Bigtable names.

func (*Group) Tables

func (g *Group) Tables() []*cbt.Table

Tables is the accessor for all the Bigtable client stubs.

func (*Group) UpdateBranchTable

func (g *Group) UpdateBranchTable(branchTable *Table)

UpdateBranchTable updates the branch Bigtable.

type Table

type Table struct {
	// contains filtered or unexported fields
}

Table holds the bigtable name and client stub.

func NewTable

func NewTable(name string, table *cbt.Table) *Table

NewTable creates a new Table struct.

func (*Table) Name

func (t *Table) Name() string

Name access the name of a table