dsutils

Documentation

Overview

Package dsutils provides beam transformations and utilities to interact with Google Datastore.

Index

• func GetAllKeysWithHexPrefix(s beam.Scope, cloudProject string, namespaces beam.PCollection, kind string, ...) beam.PCollection
• func GetAllNamespaces(s beam.Scope, cloudProject string) beam.PCollection
• type HexPosClaim
• type KeyBatch
• type NamespaceCount
• type ReadOptions

Functions

func GetAllKeysWithHexPrefix

func GetAllKeysWithHexPrefix(
	s beam.Scope,
	cloudProject string,
	namespaces beam.PCollection,
	kind string,
	opts ReadOptions,
) beam.PCollection

GetAllKeysWithHexPrefix queries all the keys from datastore for the given kind in the given namespaces (in a PCollection<string>) and returns PCollection<KeyBatch>.

func GetAllNamespaces

func GetAllNamespaces(s beam.Scope, cloudProject string) beam.PCollection

GetAllNamespaces queries all the namespaces from datastore and returns PCollection<string>.

Types

type HexPosClaim

type HexPosClaim struct {
	Value string
	End   bool
}

type KeyBatch

type KeyBatch struct {
	Namespace string
	Keys      []*datastore.Key
}

type NamespaceCount

type NamespaceCount struct {
	Namespace      string
	EstimatedCount int64
}

type ReadOptions

type ReadOptions struct {
	// HexPrefixLength is the minimum guaranteed length of hex prefix in
	// `ParentKind`'s key. The hex prefix will be used to divide the read queries
	// to make it run faster should the beam runner decides more parallelism is
	// needed.
	HexPrefixLength int
	// OutputBatchSize controls the number of datastore keys in a given output
	// batch.
	OutputBatchSize int
	// MinEstimatedCount is the count used as a floor when estimating the number
	// of entities of the specified Kind in a namespace when the stats is not
	// available or is too small. This can happen when the kind was just added to
	// the namespace and stats hasn't been updated yet.
	MinEstimatedCount int64
	// InitialSplitSize is the size used to split the query DoFn on start up.
	InitialSplitSize int64
}