bigquery

package
v0.5.9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 4, 2023 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// bigquery datastore specific configurations
	BackupConfigDataset = "dataset"
	BackupConfigPrefix  = "prefix"
)
View Source 
const (
	ExpectedDatasetNameSegments = 3
)
View Source 
const (
	ExpectedTableNameSegments = 4
)
View Source 
const (
	MaxBQClientReuse = 10
)
View Source 
const (
	// SecretName for creation and manipulation of a project resources in bigquery
	SecretName = "DATASTORE_BIGQUERY" //nolint:gosec
)
View Source 
const (
	ViewQueryFile = "view.sql"
)

Variables

View Source 
var (
	This = &BigQuery{
		ClientFac: &defaultBQClientFactory{},
	}
)

Functions

This section is empty.

Types

type BQClientFactoryMock 

type BQClientFactoryMock struct {
	mock.Mock
}

func (*BQClientFactoryMock) New 

func (fac *BQClientFactoryMock) New(ctx context.Context, svcAcc string) (bqiface.Client, error)

type BQClusteringInfo 

type BQClusteringInfo struct {
	Using []string `json:"using"`
}

BQClusteringInfo describes list of column used in table clustering

type BQDataset 

type BQDataset struct {
	Project  string
	Dataset  string
	Metadata BQDatasetMetadata
}

BQDataset is a specification for a BigQuery Dataset The dataset may or may not exist

type BQDatasetMetadata 

type BQDatasetMetadata struct {
	Description            string            `yaml:",omitempty" json:"description,omitempty"`
	DefaultTableExpiration int64             `yaml:"table_expiration,omitempty" json:"table_expiration,omitempty"`
	Labels                 map[string]string `yaml:"-" json:"-"` // will be inherited by base resource

	Location string `yaml:",omitempty" json:"location,omitempty"`
}

type BQExternalSource added in v0.0.3 

type BQExternalSource struct {
	SourceType string `yaml:"type,omitempty" json:"type"`

	// External Table URI string for the referenced spreadsheets
	SourceURIs []string `yaml:"uris,omitempty" json:"uris,omitempty"`

	// Additional configs for CSV, GoogleSheets, Bigtable, and Parquet formats.
	Config map[string]interface{} `yaml:"config,omitempty" json:"config"`
}

BQExternalSource specifies table source information for external data source

type BQField 

type BQField struct {
	Name        string `yaml:",omitempty" json:"name"`
	Type        string `yaml:",omitempty" json:"type"`
	Description string `yaml:",omitempty" json:"description,omitempty"`
	Mode        string `yaml:",omitempty" json:"mode,omitempty"`

	// optional sub-schema, if Type is set to Record
	Schema BQSchema `yaml:",omitempty" json:"schema,omitempty"`
}

BQField describes an individual field/column in a bigquery schema

type BQPartitionInfo 

type BQPartitionInfo struct {
	Field string `yaml:"field,omitempty" json:"field,omitempty"`

	// time based
	Type       string `yaml:"type,omitempty" json:"type,omitempty"`             // default day
	Expiration int64  `yaml:"expiration,omitempty" json:"expiration,omitempty"` // in hours

	// range based
	Range *BQPartitioningRange `yaml:",omitempty" json:"range,omitempty"`
}

BQPartitionInfo specifies the partitioning for a BQTable

type BQPartitioningRange 

type BQPartitioningRange struct {
	// The start value of defined range of values, inclusive of the specified value.
	Start int64 `yaml:",omitempty" json:"start,omitempty"`
	// The end of the defined range of values, exclusive of the defined value.
	End int64 `yaml:",omitempty" json:"end,omitempty"`
	// The width of each interval range.
	Interval int64 `yaml:",omitempty" json:"interval,omitempty"`
}

BQPartitioningRange defines the boundaries and width of partitioned values.

type BQSchema 

type BQSchema []BQField

BQSchema describes the schema for a field in a BigQuery table

type BQTable 

type BQTable struct {
	Project string
	Dataset string
	Table   string

	Metadata BQTableMetadata
}

BQTable is a specification for a BigQuery Table The table may or may not exist

persistent model of table

func (BQTable) FullyQualifiedName 

func (t BQTable) FullyQualifiedName() string

FullyQualifiedName returns the "full name" for a table

func (BQTable) Validate 

func (t BQTable) Validate() error

type BQTableMetadata 

type BQTableMetadata struct {
	Schema         BQSchema          `yaml:"schema" json:"schema"`
	Description    string            `yaml:",omitempty" json:"description,omitempty"`
	Cluster        *BQClusteringInfo `yaml:",omitempty" json:"cluster,omitempty"`
	Partition      *BQPartitionInfo  `yaml:",omitempty" json:"partition,omitempty"`
	ExpirationTime string            `yaml:"expiration_time,omitempty" json:"expiration_time,omitempty"`

	// external source options
	Source *BQExternalSource `yaml:",omitempty" json:"source,omitempty"`

	// regular view query
	ViewQuery string `yaml:"view_query,omitempty" json:"view_query,omitempty"`

	Location string            `yaml:",omitempty" json:"location,omitempty"`
	Labels   map[string]string `yaml:"-" json:"-"` // inherited
}

BQTableMetadata holds configuration for a table

type BigQuery 

type BigQuery struct {
	ClientFac ClientFactory
}

func (*BigQuery) BackupResource added in v0.0.3 

func (b *BigQuery) BackupResource(ctx context.Context, request models.BackupResourceRequest) (models.BackupResourceResponse, error)

func (*BigQuery) CreateResource 

func (b *BigQuery) CreateResource(ctx context.Context, request models.CreateResourceRequest) error

func (*BigQuery) DeleteResource 

func (b *BigQuery) DeleteResource(ctx context.Context, request models.DeleteResourceRequest) error

func (BigQuery) Description 

func (BigQuery) Description() string

func (BigQuery) Name 

func (BigQuery) Name() string

func (*BigQuery) ReadResource 

func (b *BigQuery) ReadResource(ctx context.Context, request models.ReadResourceRequest) (models.ReadResourceResponse, error)

func (BigQuery) Types 

func (BigQuery) Types() map[models.ResourceType]models.DatastoreTypeController

func (*BigQuery) UpdateResource 

func (b *BigQuery) UpdateResource(ctx context.Context, request models.UpdateResourceRequest) error

type BigQueryMock 

type BigQueryMock struct {
	mock.Mock
}

func (*BigQueryMock) CreateResource 

func (*BigQueryMock) CreateResource(context.Context, models.CreateResourceRequest) error

func (*BigQueryMock) DeleteResource 

func (*BigQueryMock) DeleteResource(context.Context, models.DeleteResourceRequest) error

func (*BigQueryMock) ReadResource 

func (*BigQueryMock) ReadResource(context.Context, models.ReadResourceRequest) (models.ReadResourceResponse, error)

func (*BigQueryMock) UpdateResource 

func (*BigQueryMock) UpdateResource(context.Context, models.UpdateResourceRequest) error

type BqClientMock 

type BqClientMock struct {
	mock.Mock
	bqiface.Client
}

func (*BqClientMock) Close 

func (*BqClientMock) Close() error

func (*BqClientMock) Dataset 

func (*BqClientMock) Dataset(string) bqiface.Dataset

func (*BqClientMock) DatasetInProject 

func (cli *BqClientMock) DatasetInProject(project, dataset string) bqiface.Dataset

func (*BqClientMock) Datasets 

func (*BqClientMock) Datasets(context.Context) bqiface.DatasetIterator

func (*BqClientMock) DatasetsInProject 

func (*BqClientMock) DatasetsInProject(context.Context, string) bqiface.DatasetIterator

func (*BqClientMock) JobFromID 

func (*BqClientMock) JobFromID(context.Context, string) (bqiface.Job, error)

func (*BqClientMock) JobFromIDLocation 

func (*BqClientMock) JobFromIDLocation(context.Context, string, string) (bqiface.Job, error)

func (*BqClientMock) Jobs 

func (*BqClientMock) Jobs(context.Context) bqiface.JobIterator

func (*BqClientMock) Location 

func (*BqClientMock) Location() string

func (*BqClientMock) Query 

func (*BqClientMock) Query(string) bqiface.Query

func (*BqClientMock) SetLocation 

func (*BqClientMock) SetLocation(string)

type BqCopierMock added in v0.0.3 

type BqCopierMock struct {
	mock.Mock
	bqiface.Copier
}

func (*BqCopierMock) JobIDConfig added in v0.0.3 

func (*BqCopierMock) JobIDConfig() *bigquery.JobIDConfig

func (*BqCopierMock) Run added in v0.0.3 

func (copier *BqCopierMock) Run(ctx context.Context) (bqiface.Job, error)

func (*BqCopierMock) SetCopyConfig added in v0.0.3 

func (*BqCopierMock) SetCopyConfig(bqiface.CopyConfig)

type BqDatasetMock 

type BqDatasetMock struct {
	mock.Mock
	bqiface.Dataset
}

func (*BqDatasetMock) Create 

func (ds *BqDatasetMock) Create(ctx context.Context, meta *bqiface.DatasetMetadata) error

func (*BqDatasetMock) DatasetID 

func (ds *BqDatasetMock) DatasetID() string

func (*BqDatasetMock) Delete 

func (ds *BqDatasetMock) Delete(ctx context.Context) error

func (*BqDatasetMock) DeleteWithContents 

func (*BqDatasetMock) DeleteWithContents(context.Context) error

func (*BqDatasetMock) Metadata 

func (ds *BqDatasetMock) Metadata(ctx context.Context) (*bqiface.DatasetMetadata, error)

func (*BqDatasetMock) ProjectID 

func (*BqDatasetMock) ProjectID() string

func (*BqDatasetMock) Table 

func (ds *BqDatasetMock) Table(name string) bqiface.Table

func (*BqDatasetMock) Tables 

func (*BqDatasetMock) Tables(context.Context) bqiface.TableIterator

func (*BqDatasetMock) Update 

func (ds *BqDatasetMock) Update(ctx context.Context, m bqiface.DatasetMetadataToUpdate, tag string) (*bqiface.DatasetMetadata, error)

type BqJobMock added in v0.0.3 

type BqJobMock struct {
	mock.Mock
	bqiface.Job
}

func (*BqJobMock) Cancel added in v0.0.3 

func (*BqJobMock) Cancel(context.Context) error

func (*BqJobMock) Config added in v0.0.3 

func (*BqJobMock) Config() (bigquery.JobConfig, error)

func (*BqJobMock) ID added in v0.0.3 

func (*BqJobMock) ID() string

func (*BqJobMock) LastStatus added in v0.0.3 

func (*BqJobMock) LastStatus() *bigquery.JobStatus

func (*BqJobMock) Location added in v0.0.3 

func (*BqJobMock) Location() string

func (*BqJobMock) Read added in v0.0.3 

func (*BqJobMock) Read(context.Context) (bqiface.RowIterator, error)

func (*BqJobMock) Status added in v0.0.3 

func (*BqJobMock) Status(context.Context) (*bigquery.JobStatus, error)

func (*BqJobMock) Wait added in v0.0.3 

func (job *BqJobMock) Wait(ctx context.Context) (*bigquery.JobStatus, error)

type BqTableMock 

type BqTableMock struct {
	mock.Mock
	bqiface.Table
}

func (*BqTableMock) CopierFrom 

func (table *BqTableMock) CopierFrom(t ...bqiface.Table) bqiface.Copier

func (*BqTableMock) Create 

func (table *BqTableMock) Create(ctx context.Context, meta *bigquery.TableMetadata) error

func (*BqTableMock) DatasetID 

func (*BqTableMock) DatasetID() string

func (*BqTableMock) Delete 

func (table *BqTableMock) Delete(ctx context.Context) error

func (*BqTableMock) ExtractorTo 

func (*BqTableMock) ExtractorTo(*bigquery.GCSReference) bqiface.Extractor

func (*BqTableMock) FullyQualifiedName 

func (*BqTableMock) FullyQualifiedName() string

func (*BqTableMock) LoaderFrom 

func (*BqTableMock) LoaderFrom(bigquery.LoadSource) bqiface.Loader

func (*BqTableMock) Metadata 

func (table *BqTableMock) Metadata(ctx context.Context) (*bigquery.TableMetadata, error)

func (*BqTableMock) ProjectID 

func (*BqTableMock) ProjectID() string

func (*BqTableMock) Read 

func (*BqTableMock) Read(context.Context) bqiface.RowIterator

func (*BqTableMock) TableID 

func (*BqTableMock) TableID() string

func (*BqTableMock) Update 

func (table *BqTableMock) Update(ctx context.Context, meta bigquery.TableMetadataToUpdate, etag string) (*bigquery.TableMetadata, error)

func (*BqTableMock) Uploader 

func (*BqTableMock) Uploader() bqiface.Uploader

type ClientFactory 

type ClientFactory interface {
	New(ctx context.Context, svcAccount string) (bqiface.Client, error)
}

type DatasetResourceSpec 

type DatasetResourceSpec struct {
	Version int
	Name    string
	Type    models.ResourceType
	Spec    BQDatasetMetadata
	Labels  map[string]string
}

DatasetResourceSpec is how dataset should be represented in yaml

type ExternalTableType added in v0.0.3 

type ExternalTableType string
const (
	ExternalTableTypeGoogleSheets ExternalTableType = "GOOGLE_SHEETS"
)

type TableResourceSpec 

type TableResourceSpec struct {
	Version int
	Name    string
	Type    models.ResourceType
	Spec    BQTableMetadata
	Labels  map[string]string
}

TableResourceSpec is how resource will be represented in yaml

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.