storeutils

Documentation

Index

• Variables
• func GetAndOpen(s cloudstorage.Store, o string, level cloudstorage.AccessLevel) (cloudstorage.Object, error)
• func GetObject(gc *storage.Client, bucket, name string) (*bytes.Buffer, error)
• type AwsSource
  • func (a *AwsSource) String() string
  • func (a *AwsSource) TransferSpec(bucket string) *storagetransfer.TransferSpec
• type GcsSource
  • func (g *GcsSource) String() string
  • func (g *GcsSource) TransferSpec(bucket string) *storagetransfer.TransferSpec
• type HttpSource
  • func (h *HttpSource) String() string
  • func (h *HttpSource) TransferSpec(bucket string) *storagetransfer.TransferSpec
• type Source
  • func NewAwsSource(bucket, accesskey, secret string) Source
  • func NewGcsSource(bucket string) Source
  • func NewHttpSource(url string) Source
• type Status
• type TransferConfig
  • func (t *TransferConfig) Job() (*storagetransfer.TransferJob, error)
• type Transferer
  • func NewTransferClient(client *http.Client) (*Transferer, error)
  • func (t *Transferer) GetJob(project, job string) (*storagetransfer.TransferJob, error)
  • func (t *Transferer) List(project string, statuses ...Status) ([]*storagetransfer.TransferJob, error)
  • func (t *Transferer) NewTransfer(conf *TransferConfig) (*storagetransfer.TransferJob, error)

Variables

var (
	// MaxPrefix is the maximum number of prefix filters allowed when transferring files in GCS buckets
	MaxPrefix = 20

	ErrBadFilter = errors.New("too many inclusion/exclusion prefixes")
	ErrBadConfig = errors.New("transferconfig not valid")
)

var GCSRetries = 10

Copied from cloudstorage

Functions

func GetAndOpen

func GetAndOpen(s cloudstorage.Store, o string, level cloudstorage.AccessLevel) (cloudstorage.Object, error)

GetAndOpen is a convenience method that combines Store.Get() and Object.Open() into a single call.

func GetObject

func GetObject(gc *storage.Client, bucket, name string) (*bytes.Buffer, error)

GetObject Gets a single object's bytes based on bucket and name parameters

Types

type AwsSource

type AwsSource struct {
	// contains filtered or unexported fields
}

AwsSource is an AWS S3 data source

func (*AwsSource) String

func (a *AwsSource) String() string

func (*AwsSource) TransferSpec

func (a *AwsSource) TransferSpec(bucket string) *storagetransfer.TransferSpec

type GcsSource

type GcsSource struct {
	// contains filtered or unexported fields
}

GcsSource is a Source defined by a Gcs bucket

func (*GcsSource) String

func (g *GcsSource) String() string

func (*GcsSource) TransferSpec

func (g *GcsSource) TransferSpec(bucket string) *storagetransfer.TransferSpec

type HttpSource

type HttpSource struct {
	// contains filtered or unexported fields
}

HttpSource is a Source defined by a HTTP URL data source

func (*HttpSource) String

func (h *HttpSource) String() string

func (*HttpSource) TransferSpec

func (h *HttpSource) TransferSpec(bucket string) *storagetransfer.TransferSpec

type Source

type Source interface {
	TransferSpec(destBucket string) *storagetransfer.TransferSpec
	String() string
}

Source defines the data source when transferring data to a GCS bucket. While the sink is restricted to a GCS bucket the source can either be another GCS bucket, and AWS S3 source, or a HTTP source Each source produces a storagetransfer TransferSpec

func NewAwsSource

func NewAwsSource(bucket, accesskey, secret string) Source

func NewGcsSource

func NewGcsSource(bucket string) Source

func NewHttpSource

func NewHttpSource(url string) Source

type Status

type Status string

const (
	Enabled     Status = "ENABLED"
	Disabled    Status = "DISABLED"
	Unspecified Status = "STATUS_UNSPECIFIED"
	Deleted     Status = "DELETED"
)

type TransferConfig

type TransferConfig struct {
	ProjectID       string // projectID of destination bucket
	DestBucket      string
	Src             Source
	IncludePrefixes []string
	ExcludePrefixes []string
	Schedule        *storagetransfer.Schedule
}

TransferConfig wraps all of the relevant variables for transfer jobs into a unified struct

func (*TransferConfig) Job

func (t *TransferConfig) Job() (*storagetransfer.TransferJob, error)

Job instantiates a Transfer job from the TransferConfig struct

type Transferer

type Transferer struct {
	// contains filtered or unexported fields
}

Transferer manages the transfer of data sources to GCS

func NewTransferClient

func NewTransferClient(client *http.Client) (*Transferer, error)

NewTransferClient creates a new Transferer using an authed http client

func (*Transferer) GetJob

func (t *Transferer) GetJob(project, job string) (*storagetransfer.TransferJob, error)

GetJob returns the transferJob with the specified project and job ID

func (*Transferer) List

func (t *Transferer) List(project string, statuses ...Status) ([]*storagetransfer.TransferJob, error)

List returns all of the transferJobs under a specific project. If the variadic argument "statuses" is provided, only jobs with the listed statuses are returned

func (*Transferer) NewTransfer

func (t *Transferer) NewTransfer(conf *TransferConfig) (*storagetransfer.TransferJob, error)

NewTransfer creates a new transferJob with the specified project, destination GCS bucket and source. The include/exclude arguments define the file prefixes in the source bucket to include/exclude