gcsbackend

Documentation

Overview

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• type AuthConfig
• type Client
  • func NewClient(config Config, userAuth UserAuthConfig, opts ...Option) (*Client, error)
  • func (c *Client) Download(namespace, name string, dst io.Writer) error
  • func (c *Client) List(prefix string, opts ...backend.ListOption) (*backend.ListResult, error)
  • func (c *Client) Stat(namespace, name string) (*core.BlobInfo, error)
  • func (c *Client) Upload(namespace, name string, src io.Reader) error
• type Config
• type GCS
• type GCSImpl
  • func NewGCS(ctx context.Context, bucket *storage.BucketHandle, config *Config) *GCSImpl
  • func (g *GCSImpl) Download(objectName string, w io.Writer) (int64, error)
  • func (g *GCSImpl) GetObjectIterator(prefix string) iterator.Pageable
  • func (g *GCSImpl) NextPage(pager *iterator.Pager) ([]string, string, error)
  • func (g *GCSImpl) ObjectAttrs(objectName string) (*storage.ObjectAttrs, error)
  • func (g *GCSImpl) Upload(objectName string, r io.Reader) (int64, error)
• type Option
  • func WithGCS(gcs GCS) Option
• type UserAuthConfig

Types

type AuthConfig

type AuthConfig struct {
	GCS struct {
		AccessBlob string `yaml:"access_blob"`
	} `yaml:"gcs"`
}

AuthConfig matches Langley format.

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client implements a backend.Client for GCS.

func NewClient

func NewClient(
	config Config, userAuth UserAuthConfig, opts ...Option) (*Client, error)

NewClient creates a new Client for GCS.

func (*Client) Download

func (c *Client) Download(namespace, name string, dst io.Writer) error

Download downloads the content from a configured bucket and writes the data to dst.

func (*Client) List

func (c *Client) List(prefix string, opts ...backend.ListOption) (*backend.ListResult, error)

List lists names that start with prefix.

func (*Client) Stat

func (c *Client) Stat(namespace, name string) (*core.BlobInfo, error)

Stat returns blob info for name.

func (*Client) Upload

func (c *Client) Upload(namespace, name string, src io.Reader) error

Upload uploads src to a configured bucket.

type Config

type Config struct {
	Username string `yaml:"username"` // IAM username for selecting credentials.
	Location string `yaml:"location"` // Location of the bucket. Defautls to "US".
	Bucket   string `yaml:"bucket"`   // GCS bucket

	RootDirectory   string `yaml:"root_directory"`   // GCS root directory for docker images
	UploadChunkSize int64  `yaml:"upload_part_size"` // part size gcs manager uses for upload

	// ListMaxKeys sets the max keys returned per page.
	ListMaxKeys int `yaml:"list_max_keys"`

	// BufferGuard protects download from downloading into an oversized buffer
	// when io.WriterAt is not implemented.
	BufferGuard datasize.ByteSize `yaml:"buffer_guard"`

	// NamePath identifies which namepath.Pather to use.
	NamePath string `yaml:"name_path"`
}

Config defines gcs connection specific parameters and authetication credentials

type GCS

type GCS interface {
	ObjectAttrs(objectName string) (*storage.ObjectAttrs, error)
	Download(objectName string, w io.Writer) (int64, error)
	Upload(objectName string, r io.Reader) (int64, error)
	GetObjectIterator(prefix string) iterator.Pageable
	NextPage(pager *iterator.Pager) ([]string, string, error)
}

GCS defines the operations we use in the GCS api. Useful for mocking.

type GCSImpl

type GCSImpl struct {
	// contains filtered or unexported fields
}

GCSImpl implements GCS interaface.

func NewGCS

func NewGCS(ctx context.Context, bucket *storage.BucketHandle,
	config *Config) *GCSImpl

func (*GCSImpl) Download

func (g *GCSImpl) Download(objectName string, w io.Writer) (int64, error)

func (*GCSImpl) GetObjectIterator

func (g *GCSImpl) GetObjectIterator(prefix string) iterator.Pageable

func (*GCSImpl) NextPage

func (g *GCSImpl) NextPage(pager *iterator.Pager) ([]string, string,
	error)

func (*GCSImpl) ObjectAttrs

func (g *GCSImpl) ObjectAttrs(objectName string) (*storage.ObjectAttrs, error)

func (*GCSImpl) Upload

func (g *GCSImpl) Upload(objectName string, r io.Reader) (int64, error)

type Option

type Option func(*Client)

Option allows setting optional Client parameters.

func WithGCS

func WithGCS(gcs GCS) Option

WithGCS configures a Client with a custom GCS implementation.

type UserAuthConfig

type UserAuthConfig map[string]AuthConfig

UserAuthConfig defines authentication configuration overlayed by Langley. Each key is the iam username of the credentials.