s3backend

Documentation

Overview

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright (c) 2016-2019 Uber Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• type AuthConfig
• type Client
  • func NewClient(config Config, userAuth UserAuthConfig, opts ...Option) (*Client, error)
  • func (c *Client) Download(namespace, name string, dst io.Writer) error
  • func (c *Client) List(prefix string) ([]string, error)
  • func (c *Client) Stat(namespace, name string) (*core.BlobInfo, error)
  • func (c *Client) Upload(namespace, name string, src io.Reader) error
• type Config
• type Option
  • func WithS3(s3 S3) Option
• type S3
• type UserAuthConfig

Types

type AuthConfig

type AuthConfig struct {
	S3 struct {
		AccessKeyID     string `yaml:"aws_access_key_id"`
		AccessSecretKey string `yaml:"aws_secret_access_key"`
		SessionToken    string `yaml:"aws_session_token"`
	} `yaml:"s3"`
}

AuthConfig matches Langley format.

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client implements a backend.Client for S3.

func NewClient

func NewClient(
	config Config, userAuth UserAuthConfig, opts ...Option) (*Client, error)

NewClient creates a new Client for S3.

func (*Client) Download

func (c *Client) Download(namespace, name string, dst io.Writer) error

Download downloads the content from a configured bucket and writes the data to dst.

func (*Client) List

func (c *Client) List(prefix string) ([]string, error)

List lists names with start with prefix.

func (*Client) Stat

func (c *Client) Stat(namespace, name string) (*core.BlobInfo, error)

Stat returns blob info for name.

func (*Client) Upload

func (c *Client) Upload(namespace, name string, src io.Reader) error

Upload uploads src to a configured bucket.

type Config

type Config struct {
	Username string `yaml:"username"` // IAM username for selecting credentials.
	Region   string `yaml:"region"`   // AWS S3 region
	Bucket   string `yaml:"bucket"`   // S3 bucket

	RootDirectory    string `yaml:"root_directory"`     // S3 root directory for docker images
	UploadPartSize   int64  `yaml:"upload_part_size"`   // part size s3 manager uses for upload
	DownloadPartSize int64  `yaml:"download_part_size"` // part size s3 manager uses for download

	UploadConcurrency   int `yaml:"upload_concurrency"`   // # of concurrent go-routines s3 manager uses for upload
	DownloadConcurrency int `yaml:"download_concurrency"` // # of concurrent go-routines s3 manager uses for download

	// ListMaxKeys sets the max keys returned per page.
	ListMaxKeys int `yaml:"list_max_keys"`

	// BufferGuard protects download from downloading into an oversized buffer
	// when io.WriterAt is not implemented.
	BufferGuard datasize.ByteSize `yaml:"buffer_guard"`

	// NamePath identifies which namepath.Pather to use.
	NamePath string `yaml:"name_path"`
}

Config defines s3 connection specific parameters and authetication credentials

type Option

type Option func(*Client)

Option allows setting optional Client parameters.

func WithS3

func WithS3(s3 S3) Option

WithS3 configures a Client with a custom S3 implementation.

type S3

type S3 interface {
	HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)

	Download(
		w io.WriterAt,
		input *s3.GetObjectInput,
		options ...func(*s3manager.Downloader)) (n int64, err error)

	Upload(
		input *s3manager.UploadInput,
		options ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)

	ListObjectsPages(input *s3.ListObjectsInput, fn func(*s3.ListObjectsOutput, bool) bool) error
}

S3 defines the operations we use in the s3 api. Useful for mocking.

type UserAuthConfig

type UserAuthConfig map[string]AuthConfig

UserAuthConfig defines authentication configuration overlayed by Langley. Each key is the iam username of the credentials.