obj

Documentation

Index

• func IsRetryable(client Client, err error) bool
• func NewExponentialBackOffConfig() *backoff.ExponentialBackOff
• func TestIsNotExist(c Client) error
• type AmazonCreds
• type BackoffReadCloser
  • func (b *BackoffReadCloser) Close() error
  • func (b *BackoffReadCloser) Read(data []byte) (int, error)
• type BackoffWriteCloser
  • func (b *BackoffWriteCloser) Close() error
  • func (b *BackoffWriteCloser) Write(data []byte) (int, error)
• type Client
  • func NewAmazonClient(region, bucket string, creds *AmazonCreds, distribution string) (Client, error)
  • func NewAmazonClientFromSecret(bucket string) (Client, error)
  • func NewClientFromURLAndSecret(ctx context.Context, url *ObjectStoreURL) (Client, error)
  • func NewGoogleClient(ctx context.Context, bucket string, credFile string) (Client, error)
  • func NewGoogleClientFromSecret(ctx context.Context, bucket string) (Client, error)
  • func NewLocalClient(root string) (Client, error)
  • func NewMicrosoftClient(container string, accountName string, accountKey string) (Client, error)
  • func NewMicrosoftClientFromSecret(container string) (Client, error)
  • func NewMinioClient(endpoint, bucket, id, secret string, secure, isS3V2 bool) (Client, error)
  • func NewMinioClientFromSecret(bucket string) (Client, error)
• type ObjectStoreURL
  • func ParseURL(urlStr string) (*ObjectStoreURL, error)
• type RetryError

Functions

func IsRetryable

func IsRetryable(client Client, err error) bool

IsRetryable determines if an operation should be retried given an error

func NewExponentialBackOffConfig

func NewExponentialBackOffConfig() *backoff.ExponentialBackOff

NewExponentialBackOffConfig creates an exponential back-off config with longer wait times than the default.

func TestIsNotExist

func TestIsNotExist(c Client) error

TestIsNotExist is a defensive method for checking to make sure IsNotExist is satisfying its semantics.

Types

type AmazonCreds

type AmazonCreds struct {
	// Direct credentials. Only applicable if Pachyderm is given its own permanent
	// AWS credentials
	ID     string // Access Key ID
	Secret string // Secret Access Key
	Token  string // Access token (if using temporary security credentials

	// Vault options (if getting AWS credentials from Vault)
	VaultAddress string // normally addresses come from env, but don't have vault service name
	VaultRole    string
	VaultToken   string
}

AmazonCreds are options that are applicable specifically to Pachd's credentials in an AWS deployment

type BackoffReadCloser

type BackoffReadCloser struct {
	// contains filtered or unexported fields
}

BackoffReadCloser retries with exponential backoff in the case of failures

func (*BackoffReadCloser) Close

func (b *BackoffReadCloser) Close() error

Close closes the ReaderCloser contained in b.

func (*BackoffReadCloser) Read

func (b *BackoffReadCloser) Read(data []byte) (int, error)

type BackoffWriteCloser

type BackoffWriteCloser struct {
	// contains filtered or unexported fields
}

BackoffWriteCloser retries with exponential backoff in the case of failures

func (*BackoffWriteCloser) Close

func (b *BackoffWriteCloser) Close() error

Close closes the WriteCloser contained in b.

func (*BackoffWriteCloser) Write

func (b *BackoffWriteCloser) Write(data []byte) (int, error)

type Client

type Client interface {
	// Writer returns a writer which writes to an object.
	// It should error if the object already exists or we don't have sufficient
	// permissions to write it.
	Writer(name string) (io.WriteCloser, error)
	// Reader returns a reader which reads from an object.
	// If `size == 0`, the reader should read from the offset till the end of the object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to read it.
	Reader(name string, offset uint64, size uint64) (io.ReadCloser, error)
	// Delete deletes an object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to delete it.
	Delete(name string) error
	// Walk calls `fn` with the names of objects which can be found under `prefix`.
	Walk(prefix string, fn func(name string) error) error
	// Exsits checks if a given object already exists
	Exists(name string) bool
	// IsRetryable determines if an operation should be retried given an error
	IsRetryable(err error) bool
	// IsNotExist returns true if err is a non existence error
	IsNotExist(err error) bool
	// IsIgnorable returns true if the error can be ignored
	IsIgnorable(err error) bool
}

Client is an interface to object storage.

func NewAmazonClient

func NewAmazonClient(region, bucket string, creds *AmazonCreds, distribution string) (Client, error)

NewAmazonClient creates an amazon client with the following credentials:

bucket - S3 bucket name
distribution - cloudfront distribution ID
id     - AWS access key id
secret - AWS secret access key
token  - AWS access token
region - AWS region

func NewAmazonClientFromSecret

func NewAmazonClientFromSecret(bucket string) (Client, error)

NewAmazonClientFromSecret constructs an amazon client by reading credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it will read the bucket from the secret.

func NewClientFromURLAndSecret

func NewClientFromURLAndSecret(ctx context.Context, url *ObjectStoreURL) (Client, error)

NewClientFromURLAndSecret constructs a client by parsing `URL` and then constructing the correct client for that URL using secrets.

func NewGoogleClient

func NewGoogleClient(ctx context.Context, bucket string, credFile string) (Client, error)

NewGoogleClient creates a google client with the given bucket name.

func NewGoogleClientFromSecret

func NewGoogleClientFromSecret(ctx context.Context, bucket string) (Client, error)

NewGoogleClientFromSecret creates a google client by reading credentials from a mounted GoogleSecret. You may pass "" for bucket in which case it will read the bucket from the secret.

func NewLocalClient

func NewLocalClient(root string) (Client, error)

NewLocalClient returns a Client that stores data on the local file system

func NewMicrosoftClient

func NewMicrosoftClient(container string, accountName string, accountKey string) (Client, error)

NewMicrosoftClient creates a microsoft client:

container   - Azure Blob Container name
accountName - Azure Storage Account name
accountKey  - Azure Storage Account key

func NewMicrosoftClientFromSecret

func NewMicrosoftClientFromSecret(container string) (Client, error)

NewMicrosoftClientFromSecret creates a microsoft client by reading credentials from a mounted MicrosoftSecret. You may pass "" for container in which case it will read the container from the secret.

func NewMinioClient

func NewMinioClient(endpoint, bucket, id, secret string, secure, isS3V2 bool) (Client, error)

NewMinioClient creates an s3 compatible client with the following credentials:

endpoint - S3 compatible endpoint
bucket - S3 bucket name
id     - AWS access key id
secret - AWS secret access key
secure - Set to true if connection is secure.
isS3V2 - Set to true if client follows S3V2

func NewMinioClientFromSecret

func NewMinioClientFromSecret(bucket string) (Client, error)

NewMinioClientFromSecret constructs an s3 compatible client by reading credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it will read the bucket from the secret.

type ObjectStoreURL

type ObjectStoreURL struct {
	// The object store, e.g. s3, gcs, as...
	Store string
	// The "bucket" (in AWS parlance) or the "container" (in Azure parlance).
	Bucket string
	// The object itself.
	Object string
}

ObjectStoreURL represents a parsed URL to an object in an object store.

func ParseURL

func ParseURL(urlStr string) (*ObjectStoreURL, error)

ParseURL parses an URL into ObjectStoreURL.

type RetryError

type RetryError struct {
	Err               string
	TimeTillNextRetry string
	BytesProcessed    int
}

RetryError is used to log retry attempts.