obj

Documentation

Index

• Constants
• func Copy(ctx context.Context, src, dst Client, srcPath, dstPath string) (retErr error)
• func DisableMonkeyTest()
• func EnableMonkeyTest()
• func InitMonkeyTest(seed int64)
• func IsMonkeyError(err error) bool
• func NewKeyIterator(b *blob.Bucket, prefix string) stream.Iterator[string]
• func TestEmptyWrite(t *testing.T, client Client)
• func TestInterruption(t *testing.T, client Client)
• func TestStorage(ctx context.Context, c Client) error
• func TestSuite(t *testing.T, newClient func(t testing.TB) Client)
• type AmazonAdvancedConfiguration
• type AmazonCreds
• type Client
  • func NewAmazonClient(ctx context.Context, region, bucket string, creds *AmazonCreds, ...) (c Client, err error)
  • func NewAmazonClientFromEnv(ctx context.Context) (Client, error)
  • func NewAmazonClientFromSecret(ctx context.Context, bucket string, reverse ...bool) (Client, error)
  • func NewClient(ctx context.Context, storageBackend string, storageRoot string) (Client, error)
  • func NewClientFromEnv(ctx context.Context, storageRoot string) (c Client, err error)
  • func NewClientFromURLAndSecret(ctx context.Context, url *ObjectStoreURL, reverse ...bool) (c Client, err error)
  • func NewGoogleClient(bucket string, opts []option.ClientOption) (c Client, err error)
  • func NewGoogleClientFromEnv() (Client, error)
  • func NewGoogleClientFromSecret(bucket string) (Client, error)
  • func NewLocalClient(rootDir string) (Client, error)
  • func NewMicrosoftClient(container string, accountName string, accountKey string) (c Client, err error)
  • func NewMicrosoftClientFromEnv() (Client, error)
  • func NewMicrosoftClientFromSecret(container string) (Client, error)
  • func NewMinioClient(endpoint, bucket, id, secret string, secure, isS3V2 bool) (c Client, err error)
  • func NewMinioClientFromEnv() (Client, error)
  • func NewMinioClientFromSecret(bucket string) (Client, error)
  • func TracingObjClient(provider string, c Client) Client
  • func WrapWithTestURL(c Client) Client
• type ObjectStoreURL
  • func ParseURL(urlStr string) (*ObjectStoreURL, error)
  • func (s ObjectStoreURL) BucketString() string
  • func (s ObjectStoreURL) String() string

Constants

const (
	Minio     = "MINIO"
	Amazon    = "AMAZON"
	Google    = "GOOGLE"
	Microsoft = "MICROSOFT"
	Local     = "LOCAL"
)

Valid object storage backends

const (
	GoogleBucketEnvVar = "GOOGLE_BUCKET"
	GoogleCredEnvVar   = "GOOGLE_CRED"
)

Google environment variables

const (
	MicrosoftContainerEnvVar = "MICROSOFT_CONTAINER"
	MicrosoftIDEnvVar        = "MICROSOFT_ID"
	MicrosoftSecretEnvVar    = "MICROSOFT_SECRET"
)

Microsoft environment variables

const (
	MinioBucketEnvVar    = "MINIO_BUCKET"
	MinioEndpointEnvVar  = "MINIO_ENDPOINT"
	MinioIDEnvVar        = "MINIO_ID"
	MinioSecretEnvVar    = "MINIO_SECRET"
	MinioSecureEnvVar    = "MINIO_SECURE"
	MinioSignatureEnvVar = "MINIO_SIGNATURE"
)

Minio environment variables

const (
	AmazonRegionEnvVar       = "AMAZON_REGION"
	AmazonBucketEnvVar       = "AMAZON_BUCKET"
	AmazonIDEnvVar           = "AMAZON_ID"
	AmazonSecretEnvVar       = "AMAZON_SECRET"
	AmazonTokenEnvVar        = "AMAZON_TOKEN"
	AmazonDistributionEnvVar = "AMAZON_DISTRIBUTION"
	CustomEndpointEnvVar     = "CUSTOM_ENDPOINT"
)

Amazon environment variables

const (
	StorageBackendEnvVar = "STORAGE_BACKEND"
)

Environment variables for determining storage backend and pathing

Functions

func Copy

func Copy(ctx context.Context, src, dst Client, srcPath, dstPath string) (retErr error)

Copy copys an object from src at srcPath to dst at dstPath

func DisableMonkeyTest

func DisableMonkeyTest()

DisableMonkeyTest disables sporadic request failures.

func EnableMonkeyTest

func EnableMonkeyTest()

EnableMonkeyTest enables sporadic request failures.

func InitMonkeyTest

func InitMonkeyTest(seed int64)

InitMonkeyTest sets up this package for monkey testing. Object storage clients will be wrapped with a client that sporadically fails requests.

func IsMonkeyError

func IsMonkeyError(err error) bool

IsMonkeyError checks if an error was caused by a monkey client.

func NewKeyIterator

func NewKeyIterator(b *blob.Bucket, prefix string) stream.Iterator[string]

NewKeyIterator iterates over the keys in a blob.Bucket.

func TestEmptyWrite

func TestEmptyWrite(t *testing.T, client Client)

func TestInterruption

func TestInterruption(t *testing.T, client Client)

TestInterruption Interruption is currently not implemented on the Amazon, Microsoft, and Minio clients

Amazon client - use *WithContext methods
Microsoft client - move to github.com/Azure/azure-storage-blob-go which supports contexts
Minio client - upgrade to v7 which supports contexts in all APIs
Local client - interruptible file operations are not a thing in the stdlib

func TestStorage

func TestStorage(ctx context.Context, c Client) error

TestStorage is a defensive method for checking to make sure that storage is properly configured.

func TestSuite

func TestSuite(t *testing.T, newClient func(t testing.TB) Client)

TestSuite runs tests to ensure the object returned by newClient implements Client newClient should register cleanup of the returned object using testing.T.Cleanup All of the subtests call t.Parallel, but the top level test does not.

Types

type AmazonAdvancedConfiguration

type AmazonAdvancedConfiguration struct {
	Retries int    `env:"RETRIES, default=10"`
	Timeout string `env:"TIMEOUT, default=5m"`
	// By default, objects uploaded to a bucket are only accessible to the
	// uploader, and not the owner of the bucket. Using the default ensures that
	// the owner of the bucket can access the objects as well.
	UploadACL      string `env:"UPLOAD_ACL, default=bucket-owner-full-control"`
	PartSize       int64  `env:"PART_SIZE, default=5242880"`
	MaxUploadParts int    `env:"MAX_UPLOAD_PARTS, default=10000"`
	DisableSSL     bool   `env:"DISABLE_SSL, default=false"`
	NoVerifySSL    bool   `env:"NO_VERIFY_SSL, default=false"`
	LogOptions     string `env:"OBJ_LOG_OPTS, default="`
}

AmazonAdvancedConfiguration contains the advanced configuration for the amazon client.

type AmazonCreds

type AmazonCreds struct {
	// Direct credentials. Only applicable if Pachyderm is given its own permanent
	// AWS credentials
	ID     string // Access Key ID
	Secret string // Secret Access Key
	Token  string // Access token (if using temporary security credentials
}

AmazonCreds are options that are applicable specifically to Pachd's credentials in an AWS deployment

type Client

type Client interface {
	// Put writes the data from r to an object at name
	// It should error if the object already exists or we don't have sufficient
	// permissions to write it.
	Put(ctx context.Context, name string, r io.Reader) error

	// Get writes the data for an object to w
	// If `size == 0`, the reader should read from the offset till the end of the object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to read it.
	Get(ctx context.Context, name string, w io.Writer) error

	// Delete deletes an object.
	// It should error if the object doesn't exist or we don't have sufficient
	// permission to delete it.
	Delete(ctx context.Context, name string) error

	// Walk calls `fn` with the names of objects which can be found under `prefix`.
	Walk(ctx context.Context, prefix string, fn func(name string) error) error

	// Exists checks if a given object already exists
	Exists(ctx context.Context, name string) (bool, error)

	// BucketURL returns the URL of the bucket this client uses.
	BucketURL() ObjectStoreURL
}

Client is an interface to object storage.

func NewAmazonClient

func NewAmazonClient(ctx context.Context, region, bucket string, creds *AmazonCreds, distribution string, endpoint string) (c Client, err error)

NewAmazonClient creates an amazon client with the following credentials:

bucket - S3 bucket name
distribution - cloudfront distribution ID
id     - AWS access key id
secret - AWS secret access key
token  - AWS access token
region - AWS region
endpoint - Custom endpoint (generally used for S3 compatible object stores)
reverse - Reverse object storage paths (overwrites configured value)

func NewAmazonClientFromEnv

func NewAmazonClientFromEnv(ctx context.Context) (Client, error)

NewAmazonClientFromEnv creates a Amazon client based on environment variables.

func NewAmazonClientFromSecret

func NewAmazonClientFromSecret(ctx context.Context, bucket string, reverse ...bool) (Client, error)

NewAmazonClientFromSecret constructs an amazon client by reading credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it will read the bucket from the secret.

func NewClient

func NewClient(ctx context.Context, storageBackend string, storageRoot string) (Client, error)

NewClient creates an obj.Client using the given backend and storage root (for local backends). TODO: Not sure if we want to keep the storage root configuration for non-local deployments. If so, we will need to connect it to the object path prefix for chunks.

func NewClientFromEnv

func NewClientFromEnv(ctx context.Context, storageRoot string) (c Client, err error)

NewClientFromEnv creates a client based on environment variables.

func NewClientFromURLAndSecret

func NewClientFromURLAndSecret(ctx context.Context, url *ObjectStoreURL, reverse ...bool) (c Client, err error)

NewClientFromURLAndSecret constructs a client by parsing `URL` and then constructing the correct client for that URL using secrets.

func NewGoogleClient

func NewGoogleClient(bucket string, opts []option.ClientOption) (c Client, err error)

NewGoogleClient creates a google client with the given bucket name.

func NewGoogleClientFromEnv

func NewGoogleClientFromEnv() (Client, error)

NewGoogleClientFromEnv creates a Google client based on environment variables.

func NewGoogleClientFromSecret

func NewGoogleClientFromSecret(bucket string) (Client, error)

NewGoogleClientFromSecret creates a google client by reading credentials from a mounted GoogleSecret. You may pass "" for bucket in which case it will read the bucket from the secret.

func NewLocalClient

func NewLocalClient(rootDir string) (Client, error)

NewLocalClient returns a Client that stores data on the local file system

func NewMicrosoftClient

func NewMicrosoftClient(container string, accountName string, accountKey string) (c Client, err error)

NewMicrosoftClient creates a microsoft client:

container   - Azure Blob Container name
accountName - Azure Storage Account name
accountKey  - Azure Storage Account key

func NewMicrosoftClientFromEnv

func NewMicrosoftClientFromEnv() (Client, error)

NewMicrosoftClientFromEnv creates a Microsoft client based on environment variables.

func NewMicrosoftClientFromSecret

func NewMicrosoftClientFromSecret(container string) (Client, error)

NewMicrosoftClientFromSecret creates a microsoft client by reading credentials from a mounted MicrosoftSecret. You may pass "" for container in which case it will read the container from the secret.

func NewMinioClient

func NewMinioClient(endpoint, bucket, id, secret string, secure, isS3V2 bool) (c Client, err error)

NewMinioClient creates an s3 compatible client with the following credentials:

endpoint - S3 compatible endpoint
bucket - S3 bucket name
id     - AWS access key id
secret - AWS secret access key
secure - Set to true if connection is secure.
isS3V2 - Set to true if client follows S3V2

func NewMinioClientFromEnv

func NewMinioClientFromEnv() (Client, error)

NewMinioClientFromEnv creates a Minio client based on environment variables.

func NewMinioClientFromSecret

func NewMinioClientFromSecret(bucket string) (Client, error)

NewMinioClientFromSecret constructs an s3 compatible client by reading credentials from a mounted AmazonSecret. You may pass "" for bucket in which case it will read the bucket from the secret.

func TracingObjClient

func TracingObjClient(provider string, c Client) Client

TracingObjClient wraps the given object client 'c', adding tracing and monitoring to all calls made by the returned interface.

func WrapWithTestURL

func WrapWithTestURL(c Client) Client

WrapWithTestURL marks client as a test client and will prepend test- to the url. the consturctors in this package know how to parse test urls, and assume default credentials.

type ObjectStoreURL

type ObjectStoreURL struct {
	// The object store, e.g. s3, gcs, as...
	Scheme string
	// The "bucket" (in AWS parlance) or the "container" (in Azure parlance).
	Bucket string
	// The object itself.
	Object string
	// The query parameters if defined.
	Params string
}

ObjectStoreURL represents a parsed URL to an object in an object store.

func ParseURL

func ParseURL(urlStr string) (*ObjectStoreURL, error)

ParseURL parses an URL into ObjectStoreURL.

func (ObjectStoreURL) BucketString

func (s ObjectStoreURL) BucketString() string

func (ObjectStoreURL) String

func (s ObjectStoreURL) String() string