storage

README

Package storage v1.3.3

Documentation

Index

• Constants
• func WithPartSize(size int64) s3Opt
• func WithRetryPolicy(policy RetryPolicy) s3Opt
• func WithUploadConcurrency(concurrency int) s3Opt
• type Backend
• type BackendMethod
• type ObjectNotFound
  • func (err ObjectNotFound) Error() string
• type RetryPolicy
• type S3
  • func NewS3(cfg S3Config, opts ...s3Opt) *S3
  • func (s *S3) Delete(ctx context.Context, path string) error
  • func (s *S3) Get(ctx context.Context, path string) (io.ReadCloser, error)
  • func (s *S3) Info(ctx context.Context, path string) (types.Info, error)
  • func (s *S3) List(ctx context.Context, prefix string, opts types.ListOpts) ([]string, error)
  • func (s *S3) Move(ctx context.Context, srcPath, dstPath string) error
  • func (s *S3) Size(ctx context.Context, path string) (int64, error)
  • func (s *S3) Upload(ctx context.Context, file io.Reader, path string) error
• type S3Client
• type S3Config
• type Swift
  • func NewSwift(cfg SwiftConfig) (*Swift, error)
  • func (s *Swift) Delete(ctx context.Context, path string) error
  • func (s *Swift) Get(ctx context.Context, path string) (io.ReadCloser, error)
  • func (s *Swift) Info(ctx context.Context, path string) (types.Info, error)
  • func (s *Swift) List(ctx context.Context, prefix string, opts types.ListOpts) ([]string, error)
  • func (s *Swift) Move(ctx context.Context, srcPath, dstPath string) error
  • func (s *Swift) Size(ctx context.Context, path string) (int64, error)
  • func (s *Swift) Upload(ctx context.Context, reader io.Reader, path string) error
• type SwiftConfig

Constants

const (
	NotFoundErrCode = "NotFound"
	// DefaultPartSize 16 MB part size define the size in bytes of the parts
	// uploaded in a multipart upload
	DefaultPartSize = int64(16777216)
	// DefaultUploadConcurrency defines that multipart upload will be done in
	// parallel in 2 routines
	DefaultUploadConcurrency = int(2)
	// S3ListMaxKeys is defined here https://github.com/aws/aws-sdk-go-v2/blob/v1.17.1/service/s3/api_op_ListObjectsV2.go#L107-L109
	S3ListMaxKeys = 1000
)

Functions

func WithPartSize

func WithPartSize(size int64) s3Opt

func WithRetryPolicy

func WithRetryPolicy(policy RetryPolicy) s3Opt

WithRetryPolicy is an option to constructor NewS3 to add a Retry Policy impacting GET operations

func WithUploadConcurrency

func WithUploadConcurrency(concurrency int) s3Opt

Types

type Backend

type Backend interface {
	Get(ctx context.Context, path string) (io.ReadCloser, error)
	Upload(ctx context.Context, file io.Reader, path string) error
	Size(ctx context.Context, path string) (int64, error)
	Delete(ctx context.Context, path string) error
	Info(ctx context.Context, path string) (types.Info, error)
	Move(ctx context.Context, srcPath, dstPath string) error
	List(ctx context.Context, prefix string, opts types.ListOpts) ([]string, error)
}

Backend represents something which is able to store files on an object storage service

type BackendMethod

type BackendMethod string

BackendMethod represents the name of a Method included in the Backend interface

const (
	GetMethod    BackendMethod = "Get"
	UploadMethod BackendMethod = "Upload"
	SizeMethod   BackendMethod = "Size"
	InfoMethod   BackendMethod = "Info"
	DeleteMethod BackendMethod = "Delete"
	MoveMethod   BackendMethod = "Move"
	ListMethod   BackendMethod = "List"
)

type ObjectNotFound

type ObjectNotFound struct {
	Path string
}

ObjectNotFound is a custom error in case the object is not found

func (ObjectNotFound) Error

func (err ObjectNotFound) Error() string

type RetryPolicy

type RetryPolicy struct {
	WaitDuration   time.Duration
	Attempts       int
	MethodHandlers map[BackendMethod][]string
}

type S3

type S3 struct {
	// contains filtered or unexported fields
}

func NewS3

func NewS3(cfg S3Config, opts ...s3Opt) *S3

func (*S3) Delete

func (s *S3) Delete(ctx context.Context, path string) error

func (*S3) Get

func (s *S3) Get(ctx context.Context, path string) (io.ReadCloser, error)

func (*S3) Info

func (s *S3) Info(ctx context.Context, path string) (types.Info, error)

Info returns several information contained in the header. It returns ObjectNotFound custom error if the object does not exists.

func (*S3) List

func (s *S3) List(ctx context.Context, prefix string, opts types.ListOpts) ([]string, error)

List function lists object contained in bucket up to 1,000 objects. If maxKeys > 1,000, S3 will set maxKeys to 1,000. Source: https://github.com/aws/aws-sdk-go-v2/blob/v1.17.1/service/s3/api_op_ListObjectsV2.go#L16

func (*S3) Move

func (s *S3) Move(ctx context.Context, srcPath, dstPath string) error

func (*S3) Size

func (s *S3) Size(ctx context.Context, path string) (int64, error)

Size returns the size of the content of the object. A retry mechanism is implemented because of the eventual consistency of S3 backends NotFound error are sometimes returned when the object was just uploaded.

func (*S3) Upload

func (s *S3) Upload(ctx context.Context, file io.Reader, path string) error

type S3Client

type S3Client interface {
	GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error)
	HeadObject(ctx context.Context, input *s3.HeadObjectInput, opts ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
	DeleteObject(ctx context.Context, input *s3.DeleteObjectInput, opts ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
	CopyObject(ctx context.Context, params *s3.CopyObjectInput, optFns ...func(*s3.Options)) (*s3.CopyObjectOutput, error)
	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
}

type S3Config

type S3Config struct {
	AK       string
	SK       string
	Region   string
	Endpoint string
	Bucket   string
}

type Swift

type Swift struct {
	// contains filtered or unexported fields
}

func NewSwift

func NewSwift(cfg SwiftConfig) (*Swift, error)

NewSwift instantiate a new connection to a Swift object storage. The configuration is taken from the environment. Refer to the github.com/ncw/swift documentation for more information.

func (*Swift) Delete

func (s *Swift) Delete(ctx context.Context, path string) error

func (*Swift) Get

func (s *Swift) Get(ctx context.Context, path string) (io.ReadCloser, error)

func (*Swift) Info

func (s *Swift) Info(ctx context.Context, path string) (types.Info, error)

func (*Swift) List

func (s *Swift) List(ctx context.Context, prefix string, opts types.ListOpts) ([]string, error)

func (*Swift) Move

func (s *Swift) Move(ctx context.Context, srcPath, dstPath string) error

func (*Swift) Size

func (s *Swift) Size(ctx context.Context, path string) (int64, error)

Size returns the size of the content of the object. A retry mechanism is implemented because of the eventual consistency of Swift backends NotFound error are sometimes returned when the object was just uploaded.

func (*Swift) Upload

func (s *Swift) Upload(ctx context.Context, reader io.Reader, path string) error

type SwiftConfig

type SwiftConfig struct {
	Prefix    string
	Container string
	ChunkSize int64
}