storage

package
v0.0.0-...-9c41f32 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 16, 2022 License: MIT Imports: 34 Imported by: 0

Documentation

Overview

Package storage implements operations for s3 and fs.

Index

Constants

This section is empty.

Variables

View Source
var (

	// ErrNoObjectFound indicates there are no objects found from a given directory.
	ErrNoObjectFound = fmt.Errorf("no object found")
)

Functions

func IsCancelationError

func IsCancelationError(err error) bool

IsCancelationError reports whether given error is a storage related cancelation error.

func IsGoogleEndpoint

func IsGoogleEndpoint(endpoint urlpkg.URL) bool

func ShouldProcessUrl

func ShouldProcessUrl(url *url.URL, followSymlinks bool) bool

ShouldProcessUrl returns true if follow symlinks is enabled. If follow symlinks is disabled we should not process the url. (this check is needed only for local files)

Types

type Bucket

type Bucket struct {
	CreationDate time.Time `json:"created_at"`
	Name         string    `json:"name"`
}

Bucket is a container for storage objects.

func (Bucket) JSON

func (b Bucket) JSON() string

JSON returns the JSON representation of Bucket.

func (Bucket) String

func (b Bucket) String() string

String returns the string representation of Bucket.

type ErrGivenObjectNotFound

type ErrGivenObjectNotFound struct {
	ObjectAbsPath string
}

ErrGivenObjectNotFound indicates a specified object is not found.

func (*ErrGivenObjectNotFound) Error

func (e *ErrGivenObjectNotFound) Error() string

type Filesystem

type Filesystem struct {
	// contains filtered or unexported fields
}

Filesystem is the Storage implementation of a local filesystem.

func NewLocalClient

func NewLocalClient(opts Options) *Filesystem

func (*Filesystem) Copy

func (f *Filesystem) Copy(ctx context.Context, src, dst *url.URL, _ Metadata) error

Copy copies given source to destination.

func (*Filesystem) Create

func (f *Filesystem) Create(path string) (*os.File, error)

Create creates a new os.File.

func (*Filesystem) Delete

func (f *Filesystem) Delete(ctx context.Context, url *url.URL) error

Delete deletes given file.

func (*Filesystem) List

func (f *Filesystem) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List returns the objects and directories reside in given src.

func (*Filesystem) MkdirAll

func (f *Filesystem) MkdirAll(path string) error

MkdirAll calls os.MkdirAll.

func (*Filesystem) MultiDelete

func (f *Filesystem) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete deletes all files returned from given channel.

func (*Filesystem) Open

func (f *Filesystem) Open(path string) (*os.File, error)

Open opens the given source.

func (*Filesystem) Stat

func (f *Filesystem) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat returns the Object structure describing object.

type Metadata

type Metadata map[string]string

func NewMetadata

func NewMetadata() Metadata

NewMetadata will return an empty metadata object.

func (Metadata) ACL

func (m Metadata) ACL() string

func (Metadata) CacheControl

func (m Metadata) CacheControl() string

func (Metadata) ContentEncoding

func (m Metadata) ContentEncoding() string

func (Metadata) ContentType

func (m Metadata) ContentType() string

func (Metadata) Expires

func (m Metadata) Expires() string

func (Metadata) SSE

func (m Metadata) SSE() string

func (Metadata) SSEKeyID

func (m Metadata) SSEKeyID() string

func (Metadata) SetACL

func (m Metadata) SetACL(acl string) Metadata

func (Metadata) SetCacheControl

func (m Metadata) SetCacheControl(cacheControl string) Metadata

func (Metadata) SetContentEncoding

func (m Metadata) SetContentEncoding(contentEncoding string) Metadata

func (Metadata) SetContentType

func (m Metadata) SetContentType(contentType string) Metadata

func (Metadata) SetExpires

func (m Metadata) SetExpires(expires string) Metadata

func (Metadata) SetSSE

func (m Metadata) SetSSE(sse string) Metadata

func (Metadata) SetSSEKeyID

func (m Metadata) SetSSEKeyID(kid string) Metadata

func (Metadata) SetStorageClass

func (m Metadata) SetStorageClass(class string) Metadata

func (Metadata) StorageClass

func (m Metadata) StorageClass() string

type MockStorage

type MockStorage struct {
	mock.Mock
}

MockStorage is an autogenerated mock type for the Storage type

func (*MockStorage) Copy

func (_m *MockStorage) Copy(ctx context.Context, src *url.URL, dst *url.URL, metadata Metadata) error

Copy provides a mock function with given fields: ctx, src, dst, metadata

func (*MockStorage) Delete

func (_m *MockStorage) Delete(ctx context.Context, src *url.URL) error

Delete provides a mock function with given fields: ctx, src

func (*MockStorage) List

func (_m *MockStorage) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List provides a mock function with given fields: ctx, src, followSymlinks

func (*MockStorage) MultiDelete

func (_m *MockStorage) MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

MultiDelete provides a mock function with given fields: ctx, urls

func (*MockStorage) Stat

func (_m *MockStorage) Stat(ctx context.Context, src *url.URL) (*Object, error)

Stat provides a mock function with given fields: ctx, src

type Object

type Object struct {
	URL          *url.URL     `json:"key,omitempty"`
	Etag         string       `json:"etag,omitempty"`
	ModTime      *time.Time   `json:"last_modified,omitempty"`
	Type         ObjectType   `json:"type,omitempty"`
	Size         int64        `json:"size,omitempty"`
	StorageClass StorageClass `json:"storage_class,omitempty"`
	Err          error        `json:"error,omitempty"`
	// contains filtered or unexported fields
}

Object is a generic type which contains metadata for storage items.

func (*Object) JSON

func (o *Object) JSON() string

JSON returns the JSON representation of Object.

func (*Object) String

func (o *Object) String() string

String returns the string representation of Object.

type ObjectType

type ObjectType struct {
	// contains filtered or unexported fields
}

ObjectType is the type of Object.

func (ObjectType) IsDir

func (o ObjectType) IsDir() bool

IsDir checks if the object is a directory.

func (o ObjectType) IsSymlink() bool

IsSymlink checks if the object is a symbolic link.

func (ObjectType) MarshalJSON

func (o ObjectType) MarshalJSON() ([]byte, error)

MarshalJSON returns the stringer of ObjectType as a marshalled json.

func (ObjectType) String

func (o ObjectType) String() string

String returns the string representation of ObjectType.

type Options

type Options struct {
	MaxRetries             int
	NoSuchUploadRetryCount int
	Endpoint               string
	NoVerifySSL            bool
	DryRun                 bool
	NoSignRequest          bool
	UseListObjectsV1       bool
	LogLevel               log.LogLevel
	RequestPayer           string
	Profile                string
	CredentialFile         string
	// contains filtered or unexported fields
}

Options stores configuration for storage.

func (*Options) SetRegion

func (o *Options) SetRegion(region string)

type S3

type S3 struct {
	// contains filtered or unexported fields
}

S3 is a storage type which interacts with S3API, DownloaderAPI and UploaderAPI.

func NewRemoteClient

func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error)

func (*S3) Copy

func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) error

Copy is a single-object copy operation which copies objects to S3 destination from another S3 source.

func (*S3) Delete

func (s *S3) Delete(ctx context.Context, url *url.URL) error

Delete is a single object delete operation.

func (*S3) Get

func (s *S3) Get(
	ctx context.Context,
	from *url.URL,
	to io.WriterAt,
	concurrency int,
	partSize int64,
) (int64, error)

Get is a multipart download operation which downloads S3 objects into any destination that implements io.WriterAt interface. Makes a single 'GetObject' call if 'concurrency' is 1 and ignores 'partSize'.

func (*S3) List

func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object

List is a non-blocking S3 list operation which paginates and filters S3 keys. If no object found or an error is encountered during this period, it sends these errors to object channel.

func (*S3) ListBuckets

func (s *S3) ListBuckets(ctx context.Context, prefix string) ([]Bucket, error)

ListBuckets is a blocking list-operation which gets bucket list and returns the buckets that match with given prefix.

func (*S3) MakeBucket

func (s *S3) MakeBucket(ctx context.Context, name string) error

MakeBucket creates an S3 bucket with the given name.

func (*S3) MultiDelete

func (s *S3) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete is a asynchronous removal operation for multiple objects. It reads given url channel, creates multiple chunks and run these chunks in parallel. Each chunk may have at most 1000 objects since DeleteObjects API has a limitation. See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html.

func (*S3) Put

func (s *S3) Put(
	ctx context.Context,
	reader io.Reader,
	to *url.URL,
	metadata Metadata,
	concurrency int,
	partSize int64,
) error

Put is a multipart upload operation to upload resources, which implements io.Reader interface, into S3 destination.

func (*S3) Read

func (s *S3) Read(ctx context.Context, src *url.URL) (io.ReadCloser, error)

Read fetches the remote object and returns its contents as an io.ReadCloser.

func (*S3) RemoveBucket

func (s *S3) RemoveBucket(ctx context.Context, name string) error

RemoveBucket removes an S3 bucket with the given name.

func (*S3) RequestPayer

func (s *S3) RequestPayer() *string

func (*S3) Select

func (s *S3) Select(ctx context.Context, url *url.URL, query *SelectQuery, resultCh chan<- json.RawMessage) error

func (*S3) Stat

func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat retrieves metadata from S3 object without returning the object itself.

type SelectQuery

type SelectQuery struct {
	ExpressionType  string
	Expression      string
	CompressionType string
}

type SessionCache

type SessionCache struct {
	sync.Mutex
	// contains filtered or unexported fields
}

SessionCache holds session.Session according to s3Opts and it synchronizes access/modification.

type Storage

type Storage interface {
	// Stat returns the Object structure describing object. If src is not
	// found, ErrGivenObjectNotFound is returned.
	Stat(ctx context.Context, src *url.URL) (*Object, error)

	// List the objects and directories/prefixes in the src.
	List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

	// Delete deletes the given src.
	Delete(ctx context.Context, src *url.URL) error

	// MultiDelete deletes all items returned from given urls in batches.
	MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

	// Copy src to dst, optionally setting the given metadata. Src and dst
	// arguments are of the same type. If src is a remote type, server side
	// copying will be used.
	Copy(ctx context.Context, src, dst *url.URL, metadata Metadata) error
}

Storage is an interface for storage operations that is common to local filesystem and remote object storage.

func NewClient

func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error)

type StorageClass

type StorageClass string

StorageClass represents the storage used to store an object.

func (StorageClass) IsGlacier

func (s StorageClass) IsGlacier() bool

Directories

Path Synopsis
Package url abstracts local and remote file URLs.
Package url abstracts local and remote file URLs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL