s3

package
v0.33.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 1, 2021 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BlockstoreType = "s3"

	DefaultStreamingChunkSize    = 2 << 19         // 1MiB by default per chunk
	DefaultStreamingChunkTimeout = time.Second * 1 // if we haven't read DefaultStreamingChunkSize by this duration, write whatever we have as a chunk

	ExpireObjectS3Tag = "lakefs_expire_object"
)
View Source 
const (
	StreamingSha256          = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
	StreamingContentEncoding = "aws-chunked"
)

Variables

View Source 
var (
	ErrS3          = errors.New("s3 error")
	ErrMissingETag = fmt.Errorf("%w: missing ETag", ErrS3)
)
View Source 
var ErrInventoryFilesRangesOverlap = errors.New("got s3 inventory with files covering overlapping ranges")
View Source 
var ErrInventoryNotSorted = errors.New("got unsorted s3 inventory")
View Source 
var ErrReaderTimeout = errors.New("reader timeout")

Functions

func GenerateInventory 

func GenerateInventory(logger logging.Logger, manifestURL string, s3 s3iface.S3API, inventoryReader s3inventory.IReader, shouldSort bool, prefixes []string) (block.Inventory, error)

func ReadAllWithTimeout 

func ReadAllWithTimeout(r io.Reader, buf []byte, timeout time.Duration) (n int, err error)

ReadAllWithTimeout is taken from io.ReadAtLeast and adapted to support a timeout

func WithContext 

func WithContext(ctx context.Context) func(a *Adapter)

func WithHTTPClient 

func WithHTTPClient(c *http.Client) func(a *Adapter)

func WithStreamingChunkSize 

func WithStreamingChunkSize(sz int) func(a *Adapter)

func WithStreamingChunkTimeout 

func WithStreamingChunkTimeout(d time.Duration) func(a *Adapter)

func WithTranslator 

func WithTranslator(t block.UploadIDTranslator) func(a *Adapter)

Types

type Adapter 

type Adapter struct {
	// contains filtered or unexported fields
}

func NewAdapter 

func NewAdapter(s3 s3iface.S3API, opts ...func(a *Adapter)) *Adapter

func (*Adapter) AbortMultiPartUpload 

func (a *Adapter) AbortMultiPartUpload(obj block.ObjectPointer, uploadID string) error

func (*Adapter) BlockstoreType added in v0.11.0 

func (a *Adapter) BlockstoreType() string

func (*Adapter) CompleteMultiPartUpload 

func (a *Adapter) CompleteMultiPartUpload(obj block.ObjectPointer, uploadID string, multipartList *block.MultipartUploadCompletion) (*string, int64, error)

func (*Adapter) Copy added in v0.14.0 

func (a *Adapter) Copy(sourceObj, destinationObj block.ObjectPointer) error

func (*Adapter) CreateMultiPartUpload 

func (a *Adapter) CreateMultiPartUpload(obj block.ObjectPointer, r *http.Request, opts block.CreateMultiPartUploadOpts) (string, error)

func (*Adapter) Exists added in v0.22.1 

func (a *Adapter) Exists(obj block.ObjectPointer) (bool, error)

func (*Adapter) GenerateInventory 

func (a *Adapter) GenerateInventory(ctx context.Context, logger logging.Logger, manifestURL string, shouldSort bool, prefixes []string) (block.Inventory, error)

func (*Adapter) Get 

func (a *Adapter) Get(obj block.ObjectPointer, _ int64) (io.ReadCloser, error)

func (*Adapter) GetProperties 

func (a *Adapter) GetProperties(obj block.ObjectPointer) (block.Properties, error)

func (*Adapter) GetRange 

func (a *Adapter) GetRange(obj block.ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)

func (*Adapter) GetStorageNamespaceInfo added in v0.33.0 

func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo

func (*Adapter) Put 

func (a *Adapter) Put(obj block.ObjectPointer, sizeBytes int64, reader io.Reader, opts block.PutOpts) error

func (*Adapter) Remove 

func (a *Adapter) Remove(obj block.ObjectPointer) error

func (*Adapter) UploadCopyPart added in v0.31.0 

func (a *Adapter) UploadCopyPart(sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber int64) (string, error)

func (*Adapter) UploadCopyPartRange added in v0.31.0 

func (a *Adapter) UploadCopyPartRange(sourceObj, destinationObj block.ObjectPointer, uploadID string, partNumber, startPosition, endPosition int64) (string, error)

func (*Adapter) UploadPart 

func (a *Adapter) UploadPart(obj block.ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int64) (string, error)

func (*Adapter) ValidateConfiguration 

func (a *Adapter) ValidateConfiguration(storageNamespace string) error

ValidateConfiguration on an S3 adapter checks for a usable bucket lifecycle policy: the storageNamespace bucket should expire objects marked with ExpireObjectS3Tag (with _some_ duration, even if nonzero).

func (*Adapter) Walk added in v0.22.1 

func (a *Adapter) Walk(walkOpt block.WalkOpts, walkFn block.WalkFunc) error

func (*Adapter) WithContext 

func (a *Adapter) WithContext(ctx context.Context) block.Adapter

type Inventory 

type Inventory struct {
	Manifest *Manifest
	// contains filtered or unexported fields
}

func (*Inventory) InventoryURL 

func (inv *Inventory) InventoryURL() string

func (*Inventory) Iterator 

func (inv *Inventory) Iterator() block.InventoryIterator

func (*Inventory) SourceName 

func (inv *Inventory) SourceName() string

type InventoryIterator 

type InventoryIterator struct {
	*Inventory
	// contains filtered or unexported fields
}

func NewInventoryIterator 

func NewInventoryIterator(inv *Inventory) *InventoryIterator

func (*InventoryIterator) Err 

func (it *InventoryIterator) Err() error

func (*InventoryIterator) Get 

func (it *InventoryIterator) Get() *block.InventoryObject

func (*InventoryIterator) Next 

func (it *InventoryIterator) Next() bool

func (*InventoryIterator) Progress added in v0.11.0 

func (it *InventoryIterator) Progress() []*cmdutils.Progress

type Manifest added in v0.10.0 

type Manifest struct {
	URL                string          `json:"-"`
	InventoryBucketArn string          `json:"destinationBucket"`
	SourceBucket       string          `json:"sourceBucket"`
	Files              []inventoryFile `json:"files"` // inventory list files, each contains a list of objects
	Format             string          `json:"fileFormat"`
	CreationTimestamp  string          `json:"creationTimestamp"`
	// contains filtered or unexported fields
}

type StreamingReader 

type StreamingReader struct {
	Reader       io.Reader
	Size         int
	StreamSigner *v4.StreamSigner
	Time         time.Time
	ChunkSize    int
	ChunkTimeout time.Duration
	// contains filtered or unexported fields
}

func (*StreamingReader) GetLastChunk 

func (s *StreamingReader) GetLastChunk() []byte

func (*StreamingReader) Read 

func (s *StreamingReader) Read(p []byte) (int, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.