helpers

Documentation

Index

• Constants
• Variables
• func GetRemainingBlob(client *ds3.Client, bucketName string, blob *helperModels.BlobDescription, ...) error
• func RetryGettingBlobRange(client *ds3.Client, bucketName string, objectName string, blobOffset int64, ...) (int64, error)
• type BlobDescriptionQueue
  • func NewBlobDescriptionQueue() BlobDescriptionQueue
• type BlobStrategy
• type ConditionalBool
  • func NewConditionalBool() *ConditionalBool
  • func (conditionalBool *ConditionalBool) SignalDone()
  • func (conditionalBool *ConditionalBool) Wait()
• type Consumer
• type HelperImpl
  • func (helper *HelperImpl) GetObjects(bucketName string, objects []helperModels.GetObject, ...) (string, error)
  • func (helper *HelperImpl) GetObjectsSpanningJobs(bucketName string, objects []helperModels.GetObject, ...) ([]string, error)
  • func (helper *HelperImpl) PutObjects(bucketName string, objects []helperModels.PutObject, ...) (string, error)
• type HelperInterface
  • func NewHelpers(client *ds3.Client) HelperInterface
• type IoReaderWithSizeDecorator
  • func NewIoReaderWithSizeDecorator(reader io.Reader, size int64) *IoReaderWithSizeDecorator
  • func (sizedReader *IoReaderWithSizeDecorator) Read(bytes []byte) (int, error)
  • func (sizedReader *IoReaderWithSizeDecorator) Size() (int64, error)
• type ListenerStrategy
  • func (listener *ListenerStrategy) Errored(objectName string, err error)
• type NotifyBlobDone
• type Producer
• type ReadBlobStrategy
• type ReadBulkJobOptions
• type ReadTransferStrategy
• type SimpleBlobStrategy
• type Transceiver
• type TransferOperation
• type WriteBlobStrategy
• type WriteBulkJobOptions
• type WriteTransferStrategy

Constants

const MaxQueueSize uint = 100

const MinQueueSize uint = 1

Variables

var MinUploadSize int64 = 10485760

Functions

func GetRemainingBlob

func GetRemainingBlob(client *ds3.Client, bucketName string, blob *helperModels.BlobDescription, amountAlreadyRetrieved int64, writer io.Writer, logger sdk_log.Logger) error

func RetryGettingBlobRange

func RetryGettingBlobRange(client *ds3.Client, bucketName string, objectName string, blobOffset int64, rangeStart int64, rangeEnd int64, writer io.Writer, logger sdk_log.Logger) (int64, error)

Types

type BlobDescriptionQueue

type BlobDescriptionQueue interface {
	Push(description *helperModels.BlobDescription)
	Pop() (*helperModels.BlobDescription, error)
	Size() int
}

A queue that manages descriptions of blobs Used to track blobs that are waiting to be transferred

func NewBlobDescriptionQueue

func NewBlobDescriptionQueue() BlobDescriptionQueue

type BlobStrategy

type BlobStrategy interface {
	// contains filtered or unexported methods
}

Strategy for how to blob objects, used both in writing and reading blob strategies

type ConditionalBool

type ConditionalBool struct {
	Done bool
	// contains filtered or unexported fields
}

func NewConditionalBool

func NewConditionalBool() *ConditionalBool

func (*ConditionalBool) SignalDone

func (conditionalBool *ConditionalBool) SignalDone()

func (*ConditionalBool) Wait

func (conditionalBool *ConditionalBool) Wait()

type Consumer

type Consumer interface {
	// contains filtered or unexported methods
}

type HelperImpl

type HelperImpl struct {
	// contains filtered or unexported fields
}

func (*HelperImpl) GetObjects

func (helper *HelperImpl) GetObjects(bucketName string, objects []helperModels.GetObject, strategy ReadTransferStrategy) (string, error)

func (*HelperImpl) GetObjectsSpanningJobs

func (helper *HelperImpl) GetObjectsSpanningJobs(bucketName string, objects []helperModels.GetObject, strategy ReadTransferStrategy) ([]string, error)

func (*HelperImpl) PutObjects

func (helper *HelperImpl) PutObjects(bucketName string, objects []helperModels.PutObject, strategy WriteTransferStrategy) (string, error)

type HelperInterface

type HelperInterface interface {

	// Puts the specified list of objects on the Black Pearl in the specified bucket.
	// Returns the Black Pearl bulk put Job ID and any errors that may have occurred.
	// A job ID will be returned if a BP job was successfully created, regardless of
	// whether additional errors occur.
	PutObjects(bucketName string, objects []helperModels.PutObject, strategy WriteTransferStrategy) (string, error)

	// Retrieves the list of objects from the specified bucket on the Black Pearl.
	// Returns the Black Pearl bulk get Job ID and any errors that may have occurred.
	// A job ID will be returned if a BP job was successfully created, regardless of
	// whether additional errors occur.
	GetObjects(bucketName string, objects []helperModels.GetObject, strategy ReadTransferStrategy) (string, error)

	// Retrieves the list of objects from the specified bucket on the Black Pearl.
	// If a get job cannot be created due to insufficient cache space to fulfill an
	// IN_ORDER processing guarantee, then the job is split across multiple BP jobs.
	// This allows for the IN_ORDER retrieval of objects that exceed available cache space.
	GetObjectsSpanningJobs(bucketName string, objects []helperModels.GetObject, strategy ReadTransferStrategy) ([]string, error)
}

func NewHelpers

func NewHelpers(client *ds3.Client) HelperInterface

type IoReaderWithSizeDecorator

type IoReaderWithSizeDecorator struct {
	// contains filtered or unexported fields
}

Defines a simple wrapper for an io.Reader and io.Closer which specifies size Implements ReaderWithSizeDecorator.

func NewIoReaderWithSizeDecorator

func NewIoReaderWithSizeDecorator(reader io.Reader, size int64) *IoReaderWithSizeDecorator

func (*IoReaderWithSizeDecorator) Read

func (sizedReader *IoReaderWithSizeDecorator) Read(bytes []byte) (int, error)

func (*IoReaderWithSizeDecorator) Size

func (sizedReader *IoReaderWithSizeDecorator) Size() (int64, error)

type ListenerStrategy

type ListenerStrategy struct {
	// Called when an error occurred during transfer of an object.
	// This must be a thread safe function.
	ErrorCallback func(objectName string, err error)
}

func (*ListenerStrategy) Errored

func (listener *ListenerStrategy) Errored(objectName string, err error)

type NotifyBlobDone

type NotifyBlobDone interface {
	// Waits for at least one done signal.
	Wait()

	// Sends a done signal. Multiple signals have no additional effect.
	SignalDone()
}

type Producer

type Producer interface {
	// contains filtered or unexported methods
}

type ReadBlobStrategy

type ReadBlobStrategy interface {
	BlobStrategy
}

Strategy for how to blob objects for writing

type ReadBulkJobOptions

type ReadBulkJobOptions struct {
	Aggregating                         *bool
	ChunkClientProcessingOrderGuarantee models.JobChunkClientProcessingOrderGuarantee
	ImplicitJobIdResolution             *bool
	Name                                *string
	Priority                            models.Priority
}

Defines the options to use on the get bulk job

type ReadTransferStrategy

type ReadTransferStrategy struct {
	BlobStrategy ReadBlobStrategy
	Options      ReadBulkJobOptions
	Listeners    ListenerStrategy
}

type SimpleBlobStrategy

type SimpleBlobStrategy struct {
	Delay                  time.Duration
	MaxConcurrentTransfers uint
	MaxWaitingTransfers    uint
}

type Transceiver

type Transceiver interface {
	// contains filtered or unexported methods
}

type TransferOperation

type TransferOperation func() // transfer operation that sends/gets stuff from BP

type WriteBlobStrategy

type WriteBlobStrategy interface {
	BlobStrategy
}

Strategy for how to blob objects for writing

type WriteBulkJobOptions

type WriteBulkJobOptions struct {
	Aggregating                 *bool
	ImplicitJobIdResolution     *bool
	MaxUploadSize               *int64
	MinimizeSpanningAcrossMedia *bool
	Priority                    *models.Priority
	VerifyAfterWrite            *bool
	Name                        *string
	Force                       *bool
	IgnoreNamingConflicts       *bool
	Protected                   *bool
}

Defines the options to use on the put bulk job

type WriteTransferStrategy

type WriteTransferStrategy struct {
	BlobStrategy WriteBlobStrategy
	Options      WriteBulkJobOptions
	Listeners    ListenerStrategy
}

Defines strategy for how to perform write transfers