s3

Documentation

Index

• Constants
• Variables
• func ConfigureFolder(prefix string, settings map[string]string) (storage.Folder, error)
• func CreateUploaderAPI(svc s3iface.S3API, partsize, concurrency int) s3manageriface.UploaderAPI
• func NewConfiguringError(settingName string) storage.Error
• func NewFolderError(err error, format string, args ...interface{}) storage.Error
• func NewS3Reader(body io.ReadCloser, objectPath string, retriesCount int, folder *Folder, ...) *s3Reader
• type ConnResetRetryer
  • func NewConnResetRetryer(baseRetryer request.Retryer) *ConnResetRetryer
  • func (r ConnResetRetryer) ShouldRetry(req *request.Request) bool
• type Folder
  • func NewFolder(uploader Uploader, s3API s3iface.S3API, settings map[string]string, ...) *Folder
  • func (folder *Folder) CopyObject(srcPath string, dstPath string) error
  • func (folder *Folder) DeleteObjects(objectRelativePaths []string) error
  • func (folder *Folder) Exists(objectRelativePath string) (bool, error)
  • func (folder *Folder) GetPath() string
  • func (folder *Folder) GetSubFolder(subFolderRelativePath string) storage.Folder
  • func (folder *Folder) ListFolder() (objects []storage.Object, subFolders []storage.Folder, err error)
  • func (folder *Folder) PutObject(name string, content io.Reader) error
  • func (folder *Folder) ReadObject(objectRelativePath string) (io.ReadCloser, error)
• type SseKmsIDNotSetError
  • func NewSseKmsIDNotSetError() SseKmsIDNotSetError
  • func (err SseKmsIDNotSetError) Error() string
• type Uploader
  • func NewUploader(uploaderAPI s3manageriface.UploaderAPI, ...) *Uploader

Constants

const (
	NotFoundAWSErrorCode  = "NotFound"
	NoSuchKeyAWSErrorCode = "NoSuchKey"

	EndpointSetting          = "AWS_ENDPOINT"
	RegionSetting            = "AWS_REGION"
	ForcePathStyleSetting    = "AWS_S3_FORCE_PATH_STYLE"
	AccessKeyIDSetting       = "AWS_ACCESS_KEY_ID"
	AccessKeySetting         = "AWS_ACCESS_KEY"
	SecretAccessKeySetting   = "AWS_SECRET_ACCESS_KEY"
	SecretKeySetting         = "AWS_SECRET_KEY"
	SessionTokenSetting      = "AWS_SESSION_TOKEN"
	SessionName              = "AWS_ROLE_SESSION_NAME"
	RoleARN                  = "AWS_ROLE_ARN"
	S3UseYcSessionToken      = "S3_USE_YC_SESSION_TOKEN"
	SseSetting               = "S3_SSE"
	SseCSetting              = "S3_SSE_C"
	SseKmsIDSetting          = "S3_SSE_KMS_ID"
	StorageClassSetting      = "S3_STORAGE_CLASS"
	UploadConcurrencySetting = "UPLOAD_CONCURRENCY"

	MaxPartSize            = "S3_MAX_PART_SIZE"
	EndpointSourceSetting  = "S3_ENDPOINT_SOURCE"
	EndpointPortSetting    = "S3_ENDPOINT_PORT"
	LogLevel               = "S3_LOG_LEVEL"
	UseListObjectsV1       = "S3_USE_LIST_OBJECTS_V1"
	RangeBatchEnabled      = "S3_RANGE_BATCH_ENABLED"
	RangeQueriesMaxRetries = "S3_RANGE_MAX_RETRIES"
	// MaxRetriesSetting limits retries during interaction with S3
	MaxRetriesSetting = "S3_MAX_RETRIES"

	RangeBatchEnabledDefault = false
	RangeMaxRetriesDefault   = 10
	RangeQueryMinRetryDelay  = 30 * time.Millisecond
	RangeQueryMaxRetryDelay  = 300 * time.Second
	MaxRetriesDefault        = 15
)

const (
	DefaultMaxPartSize = 20 << 20
)

const DefaultPort = "443"

const HTTP = "http"

Variables

var (
	// MaxRetries limit upload and download retries during interaction with S3
	MaxRetries  = 15
	SettingList = []string{
		EndpointPortSetting,
		EndpointSetting,
		EndpointSourceSetting,
		RegionSetting,
		ForcePathStyleSetting,
		AccessKeyIDSetting,
		AccessKeySetting,
		SecretAccessKeySetting,
		SecretKeySetting,
		SessionTokenSetting,
		SessionName,
		RoleARN,
		S3UseYcSessionToken,
		SseSetting,
		SseCSetting,
		SseKmsIDSetting,
		StorageClassSetting,
		UploadConcurrencySetting,
		s3CertFile,
		MaxPartSize,
		UseListObjectsV1,
		LogLevel,
		RangeBatchEnabled,
		RangeQueriesMaxRetries,
		MaxRetriesSetting,
	}
)

var (
	DebugLogBufferCounter = 0
)

Functions

func ConfigureFolder

func ConfigureFolder(prefix string, settings map[string]string) (storage.Folder, error)

func CreateUploaderAPI

func CreateUploaderAPI(svc s3iface.S3API, partsize, concurrency int) s3manageriface.UploaderAPI

CreateUploaderAPI returns an uploader with customizable concurrency and part size.

func NewConfiguringError

func NewConfiguringError(settingName string) storage.Error

func NewFolderError

func NewFolderError(err error, format string, args ...interface{}) storage.Error

func NewS3Reader

func NewS3Reader(body io.ReadCloser, objectPath string, retriesCount int, folder *Folder, minRetryDelay, maxRetryDelay time.Duration) *s3Reader

nolint: revive, lll

Types

type ConnResetRetryer

type ConnResetRetryer struct {
	request.Retryer
}

func NewConnResetRetryer

func NewConnResetRetryer(baseRetryer request.Retryer) *ConnResetRetryer

func (ConnResetRetryer) ShouldRetry

func (r ConnResetRetryer) ShouldRetry(req *request.Request) bool

type Folder

type Folder struct {
	S3API  s3iface.S3API
	Bucket *string
	Path   string
	// contains filtered or unexported fields
}

func NewFolder

func NewFolder(uploader Uploader, s3API s3iface.S3API, settings map[string]string, bucket, path string, useListObjectsV1 bool) *Folder

func (*Folder) CopyObject

func (folder *Folder) CopyObject(srcPath string, dstPath string) error

func (*Folder) DeleteObjects

func (folder *Folder) DeleteObjects(objectRelativePaths []string) error

func (*Folder) Exists

func (folder *Folder) Exists(objectRelativePath string) (bool, error)

func (*Folder) GetPath

func (folder *Folder) GetPath() string

func (*Folder) GetSubFolder

func (folder *Folder) GetSubFolder(subFolderRelativePath string) storage.Folder

func (*Folder) ListFolder

func (folder *Folder) ListFolder() (objects []storage.Object, subFolders []storage.Folder, err error)

func (*Folder) PutObject

func (folder *Folder) PutObject(name string, content io.Reader) error

func (*Folder) ReadObject

func (folder *Folder) ReadObject(objectRelativePath string) (io.ReadCloser, error)

type SseKmsIDNotSetError

type SseKmsIDNotSetError struct {
	// contains filtered or unexported fields
}

func NewSseKmsIDNotSetError

func NewSseKmsIDNotSetError() SseKmsIDNotSetError

func (SseKmsIDNotSetError) Error

func (err SseKmsIDNotSetError) Error() string

type Uploader

type Uploader struct {
	SSECustomerKey string
	SSEKMSKeyID    string
	StorageClass   string
	// contains filtered or unexported fields
}

func NewUploader

func NewUploader(uploaderAPI s3manageriface.UploaderAPI, serverSideEncryption, sseCustomerKey, sseKmsKeyID, storageClass string) *Uploader