filemanager

package
v0.0.0-...-0ade494 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 26, 2022 License: AGPL-3.0 Imports: 29 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetProviderConfigForBackupsFromEnv 

func GetProviderConfigForBackupsFromEnv(ctx context.Context) map[string]interface{}

GetProviderConfigForBackupsFromEnv returns the provider config

Types

type AzureBlobStorageConfig 

type AzureBlobStorageConfig struct {
	Container      string
	Prefix         string
	AccountName    string
	AccountKey     string
	EndPoint       *string
	ForcePathStyle *bool
	DisableSSL     *bool
}

func GetAzureBlogStorageConfig 

func GetAzureBlogStorageConfig(config map[string]interface{}) *AzureBlobStorageConfig

type AzureBlobStorageManager 

type AzureBlobStorageManager struct {
	Config *AzureBlobStorageConfig
	// contains filtered or unexported fields
}

func (*AzureBlobStorageManager) DeleteObjects 

func (manager *AzureBlobStorageManager) DeleteObjects(ctx context.Context, keys []string) (err error)

func (*AzureBlobStorageManager) Download 

func (manager *AzureBlobStorageManager) Download(ctx context.Context, output *os.File, key string) error

func (*AzureBlobStorageManager) GetConfiguredPrefix 

func (manager *AzureBlobStorageManager) GetConfiguredPrefix() string

func (*AzureBlobStorageManager) GetDownloadKeyFromFileLocation 

func (manager *AzureBlobStorageManager) GetDownloadKeyFromFileLocation(location string) string

func (*AzureBlobStorageManager) GetObjectNameFromLocation 

func (manager *AzureBlobStorageManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://account-name.blob.core.windows.net/container-name/key - >> key

func (*AzureBlobStorageManager) ListFilesWithPrefix 

func (manager *AzureBlobStorageManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*AzureBlobStorageManager) SetTimeout 

func (manager *AzureBlobStorageManager) SetTimeout(timeout time.Duration)

func (*AzureBlobStorageManager) Upload 

func (manager *AzureBlobStorageManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

Upload passed in file to Azure Blob Storage

type DOSpacesConfig 

type DOSpacesConfig struct {
	Bucket         string
	Prefix         string
	EndPoint       string
	AccessKeyID    string
	AccessKey      string
	Region         *string
	ForcePathStyle *bool
	DisableSSL     *bool
}

func GetDOSpacesConfig 

func GetDOSpacesConfig(config map[string]interface{}) *DOSpacesConfig

type DOSpacesManager 

type DOSpacesManager struct {
	Config *DOSpacesConfig
	// contains filtered or unexported fields
}

func (*DOSpacesManager) DeleteObjects 

func (manager *DOSpacesManager) DeleteObjects(ctx context.Context, keys []string) error

func (*DOSpacesManager) Download 

func (manager *DOSpacesManager) Download(ctx context.Context, output *os.File, key string) error

func (*DOSpacesManager) GetConfiguredPrefix 

func (manager *DOSpacesManager) GetConfiguredPrefix() string

func (*DOSpacesManager) GetDownloadKeyFromFileLocation 

func (manager *DOSpacesManager) GetDownloadKeyFromFileLocation(location string) string

func (*DOSpacesManager) GetObjectNameFromLocation 

func (manager *DOSpacesManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://rudder.sgp1.digitaloceanspaces.com/key - >> key

func (*DOSpacesManager) ListFilesWithPrefix 

func (manager *DOSpacesManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*DOSpacesManager) SetTimeout 

func (manager *DOSpacesManager) SetTimeout(timeout time.Duration)

func (*DOSpacesManager) Upload 

func (manager *DOSpacesManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

Upload passed in file to spaces

type FileManager 

type FileManager interface {
	Upload(context.Context, *os.File, ...string) (UploadOutput, error)
	Download(context.Context, *os.File, string) error
	GetObjectNameFromLocation(string) (string, error)
	GetDownloadKeyFromFileLocation(location string) string
	DeleteObjects(ctx context.Context, keys []string) error
	ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
	GetConfiguredPrefix() string
	SetTimeout(timeout time.Duration)
}

FileManager implements all upload methods

type FileManagerFactory 

type FileManagerFactory interface {
	New(settings *SettingsT) (FileManager, error)
}
var (
	DefaultFileManagerFactory FileManagerFactory
	ErrKeyNotFound            = errors.New("NoSuchKey")
)

type FileManagerFactoryT 

type FileManagerFactoryT struct{}

func (*FileManagerFactoryT) New 

func (factory *FileManagerFactoryT) New(settings *SettingsT) (FileManager, error)

New returns FileManager backed by configured provider

type FileObject 

type FileObject struct {
	Key          string
	LastModified time.Time
}

type GCSConfig 

type GCSConfig struct {
	Bucket         string
	Prefix         string
	Credentials    string
	EndPoint       *string
	ForcePathStyle *bool
	DisableSSL     *bool
}

func GetGCSConfig 

func GetGCSConfig(config map[string]interface{}) *GCSConfig

type GCSManager 

type GCSManager struct {
	Config *GCSConfig
	// contains filtered or unexported fields
}

func (*GCSManager) DeleteObjects 

func (manager *GCSManager) DeleteObjects(ctx context.Context, locations []string) (err error)

func (*GCSManager) Download 

func (manager *GCSManager) Download(ctx context.Context, output *os.File, key string) error

func (*GCSManager) GetConfiguredPrefix 

func (manager *GCSManager) GetConfiguredPrefix() string

func (*GCSManager) GetDownloadKeyFromFileLocation 

func (manager *GCSManager) GetDownloadKeyFromFileLocation(location string) string

TODO complete this

func (*GCSManager) GetObjectNameFromLocation 

func (manager *GCSManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://storage.googleapis.com/bucket-name/key - >> key

func (*GCSManager) ListFilesWithPrefix 

func (manager *GCSManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*GCSManager) SetTimeout 

func (manager *GCSManager) SetTimeout(timeout time.Duration)

func (*GCSManager) Upload 

func (manager *GCSManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

type MinioConfig 

type MinioConfig struct {
	Bucket          string
	Prefix          string
	EndPoint        string
	AccessKeyID     string
	SecretAccessKey string
	UseSSL          bool
}

func GetMinioConfig 

func GetMinioConfig(config map[string]interface{}) *MinioConfig

type MinioManager 

type MinioManager struct {
	Config *MinioConfig
	// contains filtered or unexported fields
}

func (*MinioManager) DeleteObjects 

func (manager *MinioManager) DeleteObjects(ctx context.Context, keys []string) (err error)

func (*MinioManager) Download 

func (manager *MinioManager) Download(ctx context.Context, file *os.File, key string) error

func (*MinioManager) GetConfiguredPrefix 

func (manager *MinioManager) GetConfiguredPrefix() string

func (*MinioManager) GetDownloadKeyFromFileLocation 

func (manager *MinioManager) GetDownloadKeyFromFileLocation(location string) string

func (*MinioManager) GetObjectNameFromLocation 

func (manager *MinioManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://minio-endpoint/bucket-name/key1 - >> key1
http://minio-endpoint/bucket-name/key2 - >> key2

func (*MinioManager) ListFilesWithPrefix 

func (manager *MinioManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*MinioManager) ObjectUrl 

func (manager *MinioManager) ObjectUrl(objectName string) string

func (*MinioManager) SetTimeout 

func (manager *MinioManager) SetTimeout(timeout time.Duration)

func (*MinioManager) Upload 

func (manager *MinioManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

type S3Config 

type S3Config struct {
	Bucket            string  `mapstructure:"bucketName"`
	Prefix            string  `mapstructure:"Prefix"`
	Region            *string `mapstructure:"region"`
	AccessKeyID       string  `mapstructure:"accessKeyID"`
	AccessKey         string  `mapstructure:"accessKey"`
	IAMRoleARN        string  `mapstructure:"iamRoleARN"`
	ExternalID        string  `mapstructure:"externalID"`
	Endpoint          *string `mapstructure:"endpoint"`
	S3ForcePathStyle  *bool   `mapstructure:"s3ForcePathStyle"`
	DisableSSL        *bool   `mapstructure:"disableSSL"`
	EnableSSE         bool    `mapstructure:"enableSSE"`
	RegionHint        string  `mapstructure:"regionHint"`
	ContinuationToken *string `mapstructure:"continuationToken"`
	StartAfter        string  `mapstructure:"startAfter"`
	IsTruncated       bool    `mapstructure:"isTruncated"`
	UseGlue           bool    `mapstructure:"useGlue"`
}

func GetS3Config 

func GetS3Config(config map[string]interface{}) *S3Config

type S3Manager 

type S3Manager struct {
	Config *S3Config
	// contains filtered or unexported fields
}

func (*S3Manager) DeleteObjects 

func (manager *S3Manager) DeleteObjects(ctx context.Context, keys []string) (err error)

func (*S3Manager) Download 

func (manager *S3Manager) Download(ctx context.Context, output *os.File, key string) error

func (*S3Manager) GetConfiguredPrefix 

func (manager *S3Manager) GetConfiguredPrefix() string

func (*S3Manager) GetDownloadKeyFromFileLocation 

func (manager *S3Manager) GetDownloadKeyFromFileLocation(location string) string

func (*S3Manager) GetObjectNameFromLocation 

func (manager *S3Manager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://bucket-name.s3.amazonaws.com/key - >> key

func (*S3Manager) ListFilesWithPrefix 

func (manager *S3Manager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

IMPT NOTE: `ListFilesWithPrefix` support Continuation Token. So, if you want same set of files (says 1st 1000 again) then create a new S3Manager & not use the existing one. Since, using the existing one will by default return next 1000 files.

func (*S3Manager) SetTimeout 

func (manager *S3Manager) SetTimeout(timeout time.Duration)

func (*S3Manager) Upload 

func (manager *S3Manager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

Upload passed in file to s3

type SettingsT 

type SettingsT struct {
	Provider string
	Config   map[string]interface{}
}

SettingsT sets configuration for FileManager

type UploadOutput 

type UploadOutput struct {
	Location   string
	ObjectName string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.