cloudstorage

Documentation

Index

• func GetObjectName(prefix, snapshotName string) string
• func ValidateDestination(destination string) error
• type CloudStorage
  • func NewCloudStorage(ctx context.Context, config Config, logger log.Logger) (*CloudStorage, error)
  • func NewFromDestination(ctx context.Context, destination string, logger log.Logger) (*CloudStorage, error)
  • func NewFromEnv(ctx context.Context, storageType StorageType, bucket string, logger log.Logger) (*CloudStorage, error)
  • func (c *CloudStorage) Close() error
  • func (c *CloudStorage) Delete(ctx context.Context, objectName string) error
  • func (c *CloudStorage) Download(ctx context.Context, objectName, localPath string, decompress bool) error
  • func (c *CloudStorage) Exists(ctx context.Context, objectName string) (bool, error)
  • func (c *CloudStorage) GetBucket() objstore.Bucket
  • func (c *CloudStorage) List(ctx context.Context, prefix string) ([]string, error)
  • func (c *CloudStorage) RestoreSnapshot(ctx context.Context, objectName, localDir string, decompress bool, ...) error
  • func (c *CloudStorage) SetLogSampleRate(rate int)
  • func (c *CloudStorage) SetLogVerbose(verbose bool)
  • func (c *CloudStorage) Upload(ctx context.Context, localPath, objectName string, compress bool) error
• type CompressionLevel
• type Config
• type GCSConfig
• type RangeGetter
• type S3Config
• type StorageType
  • func ParseDestination(destination string) (StorageType, string, string, error)

Functions

func GetObjectName

func GetObjectName(prefix, snapshotName string) string

GetObjectName generates an object name for a snapshot

func ValidateDestination

func ValidateDestination(destination string) error

ValidateDestination validates a destination URL

Types

type CloudStorage

type CloudStorage struct {
	// contains filtered or unexported fields
}

CloudStorage represents a cloud storage client

func NewCloudStorage

func NewCloudStorage(ctx context.Context, config Config, logger log.Logger) (*CloudStorage, error)

NewCloudStorage creates a new cloud storage client

func NewFromDestination

func NewFromDestination(ctx context.Context, destination string, logger log.Logger) (*CloudStorage, error)

func NewFromEnv

func NewFromEnv(ctx context.Context, storageType StorageType, bucket string, logger log.Logger) (*CloudStorage, error)

func (*CloudStorage) Close

func (c *CloudStorage) Close() error

Close closes the cloud storage client

func (*CloudStorage) Delete

func (c *CloudStorage) Delete(ctx context.Context, objectName string) error

Delete deletes an object from cloud storage

func (*CloudStorage) Download

func (c *CloudStorage) Download(ctx context.Context, objectName, localPath string, decompress bool) error

Download downloads a file from cloud storage

func (*CloudStorage) Exists

func (c *CloudStorage) Exists(ctx context.Context, objectName string) (bool, error)

Exists checks if an object exists in cloud storage

func (*CloudStorage) GetBucket

func (c *CloudStorage) GetBucket() objstore.Bucket

GetBucket returns the underlying objstore.Bucket

func (*CloudStorage) List

func (c *CloudStorage) List(ctx context.Context, prefix string) ([]string, error)

List lists objects in cloud storage with the given prefix

func (*CloudStorage) RestoreSnapshot

func (c *CloudStorage) RestoreSnapshot(ctx context.Context, objectName, localDir string, decompress bool, overwrite bool) error

RestoreSnapshot downloads a snapshot from cloud storage and restores it to the local filesystem

func (*CloudStorage) SetLogSampleRate

func (c *CloudStorage) SetLogSampleRate(rate int)

SetLogSampleRate sets the log sample rate (1 in n operations will be logged)

func (*CloudStorage) SetLogVerbose

func (c *CloudStorage) SetLogVerbose(verbose bool)

SetLogVerbose sets whether to log detailed information

func (*CloudStorage) Upload

func (c *CloudStorage) Upload(ctx context.Context, localPath, objectName string, compress bool) error

Upload uploads a file to cloud storage with enhanced features

type CompressionLevel

type CompressionLevel int

CompressionLevel represents the compression level for zstd

const (
	// CompressionFastest is the fastest compression level
	CompressionFastest CompressionLevel = 1
	// CompressionDefault is the default compression level
	CompressionDefault CompressionLevel = 3
	// CompressionBetter is a better compression level
	CompressionBetter CompressionLevel = 7
	// CompressionBest is the best compression level
	CompressionBest CompressionLevel = 22
)

type Config

type Config struct {
	Type StorageType

	// S3 specific configuration
	S3Config *S3Config

	// GCS specific configuration
	GCSConfig *GCSConfig

	// Common configuration
	ChunkSize        int64
	CompressionLevel CompressionLevel
	CreateBucket     bool

	// Logging configuration
	LogSampleRate int  // Only log 1 in n operations (default: 100)
	LogVerbose    bool // Whether to log detailed information (default: false)
}

Config represents the configuration for cloud storage

type GCSConfig

type GCSConfig struct {
	Bucket         string
	ServiceAccount string
}

GCSConfig represents the configuration for GCS storage

type RangeGetter

type RangeGetter interface {
	GetRange(ctx context.Context, objectName string, off, length int64) (io.ReadCloser, error)
	Size(ctx context.Context, objectName string) (int64, error)
}

RangeGetter is an optional interface that a bucket may implement to support range requests.

type S3Config

type S3Config struct {
	Endpoint       string
	AccessKey      string
	SecretKey      string
	Bucket         string
	Region         string
	Insecure       bool
	SignatureV2    bool
	ForcePathStyle bool
}

S3Config represents the configuration for S3 storage

type StorageType

type StorageType string

StorageType represents the type of cloud storage

const (
	// S3Storage represents Amazon S3 or compatible storage
	S3Storage StorageType = "s3"
	// GCSStorage represents Google Cloud Storage
	GCSStorage StorageType = "gcs"
)

func ParseDestination

func ParseDestination(destination string) (StorageType, string, string, error)

ParseDestination parses a destination URL and returns the storage type, bucket, and prefix