storage

package
v0.18.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 30, 2026 License: MIT Imports: 22 Imported by: 0

Documentation

Overview

Package storage provides interfaces and implementations for managing backup storage across primary (local), secondary (remote filesystem), and cloud (rclone) destinations.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ClassifyBackupsGFS 

func ClassifyBackupsGFS(backups []*types.BackupMetadata, config RetentionConfig) map[*types.BackupMetadata]RetentionCategory

ClassifyBackupsGFS classifies backups according to GFS (Grandfather-Father-Son) scheme Returns a map of backup -> category, allowing intelligent time-distributed retention

func GetRetentionStats 

func GetRetentionStats(classification map[*types.BackupMetadata]RetentionCategory) map[RetentionCategory]int

GetRetentionStats returns statistics about classification results

Types

type BackupLocation 

type BackupLocation string

BackupLocation represents a location where backups are stored 

const (
	LocationPrimary   BackupLocation = "primary"
	LocationSecondary BackupLocation = "secondary"
	LocationCloud     BackupLocation = "cloud"
)

type CloudStorage 

type CloudStorage struct {
	// contains filtered or unexported fields
}

func NewCloudStorage 

func NewCloudStorage(cfg *config.Config, logger *logging.Logger) (*CloudStorage, error)

NewCloudStorage creates a new cloud storage instance

func (*CloudStorage) ApplyRetention 

func (c *CloudStorage) ApplyRetention(ctx context.Context, config RetentionConfig) (deleted int, err error)

ApplyRetention removes old backups according to retention policy Supports both simple (count-based) and GFS (time-distributed) policies Uses batched deletion to avoid API rate limits

func (*CloudStorage) Delete 

func (c *CloudStorage) Delete(ctx context.Context, backupFile string) error

Delete removes a backup file from cloud storage

func (*CloudStorage) DetectFilesystem 

func (c *CloudStorage) DetectFilesystem(ctx context.Context) (info *FilesystemInfo, err error)

DetectFilesystem checks if rclone is available and the remote is accessible

func (*CloudStorage) GetStats 

func (c *CloudStorage) GetStats(ctx context.Context) (stats *StorageStats, err error)

GetStats returns storage statistics

func (*CloudStorage) IsCritical 

func (c *CloudStorage) IsCritical() bool

IsCritical returns false because cloud storage is non-critical Failures in cloud storage should NOT abort the backup

func (*CloudStorage) IsEnabled 

func (c *CloudStorage) IsEnabled() bool

IsEnabled returns true if cloud storage is configured

func (*CloudStorage) LastRetentionSummary 

func (c *CloudStorage) LastRetentionSummary() RetentionSummary

LastRetentionSummary returns the latest retention summary.

func (*CloudStorage) List 

func (c *CloudStorage) List(ctx context.Context) (backups []*types.BackupMetadata, err error)

List returns all backups in cloud storage

func (*CloudStorage) Location 

func (c *CloudStorage) Location() BackupLocation

Location returns the backup location type

func (*CloudStorage) Name 

func (c *CloudStorage) Name() string

Name returns the storage backend name

func (*CloudStorage) Store 

func (c *CloudStorage) Store(ctx context.Context, backupFile string, metadata *types.BackupMetadata) (err error)

Store uploads a backup file to cloud storage using rclone

func (*CloudStorage) UploadToRemotePath 

func (c *CloudStorage) UploadToRemotePath(ctx context.Context, localFile, remoteFile string, verify bool) error

UploadToRemotePath uploads an arbitrary file to the provided remote path using the same retry and verification logic used for backups.

func (*CloudStorage) VerifyUpload 

func (c *CloudStorage) VerifyUpload(ctx context.Context, localFile, remoteFile string) (ok bool, err error)

VerifyUpload verifies that a file was successfully uploaded to cloud storage Uses two methods: primary (rclone lsl) and alternative (rclone ls + grep)

type FilesystemDetector 

type FilesystemDetector struct {
	// contains filtered or unexported fields
}

FilesystemDetector provides methods to detect and validate filesystem types

func NewFilesystemDetector 

func NewFilesystemDetector(logger *logging.Logger) *FilesystemDetector

NewFilesystemDetector creates a new filesystem detector

func (*FilesystemDetector) DetectFilesystem 

func (d *FilesystemDetector) DetectFilesystem(ctx context.Context, path string) (*FilesystemInfo, error)

DetectFilesystem detects the filesystem type for a given path This function logs the detected filesystem type in real-time

func (*FilesystemDetector) SetPermissions 

func (d *FilesystemDetector) SetPermissions(ctx context.Context, path string, uid, gid int, mode os.FileMode, fsInfo *FilesystemInfo) error

SetPermissions sets file permissions and ownership, respecting filesystem capabilities

type FilesystemInfo 

type FilesystemInfo struct {
	Path              string
	Type              FilesystemType
	SupportsOwnership bool
	IsNetworkFS       bool
	MountPoint        string
	Device            string
}

FilesystemInfo contains information about a filesystem

type FilesystemType 

type FilesystemType string

FilesystemType represents the detected filesystem type 

const (
	// Filesystems that support Unix ownership
	FilesystemExt4     FilesystemType = "ext4"
	FilesystemExt3     FilesystemType = "ext3"
	FilesystemExt2     FilesystemType = "ext2"
	FilesystemXFS      FilesystemType = "xfs"
	FilesystemBtrfs    FilesystemType = "btrfs"
	FilesystemZFS      FilesystemType = "zfs"
	FilesystemJFS      FilesystemType = "jfs"
	FilesystemReiserFS FilesystemType = "reiserfs"
	FilesystemOverlay  FilesystemType = "overlay"
	FilesystemTmpfs    FilesystemType = "tmpfs"

	// Filesystems that do NOT support Unix ownership
	FilesystemFAT32 FilesystemType = "vfat"
	FilesystemFAT   FilesystemType = "fat"
	FilesystemExFAT FilesystemType = "exfat"
	FilesystemNTFS  FilesystemType = "ntfs"
	FilesystemFUSE  FilesystemType = "fuse"

	// Network filesystems (need testing)
	FilesystemNFS  FilesystemType = "nfs"
	FilesystemNFS4 FilesystemType = "nfs4"
	FilesystemCIFS FilesystemType = "cifs"
	FilesystemSMB  FilesystemType = "smb"

	// Unknown or unsupported
	FilesystemUnknown FilesystemType = "unknown"
)

func (FilesystemType) IsNetworkFilesystem 

func (f FilesystemType) IsNetworkFilesystem() bool

IsNetworkFilesystem returns true if the filesystem is network-based

func (FilesystemType) ShouldAutoExclude 

func (f FilesystemType) ShouldAutoExclude() bool

ShouldAutoExclude returns true if this filesystem should be automatically excluded from ownership operations (incompatible filesystems like FAT32/CIFS)

func (FilesystemType) String 

func (f FilesystemType) String() string

String returns a human-readable description of the filesystem type

func (FilesystemType) SupportsUnixOwnership 

func (f FilesystemType) SupportsUnixOwnership() bool

SupportsUnixOwnership returns true if the filesystem supports Unix ownership (chown/chmod)

type LocalStorage 

type LocalStorage struct {
	// contains filtered or unexported fields
}

LocalStorage implements the Storage interface for local filesystem storage

func NewLocalStorage 

func NewLocalStorage(cfg *config.Config, logger *logging.Logger) (*LocalStorage, error)

NewLocalStorage creates a new local storage instance

func (*LocalStorage) ApplyRetention 

func (l *LocalStorage) ApplyRetention(ctx context.Context, config RetentionConfig) (deleted int, err error)

ApplyRetention removes old backups according to retention policy Supports both simple (count-based) and GFS (time-distributed) policies

func (*LocalStorage) Delete 

func (l *LocalStorage) Delete(ctx context.Context, backupFile string) (err error)

Delete removes a backup file and its associated files

func (*LocalStorage) DetectFilesystem 

func (l *LocalStorage) DetectFilesystem(ctx context.Context) (info *FilesystemInfo, err error)

DetectFilesystem detects the filesystem type for the backup path

func (*LocalStorage) GetStats 

func (l *LocalStorage) GetStats(ctx context.Context) (stats *StorageStats, err error)

GetStats returns storage statistics

func (*LocalStorage) IsCritical 

func (l *LocalStorage) IsCritical() bool

IsCritical returns true because local storage is critical

func (*LocalStorage) IsEnabled 

func (l *LocalStorage) IsEnabled() bool

IsEnabled returns true if local storage is configured

func (*LocalStorage) LastRetentionSummary 

func (l *LocalStorage) LastRetentionSummary() RetentionSummary

LastRetentionSummary returns information about the latest retention run.

func (*LocalStorage) List 

func (l *LocalStorage) List(ctx context.Context) (backups []*types.BackupMetadata, err error)

List returns all backups in local storage

func (*LocalStorage) Location 

func (l *LocalStorage) Location() BackupLocation

Location returns the backup location type

func (*LocalStorage) Name 

func (l *LocalStorage) Name() string

Name returns the storage backend name

func (*LocalStorage) Store 

func (l *LocalStorage) Store(ctx context.Context, backupFile string, metadata *types.BackupMetadata) (err error)

Store stores a backup file to local storage For local storage, this mainly involves setting proper permissions

func (*LocalStorage) VerifyUpload 

func (l *LocalStorage) VerifyUpload(ctx context.Context, localFile, remoteFile string) (bool, error)

VerifyUpload is not applicable for local storage

type RetentionCategory 

type RetentionCategory string

RetentionCategory represents the classification of a backup 

const (
	CategoryDaily   RetentionCategory = "daily"
	CategoryWeekly  RetentionCategory = "weekly"
	CategoryMonthly RetentionCategory = "monthly"
	CategoryYearly  RetentionCategory = "yearly"
	CategoryDelete  RetentionCategory = "delete"
)

type RetentionConfig 

type RetentionConfig struct {
	// Policy type: "simple" (count-based) or "gfs" (time-distributed)
	Policy string

	// Simple retention: total number of backups to keep
	MaxBackups int

	// GFS retention: time-distributed, hierarchical (daily → weekly → monthly → yearly)
	// Daily: keep the last N backups (newest first)
	Daily   int // Keep last N backups as daily
	Weekly  int // Keep N weekly backups (one per week)
	Monthly int // Keep N monthly backups (one per month)
	Yearly  int // Keep N yearly backups (one per year, 0 = keep all)
}

RetentionConfig defines the retention policy configuration

func EffectiveGFSRetentionConfig added in v0.17.0 

func EffectiveGFSRetentionConfig(cfg RetentionConfig) RetentionConfig

EffectiveGFSRetentionConfig returns the effective GFS configuration without side effects. It applies the same value normalization used by GFS retention execution paths, but does not log. Callers are responsible for invoking it only for configurations that should use GFS semantics.

func NewRetentionConfigFromConfig 

func NewRetentionConfigFromConfig(cfg *config.Config, location BackupLocation) RetentionConfig

NewRetentionConfigFromConfig creates a RetentionConfig from main Config Auto-detects whether to use simple or GFS policy based on configuration

func NormalizeGFSRetentionConfig 

func NormalizeGFSRetentionConfig(logger *logging.Logger, backendName string, cfg RetentionConfig) RetentionConfig

NormalizeGFSRetentionConfig applies the required adjustments to the GFS configuration before running retention. Currently:

  • ensures the DAILY tier is at least 1 (minimum accepted value) when the policy is gfs and RETENTION_DAILY is <= 0.
  • emits a log line to document the adjustment.

type RetentionReporter 

type RetentionReporter interface {
	LastRetentionSummary() RetentionSummary
}

RetentionReporter can be implemented by storage backends that expose details about the most recent retention run (e.g., log counts).

type RetentionSummary 

type RetentionSummary struct {
	BackupsDeleted   int
	BackupsRemaining int
	LogsDeleted      int
	LogsRemaining    int
	HasLogInfo       bool
}

RetentionSummary captures what happened during the last retention run.

type SecondaryStorage 

type SecondaryStorage struct {
	// contains filtered or unexported fields
}

SecondaryStorage implements the Storage interface for secondary (remote) filesystem storage This is typically a network mount (NFS/CIFS) or another local path All errors from secondary storage are NON-FATAL - they log warnings but don't abort the backup

func NewSecondaryStorage 

func NewSecondaryStorage(cfg *config.Config, logger *logging.Logger) (*SecondaryStorage, error)

NewSecondaryStorage creates a new secondary storage instance

func (*SecondaryStorage) ApplyRetention 

func (s *SecondaryStorage) ApplyRetention(ctx context.Context, config RetentionConfig) (deleted int, err error)

ApplyRetention removes old backups according to retention policy Supports both simple (count-based) and GFS (time-distributed) policies

func (*SecondaryStorage) Delete 

func (s *SecondaryStorage) Delete(ctx context.Context, backupFile string) (err error)

Delete removes a backup file and its associated files

func (*SecondaryStorage) DetectFilesystem 

func (s *SecondaryStorage) DetectFilesystem(ctx context.Context) (info *FilesystemInfo, err error)

DetectFilesystem detects the filesystem type for the secondary path

func (*SecondaryStorage) GetStats 

func (s *SecondaryStorage) GetStats(ctx context.Context) (stats *StorageStats, err error)

GetStats returns storage statistics

func (*SecondaryStorage) IsCritical 

func (s *SecondaryStorage) IsCritical() bool

IsCritical returns false because secondary storage is non-critical Failures in secondary storage should NOT abort the backup

func (*SecondaryStorage) IsEnabled 

func (s *SecondaryStorage) IsEnabled() bool

IsEnabled returns true if secondary storage is configured

func (*SecondaryStorage) LastRetentionSummary 

func (s *SecondaryStorage) LastRetentionSummary() RetentionSummary

LastRetentionSummary returns the latest retention summary.

func (*SecondaryStorage) List 

func (s *SecondaryStorage) List(ctx context.Context) (backups []*types.BackupMetadata, err error)

List returns all backups in secondary storage

func (*SecondaryStorage) Location 

func (s *SecondaryStorage) Location() BackupLocation

Location returns the backup location type

func (*SecondaryStorage) Name 

func (s *SecondaryStorage) Name() string

Name returns the storage backend name

func (*SecondaryStorage) Store 

func (s *SecondaryStorage) Store(ctx context.Context, backupFile string, metadata *types.BackupMetadata) (err error)

Store copies a backup file to secondary storage using an atomic Go-based copy

func (*SecondaryStorage) VerifyUpload 

func (s *SecondaryStorage) VerifyUpload(ctx context.Context, localFile, remoteFile string) (bool, error)

VerifyUpload is not applicable for secondary storage

type Storage 

type Storage interface {
	// Name returns the human-readable name of this storage backend
	Name() string

	// Location returns the backup location type (primary/secondary/cloud)
	Location() BackupLocation

	// IsEnabled returns true if this storage backend is configured and enabled
	IsEnabled() bool

	// IsCritical returns true if failures in this storage should abort the backup
	// Primary storage is critical, secondary and cloud are non-critical
	IsCritical() bool

	// DetectFilesystem detects the filesystem type for the destination path
	// This should be called BEFORE any operations and logged in real-time
	DetectFilesystem(ctx context.Context) (*FilesystemInfo, error)

	// Store stores a backup file to this storage destination
	// For cloud storage, this includes verification and retry logic
	// Returns error only if IsCritical() is true, otherwise logs warnings
	Store(ctx context.Context, backupFile string, metadata *types.BackupMetadata) error

	// List returns all backups stored in this location
	List(ctx context.Context) ([]*types.BackupMetadata, error)

	// Delete removes a backup file and its associated files
	Delete(ctx context.Context, backupFile string) error

	// ApplyRetention removes old backups according to retention policy
	// Supports both simple (count-based) and GFS (time-distributed) policies
	// For cloud storage, uses batched deletion to avoid API rate limits
	ApplyRetention(ctx context.Context, config RetentionConfig) (int, error)

	// VerifyUpload verifies that a file was successfully uploaded (cloud only)
	VerifyUpload(ctx context.Context, localFile, remoteFile string) (bool, error)

	// GetStats returns storage statistics (space used, file count, etc.)
	GetStats(ctx context.Context) (*StorageStats, error)
}

Storage defines the interface for backup storage operations

type StorageError 

type StorageError struct {
	Location    BackupLocation
	Operation   string // "store", "delete", "verify", etc.
	Path        string
	Err         error
	IsCritical  bool
	Recoverable bool
}

StorageError represents an error from a storage operation

func (*StorageError) Error 

func (e *StorageError) Error() string

type StorageStats 

type StorageStats struct {
	TotalBackups   int
	TotalSize      int64
	OldestBackup   *time.Time
	NewestBackup   *time.Time
	AvailableSpace int64
	TotalSpace     int64
	UsedSpace      int64
	FilesystemType FilesystemType
}

StorageStats contains statistics about a storage location

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.