backup

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 15, 2020 License: GPL-3.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// MaxUint keep maximum UINT value.
	MaxUint = ^uint(0)
	// MaxInt keep maximum poistive INT value.
	MaxInt = int(MaxUint >> 1)
	// MinInt keep minimum negative INT value.
	MinInt = -MaxInt - 1
)
View Source 
const (
	MsgRsyncInfo  = "RsyncInfo"
	MsgGolangInfo = "GolangInfo"

	MsgFolderBackupTypeSkipDescription      = "FolderBackupTypeSkipDescription"
	MsgFolderBackupTypeRecursiveDescription = "FolderBackupTypeRecursiveDescription"
	MsgFolderBackupTypeContentDescription   = "FolderBackupTypeContentDescription"

	MsgLogPlanStageStarting                  = "LogPlanStageStarting"
	MsgLogPlanStageStartTime                 = "LogPlanStageStartTime"
	MsgLogPlanStageEndTime                   = "LogPlanStageEndTime"
	MsgLogPlanStartIterateViaNSources        = "LogPlanStartIterateViaNSources"
	MsgLogPlanStageInquirySource             = "LogPlanStageInquirySource"
	MsgLogPlanStageSourceFolderCountInfo     = "LogPlanStageSourceFolderCountInfo"
	MsgLogPlanStageSourceSkipFolderCountInfo = "LogPlanStageSourceSkipFolderCountInfo"
	MsgLogPlanStageSourceTotalSizeInfo       = "LogPlanStageSourceTotalSizeInfo"
	MsgLogPlanStageUseTemporaryFolder        = "LogPlanStageUseTemporaryFolder"
	MsgLogPlanStageBuildFolderError          = "LogPlanStageBuildFolderError"

	MsgLogBackupStageStarting                               = "LogBackupStageStarting"
	MsgLogBackupStageStartTime                              = "LogBackupStageStartTime"
	MsgLogBackupStageEndTime                                = "LogBackupStageEndTime"
	MsgLogBackupStageBackupToDestination                    = "LogBackupStageBackupToDestination"
	MsgLogBackupStagePreviousBackupDiscoveryPermissionError = "LogBackupStagePreviousBackupDiscoveryPermissionError"
	MsgLogBackupStagePreviousBackupDiscoveryOtherError      = "LogBackupStagePreviousBackupDiscoveryOtherError"
	MsgLogBackupStagePreviousBackupFoundAndWillBeUsed       = "LogBackupStagePreviousBackupFoundAndWillBeUsed"
	MsgLogBackupStagePreviousBackupFoundButDisabled         = "LogBackupStagePreviousBackupFoundButDisabled"
	MsgLogBackupStagePreviousBackupNotFound                 = "LogBackupStagePreviousBackupNotFound"
	MsgLogBackupStageStartToBackupFromSource                = "LogBackupStageStartToBackupFromSource"
	MsgLogBackupStageRenameDestination                      = "LogBackupStageRenameDestination"
	MsgLogBackupStageFailedToCreateFolder                   = "LogBackupStageFailedToCreateFolder"
	MsgLogBackupDetectedTotalBackupSizeGetChanged           = "LogBackupDetectedTotalBackupSizeGetChanged"
	MsgLogBackupStageProgressBackupSuccess                  = "LogBackupStageProgressBackupSuccess"
	MsgLogBackupStageProgressBackupError                    = "LogBackupStageProgressBackupError"
	MsgLogBackupStageProgressSkipBackupError                = "LogBackupStageProgressSkipBackupError"
	MsgLogBackupStageCriticalError                          = "LogBackupStageCriticalError"
	MsgLogBackupStageDiscoveringPreviousBackups             = "LogBackupStageDiscoveringPreviousBackups"
	MsgLogBackupStageRecoveredFromError                     = "LogBackupStageRecoveredFromError"
	MsgLogBackupStageSaveRsyncExtraLogTo                    = "LogBackupStageSaveRsyncExtraLogTo"
	MsgLogBackupStageSaveLogTo                              = "LogBackupStageSaveLogTo"
	MsgLogBackupStageExitMessage                            = "LogBackupStageExitMessage"

	MsgLogStatisticsSummaryCaption                            = "LogStatisticsSummaryCaption"
	MsgLogStatisticsEnvironmentCaption                        = "LogStatisticsEnvironmentCaption"
	MsgLogStatisticsResultsCaption                            = "LogStatisticsResultsCaption"
	MsgLogStatisticsStatusCaption                             = "LogStatisticsStatusCaption"
	MsgLogStatisticsStatusSuccessfullyCompleted               = "LogStatisticsStatusSuccessfullyCompleted"
	MsgLogStatisticsStatusCompletedWithErrors                 = "LogStatisticsStatusCompletedWithErrors"
	MsgLogStatisticsPlanStageCaption                          = "LogStatisticsPlanStageCaption"
	MsgLogStatisticsPlanStageSourceToBackup                   = "LogStatisticsPlanStageSourceToBackup"
	MsgLogStatisticsPlanStageTotalSize                        = "LogStatisticsPlanStageTotalSize"
	MsgLogStatisticsPlanStageFolderCount                      = "LogStatisticsPlanStageFolderCount"
	MsgLogStatisticsPlanStageFolderSkipCount                  = "LogStatisticsPlanStageFolderSkipCount"
	MsgLogStatisticsPlanStageTimeTaken                        = "LogStatisticsPlanStageTimeTaken"
	MsgLogStatisticsBackupStageCaption                        = "LogStatisticsBackupStageCaption"
	MsgLogStatisticsBackupStageDestinationPath                = "LogStatisticsBackupStageDestinationPath"
	MsgLogStatisticsBackupStagePreviousBackupFound            = "LogStatisticsBackupStagePreviousBackupFound"
	MsgLogStatisticsBackupStagePreviousBackupFoundButDisabled = "LogStatisticsBackupStagePreviousBackupFoundButDisabled"
	MsgLogStatisticsBackupStageNoValidPreviousBackupFound     = "LogStatisticsBackupStageNoValidPreviousBackupFound"
	MsgLogStatisticsBackupStageTotalSize                      = "LogStatisticsBackupStageTotalSize"
	MsgLogStatisticsBackupStageSkippedSize                    = "LogStatisticsBackupStageSkippedSize"
	MsgLogStatisticsBackupStageFailedToBackupSize             = "LogStatisticsBackupStageFailedToBackupSize"
	MsgLogStatisticsBackupStageTimeTaken                      = "LogStatisticsBackupStageTimeTaken"
)
View Source 
const TAB_RUNE = '\t'

TAB_RUNE keep tab character.

Variables

View Source 
var (
	// DoubleSplitLogLine keep continuous line build with specific char.
	DoubleSplitLogLine string = strings.Repeat("=", 100)
	// SingleSplitLogLine keep continuous line build with specific char.
	SingleSplitLogLine string = strings.Repeat("-", 100)
)
View Source 
var LocalLog = logger.NewPackageLogger("backup",

	logger.InfoLevel,
)

You can manage verbosity of log output in the package by changing last parameter value (comment/uncomment corresponding lines).

Functions

func BuildBackupPlan 

func BuildBackupPlan(ctx context.Context, lg logger.PackageLog, config *Config,
	modules []Module, notifier Notifier) (*Plan, *Progress, error)

BuildBackupPlan perform 1st stage (plan stage) to measure RSYNC source volume to backup and find optimal traverse path of source directory tree. Use plan built in 1st stage later in 2nd stage.

func CreateMetadataSignatureFile 

func CreateMetadataSignatureFile(modules []Module, destPath string) error

CreateMetadataSignatureFile serialize RSYNC sources plus destination subpaths to the special "backup session signature" file.

func EncodeSignatures 

func EncodeSignatures(signs NodeSignatures) (string, error)

EncodeSignatures encode NodeSignatures object to self-describing binary format.

func GenerateSourceID added in v0.3.2 

func GenerateSourceID(rsyncSource string) string

GenerateSourceID convert RSYNC source URL to unique identifier.

func GetBackupFolderName 

func GetBackupFolderName(incomplete bool, date *time.Time) string

GetBackupFolderName return new folder name for ongoing backup process.

func GetBackupTypeDescription 

func GetBackupTypeDescription(backupType core.FolderBackupType) string

GetBackupTypeDescription return localized description of how application will backup specific directory described by core.Dir object. It could be 3 options: 1) full backup: backup full content include all nested folders; 2) flat backup: backup only direct files in folder, ignore nested folders; 3) skip backup: skip folder backup (this happens when specific signature file found).

func GetLogFileName 

func GetLogFileName() string

GetLogFileName return the name of general backup process log.

func GetMetadataSignatureFileName 

func GetMetadataSignatureFileName() string

GetMetadataSignatureFileName return the name of specific file which describe all sources used in backup process.

func GetRsyncLogFileName 

func GetRsyncLogFileName() string

GetRsyncLogFileName return the name of specific low-level RSYNC utility log.

func GetRsyncParams added in v0.3.3 

func GetRsyncParams(conf *Config, module *Module, addExtraParams []string) []string

GetRsyncParams prepare RSYNC CLI parameters to run console RSYNC process.

func MeasureDir 

func MeasureDir(ctx context.Context, password *string, dir *core.Dir, retryCount *int,
	rsyncProtocol string, log *rsync.Logging, blockSize *backupBlockSizeSettings) (int, error)

MeasureDir is a main heuristic function for planing backup process. MeasureDir walks through directory tree taken from dir variable, and find size which match size criteria optimal for splitting backup process to the pieces. As a result folders marked with corresponding type of processing, like core.FBT_RECURSIVE, core.FBT_CONTENT or core.FBT_SKIP, which lately used in backup stage as a direct instruction what to do. Returning totalCount contains statistics how many times application call RSYNC utility to measure folder size on remote server (with all content).

Types

type Config 

type Config struct {
	SigFileIgnoreBackup                string `toml:"sig_file_ignore_backup"`
	RsyncRetryCount                    *int   `toml:"retry_count"`
	AutoManageBackupBlockSize          *bool  `toml:"auto_manage_backup_block_size"`
	MaxBackupBlockSizeMb               *int   `toml:"max_backup_block_size_mb"`
	UsePreviousBackup                  *bool  `toml:"use_previous_backup"`
	NumberOfPreviousBackupToUse        *int   `toml:"number_of_previous_backup_to_use"`
	EnableLowLevelLogForRsync          *bool  `toml:"enable_low_level_log_rsync"`
	EnableIntensiveLowLevelLogForRsync *bool  `toml:"enable_intensive_low_level_log_rsync"`

	RsyncTransferSourceOwner       *bool `toml:"rsync_transfer_source_owner"`       // rsync --owner
	RsyncTransferSourceGroup       *bool `toml:"rsync_transfer_source_group"`       // rsync --group
	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"` // rsync --perms
	RsyncRecreateSymlinks          *bool `toml:"rsync_recreate_symlinks"`           // rsync --links
	RsyncTransferDeviceFiles       *bool `toml:"rsync_transfer_device_files"`       // rsync --devices
	RsyncTransferSpecialFiles      *bool `toml:"rsync_transfer_special_files"`      // rsync --specials
	RsyncCompressFileTransfer      *bool `toml:"rsync_compress_file_transfer"`      // rsync --compress

}

Config keeps backup session configuration. Config instance is initialized mainly from GLIB GSettings in ui/gtkui package.

type LogFiles 

type LogFiles struct {
	// contains filtered or unexported fields
}

LogFiles track log files during backup session. It has functionality to relocate log files from one storage to another: used when log files moved from /tmp partition to permanent destination location.

func NewLogFiles 

func NewLogFiles() *LogFiles

NewLogFiles create new LogFiles instance.

func (*LogFiles) ChangeRootPath 

func (v *LogFiles) ChangeRootPath(newRootPath string) error

ChangeRootPath relocate log files from one storage to another. Used to move from 1st backup stage (plan stage) to 2nd (backup stage). In 1st backup stage we keep log files in /tmp partition, in 2nd stage we relocate and save them in destination location.

func (*LogFiles) Close 

func (v *LogFiles) Close() error

Close will close all os.File instances found in the object.

func (*LogFiles) CreateOrGetLogFile added in v0.4.0 

func (v *LogFiles) CreateOrGetLogFile(suffixPath string) (*os.File, error)

CreateOrGetLogFile return os.File by file name identifier. This allow to control and operate multiple log files in one place.

type Module added in v0.3.2 

type Module struct {
	SourceRsync string `toml:"src_rsync"`
	DestSubPath string `toml:"dst_subpath"`

	ChangeFilePermission string  `toml:"rsync_change_file_permission"`
	AuthPassword         *string `toml:"module_auth_password"`

	RsyncTransferSourceOwner       *bool `toml:"rsync_transfer_source_owner"`       // rsync --owner
	RsyncTransferSourceGroup       *bool `toml:"rsync_transfer_source_group"`       // rsync --group
	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"` // rsync --perms
	RsyncRecreateSymlinks          *bool `toml:"rsync_recreate_symlinks"`           // rsync --links
	RsyncTransferDeviceFiles       *bool `toml:"rsync_transfer_device_files"`       // rsync --devices
	RsyncTransferSpecialFiles      *bool `toml:"rsync_transfer_special_files"`      // rsync --specials
}

Module signify RSYNC source/destination block, with source/destination URLs and other auxiliary options. Used as configuration data in the backup session code.

type Node added in v0.3.2 

type Node struct {
	Module  Module
	RootDir *core.Dir
}

Node contain information about single RSYNC source backup.

type NodeSignature 

type NodeSignature struct {
	SourceRsyncCipher string
	DestSubPath       string
}

NodeSignature keep RSYNC source path crypted with hash function and destination subpath. RSYNC source path crypted with hash function is used as "source identifier" to search repeated backup sessions to use for deduplication. Content of this object is serialized to the file stored in backup session root folder.

func GetSignature 

func GetSignature(module Module) NodeSignature

GetSignature builds NodeSignature object on the basis of BackupNodePath data.

type NodeSignatures 

type NodeSignatures struct {
	Signatures []NodeSignature
}

NodeSignatures keeps list of RSYNC source to backup in one session.

func DecodeSignatures 

func DecodeSignatures(str string) (*NodeSignatures, error)

DecodeSignatures decode NodeSignatures object from self-describing binary format.

func GetNodeSignatures 

func GetNodeSignatures(modules []Module) NodeSignatures

GetNodeSignatures convert RSYNC module source URLs to unique identifiers built with use chiper function. These signatures used lately to find previous backup session for deduplication purpose.

func (NodeSignatures) FindFirstSignature 

func (v NodeSignatures) FindFirstSignature(signature string) *NodeSignature

FindFirstSignature find first item which match signature parameter.

type Notifier 

type Notifier interface {

	// Pair of calls to report about 1st pass start and completion.
	NotifyPlanStage_NodeStructureStartInquiry(sourceID int,
		sourceRsync string) error
	NotifyPlanStage_NodeStructureDoneInquiry(sourceID int,
		sourceRsync string, dir *core.Dir) error

	// Pair of calls to report about 2nd pass start and completion.
	NotifyBackupStage_FolderStartBackup(rootDest string,
		paths core.SrcDstPath, backupType core.FolderBackupType,
		leftToBackup core.FolderSize,
		timePassed time.Duration, eta *time.Duration,
	) error
	NotifyBackupStage_FolderDoneBackup(rootDest string,
		paths core.SrcDstPath, backupType core.FolderBackupType,
		leftToBackup core.FolderSize, sizeDone core.SizeProgress,
		timePassed time.Duration, eta *time.Duration,
		sessionErr error) error
}

Notifier interface is used as a contract to provide event-driven mechanism, to map backup process steps with, for instance, user interface.

type Plan added in v0.3.2 

type Plan struct {
	Config     *Config
	Nodes      []Node
	BackupSize core.FolderSize
}

Plan keep all necessary information obtained from preferences and 1st backup pass to start backup process.

func (*Plan) GetModules added in v0.3.2 

func (v *Plan) GetModules() []Module

GetModules returns all RSYNC source/destination blocks defined in single (specific) backup profile.

func (*Plan) RunBackup added in v0.3.2 

func (plan *Plan) RunBackup(progress *Progress, destPath string,
	errorHookCall rsync.ErrorHookCall) error

RunBackup perform whole 2nd stage (backup stage) here, then save and report about completion to session logs.

type PrevBackup 

type PrevBackup struct {
	// Full path to signature file name
	SignatureFileName string
	Signature         NodeSignature
}

PrevBackup describe previous backup found, which contain same RSYNC source. Such previous backups used for RSYNC utility deduplication, which significantly decrease size and time for new backup session.

func (PrevBackup) GetDirPath 

func (v PrevBackup) GetDirPath() string

GetDirPath returns full path to data copied in previous successful backup session.

type PreviousBackups added in v0.4.0 

type PreviousBackups struct {
	Backups []PrevBackup
}

PreviousBackups keeps list of previous backup found. See description of PrevBackup.

func FindPrevBackupPathsByNodeSignatures 

func FindPrevBackupPathsByNodeSignatures(lg logger.PackageLog, destPath string,
	signs NodeSignatures, lastN int) (*PreviousBackups, error)

FindPrevBackupPathsByNodeSignatures search for previous backup sessions which might significantly decrease backup size and speed up process. In the end it should return list of previous backup sessions sorted by date/time in descending order (recent go first).

func (*PreviousBackups) FilterBySourceID added in v0.4.0 

func (v *PreviousBackups) FilterBySourceID(sourceID string) *PreviousBackups

FilterBySourceID choose backup sessions which contains same source as specified by sourceID.

func (*PreviousBackups) GetDirPaths added in v0.4.0 

func (v *PreviousBackups) GetDirPaths() []string

GetDirPaths provide file system paths to previous backup sessions found.

type Progress 

type Progress struct {
	Context       context.Context
	LogFiles      *LogFiles
	Log           logger.PackageLog
	RsyncLog      *rsync.Logging
	Progress      *core.SizeProgress
	TotalProgress *core.SizeProgress

	// Notifier interface used to plug external objects to notify
	Notifier Notifier

	// Time stamps in 1st stage
	StartPlanTime time.Time
	EndPlanTime   time.Time

	// Time stamps in 2nd stage
	StartBackupTime time.Time
	EndBackupTime   time.Time

	// Previous backup sessions found to use for deduplicaton
	PreviousBackups *PreviousBackups

	RootDest     string
	BackupFolder string

	// Notify only once (theoretically it never happens)
	SizeChangedNotified bool
}

Progress tracks all aspects of backup session progress including time points, volume processed and so on. Progress employ Notifier interface to report progress to plug-in interface.

func (*Progress) CalcTimePassedAndETA 

func (v *Progress) CalcTimePassedAndETA(plan *Plan) (time.Duration, *time.Duration)

CalcTimePassedAndETA count total time passed in backup stage (2nd stage) and compute ETA (estimated time of arrival) - time left.

func (*Progress) Close 

func (v *Progress) Close() error

Close release any resources occupied.

func (*Progress) EventBackupStage_FolderDoneBackup 

func (v *Progress) EventBackupStage_FolderDoneBackup(paths core.SrcDstPath,
	backupType core.FolderBackupType, plan *Plan,
	sizeDone core.SizeProgress, sessionErr error) error

EventBackupStage_FolderDoneBackup report about backup folder end (2nd stage).

func (*Progress) EventBackupStage_FolderStartBackup 

func (v *Progress) EventBackupStage_FolderStartBackup(paths core.SrcDstPath,
	backupType core.FolderBackupType, plan *Plan) error

EventBackupStage_FolderStartBackup report about backup folder start (2nd stage).

func (*Progress) EventPlanStage_NodeStructureDoneInquiry 

func (v *Progress) EventPlanStage_NodeStructureDoneInquiry(sourceID int,
	sourceRsync string, dir *core.Dir) error

EventPlanStage_NodeStructureDoneInquiry report about end inquiry of RSYNC source (1st stage).

func (*Progress) EventPlanStage_NodeStructureStartInquiry 

func (v *Progress) EventPlanStage_NodeStructureStartInquiry(sourceID int,
	sourceRsync string) error

EventPlanStage_NodeStructureStartInquiry report about start inquiry of RSYNC source (1st stage).

func (*Progress) FinishBackupStage 

func (v *Progress) FinishBackupStage()

FinishBackupStage save the end time of 2nd stage.

func (*Progress) FinishPlanStage 

func (v *Progress) FinishPlanStage()

FinishPlanStage save the end time of 1st stage.

func (*Progress) GetBackupFullPath 

func (v *Progress) GetBackupFullPath(backupFolder string) string

GetBackupFullPath return absolute destination path with backup session subfolder concatenated.

func (*Progress) GetTotalTimeTaken 

func (v *Progress) GetTotalTimeTaken() time.Duration

GetTotalTimeTaken count up total time of backup session execution.

func (*Progress) LeftToBackup 

func (v *Progress) LeftToBackup(plan *Plan) core.FolderSize

LeftToBackup return size left to process in 2nd stage.

func (*Progress) PreviousBackupsUsed added in v0.4.0 

func (v *Progress) PreviousBackupsUsed(prevBackups *PreviousBackups)

PreviousBackupsUsed save previous backup sessions found for deduplication to activate.

func (*Progress) PrintTotalStatistics 

func (v *Progress) PrintTotalStatistics(lg logger.PackageLog, plan *Plan) error

PrintTotalStatistics print results on backup session completion. Print all statistics including time taken, volume processed, errors happens and so on.

func (*Progress) SayGoodbye 

func (v *Progress) SayGoodbye(lg logger.PackageLog)

SayGoodbye report completion to the backup session log.

func (*Progress) SetBackupFolder 

func (v *Progress) SetBackupFolder(backupFolder string) error

SetBackupFolder set newly created backup session subfolder, where copied data and logs will be stored.

func (*Progress) SetRootDestination 

func (v *Progress) SetRootDestination(rootDestPath string)

SetRootDestination set absolute destination path, where backup session will create it new subfolder and store data.

func (*Progress) SizeBackedUp 

func (v *Progress) SizeBackedUp() core.FolderSize

SizeBackedUp return total size processed during 2nd stage.

func (*Progress) StartBackupStage 

func (v *Progress) StartBackupStage()

StartBackupStage save the start time of 2nd stage.

func (*Progress) StartPlanStage 

func (v *Progress) StartPlanStage()

StartPlanStage save the start time of 1st stage.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.