internal

package
v1.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 12, 2021 License: Apache-2.0 Imports: 48 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	PG        = "PG"
	SQLSERVER = "SQLSERVER"
	MYSQL     = "MYSQL"
	REDIS     = "REDIS"
	FDB       = "FDB"
	MONGO     = "MONGO"
	GP        = "GP"

	DownloadConcurrencySetting   = "WALG_DOWNLOAD_CONCURRENCY"
	UploadConcurrencySetting     = "WALG_UPLOAD_CONCURRENCY"
	UploadDiskConcurrencySetting = "WALG_UPLOAD_DISK_CONCURRENCY"
	UploadQueueSetting           = "WALG_UPLOAD_QUEUE"
	SentinelUserDataSetting      = "WALG_SENTINEL_USER_DATA"
	PreventWalOverwriteSetting   = "WALG_PREVENT_WAL_OVERWRITE"
	UploadWalMetadata            = "WALG_UPLOAD_WAL_METADATA"
	DeltaMaxStepsSetting         = "WALG_DELTA_MAX_STEPS"
	DeltaOriginSetting           = "WALG_DELTA_ORIGIN"
	CompressionMethodSetting     = "WALG_COMPRESSION_METHOD"
	StoragePrefixSetting         = "WALG_STORAGE_PREFIX"
	DiskRateLimitSetting         = "WALG_DISK_RATE_LIMIT"
	NetworkRateLimitSetting      = "WALG_NETWORK_RATE_LIMIT"
	UseWalDeltaSetting           = "WALG_USE_WAL_DELTA"
	UseReverseUnpackSetting      = "WALG_USE_REVERSE_UNPACK"
	SkipRedundantTarsSetting     = "WALG_SKIP_REDUNDANT_TARS"
	VerifyPageChecksumsSetting   = "WALG_VERIFY_PAGE_CHECKSUMS"
	StoreAllCorruptBlocksSetting = "WALG_STORE_ALL_CORRUPT_BLOCKS"
	UseRatingComposerSetting     = "WALG_USE_RATING_COMPOSER"
	UseCopyComposerSetting       = "WALG_USE_COPY_COMPOSER"
	DeltaFromNameSetting         = "WALG_DELTA_FROM_NAME"
	DeltaFromUserDataSetting     = "WALG_DELTA_FROM_USER_DATA"
	FetchTargetUserDataSetting   = "WALG_FETCH_TARGET_USER_DATA"
	LogLevelSetting              = "WALG_LOG_LEVEL"
	TarSizeThresholdSetting      = "WALG_TAR_SIZE_THRESHOLD"
	CseKmsIDSetting              = "WALG_CSE_KMS_ID"
	CseKmsRegionSetting          = "WALG_CSE_KMS_REGION"
	LibsodiumKeySetting          = "WALG_LIBSODIUM_KEY"
	LibsodiumKeyPathSetting      = "WALG_LIBSODIUM_KEY_PATH"
	GpgKeyIDSetting              = "GPG_KEY_ID"
	PgpKeySetting                = "WALG_PGP_KEY"
	PgpKeyPathSetting            = "WALG_PGP_KEY_PATH"
	PgpKeyPassphraseSetting      = "WALG_PGP_KEY_PASSPHRASE"
	PgDataSetting                = "PGDATA"
	UserSetting                  = "USER" // TODO : do something with it
	PgPortSetting                = "PGPORT"
	PgUserSetting                = "PGUSER"
	PgHostSetting                = "PGHOST"
	PgPasswordSetting            = "PGPASSWORD"
	PgDatabaseSetting            = "PGDATABASE"
	PgSslModeSetting             = "PGSSLMODE"
	PgSlotName                   = "WALG_SLOTNAME"
	PgWalSize                    = "WALG_PG_WAL_SIZE"
	TotalBgUploadedLimit         = "TOTAL_BG_UPLOADED_LIMIT"
	NameStreamCreateCmd          = "WALG_STREAM_CREATE_COMMAND"
	NameStreamRestoreCmd         = "WALG_STREAM_RESTORE_COMMAND"
	MaxDelayedSegmentsCount      = "WALG_INTEGRITY_MAX_DELAYED_WALS"
	PrefetchDir                  = "WALG_PREFETCH_DIR"
	PgReadyRename                = "PG_READY_RENAME"
	SerializerTypeSetting        = "WALG_SERIALIZER_TYPE"

	MongoDBUriSetting               = "MONGODB_URI"
	MongoDBLastWriteUpdateInterval  = "MONGODB_LAST_WRITE_UPDATE_INTERVAL"
	OplogArchiveAfterSize           = "OPLOG_ARCHIVE_AFTER_SIZE"
	OplogArchiveTimeoutInterval     = "OPLOG_ARCHIVE_TIMEOUT_INTERVAL"
	OplogPITRDiscoveryInterval      = "OPLOG_PITR_DISCOVERY_INTERVAL"
	OplogPushStatsEnabled           = "OPLOG_PUSH_STATS_ENABLED"
	OplogPushStatsLoggingInterval   = "OPLOG_PUSH_STATS_LOGGING_INTERVAL"
	OplogPushStatsUpdateInterval    = "OPLOG_PUSH_STATS_UPDATE_INTERVAL"
	OplogPushStatsExposeHTTP        = "OPLOG_PUSH_STATS_EXPOSE_HTTP"
	OplogPushWaitForBecomePrimary   = "OPLOG_PUSH_WAIT_FOR_BECOME_PRIMARY"
	OplogPushPrimaryCheckInterval   = "OPLOG_PUSH_PRIMARY_CHECK_INTERVAL"
	OplogReplayOplogAlwaysUpsert    = "OPLOG_REPLAY_OPLOG_ALWAYS_UPSERT"
	OplogReplayOplogApplicationMode = "OPLOG_REPLAY_OPLOG_APPLICATION_MODE"
	OplogReplayIgnoreErrorCodes     = "OPLOG_REPLAY_IGNORE_ERROR_CODES"

	MysqlDatasourceNameSetting = "WALG_MYSQL_DATASOURCE_NAME"
	MysqlSslCaSetting          = "WALG_MYSQL_SSL_CA"
	MysqlBinlogReplayCmd       = "WALG_MYSQL_BINLOG_REPLAY_COMMAND"
	MysqlBinlogDstSetting      = "WALG_MYSQL_BINLOG_DST"
	MysqlBackupPrepareCmd      = "WALG_MYSQL_BACKUP_PREPARE_COMMAND"
	MysqlTakeBinlogsFromMaster = "WALG_MYSQL_TAKE_BINLOGS_FROM_MASTER"
	MysqlCheckGTIDs            = "WALG_MYSQL_CHECK_GTIDS"

	RedisPassword = "WALG_REDIS_PASSWORD"

	GPLogsDirectory = "WALG_GP_LOGS_DIR"

	GoMaxProcs = "GOMAXPROCS"

	HTTPListen       = "HTTP_LISTEN"
	HTTPExposePprof  = "HTTP_EXPOSE_PPROF"
	HTTPExposeExpVar = "HTTP_EXPOSE_EXPVAR"

	SQLServerBlobHostname     = "SQLSERVER_BLOB_HOSTNAME"
	SQLServerBlobCertFile     = "SQLSERVER_BLOB_CERT_FILE"
	SQLServerBlobKeyFile      = "SQLSERVER_BLOB_KEY_FILE"
	SQLServerBlobLockFile     = "SQLSERVER_BLOB_LOCK_FILE"
	SQLServerConnectionString = "SQLSERVER_CONNECTION_STRING"
	SQLServerDBConcurrency    = "SQLSERVER_DB_CONCURRENCY"
	SQLServerReuseProxy       = "SQLSERVER_REUSE_PROXY"

	EndpointSourceSetting = "S3_ENDPOINT_SOURCE"
	EndpointPortSetting   = "S3_ENDPOINT_PORT"

	AwsAccessKeyID     = "AWS_ACCESS_KEY_ID"
	AwsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"

	YcKmsKeyIDSetting  = "YC_CSE_KMS_KEY_ID"
	YcSaKeyFileSetting = "YC_SERVICE_ACCOUNT_KEY_FILE"
)
View Source 
const (
	DefaultDataBurstRateLimit = 8 * pgDefaultDatabasePageSize
	DefaultDataFolderPath     = "/tmp"
	WaleFileHost              = "file://localhost"
)
View Source 
const (
	NoDeleteModifier = iota
	FullDeleteModifier
	FindFullDeleteModifier
	ForceDeleteModifier
	ConfirmFlag            = "confirm"
	DeleteShortDescription = "Clears old backups and WALs"

	DeleteRetainExamples = `` /* 321-byte string literal not displayed */

	DeleteBeforeExamples = `` /* 156-byte string literal not displayed */

	DeleteEverythingExamples = `` /* 154-byte string literal not displayed */

	DeleteTargetExamples = ``

	//nolint:lll
	/* 420-byte string literal not displayed */

	DeleteEverythingUsageExample = "everything [FORCE]"
	DeleteRetainUsageExample     = "retain [FULL|FIND_FULL] backup_count"
	DeleteBeforeUsageExample     = "before [FIND_FULL] backup_name|timestamp"
	DeleteTargetUsageExample     = "target [FIND_FULL] backup_name | --target-user-data <data>"

	DeleteTargetUserDataFlag        = "target-user-data"
	DeleteTargetUserDataDescription = "delete storage backup which has the specified user data"
)
View Source 
const (
	StreamPrefix           = "stream_"
	StreamBackupNameLength = 23 // len(StreamPrefix) + len(utility.BackupTimeFormat)
)
View Source 
const LatestString = "LATEST"
View Source 
const MaxCorruptBlocksInFileDesc int = 10
View Source 
const MinAllowedConcurrency = 1
View Source 
const TarPartitionFolderName = "/tar_partitions/"

Variables

View Source 
var (
	CfgFile string

	MongoDefaultSettings = map[string]string{
		OplogPushStatsLoggingInterval:  "30s",
		OplogPushStatsUpdateInterval:   "30s",
		OplogPushWaitForBecomePrimary:  "false",
		OplogPushPrimaryCheckInterval:  "30s",
		OplogArchiveTimeoutInterval:    "60s",
		OplogArchiveAfterSize:          "16777216",
		MongoDBLastWriteUpdateInterval: "3s",
	}

	SQLServerDefaultSettings = map[string]string{
		SQLServerDBConcurrency: "10",
	}

	PGDefaultSettings = map[string]string{
		PgWalSize: "16",
	}

	GPDefaultSettings = map[string]string{
		GPLogsDirectory: "",
	}

	AllowedSettings map[string]bool

	CommonAllowedSettings = map[string]bool{

		DownloadConcurrencySetting:   true,
		UploadConcurrencySetting:     true,
		UploadDiskConcurrencySetting: true,
		UploadQueueSetting:           true,
		SentinelUserDataSetting:      true,
		PreventWalOverwriteSetting:   true,
		UploadWalMetadata:            true,
		DeltaMaxStepsSetting:         true,
		DeltaOriginSetting:           true,
		CompressionMethodSetting:     true,
		StoragePrefixSetting:         true,
		DiskRateLimitSetting:         true,
		NetworkRateLimitSetting:      true,
		UseWalDeltaSetting:           true,
		LogLevelSetting:              true,
		TarSizeThresholdSetting:      true,
		"WALG_" + GpgKeyIDSetting:    true,
		"WALE_" + GpgKeyIDSetting:    true,
		PgpKeySetting:                true,
		PgpKeyPathSetting:            true,
		PgpKeyPassphraseSetting:      true,
		LibsodiumKeySetting:          true,
		LibsodiumKeyPathSetting:      true,
		TotalBgUploadedLimit:         true,
		NameStreamCreateCmd:          true,
		NameStreamRestoreCmd:         true,
		UseReverseUnpackSetting:      true,
		SkipRedundantTarsSetting:     true,
		VerifyPageChecksumsSetting:   true,
		StoreAllCorruptBlocksSetting: true,
		UseRatingComposerSetting:     true,
		UseCopyComposerSetting:       true,
		MaxDelayedSegmentsCount:      true,
		DeltaFromNameSetting:         true,
		DeltaFromUserDataSetting:     true,
		FetchTargetUserDataSetting:   true,
		SerializerTypeSetting:        true,

		"WALG_SWIFT_PREFIX": true,
		"OS_AUTH_URL":       true,
		"OS_USERNAME":       true,
		"OS_PASSWORD":       true,
		"OS_TENANT_NAME":    true,
		"OS_REGION_NAME":    true,

		"WALG_S3_PREFIX":              true,
		"WALE_S3_PREFIX":              true,
		AwsAccessKeyID:                true,
		AwsSecretAccessKey:            true,
		"AWS_SESSION_TOKEN":           true,
		"AWS_DEFAULT_REGION":          true,
		"AWS_DEFAULT_OUTPUT":          true,
		"AWS_PROFILE":                 true,
		"AWS_ROLE_SESSION_NAME":       true,
		"AWS_CA_BUNDLE":               true,
		"AWS_SHARED_CREDENTIALS_FILE": true,
		"AWS_CONFIG_FILE":             true,
		"AWS_REGION":                  true,
		"AWS_ENDPOINT":                true,
		"AWS_S3_FORCE_PATH_STYLE":     true,
		"WALG_S3_CA_CERT_FILE":        true,
		"WALG_S3_STORAGE_CLASS":       true,
		"WALG_S3_SSE":                 true,
		"WALG_S3_SSE_C":               true,
		"WALG_S3_SSE_KMS_ID":          true,
		"WALG_CSE_KMS_ID":             true,
		"WALG_CSE_KMS_REGION":         true,
		"WALG_S3_MAX_PART_SIZE":       true,
		"S3_ENDPOINT_SOURCE":          true,
		"S3_ENDPOINT_PORT":            true,
		"S3_USE_LIST_OBJECTS_V1":      true,
		"S3_RANGE_BATCH_ENABLED":      true,
		"S3_RANGE_MAX_RETRIES":        true,

		"WALG_AZ_PREFIX":           true,
		"AZURE_STORAGE_ACCOUNT":    true,
		"AZURE_STORAGE_ACCESS_KEY": true,
		"AZURE_STORAGE_SAS_TOKEN":  true,
		"AZURE_ENVIRONMENT_NAME":   true,
		"WALG_AZURE_BUFFER_SIZE":   true,
		"WALG_AZURE_MAX_BUFFERS":   true,

		"WALG_GS_PREFIX":                 true,
		"GOOGLE_APPLICATION_CREDENTIALS": true,

		YcSaKeyFileSetting: true,
		YcKmsKeyIDSetting:  true,

		"WALG_SSH_PREFIX":      true,
		"SSH_PORT":             true,
		"SSH_PASSWORD":         true,
		"SSH_USERNAME":         true,
		"SSH_PRIVATE_KEY_PATH": true,

		"WALG_FILE_PREFIX": true,

		GoMaxProcs: true,

		HTTPListen:       true,
		HTTPExposePprof:  true,
		HTTPExposeExpVar: true,
	}

	PGAllowedSettings = map[string]bool{

		PgPortSetting:     true,
		PgUserSetting:     true,
		PgHostSetting:     true,
		PgDataSetting:     true,
		PgPasswordSetting: true,
		PgDatabaseSetting: true,
		PgSslModeSetting:  true,
		PgSlotName:        true,
		PgWalSize:         true,
		"PGPASSFILE":      true,
		PrefetchDir:       true,
		PgReadyRename:     true,
	}

	MongoAllowedSettings = map[string]bool{

		MongoDBUriSetting:              true,
		MongoDBLastWriteUpdateInterval: true,
		OplogArchiveTimeoutInterval:    true,
		OplogArchiveAfterSize:          true,
		OplogPushStatsEnabled:          true,
		OplogPushStatsLoggingInterval:  true,
		OplogPushStatsUpdateInterval:   true,
		OplogPushStatsExposeHTTP:       true,
		OplogPushWaitForBecomePrimary:  true,
		OplogPushPrimaryCheckInterval:  true,
		OplogPITRDiscoveryInterval:     true,
	}

	SQLServerAllowedSettings = map[string]bool{

		SQLServerBlobHostname:     true,
		SQLServerBlobCertFile:     true,
		SQLServerBlobKeyFile:      true,
		SQLServerBlobLockFile:     true,
		SQLServerConnectionString: true,
		SQLServerDBConcurrency:    true,
		SQLServerReuseProxy:       true,
	}

	MysqlAllowedSettings = map[string]bool{

		MysqlDatasourceNameSetting: true,
		MysqlSslCaSetting:          true,
		MysqlBinlogReplayCmd:       true,
		MysqlBinlogDstSetting:      true,
		MysqlBackupPrepareCmd:      true,
		MysqlTakeBinlogsFromMaster: true,
		MysqlCheckGTIDs:            true,
	}

	RedisAllowedSettings = map[string]bool{

		RedisPassword: true,
	}

	GPAllowedSettings = map[string]bool{
		GPLogsDirectory: true,
	}

	RequiredSettings       = make(map[string]bool)
	HTTPSettingExposeFuncs = map[string]func(webserver.WebServer){
		HTTPExposePprof:          webserver.EnablePprofEndpoints,
		HTTPExposeExpVar:         webserver.EnableExpVarEndpoints,
		OplogPushStatsExposeHTTP: nil,
	}
	Turbo bool
)
View Source 
var DeprecatedExternalGpgMessage = fmt.Sprintf(
	`You are using deprecated functionality that uses an external gpg library.
It will be removed in next major version.
Please set GPG key using environment variables %s or %s.
`, PgpKeySetting, PgpKeyPathSetting)
View Source 
var ErrorSizeTrackingDisabled = fmt.Errorf("size tracking disabled by DisableSizeTracking method")
View Source 
var MaxExtractRetryWait = 5 * time.Minute
View Source 
var MinExtractRetryWait = time.Minute
View Source 
var StorageAdapters = []StorageAdapter{
	{"S3_PREFIX", s3.SettingList, s3.ConfigureFolder, nil},
	{"FILE_PREFIX", nil, fs.ConfigureFolder, preprocessFilePrefix},
	{"GS_PREFIX", gcs.SettingList, gcs.ConfigureFolder, nil},
	{"AZ_PREFIX", azure.SettingList, azure.ConfigureFolder, nil},
	{"SWIFT_PREFIX", swift.SettingList, swift.ConfigureFolder, nil},
	{"SSH_PREFIX", sh.SettingsList, sh.ConfigureFolder, nil},
}
View Source 
var StringModifiers = []string{"FULL", "FIND_FULL"}
View Source 
var StringModifiersDeleteEverything = []string{"FORCE"}

Functions

func AddConfigFlags added in v0.2.16 

func AddConfigFlags(Cmd *cobra.Command)

func AssertRequiredSettingsSet added in v0.2.16 

func AssertRequiredSettingsSet() error

func CheckAllowedSettings added in v0.2.17 

func CheckAllowedSettings(config *viper.Viper)

CheckAllowedSettings warnings if a viper instance's setting not allowed

func CompressAndEncrypt added in v0.2.10 

func CompressAndEncrypt(source io.Reader, compressor compression.Compressor, crypter crypto.Crypter) io.Reader

CompressAndEncrypt compresses input to a pipe reader. Output must be used or pipe will block.

func Configure 

func Configure()

func ConfigureAndRunDefaultWebServer added in v0.2.17 

func ConfigureAndRunDefaultWebServer() error

ConfigureAndRunDefaultWebServer configures and runs web server

func ConfigureArchiveStatusManager added in v0.2.14 

func ConfigureArchiveStatusManager() (fsutil.DataFolder, error)

TODO : unit tests

func ConfigureCompressor added in v0.2.22 

func ConfigureCompressor() (compression.Compressor, error)

TODO : unit tests

func ConfigureCrypter added in v0.2.10 

func ConfigureCrypter() crypto.Crypter

ConfigureCrypter uses environment variables to create and configure a crypter. In case no configuration in environment variables found, return `<nil>` value.

func ConfigureFolder added in v0.2.8 

func ConfigureFolder() (storage.Folder, error)

TODO : unit tests

func ConfigureFolderForSpecificConfig added in v0.2.17 

func ConfigureFolderForSpecificConfig(config *viper.Viper) (storage.Folder, error)

TODO: something with that when provided multiple 'keys' in the config, this function will always return only one concrete 'folder'. Chosen folder depends only on 'StorageAdapters' order

func ConfigureLogging added in v0.2.8 

func ConfigureLogging() error

func ConfigurePGArchiveStatusManager added in v1.1.1 

func ConfigurePGArchiveStatusManager() (fsutil.DataFolder, error)

func ConfigureSettings added in v0.2.22 

func ConfigureSettings(currentType string)

nolint: gocyclo

func ConfigureStoragePrefix added in v1.1.1 

func ConfigureStoragePrefix(folder storage.Folder) storage.Folder

func DecompressDecryptBytes added in v0.2.15 

func DecompressDecryptBytes(dst io.Writer, archiveReader io.ReadCloser, decompressor compression.Decompressor) error

TODO : unit tests

func DecryptAndDecompressTar 

func DecryptAndDecompressTar(writer io.Writer, readerMaker ReaderMaker, crypter crypto.Crypter) error

DecryptAndDecompressTar decrypts file and checks its extension. If it's tar, a decompression is not needed. Otherwise it uses corresponding decompressor. If none found an error will be returned.

func DecryptBytes added in v1.1.1 

func DecryptBytes(archiveReader io.ReadCloser) (io.ReadCloser, error)

func DefaultHandleBackupList added in v0.2.15 

func DefaultHandleBackupList(folder storage.Folder, pretty, json bool)

func DeleteBackups added in v1.1.1 

func DeleteBackups(folder storage.Folder, backups []string) error

DeleteBackups purges given backups files TODO: extract BackupLayout abstraction and provide DataPath(), SentinelPath(), Exists() methods

func DeleteBeforeArgsValidator added in v0.2.10 

func DeleteBeforeArgsValidator(cmd *cobra.Command, args []string) error

func DeleteEverythingArgsValidator added in v0.2.13 

func DeleteEverythingArgsValidator(cmd *cobra.Command, args []string) error

func DeleteGarbage added in v1.1.1 

func DeleteGarbage(folder storage.Folder, garbage []string) error

DeleteGarbage purges given garbage keys

func DeleteRetainAfterArgsValidator added in v0.2.15 

func DeleteRetainAfterArgsValidator(cmd *cobra.Command, args []string) error

func DeleteRetainArgsValidator added in v0.2.10 

func DeleteRetainArgsValidator(cmd *cobra.Command, args []string) error

func DeleteTargetArgsValidator added in v0.2.22 

func DeleteTargetArgsValidator(cmd *cobra.Command, args []string) error

func DownloadAndDecompressStorageFile added in v0.2.18 

func DownloadAndDecompressStorageFile(folder storage.Folder, fileName string) (io.ReadCloser, error)

TODO : unit tests

func DownloadFile added in v0.2.15 

func DownloadFile(folder storage.Folder, filename, ext string, writeCloser io.WriteCloser) error

DownloadFile downloads, decompresses and decrypts

func DownloadFileTo added in v0.2.22 

func DownloadFileTo(folder storage.Folder, fileName string, dstPath string) error

TODO : unit tests DownloadFileTo downloads a file and writes it to local file

func ExtractAll 

func ExtractAll(tarInterpreter TarInterpreter, files []ReaderMaker) error

ExtractAll Handles all files passed in. Supports `.lzo`, `.lz4`, `.lzma`, and `.tar`. File type `.nop` is used for testing purposes. Each file is extracted in its own goroutine and ExtractAll will wait for all goroutines to finish. Retries unsuccessful attempts log2(MaxConcurrency) times, dividing concurrency by two each time.

func ExtractAllWithSleeper added in v1.1.1 

func ExtractAllWithSleeper(tarInterpreter TarInterpreter, files []ReaderMaker, sleeper Sleeper) error

func ExtractDeleteEverythingModifierFromArgs added in v0.2.20 

func ExtractDeleteEverythingModifierFromArgs(args []string) int

ExtractDeleteEverythingModifierFromArgs extracts the args for the "delete everything" command

func ExtractDeleteModifierFromArgs added in v1.1.1 

func ExtractDeleteModifierFromArgs(args []string) (int, string)

ExtractDeleteModifierFromArgs extracts the delete modifier the "delete retain"/"delete before" commands

func ExtractDeleteRetainAfterModifierFromArgs added in v1.1.1 

func ExtractDeleteRetainAfterModifierFromArgs(args []string) (int, string, string)

ExtractDeleteRetainAfterModifierFromArgs extracts the args for the "delete retain --after" command

func ExtractDeleteTargetModifierFromArgs added in v0.2.22 

func ExtractDeleteTargetModifierFromArgs(args []string) int

ExtractDeleteTargetModifierFromArgs extracts the args for the "delete target" command

func FileIsPiped added in v0.2.10 

func FileIsPiped(stream *os.File) bool

FileIsPiped Check if file is piped

func FindPermanentBackups added in v1.1.1 

func FindPermanentBackups(folder storage.Folder, metaFetcher GenericMetaFetcher) map[string]bool

func FolderFromConfig added in v0.2.18 

func FolderFromConfig(configFile string) (storage.Folder, error)

FolderFromConfig prefers the config parameters instead of the current environment variables

func FolderSize added in v0.2.22 

func FolderSize(folder storage.Folder, path string) (int64, error)

func FormatTime added in v1.1.1 

func FormatTime(backupTime time.Time) string

func FormatTimeInner added in v1.1.1 

func FormatTimeInner(backupTime time.Time, timeFormat string) string

func GetBackupSentinelObjects added in v0.2.20 

func GetBackupSentinelObjects(folder storage.Folder) ([]storage.Object, error)

func GetBoolSetting added in v0.2.17 

func GetBoolSetting(setting string) (val bool, ok bool, err error)

func GetBoolSettingDefault added in v0.2.18 

func GetBoolSettingDefault(setting string, def bool) (bool, error)

func GetCommandSetting added in v0.2.15 

func GetCommandSetting(variableName string) (*exec.Cmd, error)

func GetCommandSettingContext added in v0.2.15 

func GetCommandSettingContext(ctx context.Context, variableName string) (*exec.Cmd, error)

func GetCommandStreamFetcher added in v0.2.15 

func GetCommandStreamFetcher(cmd *exec.Cmd) func(folder storage.Folder, backup Backup)

func GetDataFolderPath added in v0.2.10 

func GetDataFolderPath() string

func GetDurationSetting added in v0.2.17 

func GetDurationSetting(setting string) (time.Duration, error)

func GetGarbageFromPrefix added in v1.1.1 

func GetGarbageFromPrefix(folders []storage.Folder, nonGarbage []BackupTime) []string

func GetLastDecompressor added in v0.2.10 

func GetLastDecompressor() (compression.Decompressor, error)

func GetLatestBackupName added in v0.2.7 

func GetLatestBackupName(folder storage.Folder) (string, error)

TODO : unit tests

func GetLogsDstSettings added in v0.2.14 

func GetLogsDstSettings(operationLogsDstEnvVariable string) (dstFolder string, err error)

TODO : unit tests GetLogsDstSettings reads from the environment variables fetch settings

func GetMaxConcurrency added in v0.2.10 

func GetMaxConcurrency(concurrencyType string) (int, error)

func GetMaxDownloadConcurrency added in v0.2.10 

func GetMaxDownloadConcurrency() (int, error)

func GetMaxUploadConcurrency added in v0.2.10 

func GetMaxUploadConcurrency() (int, error)

func GetMaxUploadDiskConcurrency added in v0.2.10 

func GetMaxUploadDiskConcurrency() (int, error)

func GetOplogArchiveAfterSize added in v0.2.15 

func GetOplogArchiveAfterSize() (int, error)

func GetOplogPITRDiscoveryIntervalSetting added in v0.2.17 

func GetOplogPITRDiscoveryIntervalSetting() (*time.Duration, error)

func GetPermanentBackups added in v0.2.22 

func GetPermanentBackups(folder storage.Folder, metaFetcher GenericMetaFetcher) map[string]bool

func GetPgSlotName added in v0.2.18 

func GetPgSlotName() (pgSlotName string)

GetPgSlotName reads the slot name from the environment

func GetRelativeArchiveDataFolderPath added in v0.2.22 

func GetRelativeArchiveDataFolderPath() string

func GetRequiredSetting added in v0.2.15 

func GetRequiredSetting(setting string) (string, error)

func GetSentinelUserData 

func GetSentinelUserData() (interface{}, error)

func GetSetting added in v0.2.10 

func GetSetting(key string) (value string, ok bool)

GetSetting extract setting by key if key is set, return empty string otherwise

func GetStreamName added in v0.2.17 

func GetStreamName(backupName string, extension string) string

func HandleBackupFetch 

func HandleBackupFetch(folder storage.Folder,
	targetBackupSelector BackupSelector,
	fetcher func(folder storage.Folder, backup Backup))

TODO : unit tests HandleBackupFetch is invoked to perform wal-g backup-fetch

func HandleBackupList 

func HandleBackupList(
	getBackupsFunc func() ([]BackupTime, error),
	writeBackupListFunc func([]BackupTime),
	logging Logging,
)

func HandleBackupMark added in v0.2.12 

func HandleBackupMark(uploader *Uploader, backupName string, toPermanent bool, metaInteractor GenericMetaInteractor)

func InitConfig added in v0.2.10 

func InitConfig()

InitConfig reads config file and ENV variables if set.

func IsPermanent added in v0.2.20 

func IsPermanent(objectName string, permanentBackups map[string]bool, backupNameLength int) bool

IsPermanent is a generic function to determine if the storage object is permanent. It does not support permanent WALs or binlogs.

func PackFileTo 

func PackFileTo(tarBall TarBall, fileInfoHeader *tar.Header, fileContent io.Reader) (fileSize int64, err error)

func ParseTS added in v0.2.14 

func ParseTS(endTSEnvVar string) (endTS *time.Time, err error)

func PrettyFormatTime added in v1.1.1 

func PrettyFormatTime(backupTime time.Time) string

func ReadConfigFromFile added in v0.2.17 

func ReadConfigFromFile(config *viper.Viper, configFile string)

ReadConfigFromFile read config to the viper instance

func SentinelNameFromBackup added in v0.2.17 

func SentinelNameFromBackup(backupName string) string

func SetDefaultValues added in v0.2.17 

func SetDefaultValues(config *viper.Viper)

SetDefaultValues set default settings to the viper instance

func SetLastDecompressor added in v0.2.10 

func SetLastDecompressor(decompressor compression.Decompressor) error

func SortBackupTimeSlices added in v1.1.1 

func SortBackupTimeSlices(backupTimes []BackupTime)

func SortTimedBackup added in v1.1.1 

func SortTimedBackup(backups []TimedBackup)

func SplitPurgingBackups added in v1.1.1 

func SplitPurgingBackups(backups []TimedBackup,
	retainCount *int,
	retainAfter *time.Time) (purge, retain map[string]bool, err error)

SplitPurgingBackups partitions backups to delete and retain

func StreamBackupToCommandStdin added in v0.2.17 

func StreamBackupToCommandStdin(cmd *exec.Cmd, backup Backup) error

StreamBackupToCommandStdin downloads and decompresses backup stream to cmd stdin.

func TryDownloadFile added in v0.2.15 

func TryDownloadFile(folder storage.Folder, path string) (walFileReader io.ReadCloser, exists bool, err error)

func UnmarshalSentinelUserData added in v0.2.21 

func UnmarshalSentinelUserData(userDataStr string) (interface{}, error)

func UnwrapLatestModifier added in v0.2.22 

func UnwrapLatestModifier(backupName string, folder storage.Folder) (string, error)

UnwrapLatestModifier checks if LATEST is provided instead of backupName if so, replaces it with the name of the latest backup

func UploadSentinel added in v0.2.7 

func UploadSentinel(uploader UploaderProvider, sentinelDto interface{}, backupName string) error

TODO : unit tests

func WriteAsJSON added in v0.2.22 

func WriteAsJSON(data interface{}, output io.Writer, pretty bool) error

func WriteBackupList added in v0.2.10 

func WriteBackupList(backups []BackupTime, output io.Writer)

func WritePrettyBackupList added in v0.2.10 

func WritePrettyBackupList(backups []BackupTime, output io.Writer)

Types

type ArchiveNonExistenceError 

type ArchiveNonExistenceError struct {
	// contains filtered or unexported fields
}

func (ArchiveNonExistenceError) Error 

func (err ArchiveNonExistenceError) Error() string

type Backup 

type Backup struct {
	Name string
	// base backup folder or catchup backup folder
	Folder storage.Folder
}

Backup provides basic functionality to fetch backup-related information from storage

WAL-G stores information about single backup in the following files:

Sentinel file - contains useful information, such as backup start time, backup size, etc. see FetchSentinel, UploadSentinel

Metadata file (only in Postgres) - Postgres sentinel files can be quite large (> 1GB), so the metadata file is useful for the quick fetch of backup-related information. see FetchMetadata, UploadMetadata

func GetBackupByName 

func GetBackupByName(backupName, subfolder string, folder storage.Folder) (Backup, error)

func NewBackup 

func NewBackup(folder storage.Folder, name string) Backup

func (*Backup) AssureExists added in v0.2.22 

func (backup *Backup) AssureExists() error

AssureExists is similar to CheckExistence, but returns an error in two cases: 1. Backup does not exist 2. Failed to check if backup exist

func (*Backup) CheckExistence 

func (backup *Backup) CheckExistence() (bool, error)

func (*Backup) FetchDto added in v1.1.1 

func (backup *Backup) FetchDto(dto interface{}, path string) error

FetchDto gets data from path and de-serializes it to given object

func (*Backup) FetchMetadata added in v0.2.22 

func (backup *Backup) FetchMetadata(metadataDto interface{}) error

TODO : unit tests

func (*Backup) FetchSentinel added in v0.2.4 

func (backup *Backup) FetchSentinel(sentinelDto interface{}) error

TODO : unit tests

func (*Backup) SentinelExists added in v0.2.22 

func (backup *Backup) SentinelExists() (bool, error)

SentinelExists checks that the sentinel file of the specified backup exists.

func (*Backup) UploadMetadata added in v0.2.22 

func (backup *Backup) UploadMetadata(metadataDto interface{}) error

func (*Backup) UploadSentinel added in v0.2.22 

func (backup *Backup) UploadSentinel(sentinelDto interface{}) error

type BackupFileDescription 

type BackupFileDescription struct {
	IsIncremented bool // should never be both incremented and Skipped
	IsSkipped     bool
	MTime         time.Time
	CorruptBlocks *CorruptBlocksInfo `json:",omitempty"`
	UpdatesCount  uint64
}

func NewBackupFileDescription 

func NewBackupFileDescription(isIncremented, isSkipped bool, modTime time.Time) *BackupFileDescription

func (*BackupFileDescription) SetCorruptBlocks added in v0.2.18 

func (desc *BackupFileDescription) SetCorruptBlocks(corruptBlockNumbers []uint32, storeAllBlocks bool)

type BackupFileList 

type BackupFileList map[string]BackupFileDescription

type BackupHasPermanentBackupInFutureError added in v0.2.12 

type BackupHasPermanentBackupInFutureError struct {
	// contains filtered or unexported fields
}

type BackupMarkHandler added in v0.2.22 

type BackupMarkHandler struct {
	// contains filtered or unexported fields
}

func NewBackupMarkHandler added in v0.2.22 

func NewBackupMarkHandler(metaInteractor GenericMetaInteractor, storageRootFolder storage.Folder) BackupMarkHandler

func (*BackupMarkHandler) GetBackupsToMark added in v0.2.22 

func (h *BackupMarkHandler) GetBackupsToMark(backupName string, toPermanent bool) ([]string, error)

GetBackupsToMark retrieves all previous permanent or impermanent backups, including itself, any previous delta backups and initial full backup, in increasing order beginning from full backup, returning backups ready to be marked

For example, when marking backups from impermanent to permanent, we retrieve all currently impermanent backups and return them as a slice

func (*BackupMarkHandler) MarkBackup added in v0.2.22 

func (h *BackupMarkHandler) MarkBackup(backupName string, toPermanent bool)

MarkBackup marks a backup as permanent or impermanent

type BackupNameSelector added in v0.2.21 

type BackupNameSelector struct {
	// contains filtered or unexported fields
}

Select backup by provided backup name

func NewBackupNameSelector added in v0.2.21 

func NewBackupNameSelector(backupName string, checkExistence bool) (BackupNameSelector, error)

func (BackupNameSelector) Select added in v0.2.21 

func (s BackupNameSelector) Select(folder storage.Folder) (string, error)

type BackupNonExistenceError 

type BackupNonExistenceError struct {
	// contains filtered or unexported fields
}

func NewBackupNonExistenceError 

func NewBackupNonExistenceError(backupName string) BackupNonExistenceError

func (BackupNonExistenceError) Error 

func (err BackupNonExistenceError) Error() string

type BackupObject added in v0.2.20 

type BackupObject interface {
	storage.Object
	GetBackupTime() time.Time
	GetBackupName() string

	// TODO: move increment info into separate struct (in backup.go)
	IsFullBackup() bool
	GetBaseBackupName() string
	GetIncrementFromName() string
}

BackupObject represents the backup sentinel object uploaded on storage

func FindBackupObjects added in v1.1.1 

func FindBackupObjects(folder storage.Folder) ([]BackupObject, error)

func NewDefaultBackupObject added in v0.2.22 

func NewDefaultBackupObject(object storage.Object) BackupObject

type BackupSelector added in v0.2.21 

type BackupSelector interface {
	Select(folder storage.Folder) (string, error)
}

Select the name of storage backup chosen according to the internal rules

func CreateTargetDeleteBackupSelector added in v1.1.1 

func CreateTargetDeleteBackupSelector(cmd *cobra.Command,
	args []string, targetUserData string, metaFetcher GenericMetaFetcher) (BackupSelector, error)

create the BackupSelector to select the backup to delete

func NewTargetBackupSelector added in v0.2.22 

func NewTargetBackupSelector(targetUserData, targetName string, metaFetcher GenericMetaFetcher) (BackupSelector, error)

type BackupTime 

type BackupTime struct {
	BackupName  string    `json:"backup_name"`
	Time        time.Time `json:"time"`
	WalFileName string    `json:"wal_file_name"`
}

BackupTime is used to sort backups by latest modified time.

func GetBackupTimeSlices added in v0.2.20 

func GetBackupTimeSlices(backups []storage.Object) []BackupTime

func GetBackups added in v0.2.20 

func GetBackups(folder storage.Folder) (backups []BackupTime, err error)

TODO : unit tests GetBackups receives backup descriptions and sorts them by time

func GetBackupsAndGarbage added in v0.2.17 

func GetBackupsAndGarbage(folder storage.Folder) (backups []BackupTime, garbage []string, err error)

TODO : unit tests

type CachedDecompressor added in v0.2.10 

type CachedDecompressor struct {
	FileExtension string
}

CachedDecompressor is the file extension describing decompressor

type ComposeRatingEvaluator added in v0.2.18 

type ComposeRatingEvaluator interface {
	Evaluate(path string, updatesCount uint64, wasInBase bool) uint64
}

type CompressAndEncryptError added in v0.2.10 

type CompressAndEncryptError struct {
	// contains filtered or unexported fields
}

CompressAndEncryptError is used to catch specific errors from CompressAndEncrypt when uploading to Storage. Will not retry upload if this error occurs.

func (CompressAndEncryptError) Error added in v0.2.10 

func (err CompressAndEncryptError) Error() string

type CorruptBlocksInfo added in v0.2.18 

type CorruptBlocksInfo struct {
	CorruptBlocksCount int
	SomeCorruptBlocks  []uint32
}

type DecompressionError added in v0.2.15 

type DecompressionError struct {
	// contains filtered or unexported fields
}

type DefaultBackupObject added in v0.2.22 

type DefaultBackupObject struct {
	storage.Object
}

func (DefaultBackupObject) GetBackupName added in v0.2.22 

func (o DefaultBackupObject) GetBackupName() string

func (DefaultBackupObject) GetBackupTime added in v0.2.22 

func (o DefaultBackupObject) GetBackupTime() time.Time

func (DefaultBackupObject) GetBaseBackupName added in v0.2.22 

func (o DefaultBackupObject) GetBaseBackupName() string

func (DefaultBackupObject) GetIncrementFromName added in v0.2.22 

func (o DefaultBackupObject) GetIncrementFromName() string

func (DefaultBackupObject) IsFullBackup added in v0.2.22 

func (o DefaultBackupObject) IsFullBackup() bool

type DefaultComposeRatingEvaluator added in v0.2.18 

type DefaultComposeRatingEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultComposeRatingEvaluator added in v0.2.18 

func NewDefaultComposeRatingEvaluator(incrementFromFiles BackupFileList) *DefaultComposeRatingEvaluator

func (*DefaultComposeRatingEvaluator) Evaluate added in v0.2.18 

func (evaluator *DefaultComposeRatingEvaluator) Evaluate(path string, updatesCount uint64, wasInBase bool) uint64

type DeleteHandler added in v0.2.20 

type DeleteHandler struct {
	Folder storage.Folder
	// contains filtered or unexported fields
}

func NewDeleteHandler added in v0.2.20 

func NewDeleteHandler(
	folder storage.Folder,
	backups []BackupObject,
	less func(object1, object2 storage.Object) bool,
	options ...DeleteHandlerOption,
) *DeleteHandler

func (*DeleteHandler) DeleteBeforeTarget added in v0.2.20 

func (h *DeleteHandler) DeleteBeforeTarget(target BackupObject, confirmed bool) error

func (*DeleteHandler) DeleteBeforeTargetWhere added in v1.1.1 

func (h *DeleteHandler) DeleteBeforeTargetWhere(target BackupObject, confirmed bool, selector func(object storage.Object) bool) error

func (*DeleteHandler) DeleteEverything added in v0.2.20 

func (h *DeleteHandler) DeleteEverything(confirmed bool)

func (*DeleteHandler) DeleteTargets added in v0.2.22 

func (h *DeleteHandler) DeleteTargets(targets []BackupObject, confirmed bool) error

func (*DeleteHandler) FindTargetBefore added in v1.1.1 

func (h *DeleteHandler) FindTargetBefore(beforeStr string, modifier int) (BackupObject, error)

func (*DeleteHandler) FindTargetBeforeName added in v0.2.20 

func (h *DeleteHandler) FindTargetBeforeName(name string, modifier int) (BackupObject, error)

func (*DeleteHandler) FindTargetBeforeTime added in v0.2.20 

func (h *DeleteHandler) FindTargetBeforeTime(timeLine time.Time, modifier int) (BackupObject, error)

func (*DeleteHandler) FindTargetByName added in v0.2.20 

func (h *DeleteHandler) FindTargetByName(bname string) (BackupObject, error)

func (*DeleteHandler) FindTargetRetain added in v0.2.20 

func (h *DeleteHandler) FindTargetRetain(retentionCount, modifier int) (BackupObject, error)

func (*DeleteHandler) FindTargetRetainAfter added in v1.1.1 

func (h *DeleteHandler) FindTargetRetainAfter(retentionCount int, afterStr string, modifier int) (BackupObject, error)

func (*DeleteHandler) FindTargetRetainAfterName added in v0.2.20 

func (h *DeleteHandler) FindTargetRetainAfterName(
	retentionCount int, name string, modifier int) (BackupObject, error)

func (*DeleteHandler) FindTargetRetainAfterTime added in v0.2.20 

func (h *DeleteHandler) FindTargetRetainAfterTime(retentionCount int, timeLine time.Time, modifier int,
) (BackupObject, error)

func (*DeleteHandler) HandleDeleteBefore added in v0.2.20 

func (h *DeleteHandler) HandleDeleteBefore(args []string, confirmed bool)

func (*DeleteHandler) HandleDeleteEverything added in v0.2.22 

func (h *DeleteHandler) HandleDeleteEverything(args []string, permanentBackups map[string]bool, confirmed bool)

func (*DeleteHandler) HandleDeleteRetain added in v0.2.20 

func (h *DeleteHandler) HandleDeleteRetain(args []string, confirmed bool)

func (*DeleteHandler) HandleDeleteRetainAfter added in v0.2.20 

func (h *DeleteHandler) HandleDeleteRetainAfter(args []string, confirmed bool)

func (*DeleteHandler) HandleDeleteTarget added in v0.2.22 

func (h *DeleteHandler) HandleDeleteTarget(targetSelector BackupSelector, confirmed, findFull bool)

type DeleteHandlerOption added in v0.2.20 

type DeleteHandlerOption func(h *DeleteHandler)

func IsIgnoredFunc added in v1.1.1 

func IsIgnoredFunc(isIgnored func(storage.Object) bool) DeleteHandlerOption

func IsPermanentFunc added in v0.2.20 

func IsPermanentFunc(isPermanent func(storage.Object) bool) DeleteHandlerOption

type DtoSerializer added in v1.1.1 

type DtoSerializer interface {
	Marshal(dto interface{}) (io.Reader, error)
	Unmarshal(reader io.Reader, dto interface{}) error
}

func NewDtoSerializer added in v1.1.1 

func NewDtoSerializer() (DtoSerializer, error)

TODO: unit test

type DtoSerializerType added in v1.1.1 

type DtoSerializerType string
const (
	RegularJSONSerializer  DtoSerializerType = "json_default"
	StreamedJSONSerializer DtoSerializerType = "json_streamed"
)

type EmptyWriteIgnorer 

type EmptyWriteIgnorer struct {
	io.WriteCloser
}

EmptyWriteIgnorer handles 0 byte write in LZ4 package to stop pipe reader/writer from blocking.

func (EmptyWriteIgnorer) Write 

func (e EmptyWriteIgnorer) Write(p []byte) (int, error)

type ErrWaiter added in v0.2.22 

type ErrWaiter interface {
	Wait() error
}

type ErrorLogger added in v0.2.15 

type ErrorLogger interface {
	FatalOnError(err error)
}

type ExponentialSleeper added in v1.1.1 

type ExponentialSleeper struct {
	// contains filtered or unexported fields
}

func NewExponentialSleeper added in v1.1.1 

func NewExponentialSleeper(startSleepDuration, sleepDurationBound time.Duration) *ExponentialSleeper

func (*ExponentialSleeper) Sleep added in v1.1.1 

func (sleeper *ExponentialSleeper) Sleep()

type GenericMetaFetcher added in v0.2.22 

type GenericMetaFetcher interface {
	Fetch(backupName string, backupFolder storage.Folder) (GenericMetadata, error)
}

type GenericMetaInteractor added in v0.2.22 

type GenericMetaInteractor interface {
	GenericMetaFetcher
	GenericMetaSetter
}

GenericMetaInteractor is a combination of GenericMetaFetcher and GenericMetaSetter. It can be useful when need both.

type GenericMetaSetter added in v0.2.22 

type GenericMetaSetter interface {
	SetUserData(backupName string, backupFolder storage.Folder, userData interface{}) error
	SetIsPermanent(backupName string, backupFolder storage.Folder, isPermanent bool) error
}

type GenericMetadata added in v0.2.22 

type GenericMetadata struct {
	BackupName       string
	UncompressedSize int64
	CompressedSize   int64
	Hostname         string
	StartTime        time.Time
	FinishTime       time.Time

	IsPermanent   bool
	IsIncremental bool

	// need to use separate fetcher
	// to avoid useless sentinel load (in Postgres)
	IncrementDetails IncrementDetailsFetcher

	UserData interface{}
}

GenericMetadata allows to obtain some basic information about existing backup in storage. It is useful when creating a functionality that is common to all databases, for example backup-list or backup-mark.

To support the GenericMetadata in some particular database, one should write its own GenericMetaFetcher and GenericMetaSetter.

type IncrementDetails added in v0.2.22 

type IncrementDetails struct {
	IncrementFrom     string
	IncrementFullName string
	IncrementCount    int
}

IncrementDetails is useful to fetch information about dependencies of some incremental backup

type IncrementDetailsFetcher added in v0.2.22 

type IncrementDetailsFetcher interface {
	Fetch() (isIncremental bool, details IncrementDetails, err error)
}

type InfoLogger added in v0.2.15 

type InfoLogger interface {
	Println(v ...interface{})
}

type InvalidConcurrencyValueError added in v0.2.10 

type InvalidConcurrencyValueError struct {
	// contains filtered or unexported fields
}

func (InvalidConcurrencyValueError) Error added in v0.2.10 

func (err InvalidConcurrencyValueError) Error() string

type LatestBackupSelector added in v0.2.21 

type LatestBackupSelector struct {
}

Select the latest backup from storage

func NewLatestBackupSelector added in v0.2.21 

func NewLatestBackupSelector() LatestBackupSelector

func (LatestBackupSelector) Select added in v0.2.21 

func (s LatestBackupSelector) Select(folder storage.Folder) (string, error)

type LazyCache 

type LazyCache struct {
	// contains filtered or unexported fields
}

func NewLazyCache 

func NewLazyCache(load func(key interface{}) (value interface{}, err error)) *LazyCache

func (*LazyCache) Load 

func (lazyCache *LazyCache) Load(key interface{}) (value interface{}, exists bool, err error)

func (*LazyCache) LoadExisting 

func (lazyCache *LazyCache) LoadExisting(key interface{}) (value interface{}, exists bool)

func (*LazyCache) Range 

func (lazyCache *LazyCache) Range(reduce func(key, value interface{}) bool)

Range calls reduce sequentially for each key and value present in the cache. If reduce returns false, range stops the iteration.

func (*LazyCache) Store 

func (lazyCache *LazyCache) Store(key, value interface{})

type Logging added in v0.2.15 

type Logging struct {
	InfoLogger  InfoLogger
	ErrorLogger ErrorLogger
}

type MetaConstructor added in v0.2.22 

type MetaConstructor interface {
	Init() error
	Finalize(backupName string) error
	MetaInfo() interface{}
}

MetaConstructor - interface that helps with building meta-info about backup and generate MetaInfo see MongoMetaConstructor see RedisMetaConstructor

type NOPTarBall 

type NOPTarBall struct {
	// contains filtered or unexported fields
}

NOPTarBall mocks a tarball. Used for prefault logic.

func (*NOPTarBall) AddSize 

func (tarBall *NOPTarBall) AddSize(i int64)

func (*NOPTarBall) AwaitUploads 

func (tarBall *NOPTarBall) AwaitUploads()

func (*NOPTarBall) CloseTar 

func (tarBall *NOPTarBall) CloseTar() error

func (*NOPTarBall) Name added in v0.2.18 

func (tarBall *NOPTarBall) Name() string

func (*NOPTarBall) SetUp 

func (tarBall *NOPTarBall) SetUp(crypter crypto.Crypter, params ...string)

func (*NOPTarBall) Size 

func (tarBall *NOPTarBall) Size() int64

func (*NOPTarBall) TarWriter 

func (tarBall *NOPTarBall) TarWriter() *tar.Writer

type NOPTarBallMaker 

type NOPTarBallMaker struct {
	// contains filtered or unexported fields
}

NOPTarBallMaker creates a new NOPTarBall. Used for testing purposes.

func (*NOPTarBallMaker) Make 

func (tarBallMaker *NOPTarBallMaker) Make(inheritState bool) TarBall

Make creates a new NOPTarBall.

type NoBackupsFoundError 

type NoBackupsFoundError struct {
	// contains filtered or unexported fields
}

func NewNoBackupsFoundError 

func NewNoBackupsFoundError() NoBackupsFoundError

func (NoBackupsFoundError) Error 

func (err NoBackupsFoundError) Error() string

type NoFilesToExtractError 

type NoFilesToExtractError struct {
	// contains filtered or unexported fields
}

func (NoFilesToExtractError) Error 

func (err NoFilesToExtractError) Error() string

type NopIncrementDetailsFetcher added in v0.2.22 

type NopIncrementDetailsFetcher struct{}

NopIncrementDetailsFetcher is useful for databases without incremental backup support

func (*NopIncrementDetailsFetcher) Fetch added in v0.2.22 

func (idf *NopIncrementDetailsFetcher) Fetch() (bool, IncrementDetails, error)

type ReaderMaker 

type ReaderMaker interface {
	Reader() (io.ReadCloser, error)
	Path() string
}

ReaderMaker is the generic interface used by extract. It allows for ease of handling different file formats.

type RegularJSON added in v1.1.1 

type RegularJSON struct{}

func (RegularJSON) Marshal added in v1.1.1 

func (r RegularJSON) Marshal(dto interface{}) (io.Reader, error)

func (RegularJSON) Unmarshal added in v1.1.1 

func (r RegularJSON) Unmarshal(reader io.Reader, dto interface{}) error

type Sentinel 

type Sentinel struct {
	Info os.FileInfo
	Path string
}

Sentinel is used to signal completion of a walked directory.

type SentinelMarshallingError added in v0.2.7 

type SentinelMarshallingError struct {
	// contains filtered or unexported fields
}

region errors

func NewSentinelMarshallingError added in v0.2.7 

func NewSentinelMarshallingError(sentinelName string, err error) SentinelMarshallingError

func (SentinelMarshallingError) Error added in v0.2.7 

func (err SentinelMarshallingError) Error() string

type Sleeper added in v1.1.1 

type Sleeper interface {
	Sleep()
}

type StorageAdapter added in v0.2.7 

type StorageAdapter struct {
	// contains filtered or unexported fields
}

type StorageReaderMaker 

type StorageReaderMaker struct {
	Folder       storage.Folder
	RelativePath string
}

StorageReaderMaker creates readers for downloading from storage

func NewStorageReaderMaker 

func NewStorageReaderMaker(folder storage.Folder, relativePath string) *StorageReaderMaker

func (*StorageReaderMaker) Path 

func (readerMaker *StorageReaderMaker) Path() string

func (*StorageReaderMaker) Reader 

func (readerMaker *StorageReaderMaker) Reader() (io.ReadCloser, error)

type StorageTarBall 

type StorageTarBall struct {
	// contains filtered or unexported fields
}

StorageTarBall represents a tar file that is going to be uploaded to storage.

func (*StorageTarBall) AddSize 

func (tarBall *StorageTarBall) AddSize(i int64)

AddSize to total Size

func (*StorageTarBall) AwaitUploads 

func (tarBall *StorageTarBall) AwaitUploads()

func (*StorageTarBall) CloseTar 

func (tarBall *StorageTarBall) CloseTar() error

CloseTar closes the tar writer, flushing any unwritten data to the underlying writer before also closing the underlying writer.

func (*StorageTarBall) Name added in v0.2.18 

func (tarBall *StorageTarBall) Name() string

func (*StorageTarBall) SetUp 

func (tarBall *StorageTarBall) SetUp(crypter crypto.Crypter, names ...string)

SetUp creates a new tar writer and starts upload to storage. Upload will block until the tar file is finished writing. If a name for the file is not given, default name is of the form `part_....tar.[Compressor file extension]`.

func (*StorageTarBall) Size 

func (tarBall *StorageTarBall) Size() int64

Size accumulated in this tarball

func (*StorageTarBall) TarWriter 

func (tarBall *StorageTarBall) TarWriter() *tar.Writer

type StorageTarBallMaker 

type StorageTarBallMaker struct {
	// contains filtered or unexported fields
}

StorageTarBallMaker creates tarballs that are uploaded to storage.

func NewStorageTarBallMaker 

func NewStorageTarBallMaker(backupName string, uploader *Uploader) *StorageTarBallMaker

func (*StorageTarBallMaker) Make 

func (tarBallMaker *StorageTarBallMaker) Make(dedicatedUploader bool) TarBall

Make returns a tarball with required storage fields.

type StreamedJSON added in v1.1.1 

type StreamedJSON struct{}

func (StreamedJSON) Marshal added in v1.1.1 

func (s StreamedJSON) Marshal(dto interface{}) (io.Reader, error)

func (StreamedJSON) Unmarshal added in v1.1.1 

func (s StreamedJSON) Unmarshal(reader io.Reader, dto interface{}) error

type TarBall 

type TarBall interface {
	SetUp(crypter crypto.Crypter, args ...string)
	CloseTar() error
	Size() int64
	AddSize(int64)
	TarWriter() *tar.Writer
	AwaitUploads()
	Name() string
}

A TarBall represents one tar file.

type TarBallMaker 

type TarBallMaker interface {
	Make(dedicatedUploader bool) TarBall
}

TarBallMaker is used to allow for flexible creation of different TarBalls.

func NewNopTarBallMaker 

func NewNopTarBallMaker() TarBallMaker

type TarBallQueue added in v0.2.18 

type TarBallQueue struct {
	TarSizeThreshold   int64
	AllTarballsSize    *int64
	TarBallMaker       TarBallMaker
	LastCreatedTarball TarBall
	// contains filtered or unexported fields
}

TarBallQueue is used to process multiple tarballs concurrently

func NewTarBallQueue added in v0.2.22 

func NewTarBallQueue(tarSizeThreshold int64, tarBallMaker TarBallMaker) *TarBallQueue

func (*TarBallQueue) CheckSizeAndEnqueueBack added in v0.2.18 

func (tarQueue *TarBallQueue) CheckSizeAndEnqueueBack(tarBall TarBall) error

func (*TarBallQueue) CloseTarball added in v0.2.18 

func (tarQueue *TarBallQueue) CloseTarball(tarBall TarBall) error

func (*TarBallQueue) Deque added in v0.2.18 

func (tarQueue *TarBallQueue) Deque() TarBall

func (*TarBallQueue) DequeCtx added in v0.2.22 

func (tarQueue *TarBallQueue) DequeCtx(ctx context.Context) (TarBall, error)

DequeCtx returns a TarBall from the queue. If the context finishes before it can do so, it returns the result of ctx.Err().

func (*TarBallQueue) EnqueueBack added in v0.2.18 

func (tarQueue *TarBallQueue) EnqueueBack(tarBall TarBall)

func (*TarBallQueue) FinishQueue added in v0.2.18 

func (tarQueue *TarBallQueue) FinishQueue() error

func (*TarBallQueue) FinishTarBall added in v0.2.18 

func (tarQueue *TarBallQueue) FinishTarBall(tarBall TarBall) error

func (*TarBallQueue) NewTarBall added in v0.2.18 

func (tarQueue *TarBallQueue) NewTarBall(dedicatedUploader bool) TarBall

NewTarBall starts writing new tarball

func (*TarBallQueue) StartQueue added in v0.2.18 

func (tarQueue *TarBallQueue) StartQueue() error

type TarInterpreter 

type TarInterpreter interface {
	Interpret(reader io.Reader, header *tar.Header) error
}

TarInterpreter behaves differently for different file types.

type TimedBackup added in v1.1.1 

type TimedBackup interface {
	Name() string
	StartTime() time.Time
	IsPermanent() bool
}

type UnconfiguredStorageError added in v0.2.8 

type UnconfiguredStorageError struct {
	// contains filtered or unexported fields
}

func (UnconfiguredStorageError) Error added in v0.2.8 

func (err UnconfiguredStorageError) Error() string

type UnknownCompressionMethodError 

type UnknownCompressionMethodError struct {
	// contains filtered or unexported fields
}

func (UnknownCompressionMethodError) Error 

func (err UnknownCompressionMethodError) Error() string

type UnknownSerializerTypeError added in v1.1.1 

type UnknownSerializerTypeError struct {
	// contains filtered or unexported fields
}

func NewUnknownSerializerTypeError added in v1.1.1 

func NewUnknownSerializerTypeError(serializerType DtoSerializerType) UnknownSerializerTypeError

func (UnknownSerializerTypeError) Error added in v1.1.1 

func (err UnknownSerializerTypeError) Error() string

type UnmarshallingError added in v0.2.10 

type UnmarshallingError struct {
	// contains filtered or unexported fields
}

func (UnmarshallingError) Error added in v0.2.10 

func (err UnmarshallingError) Error() string

type UnsetRequiredSettingError added in v0.2.10 

type UnsetRequiredSettingError struct {
	// contains filtered or unexported fields
}

func NewUnsetRequiredSettingError added in v0.2.10 

func NewUnsetRequiredSettingError(settingName string) UnsetRequiredSettingError

func (UnsetRequiredSettingError) Error added in v0.2.10 

func (err UnsetRequiredSettingError) Error() string

type UnsupportedFileTypeError 

type UnsupportedFileTypeError struct {
	// contains filtered or unexported fields
}

UnsupportedFileTypeError is used to signal file types that are unsupported by WAL-G.

func (UnsupportedFileTypeError) Error added in v0.2.8 

func (err UnsupportedFileTypeError) Error() string

type UploadObject added in v0.2.10 

type UploadObject struct {
	Path    string
	Content io.Reader
}

UploadObject

type Uploader 

type Uploader struct {
	UploadingFolder storage.Folder
	Compressor      compression.Compressor

	ArchiveStatusManager   asm.ArchiveStatusManager
	PGArchiveStatusManager asm.ArchiveStatusManager
	Failed                 atomic.Value
	// contains filtered or unexported fields
}

Uploader contains fields associated with uploading tarballs. Multiple tarballs can share one uploader.

func ConfigureUploader added in v0.2.8 

func ConfigureUploader() (uploader *Uploader, err error)

ConfigureUploader connects to storage and creates an uploader. It makes sure that a valid session has started; if invalid, returns AWS error and `<nil>` values.

func ConfigureUploaderWithoutCompressMethod added in v0.2.15 

func ConfigureUploaderWithoutCompressMethod() (uploader *Uploader, err error)

func NewUploader 

func NewUploader(
	compressor compression.Compressor,
	uploadingLocation storage.Folder,
) *Uploader

func (*Uploader) Clone 

func (uploader *Uploader) Clone() *Uploader

Clone creates similar Uploader with new WaitGroup

func (*Uploader) Compression added in v0.2.17 

func (uploader *Uploader) Compression() compression.Compressor

Compression returns configured compressor

func (*Uploader) DisableSizeTracking added in v0.2.17 

func (uploader *Uploader) DisableSizeTracking()

DisableSizeTracking stops bandwidth tracking

func (*Uploader) Finish added in v0.2.22 

func (uploader *Uploader) Finish()

Finish waits for all waiting parts to be uploaded. If an error occurs, prints alert to stderr.

func (*Uploader) PushStream added in v0.2.10 

func (uploader *Uploader) PushStream(stream io.Reader) (string, error)

TODO : unit tests PushStream compresses a stream and push it

func (*Uploader) PushStreamToDestination added in v0.2.10 

func (uploader *Uploader) PushStreamToDestination(stream io.Reader, dstPath string) error

TODO : unit tests PushStreamToDestination compresses a stream and push it to specifyed destination

func (*Uploader) RawDataSize added in v0.2.22 

func (uploader *Uploader) RawDataSize() (int64, error)

RawDataSize returns 0 and error when SizeTracking disabled (see DisableSizeTracking)

func (*Uploader) Upload added in v0.2.7 

func (uploader *Uploader) Upload(path string, content io.Reader) error

TODO : unit tests

func (*Uploader) UploadFile 

func (uploader *Uploader) UploadFile(file ioextensions.NamedReader) error

TODO : unit tests UploadFile compresses a file and uploads it.

func (*Uploader) UploadMultiple added in v0.2.10 

func (uploader *Uploader) UploadMultiple(objects []UploadObject) error

UploadMultiple uploads multiple objects from the start of the slice, returning the first error if any. Note that this operation is not atomic TODO : unit tests

func (*Uploader) UploadedDataSize added in v0.2.22 

func (uploader *Uploader) UploadedDataSize() (int64, error)

UploadedDataSize returns 0 and error when SizeTracking disabled (see DisableSizeTracking)

type UploaderProvider added in v0.2.17 

type UploaderProvider interface {
	Upload(path string, content io.Reader) error
	UploadFile(file ioextensions.NamedReader) error
	PushStream(stream io.Reader) (string, error)
	PushStreamToDestination(stream io.Reader, dstPath string) error
	Compression() compression.Compressor
	DisableSizeTracking()
	UploadedDataSize() (int64, error)
	RawDataSize() (int64, error)
}

type UserDataBackupSelector added in v0.2.21 

type UserDataBackupSelector struct {
	// contains filtered or unexported fields
}

Select backup which has the provided user data

func NewUserDataBackupSelector added in v0.2.21 

func NewUserDataBackupSelector(userDataRaw string, metaFetcher GenericMetaFetcher) (UserDataBackupSelector, error)

func (UserDataBackupSelector) Select added in v0.2.21 

func (s UserDataBackupSelector) Select(folder storage.Folder) (string, error)

type WithSizeReader added in v0.2.13 

type WithSizeReader struct {
	// contains filtered or unexported fields
}

func NewWithSizeReader added in v0.2.22 

func NewWithSizeReader(underlying io.Reader, readSize *int64) *WithSizeReader

func (*WithSizeReader) Read added in v0.2.13 

func (reader *WithSizeReader) Read(p []byte) (n int, err error)

type WrongTypeError 

type WrongTypeError struct {
	// contains filtered or unexported fields
}

func NewWrongTypeError 

func NewWrongTypeError(desiredType string) WrongTypeError

func (WrongTypeError) Error 

func (err WrongTypeError) Error() string

Source Files

View all Source files

Directories

Path Synopsis
Package abool provides atomic Boolean type for cleaner code and better performance.
 Package abool provides atomic Boolean type for cleaner code and better performance.
computils
lz4
lzma
zstd
awskms
libsodium
openpgp
yckms
databases
fdb
greenplum
mongo
mongo/archive
mongo/archive/mocks
mongo/client
mongo/client/mocks
mongo/discovery
mongo/models
mongo/oplog
mongo/stages
mongo/stages/mocks
mongo/stats
mysql
postgres
redis
redis/archive
sqlserver
sqlserver/blob
testtools
Package mock_internal is a generated GoMock package.
 Package mock_internal is a generated GoMock package.
parsingutil

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.