internal

package
v0.2.18 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 6, 2020 License: Apache-2.0 Imports: 57 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	TarPartitionFolderName = "/tar_partitions/"
	PgControlPath          = "/global/pg_control"
)
View Source 
const (
	PgControl             = "pg_control"
	BackupLabelFilename   = "backup_label"
	TablespaceMapFilename = "tablespace_map"
	TablespaceFolder      = "pg_tblspc"
)
View Source 
const (
	DownloadConcurrencySetting   = "WALG_DOWNLOAD_CONCURRENCY"
	UploadConcurrencySetting     = "WALG_UPLOAD_CONCURRENCY"
	UploadDiskConcurrencySetting = "WALG_UPLOAD_DISK_CONCURRENCY"
	UploadQueueSetting           = "WALG_UPLOAD_QUEUE"
	SentinelUserDataSetting      = "WALG_SENTINEL_USER_DATA"
	PreventWalOverwriteSetting   = "WALG_PREVENT_WAL_OVERWRITE"
	DeltaMaxStepsSetting         = "WALG_DELTA_MAX_STEPS"
	DeltaOriginSetting           = "WALG_DELTA_ORIGIN"
	CompressionMethodSetting     = "WALG_COMPRESSION_METHOD"
	DiskRateLimitSetting         = "WALG_DISK_RATE_LIMIT"
	NetworkRateLimitSetting      = "WALG_NETWORK_RATE_LIMIT"
	UseWalDeltaSetting           = "WALG_USE_WAL_DELTA"
	UseReverseUnpackSetting      = "WALG_USE_REVERSE_UNPACK"
	SkipRedundantTarsSetting     = "WALG_SKIP_REDUNDANT_TARS"
	VerifyPageChecksumsSetting   = "WALG_VERIFY_PAGE_CHECKSUMS"
	StoreAllCorruptBlocksSetting = "WALG_STORE_ALL_CORRUPT_BLOCKS"
	UseRatingComposerSetting     = "WALG_USE_RATING_COMPOSER"
	LogLevelSetting              = "WALG_LOG_LEVEL"
	TarSizeThresholdSetting      = "WALG_TAR_SIZE_THRESHOLD"
	CseKmsIDSetting              = "WALG_CSE_KMS_ID"
	CseKmsRegionSetting          = "WALG_CSE_KMS_REGION"
	LibsodiumKeySetting          = "WALG_LIBSODIUM_KEY"
	LibsodiumKeyPathSetting      = "WALG_LIBSODIUM_KEY_PATH"
	GpgKeyIDSetting              = "GPG_KEY_ID"
	PgpKeySetting                = "WALG_PGP_KEY"
	PgpKeyPathSetting            = "WALG_PGP_KEY_PATH"
	PgpKeyPassphraseSetting      = "WALG_PGP_KEY_PASSPHRASE"
	PgDataSetting                = "PGDATA"
	UserSetting                  = "USER" // TODO : do something with it
	PgPortSetting                = "PGPORT"
	PgUserSetting                = "PGUSER"
	PgHostSetting                = "PGHOST"
	PgPasswordSetting            = "PGPASSWORD"
	PgDatabaseSetting            = "PGDATABASE"
	PgSslModeSetting             = "PGSSLMODE"
	PgSlotName                   = "WALG_SLOTNAME"
	TotalBgUploadedLimit         = "TOTAL_BG_UPLOADED_LIMIT"
	NameStreamCreateCmd          = "WALG_STREAM_CREATE_COMMAND"
	NameStreamRestoreCmd         = "WALG_STREAM_RESTORE_COMMAND"

	MongoDBUriSetting               = "MONGODB_URI"
	MongoDBLastWriteUpdateInterval  = "MONGODB_LAST_WRITE_UPDATE_INTERVAL"
	OplogArchiveAfterSize           = "OPLOG_ARCHIVE_AFTER_SIZE"
	OplogArchiveTimeoutInterval     = "OPLOG_ARCHIVE_TIMEOUT_INTERVAL"
	OplogPITRDiscoveryInterval      = "OPLOG_PITR_DISCOVERY_INTERVAL"
	OplogPushStatsEnabled           = "OPLOG_PUSH_STATS_ENABLED"
	OplogPushStatsLoggingInterval   = "OPLOG_PUSH_STATS_LOGGING_INTERVAL"
	OplogPushStatsUpdateInterval    = "OPLOG_PUSH_STATS_UPDATE_INTERVAL"
	OplogPushStatsExposeHttp        = "OPLOG_PUSH_STATS_EXPOSE_HTTP"
	OplogPushWaitForBecomePrimary   = "OPLOG_PUSH_WAIT_FOR_BECOME_PRIMARY"
	OplogPushPrimaryCheckInterval   = "OPLOG_PUSH_PRIMARY_CHECK_INTERVAL"
	OplogReplayOplogAlwaysUpsert    = "OPLOG_REPLAY_OPLOG_ALWAYS_UPSERT"
	OplogReplayOplogApplicationMode = "OPLOG_REPLAY_OPLOG_APPLICATION_MODE"
	OplogReplayIgnoreErrorCodes     = "OPLOG_REPLAY_IGNORE_ERROR_CODES"

	MysqlDatasourceNameSetting = "WALG_MYSQL_DATASOURCE_NAME"
	MysqlSslCaSetting          = "WALG_MYSQL_SSL_CA"
	MysqlBinlogReplayCmd       = "WALG_MYSQL_BINLOG_REPLAY_COMMAND"
	MysqlBinlogDstSetting      = "WALG_MYSQL_BINLOG_DST"
	MysqlBackupPrepareCmd      = "WALG_MYSQL_BACKUP_PREPARE_COMMAND"
	MysqlTakeBinlogsFromMaster = "WALG_MYSQL_TAKE_BINLOGS_FROM_MASTER"

	GoMaxProcs = "GOMAXPROCS"

	HttpListen       = "HTTP_LISTEN"
	HttpExposePprof  = "HTTP_EXPOSE_PPROF"
	HttpExposeExpVar = "HTTP_EXPOSE_EXPVAR"

	SQLServerBlobHostname     = "SQLSERVER_BLOB_HOSTNAME"
	SQLServerBlobCertFile     = "SQLSERVER_BLOB_CERT_FILE"
	SQLServerBlobKeyFile      = "SQLSERVER_BLOB_KEY_FILE"
	SQLServerConnectionString = "SQLSERVER_CONNECTION_STRING"

	EndpointSourceSetting = "S3_ENDPOINT_SOURCE"
	EndpointPortSetting   = "S3_ENDPOINT_PORT"

	AwsAccessKeyId     = "AWS_ACCESS_KEY_ID"
	AwsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
)
View Source 
const (
	DefaultDataBurstRateLimit = 8 * DatabasePageSize
	DefaultDataFolderPath     = "/tmp"
	WaleFileHost              = "file://localhost"
)
View Source 
const (
	NoDeleteModifier = iota
	FullDeleteModifier
	FindFullDeleteModifier
	ForceDeleteModifier
	ConfirmFlag            = "confirm"
	DeleteShortDescription = "Clears old backups and WALs"

	DeleteRetainExamples = `` /* 321-byte string literal not displayed */

	DeleteBeforeExamples = `` /* 156-byte string literal not displayed */

	DeleteEverythingExamples = `` /* 152-byte string literal not displayed */

	DeleteEverythingUsageExample = "everything [FORCE]"
	DeleteRetainUsageExample     = "retain [FULL|FIND_FULL] backup_count"
	DeleteBeforeUsageExample     = "before [FIND_FULL] backup_name|timestamp"
)
View Source 
const (
	RelFileSizeBound               = 1 << 30
	BlocksInRelFile                = int(RelFileSizeBound / DatabasePageSize)
	DefaultSpcNode   walparser.Oid = 1663
)
View Source 
const (
	// number of checksums to calculate in parallel
	NSums int = 32
	// prime multiplier of FNV-1a hash
	FnvPrime uint32 = 16777619
	// page header checksum offset
	PdChecksumOffset = 8
	// page header checksum length (in bytes)
	PdChecksumLen = 2
)
View Source 
const (
	DatabasePageSize = int64(walparser.BlockSize)

	SignatureMagicNumber byte = 0x55

	DefaultTablespace    = "base"
	GlobalTablespace     = "global"
	NonDefaultTablespace = "pg_tblspc"
)
View Source 
const (
	BasePrefix  = "base_prefix"
	Tablespaces = "tablespaces"
)
View Source 
const (
	WalFileInDelta      uint64 = 16
	DeltaFilenameSuffix        = "_delta"
	PartFilenameSuffix         = "_part"
)
View Source 
const (
	TimelineOkStatus          = "OK"
	TimelineLostSegmentStatus = "LOST_SEGMENTS"
)
View Source 
const LatestString = "LATEST"
View Source 
const MaxCorruptBlocksInFileDesc int = 10
View Source 
const MinAllowedConcurrency = 1
View Source 
const (
	RecordPartFilename = "currentRecord.part"
)
View Source 
const (
	StreamPrefix = "stream_"
)
View Source 
const (
	// WalSegmentSize is the size of one WAL file
	WalSegmentSize = uint64(16 * 1024 * 1024) // xlog.info line 113ß

)

Variables

View Source 
var (
	CfgFile string

	AllowedSettings = map[string]bool{}/* 105 elements not displayed */

	RequiredSettings       = make(map[string]bool)
	HttpSettingExposeFuncs = map[string]func(webserver.WebServer){
		HttpExposePprof:          webserver.EnablePprofEndpoints,
		HttpExposeExpVar:         webserver.EnableExpVarEndpoints,
		OplogPushStatsExposeHttp: nil,
	}
)
View Source 
var BasePrefixMissingError = fmt.Errorf("Base prefix not set while working with tablespaces.\n")
View Source 
var DeprecatedExternalGpgMessage = fmt.Sprintf(
	`You are using deprecated functionality that uses an external gpg library.
It will be removed in next major version.
Please set GPG key using environment variables %s or %s.
`, PgpKeySetting, PgpKeyPathSetting)
View Source 
var DiskLimiter *rate.Limiter
View Source 
var ExcludedFilenames = make(map[string]utility.Empty)

ExcludedFilenames is a list of excluded members from the bundled backup.

View Source 
var Extensions []Extension
View Source 
var IncrementFileHeader = []byte{'w', 'i', '1', SignatureMagicNumber}

"wi" at the head stands for "wal-g increment" format version "1", signature magic number

View Source 
var MaxExtractRetryWait = 5 * time.Minute
View Source 
var MaxTime = time.Unix(1<<63-62135596801, 999999999)
View Source 
var MinExtractRetryWait = time.Minute
View Source 
var NetworkLimiter *rate.Limiter
View Source 
var StorageAdapters = []StorageAdapter{
	{"S3_PREFIX", s3.SettingList, s3.ConfigureFolder, nil},
	{"FILE_PREFIX", nil, fs.ConfigureFolder, preprocessFilePrefix},
	{"GS_PREFIX", gcs.SettingList, gcs.ConfigureFolder, nil},
	{"AZ_PREFIX", azure.SettingList, azure.ConfigureFolder, nil},
	{"SWIFT_PREFIX", swift.SettingList, swift.ConfigureFolder, nil},
	{"SSH_PREFIX", sh.SettingsList, sh.ConfigureFolder, nil},
}
View Source 
var StringModifiers = []string{"FULL", "FIND_FULL"}
View Source 
var StringModifiersDeleteEverything = []string{"FORCE"}
View Source 
var TerminalLocation = *walparser.NewBlockLocation(0, 0, 0, 0)
View Source 
var UnwrapAll map[string]bool = nil
View Source 
var UtilityFilePaths = map[string]bool{
	PgControlPath:         true,
	BackupLabelFilename:   true,
	TablespaceMapFilename: true,
}

Functions

func AddConfigFlags added in v0.2.16 

func AddConfigFlags(Cmd *cobra.Command)

func ApplyFileIncrement 

func ApplyFileIncrement(fileName string, increment io.Reader, createNewIncrementalFiles bool) error

ApplyFileIncrement changes pages according to supplied change map file

func AssertRequiredSettingsSet added in v0.2.16 

func AssertRequiredSettingsSet() error

func CheckAllowedSettings added in v0.2.17 

func CheckAllowedSettings(config *viper.Viper)

CheckAllowedSettings warnings if a viper instance's setting not allowed

func CleanupPrefetchDirectories 

func CleanupPrefetchDirectories(walFileName string, location string, cleaner Cleaner)

func CompressAndEncrypt added in v0.2.10 

func CompressAndEncrypt(source io.Reader, compressor compression.Compressor, crypter crypto.Crypter) io.Reader

CompressAndEncrypt compresses input to a pipe reader. Output must be used or pipe will block.

func Configure 

func Configure()

func ConfigureAndRunDefaultWebServer added in v0.2.17 

func ConfigureAndRunDefaultWebServer() error

ConfigureAndRunDefaultWebServer configures and runs web server

func ConfigureCrypter added in v0.2.10 

func ConfigureCrypter() crypto.Crypter

ConfigureCrypter uses environment variables to create and configure a crypter. In case no configuration in environment variables found, return `<nil>` value.

func ConfigureFolder added in v0.2.8 

func ConfigureFolder() (storage.Folder, error)

TODO : unit tests

func ConfigureFolderForSpecificConfig added in v0.2.17 

func ConfigureFolderForSpecificConfig(config *viper.Viper) (storage.Folder, error)

func ConfigureLogging added in v0.2.8 

func ConfigureLogging() error

func Connect 

func Connect(configOptions ...func(config *pgx.ConnConfig) error) (*pgx.Conn, error)

Connect establishes a connection to postgres using a UNIX socket. Must export PGHOST and run with `sudo -E -u postgres`. If PGHOST is not set or if the connection fails, an error is returned and the connection is `<nil>`.

Example: PGHOST=/var/run/postgresql or PGHOST=10.0.0.1

func CreateFileFromIncrement added in v0.2.17 

func CreateFileFromIncrement(increment io.Reader, target ReadWriterAt) (int64, error)

CreateFileFromIncrement writes the pages from the increment to local file and write empty blocks in place of pages which are not present in the increment

func DecompressDecryptBytes added in v0.2.15 

func DecompressDecryptBytes(dst io.Writer, archiveReader io.ReadCloser, decompressor compression.Decompressor) error

TODO : unit tests

func DecryptAndDecompressTar 

func DecryptAndDecompressTar(writer io.Writer, readerMaker ReaderMaker, crypter crypto.Crypter) error

DecryptAndDecompressTar decrypts file and checks its extension. If it's tar, a decompression is not needed. Otherwise it uses corresponding decompressor. If none found an error will be returned.

func DefaultHandleBackupList added in v0.2.15 

func DefaultHandleBackupList(folder storage.Folder)

func DeleteBeforeArgsValidator added in v0.2.10 

func DeleteBeforeArgsValidator(cmd *cobra.Command, args []string) error

func DeleteBeforeTarget added in v0.2.10 

func DeleteBeforeTarget(folder storage.Folder, target storage.Object,
	confirmed bool,
	isFullBackup func(object storage.Object) bool,
	less func(object1, object2 storage.Object) bool) error

func DeleteEverything added in v0.2.13 

func DeleteEverything(folder storage.Folder,
	confirmed bool,
	args []string)

func DeleteEverythingArgsValidator added in v0.2.13 

func DeleteEverythingArgsValidator(cmd *cobra.Command, args []string) error

func DeleteRetainAfterArgsValidator added in v0.2.15 

func DeleteRetainAfterArgsValidator(cmd *cobra.Command, args []string) error

func DeleteRetainArgsValidator added in v0.2.10 

func DeleteRetainArgsValidator(cmd *cobra.Command, args []string) error

func DownloadAndDecompressStorageFile added in v0.2.18 

func DownloadAndDecompressStorageFile(folder storage.Folder, fileName string) (io.ReadCloser, error)

TODO : unit tests

func DownloadFile added in v0.2.15 

func DownloadFile(folder storage.Folder, filename, ext string, writeCloser io.WriteCloser) error

DownloadFile downloads, decompresses and decrypts

func DownloadWALFileTo added in v0.2.7 

func DownloadWALFileTo(folder storage.Folder, walFileName string, dstPath string) error

TODO : unit tests downloadWALFileTo downloads a file and writes it to local file

func ExtractAll 

func ExtractAll(tarInterpreter TarInterpreter, files []ReaderMaker) error

TODO : unit tests ExtractAll Handles all files passed in. Supports `.lzo`, `.lz4`, `.lzma`, and `.tar`. File type `.nop` is used for testing purposes. Each file is extracted in its own goroutine and ExtractAll will wait for all goroutines to finish. Returns the first error encountered.

func ExtractBlockLocations 

func ExtractBlockLocations(records []walparser.XLogRecord) []walparser.BlockLocation

func FetchStreamSentinel added in v0.2.12 

func FetchStreamSentinel(backup *Backup, sentinelDto interface{}) error

TODO : unit tests

func FileIsPiped added in v0.2.10 

func FileIsPiped(stream *os.File) bool

FileIsPiped Check if file is piped

func FindTargetBeforeName added in v0.2.8 

func FindTargetBeforeName(folder storage.Folder,
	name string, modifier int,
	isFullBackup func(object storage.Object) bool,
	greater func(object1, object2 storage.Object) bool) (storage.Object, error)

func FindTargetBeforeTime added in v0.2.10 

func FindTargetBeforeTime(folder storage.Folder,
	timeLine time.Time, modifier int,
	isFullBackup func(object storage.Object) bool,
	less func(object1, object2 storage.Object) bool) (storage.Object, error)

func FindTargetRetain added in v0.2.10 

func FindTargetRetain(folder storage.Folder,
	retentionCount, modifier int,
	isFullBackup func(object storage.Object) bool,
	greater func(object1, object2 storage.Object) bool) (storage.Object, error)

func FindTargetRetainAfterName added in v0.2.15 

func FindTargetRetainAfterName(folder storage.Folder,
	retentionCount int, name string, modifier int,
	isFullBackup func(object storage.Object) bool,
	greater func(object1, object2 storage.Object) bool) (storage.Object, error)

func FindTargetRetainAfterTime added in v0.2.15 

func FindTargetRetainAfterTime(folder storage.Folder,
	retentionCount int,
	timeLine time.Time,
	modifier int,
	isFullBackup func(object storage.Object) bool,
	greater func(object1, object2 storage.Object) bool) (storage.Object, error)

func FolderFromConfig added in v0.2.18 

func FolderFromConfig(configFile string) (storage.Folder, error)

func GetBaseFilesToUnwrap 

func GetBaseFilesToUnwrap(backupFileStates BackupFileList, currentFilesToUnwrap map[string]bool) (map[string]bool, error)

func GetBoolSetting added in v0.2.17 

func GetBoolSetting(setting string) (val bool, ok bool, err error)

func GetBoolSettingDefault added in v0.2.18 

func GetBoolSettingDefault(setting string, def bool) (bool, error)

func GetCommandSetting added in v0.2.15 

func GetCommandSetting(variableName string) (*exec.Cmd, error)

func GetCommandSettingContext added in v0.2.15 

func GetCommandSettingContext(ctx context.Context, variableName string) (*exec.Cmd, error)

func GetCommandStreamFetcher added in v0.2.15 

func GetCommandStreamFetcher(cmd *exec.Cmd) func(folder storage.Folder, backup Backup)

func GetDataFolderPath added in v0.2.10 

func GetDataFolderPath() string

func GetDeltaFilenameFor 

func GetDeltaFilenameFor(walFilename string) (string, error)

func GetDurationSetting added in v0.2.17 

func GetDurationSetting(setting string) (time.Duration, error)

func GetIncrementHeaderFields added in v0.2.17 

func GetIncrementHeaderFields(increment io.Reader) (uint64, uint32, []byte, error)

func GetLastDecompressor added in v0.2.10 

func GetLastDecompressor() (compression.Decompressor, error)

func GetLastWalFilename added in v0.2.18 

func GetLastWalFilename(backup *Backup) (string, error)

func GetLogsDstSettings added in v0.2.14 

func GetLogsDstSettings(operationLogsDstEnvVariable string) (dstFolder string, err error)

TODO : unit tests GetLogsDstSettings reads from the environment variables fetch settings

func GetMaxConcurrency added in v0.2.10 

func GetMaxConcurrency(concurrencyType string) (int, error)

func GetNextWalFilename 

func GetNextWalFilename(name string) (string, error)

GetNextWalFilename computes name of next WAL segment

func GetOplogArchiveAfterSize added in v0.2.15 

func GetOplogArchiveAfterSize() (int, error)

func GetOplogPITRDiscoveryIntervalSetting added in v0.2.17 

func GetOplogPITRDiscoveryIntervalSetting() (*time.Duration, error)

func GetPgFetcherNew added in v0.2.17 

func GetPgFetcherNew(dbDataDirectory, fileMask, restoreSpecPath string, skipRedundantTars bool) func(folder storage.Folder, backup Backup)

func GetPgFetcherOld added in v0.2.17 

func GetPgFetcherOld(dbDataDirectory, fileMask, restoreSpecPath string) func(folder storage.Folder, backup Backup)

func GetPgSlotName added in v0.2.18 

func GetPgSlotName() (pgSlotName string)

GetPgSlotName reads the slot name from the environment

func GetPositionInDelta 

func GetPositionInDelta(walFilename string) int

func GetRelFileIdFrom 

func GetRelFileIdFrom(filePath string) (int, error)

func GetRelFileNodeFrom 

func GetRelFileNodeFrom(filePath string) (*walparser.RelFileNode, error)

func GetRequiredSetting added in v0.2.15 

func GetRequiredSetting(setting string) (string, error)

func GetSentinelUserData 

func GetSentinelUserData() interface{}

func GetSetting added in v0.2.10 

func GetSetting(key string) (value string, ok bool)

GetSetting extract setting by key if key is set, return empty string otherwise

func GetStreamName added in v0.2.17 

func GetStreamName(backupName string, extension string) string

func HandleBackupFetch 

func HandleBackupFetch(folder storage.Folder, backupName string, fetcher func(folder storage.Folder, backup Backup))

TODO : unit tests HandleBackupFetch is invoked to perform wal-g backup-fetch

func HandleBackupList 

func HandleBackupList(
	getBackupsFunc func() ([]BackupTime, error),
	writeBackupListFunc func([]BackupTime),
	logging Logging,
)

func HandleBackupListWithFlags added in v0.2.10 

func HandleBackupListWithFlags(folder storage.Folder, pretty bool, json bool, detail bool)

TODO : unit tests

func HandleBackupMark added in v0.2.12 

func HandleBackupMark(uploader *Uploader, backupName string, toPermanent bool)

func HandleBackupPush 

func HandleBackupPush(uploader *WalUploader, archiveDirectory string, isPermanent, isFullBackup,
	verifyPageChecksums, storeAllCorruptBlocks bool, tarBallComposerType TarBallComposerType)

TODO : unit tests HandleBackupPush is invoked to perform a wal-g backup-push

func HandleCatchupFetch added in v0.2.15 

func HandleCatchupFetch(folder storage.Folder, dbDirectory, backupName string, useNewUnwrap bool)

HandleCatchupFetch is invoked to perform wal-g catchup-fetch

func HandleCatchupPush added in v0.2.15 

func HandleCatchupPush(uploader *WalUploader, archiveDirectory string, fromLSN uint64)

HandleCatchupPush is invoked to perform a wal-g catchup-push

func HandleDeletaRetainAfter added in v0.2.15 

func HandleDeletaRetainAfter(folder storage.Folder, args []string, confirmed bool,
	isFullBackup func(object storage.Object) bool,
	less func(object1, object2 storage.Object) bool)

func HandleDeleteBefore added in v0.2.10 

func HandleDeleteBefore(folder storage.Folder, args []string, confirmed bool,
	isFullBackup func(object storage.Object) bool,
	less func(object1, object2 storage.Object) bool)

func HandleDeleteRetain added in v0.2.8 

func HandleDeleteRetain(folder storage.Folder, args []string, confirmed bool,
	isFullBackup func(object storage.Object) bool,
	less func(object1, object2 storage.Object) bool)

func HandleWALFetch 

func HandleWALFetch(folder storage.Folder, walFileName string, location string, triggerPrefetch bool)

TODO : unit tests HandleWALFetch is invoked to performa wal-g wal-fetch

func HandleWALPrefetch 

func HandleWALPrefetch(uploader *WalUploader, walFileName string, location string)

TODO : unit tests HandleWALPrefetch is invoked by wal-fetch command to speed up database restoration

func HandleWALPush 

func HandleWALPush(uploader *WalUploader, walFilePath string)

TODO : unit tests HandleWALPush is invoked to perform wal-g wal-push

func HandleWALReceive added in v0.2.18 

func HandleWALReceive(uploader *WalUploader)

HandleWALReceive is invoked to receive wal with a replication connection and push

func HandleWalShow added in v0.2.18 

func HandleWalShow(rootFolder storage.Folder, showBackups bool, outputWriter WalShowOutputWriter)

HandleWalShow gets the list of files inside WAL folder, detects the available WAL segments, groups WAL segments by the timeline and shows detailed info about each timeline stored in storage

func HandleWalVerify added in v0.2.18 

func HandleWalVerify(rootFolder storage.Folder, startWalSegment WalSegmentDescription, outputWriter WalVerifyOutputWriter)

HandleWalVerify queries the current cluster WAL segment and timeline and travels through WAL segments in storage in reversed chronological order (starting from that segment) to find any missing WAL segments that could potentially fail the PITR procedure

func InitConfig added in v0.2.10 

func InitConfig()

InitConfig reads config file and ENV variables if set.

func IsPgControlRequired added in v0.2.4 

func IsPgControlRequired(backup *Backup, sentinelDto BackupSentinelDto) bool

func NewDiskLimitReader 

func NewDiskLimitReader(r io.Reader) io.Reader

NewDiskLimitReader returns a reader that is rate limited by disk limiter

func NewNetworkLimitReader 

func NewNetworkLimitReader(r io.Reader) io.Reader

NewNetworkLimitReader returns a reader that is rate limited by network limiter

func PackFileTo 

func PackFileTo(tarBall TarBall, fileInfoHeader *tar.Header, fileContent io.Reader) (fileSize int64, err error)

func ParseTS added in v0.2.14 

func ParseTS(endTSEnvVar string) (endTS *time.Time, err error)

func ParseWALFilename 

func ParseWALFilename(name string) (timelineID uint32, logSegNo uint64, err error)

ParseWALFilename extracts numeric parts from WAL file name

func PrepareDirs added in v0.2.15 

func PrepareDirs(fileName string, targetPath string) error

PrepareDirs makes sure all dirs exist

func ReadConfigFromFile added in v0.2.17 

func ReadConfigFromFile(config *viper.Viper, configFile string)

ReadConfigFromFile read config to the viper instance

func ReadIncrementFileHeader 

func ReadIncrementFileHeader(reader io.Reader) error

func ReadIncrementLocations added in v0.2.18 

func ReadIncrementLocations(filePath string, fileSize int64, lsn uint64) ([]walparser.BlockLocation, error)

func ReadIncrementalFile 

func ReadIncrementalFile(filePath string, fileSize int64, lsn uint64, deltaBitmap *roaring.Bitmap) (fileReader io.ReadCloser, size int64, err error)

func ReadLocationsFrom 

func ReadLocationsFrom(reader io.Reader) ([]walparser.BlockLocation, error)

func RestoreMissingPages added in v0.2.17 

func RestoreMissingPages(base io.Reader, target ReadWriterAt) error

RestoreMissingPages restores missing pages (zero blocks) of local file with their base backup version

func SelectRelFileBlocks 

func SelectRelFileBlocks(bitmap *roaring.Bitmap, relFileId int) *roaring.Bitmap

func SentinelNameFromBackup added in v0.2.17 

func SentinelNameFromBackup(backupName string) string

func SetDefaultValues added in v0.2.17 

func SetDefaultValues(config *viper.Viper)

SetDefaultValues set default settings to the viper instance

func SetLastDecompressor added in v0.2.10 

func SetLastDecompressor(decompressor compression.Decompressor) error

func StreamBackupToCommandStdin added in v0.2.17 

func StreamBackupToCommandStdin(cmd *exec.Cmd, backup *Backup) error

StreamBackupToCommandStdin downloads and decompresses backup stream to cmd stdin.

func ToPartFilename 

func ToPartFilename(deltaFilename string) string

func TryDownloadFile added in v0.2.15 

func TryDownloadFile(folder storage.Folder, path string) (walFileReader io.ReadCloser, exists bool, err error)

func UploadSentinel added in v0.2.7 

func UploadSentinel(uploader UploaderProvider, sentinelDto interface{}, backupName string) error

TODO : unit tests

func ValidateSlotName added in v0.2.18 

func ValidateSlotName(pgSlotName string) (err error)

ValidateSlotName validates pgSlotName to be a valid slot name

func VerifyPagedFileBase added in v0.2.18 

func VerifyPagedFileBase(path string, fileInfo os.FileInfo, pagedFile io.Reader) ([]uint32, error)

VerifyPagedFileBase verifies pages of a standard paged file

func VerifyPagedFileIncrement added in v0.2.18 

func VerifyPagedFileIncrement(path string, fileInfo os.FileInfo, increment io.Reader) ([]uint32, error)

VerifyPagedFileIncrement verifies pages of an increment

func WriteAsJson added in v0.2.12 

func WriteAsJson(data interface{}, output io.Writer, pretty bool) error

func WriteBackupList added in v0.2.10 

func WriteBackupList(backups []BackupTime, output io.Writer)

TODO : unit tests

func WriteLocationsTo 

func WriteLocationsTo(writer io.Writer, locations []walparser.BlockLocation) error

func WritePagesFromIncrement added in v0.2.17 

func WritePagesFromIncrement(increment io.Reader, target ReadWriterAt, overwriteExisting bool) (int64, error)

WritePagesFromIncrement writes pages from delta backup according to diffMap

func WritePrettyBackupList added in v0.2.10 

func WritePrettyBackupList(backups []BackupTime, output io.Writer)

Types

type ArchiveNonExistenceError 

type ArchiveNonExistenceError struct {
	// contains filtered or unexported fields
}

func (ArchiveNonExistenceError) Error 

func (err ArchiveNonExistenceError) Error() string

type ArchiveStatusManager added in v0.2.14 

type ArchiveStatusManager interface {
	// contains filtered or unexported methods
}

func NewNopASM added in v0.2.14 

func NewNopASM() ArchiveStatusManager

type Backup 

type Backup struct {
	BaseBackupFolder storage.Folder
	Name             string
	SentinelDto      *BackupSentinelDto // used for storage query caching
}

Backup contains information about a valid backup generated and uploaded by WAL-G.

func GetBackupByName 

func GetBackupByName(backupName, subfolder string, folder storage.Folder) (*Backup, error)

func NewBackup 

func NewBackup(baseBackupFolder storage.Folder, name string) *Backup

func (*Backup) CheckExistence 

func (backup *Backup) CheckExistence() (bool, error)

CheckExistence checks that the specified backup exists.

func (*Backup) GetFilesToUnwrap added in v0.2.10 

func (backup *Backup) GetFilesToUnwrap(fileMask string) (map[string]bool, error)

func (*Backup) GetSentinel added in v0.2.10 

func (backup *Backup) GetSentinel() (BackupSentinelDto, error)

func (*Backup) GetStopSentinelPath added in v0.2.7 

func (backup *Backup) GetStopSentinelPath() string

GetStopSentinelPath returns sentinel path.

func (*Backup) GetTarNames 

func (backup *Backup) GetTarNames() ([]string, error)

type BackupAlreadyThisTypePermanentError added in v0.2.12 

type BackupAlreadyThisTypePermanentError struct {
	// contains filtered or unexported fields
}

type BackupDetail added in v0.2.10 

type BackupDetail struct {
	BackupTime
	ExtendedMetadataDto
}

BackupDetails is used to append ExtendedMetadataDto details to BackupTime struct

type BackupFileDescription 

type BackupFileDescription struct {
	IsIncremented bool // should never be both incremented and Skipped
	IsSkipped     bool
	MTime         time.Time
	CorruptBlocks *CorruptBlocksInfo `json:",omitempty"`
	UpdatesCount  uint64
}

func NewBackupFileDescription 

func NewBackupFileDescription(isIncremented, isSkipped bool, modTime time.Time) *BackupFileDescription

func (*BackupFileDescription) SetCorruptBlocks added in v0.2.18 

func (desc *BackupFileDescription) SetCorruptBlocks(corruptBlockNumbers []uint32, storeAllBlocks bool)

type BackupFileList 

type BackupFileList map[string]BackupFileDescription

type BackupFileOptions added in v0.2.17 

type BackupFileOptions struct {
	// contains filtered or unexported fields
}

type BackupFileUnwrapper added in v0.2.17 

type BackupFileUnwrapper struct {
	// contains filtered or unexported fields
}

type BackupFromFuture added in v0.2.13 

type BackupFromFuture struct {
	// contains filtered or unexported fields
}

func (BackupFromFuture) Error added in v0.2.13 

func (err BackupFromFuture) Error() string

type BackupFromOtherBD added in v0.2.13 

type BackupFromOtherBD struct {
	// contains filtered or unexported fields
}

func (BackupFromOtherBD) Error added in v0.2.13 

func (err BackupFromOtherBD) Error() string

type BackupHasPermanentBackupInFutureError added in v0.2.12 

type BackupHasPermanentBackupInFutureError struct {
	// contains filtered or unexported fields
}

type BackupNonExistenceError 

type BackupNonExistenceError struct {
	// contains filtered or unexported fields
}

func NewBackupNonExistenceError 

func NewBackupNonExistenceError(backupName string) BackupNonExistenceError

func (BackupNonExistenceError) Error 

func (err BackupNonExistenceError) Error() string

type BackupSentinelDto 

type BackupSentinelDto struct {
	BackupStartLSN    *uint64 `json:"LSN"`
	IncrementFromLSN  *uint64 `json:"DeltaFromLSN,omitempty"`
	IncrementFrom     *string `json:"DeltaFrom,omitempty"`
	IncrementFullName *string `json:"DeltaFullName,omitempty"`
	IncrementCount    *int    `json:"DeltaCount,omitempty"`

	Files       BackupFileList `json:"Files"`
	TarFileSets TarFileSets    `json:"TarFileSets"`

	PgVersion        int     `json:"PgVersion"`
	BackupFinishLSN  *uint64 `json:"FinishLSN"`
	SystemIdentifier *uint64 `json:"SystemIdentifier,omitempty"`

	UncompressedSize int64           `json:"UncompressedSize"`
	CompressedSize   int64           `json:"CompressedSize"`
	TablespaceSpec   *TablespaceSpec `json:"Spec"`

	UserData interface{} `json:"UserData,omitempty"`
}

BackupSentinelDto describes file structure of json sentinel

func (*BackupSentinelDto) IsIncremental added in v0.2.10 

func (dto *BackupSentinelDto) IsIncremental() bool

TODO : unit tests TODO : get rid of panic here IsIncremental checks that sentinel represents delta backup

type BackupTime 

type BackupTime struct {
	BackupName  string    `json:"backup_name"`
	Time        time.Time `json:"time"`
	WalFileName string    `json:"wal_file_name"`
}

BackupTime is used to sort backups by latest modified time.

func GetBackupsAndGarbage added in v0.2.17 

func GetBackupsAndGarbage(folder storage.Folder) (backups []BackupTime, garbage []string, err error)

TODO : unit tests

type BgUploader 

type BgUploader struct {
	// contains filtered or unexported fields
}

BgUploader represents the state of concurrent WAL upload

func NewBgUploader 

func NewBgUploader(walFilePath string, maxParallelWorkers int32, maxNumUploaded int32, uploader *WalUploader, preventWalOverwrite bool) *BgUploader

NewBgUploader creates a new BgUploader which looks for WAL files adjacent to walFilePath. maxParallelWorkers and maxNumUploaded limits maximum concurrency and total work done by this BgUploader respectively.

func (*BgUploader) Start 

func (b *BgUploader) Start()

Start up checking what's inside archive_status

func (*BgUploader) Stop 

func (b *BgUploader) Stop()

Stop pipeline. Stop can be safely called concurrently and repeatedly.

type BlockLocationReader 

type BlockLocationReader struct {
	// contains filtered or unexported fields
}

func NewBlockLocationReader 

func NewBlockLocationReader(underlying io.Reader) *BlockLocationReader

func (*BlockLocationReader) ReadNextLocation 

func (reader *BlockLocationReader) ReadNextLocation() (*walparser.BlockLocation, error)

ReadNextLocation returns any reader error wrapped with errors.Wrap

type BlockLocationWriter 

type BlockLocationWriter struct {
	Underlying io.Writer
}

func NewBlockLocationWriter 

func NewBlockLocationWriter(underlying io.Writer) *BlockLocationWriter

func (*BlockLocationWriter) WriteLocation 

func (locationWriter *BlockLocationWriter) WriteLocation(location walparser.BlockLocation) error

type Bundle 

type Bundle struct {
	ArchiveDirectory string
	Sentinel         *Sentinel

	TarBallComposer TarBallComposer
	TarBallQueue    *TarBallQueue

	Crypter            crypto.Crypter
	Timeline           uint32
	Replica            bool
	IncrementFromLsn   *uint64
	IncrementFromFiles BackupFileList
	DeltaMap           PagedFileDeltaMap
	TablespaceSpec     TablespaceSpec

	TarSizeThreshold int64
	// contains filtered or unexported fields
}

A Bundle represents the directory to be walked. Contains at least one TarBall if walk has started. Each TarBall except for the last one will be at least TarSizeThreshold bytes. The Sentinel is used to ensure complete uploaded backups; in this case, pg_control is used as the sentinel.

func NewBundle 

func NewBundle(
	archiveDirectory string, crypter crypto.Crypter,
	incrementFromLsn *uint64, incrementFromFiles BackupFileList,
	forceIncremental bool, tarSizeThreshold int64,
) *Bundle

TODO: use DiskDataFolder

func (*Bundle) DownloadDeltaMap 

func (bundle *Bundle) DownloadDeltaMap(folder storage.Folder, backupStartLSN uint64) error

func (*Bundle) FinishQueue 

func (bundle *Bundle) FinishQueue() error

func (*Bundle) GetFiles 

func (bundle *Bundle) GetFiles() *sync.Map

func (*Bundle) HandleWalkedFSObject 

func (bundle *Bundle) HandleWalkedFSObject(path string, info os.FileInfo, err error) error

TODO : unit tests HandleWalkedFSObject walks files provided by the passed in directory and creates compressed tar members labeled as `part_00i.tar.*`, where '*' is compressor file extension.

To see which files and directories are Skipped, please consult ExcludedFilenames. Excluded directories will be created but their contents will not be included in the tar bundle.

func (*Bundle) NewTarBall 

func (bundle *Bundle) NewTarBall(dedicatedUploader bool) TarBall

NewTarBall starts writing new tarball

func (*Bundle) PackTarballs added in v0.2.18 

func (bundle *Bundle) PackTarballs() (TarFileSets, error)

func (*Bundle) SetupComposer added in v0.2.18 

func (bundle *Bundle) SetupComposer(composerMaker TarBallComposerMaker) (err error)

func (*Bundle) StartBackup 

func (bundle *Bundle) StartBackup(conn *pgx.Conn, backup string) (backupName string, lsn uint64, version int, dataDir string, systemIdentifier *uint64, err error)

TODO : unit tests StartBackup starts a non-exclusive base backup immediately. When finishing the backup, `backup_label` and `tablespace_map` contents are not immediately written to a file but returned instead. Returns empty string and an error if backup fails.

func (*Bundle) StartQueue 

func (bundle *Bundle) StartQueue(tarBallMaker TarBallMaker) error

func (*Bundle) UploadPgControl 

func (bundle *Bundle) UploadPgControl(compressorFileExtension string) error

TODO : unit tests UploadPgControl should only be called after the rest of the backup is successfully uploaded to S3.

type BundleFiles added in v0.2.18 

type BundleFiles interface {
	AddSkippedFile(tarHeader *tar.Header, fileInfo os.FileInfo)
	AddFile(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool)
	AddFileWithCorruptBlocks(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool,
		corruptedBlocks []uint32, storeAllBlocks bool)
	GetUnderlyingMap() *sync.Map
}

BundleFiles represents the files in the backup that is going to be created

type BytesPerWalSegmentError 

type BytesPerWalSegmentError struct {
	// contains filtered or unexported fields
}

func (BytesPerWalSegmentError) Error 

func (err BytesPerWalSegmentError) Error() string

type CachedDecompressor added in v0.2.10 

type CachedDecompressor struct {
	FileExtension string
}

CachedDecompressor is the file extension describing decompressor

type CantDiscardWalDataError 

type CantDiscardWalDataError struct {
	// contains filtered or unexported fields
}

func (CantDiscardWalDataError) Error 

func (err CantDiscardWalDataError) Error() string

type CantOverwriteWalFileError 

type CantOverwriteWalFileError struct {
	// contains filtered or unexported fields
}

func (CantOverwriteWalFileError) Error 

func (err CantOverwriteWalFileError) Error() string

type CascadeWriteCloser 

type CascadeWriteCloser struct {
	io.WriteCloser
	Underlying io.Closer
}

CascadeWriteCloser bundles multiple closures into one function. Calling Close() will close the main and underlying writers.

func (*CascadeWriteCloser) Close 

func (cascadeCloser *CascadeWriteCloser) Close() error

Close returns the first encountered error from closing main or underlying writer.

type CatchupFileUnwrapper added in v0.2.17 

type CatchupFileUnwrapper struct {
	BackupFileUnwrapper
}

CatchupFileUnwrapper is used for catchup (catchup-push) backups

func (*CatchupFileUnwrapper) UnwrapExistingFile added in v0.2.17 

func (u *CatchupFileUnwrapper) UnwrapExistingFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

func (*CatchupFileUnwrapper) UnwrapNewFile added in v0.2.17 

func (u *CatchupFileUnwrapper) UnwrapNewFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

type Cleaner 

type Cleaner interface {
	GetFiles(directory string) ([]string, error)
	Remove(file string)
}

Cleaner interface serves to separate file system logic from prefetch clean logic to make it testable

type ComposeFileInfo added in v0.2.18 

type ComposeFileInfo struct {
	// contains filtered or unexported fields
}

ComposeFileInfo holds data which is required to pack a file to some tarball

func NewComposeFileInfo added in v0.2.18 

func NewComposeFileInfo(path string, fileInfo os.FileInfo, wasInBase, isIncremented bool,
	header *tar.Header) *ComposeFileInfo

type ComposeRatingEvaluator added in v0.2.18 

type ComposeRatingEvaluator interface {
	Evaluate(path string, updatesCount uint64, wasInBase bool) uint64
}

type CompressAndEncryptError added in v0.2.10 

type CompressAndEncryptError struct {
	// contains filtered or unexported fields
}

CompressAndEncryptError is used to catch specific errors from CompressAndEncrypt when uploading to Storage. Will not retry upload if this error occurs.

func (CompressAndEncryptError) Error added in v0.2.10 

func (err CompressAndEncryptError) Error() string

type CorruptBlocksInfo added in v0.2.18 

type CorruptBlocksInfo struct {
	CorruptBlocksCount int
	SomeCorruptBlocks  []uint32
}

type DataFolder 

type DataFolder interface {
	// OpenReadonlyFile should return NoSuchFileError if it cannot find desired file
	OpenReadonlyFile(filename string) (io.ReadCloser, error)
	OpenWriteOnlyFile(filename string) (io.WriteCloser, error)
	CleanFolder() error
	FileExists(filename string) bool
	DeleteFile(filename string) error
	CreateFile(filename string) error
}

func ConfigureArchiveStatusManager added in v0.2.14 

func ConfigureArchiveStatusManager() (DataFolder, error)

TODO : unit tests

type DataFolderASM added in v0.2.14 

type DataFolderASM struct {
	// contains filtered or unexported fields
}

func NewDataFolderASM added in v0.2.14 

func NewDataFolderASM(folder DataFolder) DataFolderASM

type DecompressionError added in v0.2.15 

type DecompressionError struct {
	// contains filtered or unexported fields
}

type DefaultComposeRatingEvaluator added in v0.2.18 

type DefaultComposeRatingEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultComposeRatingEvaluator added in v0.2.18 

func NewDefaultComposeRatingEvaluator(incrementFromFiles BackupFileList) *DefaultComposeRatingEvaluator

func (*DefaultComposeRatingEvaluator) Evaluate added in v0.2.18 

func (evaluator *DefaultComposeRatingEvaluator) Evaluate(path string, updatesCount uint64, wasInBase bool) uint64

type DefaultFileUnwrapper added in v0.2.17 

type DefaultFileUnwrapper struct {
	BackupFileUnwrapper
}

DefaultFileUnwrapper is used for default (backup-push) backups

func (*DefaultFileUnwrapper) UnwrapExistingFile added in v0.2.17 

func (u *DefaultFileUnwrapper) UnwrapExistingFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

func (*DefaultFileUnwrapper) UnwrapNewFile added in v0.2.17 

func (u *DefaultFileUnwrapper) UnwrapNewFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

type DeltaFile 

type DeltaFile struct {
	Locations []walparser.BlockLocation
	WalParser *walparser.WalParser
}

func LoadDeltaFile 

func LoadDeltaFile(reader io.Reader) (*DeltaFile, error)

func NewDeltaFile 

func NewDeltaFile(walParser *walparser.WalParser) (*DeltaFile, error)

func (*DeltaFile) Save 

func (deltaFile *DeltaFile) Save(writer io.Writer) error

type DeltaFileChanWriter 

type DeltaFileChanWriter struct {
	DeltaFile             *DeltaFile
	BlockLocationConsumer chan walparser.BlockLocation
}

func NewDeltaFileChanWriter 

func NewDeltaFileChanWriter(deltaFile *DeltaFile) *DeltaFileChanWriter

func (*DeltaFileChanWriter) Consume 

func (writer *DeltaFileChanWriter) Consume(waitGroup *sync.WaitGroup)

type DeltaFileManager 

type DeltaFileManager struct {
	PartFiles        *LazyCache
	DeltaFileWriters *LazyCache

	CanceledDeltaFiles map[string]bool
	// contains filtered or unexported fields
}

func NewDeltaFileManager 

func NewDeltaFileManager(dataFolder DataFolder) *DeltaFileManager

func (*DeltaFileManager) CancelRecording 

func (manager *DeltaFileManager) CancelRecording(walFilename string)

func (*DeltaFileManager) CombinePartFile 

func (manager *DeltaFileManager) CombinePartFile(deltaFilename string, partFile *WalPartFile) error

func (*DeltaFileManager) FlushDeltaFiles 

func (manager *DeltaFileManager) FlushDeltaFiles(uploader *Uploader, completedPartFiles map[string]bool)

func (*DeltaFileManager) FlushFiles 

func (manager *DeltaFileManager) FlushFiles(uploader *Uploader)

func (*DeltaFileManager) FlushPartFiles 

func (manager *DeltaFileManager) FlushPartFiles() (completedPartFiles map[string]bool)

func (*DeltaFileManager) GetBlockLocationConsumer 

func (manager *DeltaFileManager) GetBlockLocationConsumer(deltaFilename string) (chan walparser.BlockLocation, error)

func (*DeltaFileManager) GetPartFile 

func (manager *DeltaFileManager) GetPartFile(deltaFilename string) (*WalPartFile, error)

type DeltaFileWriterNotFoundError 

type DeltaFileWriterNotFoundError struct {
	// contains filtered or unexported fields
}

func (DeltaFileWriterNotFoundError) Error 

func (err DeltaFileWriterNotFoundError) Error() string

type DeltaNo added in v0.2.12 

type DeltaNo uint64

type DiskDataFolder 

type DiskDataFolder struct {
	// contains filtered or unexported fields
}

func (*DiskDataFolder) CleanFolder 

func (folder *DiskDataFolder) CleanFolder() error

func (*DiskDataFolder) CreateFile added in v0.2.10 

func (folder *DiskDataFolder) CreateFile(filename string) error

func (*DiskDataFolder) DeleteFile added in v0.2.10 

func (folder *DiskDataFolder) DeleteFile(filename string) error

func (*DiskDataFolder) FileExists added in v0.2.10 

func (folder *DiskDataFolder) FileExists(filename string) bool

func (*DiskDataFolder) OpenReadonlyFile 

func (folder *DiskDataFolder) OpenReadonlyFile(filename string) (io.ReadCloser, error)

func (*DiskDataFolder) OpenWriteOnlyFile 

func (folder *DiskDataFolder) OpenWriteOnlyFile(filename string) (io.WriteCloser, error)

type EmptyWriteIgnorer 

type EmptyWriteIgnorer struct {
	io.WriteCloser
}

EmptyWriteIgnorer handles 0 byte write in LZ4 package to stop pipe reader/writer from blocking.

func (EmptyWriteIgnorer) Write 

func (e EmptyWriteIgnorer) Write(p []byte) (int, error)

type ErrorLogger added in v0.2.15 

type ErrorLogger interface {
	FatalOnError(err error)
}

type ExponentialRetrier 

type ExponentialRetrier struct {
	// contains filtered or unexported fields
}

type ExtendedMetadataDto added in v0.2.10 

type ExtendedMetadataDto struct {
	StartTime        time.Time `json:"start_time"`
	FinishTime       time.Time `json:"finish_time"`
	DatetimeFormat   string    `json:"date_fmt"`
	Hostname         string    `json:"hostname"`
	DataDir          string    `json:"data_dir"`
	PgVersion        int       `json:"pg_version"`
	StartLsn         uint64    `json:"start_lsn"`
	FinishLsn        uint64    `json:"finish_lsn"`
	IsPermanent      bool      `json:"is_permanent"`
	SystemIdentifier *uint64   `json:"system_identifier"`

	UncompressedSize int64 `json:"uncompressed_size"`
	CompressedSize   int64 `json:"compressed_size"`

	UserData interface{} `json:"user_data,omitempty"`
}

Extended metadata should describe backup in more details, but be small enough to be downloaded often

type Extension added in v0.2.7 

type Extension interface {
	RegisterCommands(cmd *cobra.Command)
	GetAllowedConfigKeys() map[string]*string
}

type FakeASM added in v0.2.16 

type FakeASM struct {
	// contains filtered or unexported fields
}

func NewFakeASM added in v0.2.16 

func NewFakeASM() *FakeASM

func (*FakeASM) IsWalAlreadyUploaded added in v0.2.16 

func (asm *FakeASM) IsWalAlreadyUploaded(walFilePath string) bool

IsWalAlreadyUploaded is used for testing

type FetchConfig added in v0.2.17 

type FetchConfig struct {
	// contains filtered or unexported fields
}

func NewFetchConfig added in v0.2.17 

func NewFetchConfig(backupName, dbDataDirectory string, folder storage.Folder, spec *TablespaceSpec,
	filesToUnwrap map[string]bool, skipRedundantTars bool) *FetchConfig

func (*FetchConfig) SkipRedundantFiles added in v0.2.17 

func (fc *FetchConfig) SkipRedundantFiles(unwrapResult *UnwrapResult)

type FileSystemCleaner 

type FileSystemCleaner struct{}

FileSystemCleaner actually performs it's functions on file system

func (FileSystemCleaner) GetFiles 

func (cleaner FileSystemCleaner) GetFiles(directory string) (files []string, err error)

TODO : unit tests GetFiles of a directory

func (FileSystemCleaner) Remove 

func (cleaner FileSystemCleaner) Remove(file string)

Remove file

type FileTarInterpreter 

type FileTarInterpreter struct {
	DBDataDirectory string
	Sentinel        BackupSentinelDto
	FilesToUnwrap   map[string]bool
	UnwrapResult    *UnwrapResult
	// contains filtered or unexported fields
}

FileTarInterpreter extracts input to disk.

func NewFileTarInterpreter 

func NewFileTarInterpreter(
	dbDataDirectory string, sentinel BackupSentinelDto, filesToUnwrap map[string]bool, createNewIncrementalFiles bool,
) *FileTarInterpreter

func (*FileTarInterpreter) AddFileUnwrapResult added in v0.2.17 

func (tarInterpreter *FileTarInterpreter) AddFileUnwrapResult(result *FileUnwrapResult, fileName string)

func (*FileTarInterpreter) Interpret 

func (tarInterpreter *FileTarInterpreter) Interpret(fileReader io.Reader, fileInfo *tar.Header) error

Interpret extracts a tar file to disk and creates needed directories. Returns the first error encountered. Calls fsync after each file is written successfully.

type FileUnwrapResult added in v0.2.17 

type FileUnwrapResult struct {
	FileUnwrapResultType
	// contains filtered or unexported fields
}

func NewCompletedResult added in v0.2.17 

func NewCompletedResult() *FileUnwrapResult

func NewCreatedFromIncrementResult added in v0.2.17 

func NewCreatedFromIncrementResult(missingBlockCount int64) *FileUnwrapResult

func NewSkippedResult added in v0.2.17 

func NewSkippedResult() *FileUnwrapResult

func NewWroteIncrementBlocksResult added in v0.2.17 

func NewWroteIncrementBlocksResult(restoredBlockCount int64) *FileUnwrapResult

type FileUnwrapResultType added in v0.2.17 

type FileUnwrapResultType int
const (
	Completed FileUnwrapResultType = iota + 1
	CreatedFromIncrement
	WroteIncrementBlocks
	Skipped
)

type FileUnwrapperType added in v0.2.17 

type FileUnwrapperType int
const (
	DefaultBackupFileUnwrapper FileUnwrapperType = iota + 1
	CatchupBackupFileUnwrapper
)

type HistoryFileNotFoundError added in v0.2.18 

type HistoryFileNotFoundError struct {
	// contains filtered or unexported fields
}

func (HistoryFileNotFoundError) Error added in v0.2.18 

func (err HistoryFileNotFoundError) Error() string

type IBackupFileUnwrapper added in v0.2.17 

type IBackupFileUnwrapper interface {
	UnwrapNewFile(reader io.Reader, header *tar.Header, file *os.File) (*FileUnwrapResult, error)
	UnwrapExistingFile(reader io.Reader, header *tar.Header, file *os.File) (*FileUnwrapResult, error)
}

func NewFileUnwrapper added in v0.2.17 

func NewFileUnwrapper(unwrapperType FileUnwrapperType, options *BackupFileOptions) IBackupFileUnwrapper

type IgnoredFileError added in v0.2.18 

type IgnoredFileError struct {
	// contains filtered or unexported fields
}

func (IgnoredFileError) Error added in v0.2.18 

func (err IgnoredFileError) Error() string

type IncorrectLogSegNoError 

type IncorrectLogSegNoError struct {
	// contains filtered or unexported fields
}

func (IncorrectLogSegNoError) Error 

func (err IncorrectLogSegNoError) Error() string

type IncrementalPageReader 

type IncrementalPageReader struct {
	PagedFile ioextensions.ReadSeekCloser
	FileSize  int64
	Lsn       uint64
	Next      []byte
	Blocks    []uint32
}

IncrementalPageReader constructs difference map during initialization and than re-read file Diff map may consist of 1Gb/PostgresBlockSize elements == 512Kb

func (*IncrementalPageReader) AdvanceFileReader 

func (pageReader *IncrementalPageReader) AdvanceFileReader() error

func (*IncrementalPageReader) Close 

func (pageReader *IncrementalPageReader) Close() error

Close IncrementalPageReader

func (*IncrementalPageReader) DeltaBitmapInitialize 

func (pageReader *IncrementalPageReader) DeltaBitmapInitialize(deltaBitmap *roaring.Bitmap)

func (*IncrementalPageReader) DrainMoreData 

func (pageReader *IncrementalPageReader) DrainMoreData() (succeed bool, err error)

func (*IncrementalPageReader) FullScanInitialize 

func (pageReader *IncrementalPageReader) FullScanInitialize() error

func (*IncrementalPageReader) Read 

func (pageReader *IncrementalPageReader) Read(p []byte) (n int, err error)

func (*IncrementalPageReader) SelectNewValidPage 

func (pageReader *IncrementalPageReader) SelectNewValidPage(pageBytes []byte, blockNo uint32) (valid bool)

SelectNewValidPage checks whether page is valid and if it so, then blockNo is appended to Blocks list

func (*IncrementalPageReader) WriteDiffMapToHeader 

func (pageReader *IncrementalPageReader) WriteDiffMapToHeader(headerWriter io.Writer)

WriteDiffMapToHeader is currently used only with buffers, so we don't handle any writing errors

type InfoLogger added in v0.2.15 

type InfoLogger interface {
	Println(v ...interface{})
}

type InvalidBlockError 

type InvalidBlockError struct {
	// contains filtered or unexported fields
}

InvalidBlockError indicates that file contain invalid page and cannot be archived incrementally

func (InvalidBlockError) Error 

func (err InvalidBlockError) Error() string

type InvalidConcurrencyValueError added in v0.2.10 

type InvalidConcurrencyValueError struct {
	// contains filtered or unexported fields
}

func (InvalidConcurrencyValueError) Error added in v0.2.10 

func (err InvalidConcurrencyValueError) Error() string

type InvalidIncrementFileHeaderError 

type InvalidIncrementFileHeaderError struct {
	// contains filtered or unexported fields
}

func (InvalidIncrementFileHeaderError) Error 

func (err InvalidIncrementFileHeaderError) Error() string

type InvalidWalFileMagicError 

type InvalidWalFileMagicError struct {
	// contains filtered or unexported fields
}

func (InvalidWalFileMagicError) Error 

func (err InvalidWalFileMagicError) Error() string

type LazyCache 

type LazyCache struct {
	// contains filtered or unexported fields
}

func (*LazyCache) Load 

func (lazyCache *LazyCache) Load(key interface{}) (value interface{}, exists bool, err error)

func (*LazyCache) LoadExisting 

func (lazyCache *LazyCache) LoadExisting(key interface{}) (value interface{}, exists bool)

func (*LazyCache) Range 

func (lazyCache *LazyCache) Range(reduce func(key, value interface{}) bool)

Range calls reduce sequentially for each key and value present in the cache. If reduce returns false, range stops the iteration.

func (*LazyCache) Store 

func (lazyCache *LazyCache) Store(key, value interface{})

type Logging added in v0.2.15 

type Logging struct {
	InfoLogger  InfoLogger
	ErrorLogger ErrorLogger
}

type NOPTarBall 

type NOPTarBall struct {
	// contains filtered or unexported fields
}

NOPTarBall mocks a tarball. Used for prefault logic.

func (*NOPTarBall) AddSize 

func (tarBall *NOPTarBall) AddSize(i int64)

func (*NOPTarBall) AwaitUploads 

func (tarBall *NOPTarBall) AwaitUploads()

func (*NOPTarBall) CloseTar 

func (tarBall *NOPTarBall) CloseTar() error

func (*NOPTarBall) Name added in v0.2.18 

func (tarBall *NOPTarBall) Name() string

func (*NOPTarBall) SetUp 

func (tarBall *NOPTarBall) SetUp(crypter crypto.Crypter, params ...string)

func (*NOPTarBall) Size 

func (tarBall *NOPTarBall) Size() int64

func (*NOPTarBall) TarWriter 

func (tarBall *NOPTarBall) TarWriter() *tar.Writer

type NOPTarBallMaker 

type NOPTarBallMaker struct {
	// contains filtered or unexported fields
}

NOPTarBallMaker creates a new NOPTarBall. Used for testing purposes.

func (*NOPTarBallMaker) Make 

func (tarBallMaker *NOPTarBallMaker) Make(inheritState bool) TarBall

Make creates a new NOPTarBall.

type NamedReader 

type NamedReader interface {
	io.Reader
	Name() string
}

type NamedReaderImpl 

type NamedReaderImpl struct {
	io.Reader
	// contains filtered or unexported fields
}

func (*NamedReaderImpl) Name 

func (reader *NamedReaderImpl) Name() string

type NilWalParserError 

type NilWalParserError struct {
	// contains filtered or unexported fields
}

func (NilWalParserError) Error 

func (err NilWalParserError) Error() string

type NoBackupsFoundError 

type NoBackupsFoundError struct {
	// contains filtered or unexported fields
}

func NewNoBackupsFoundError 

func NewNoBackupsFoundError() NoBackupsFoundError

func (NoBackupsFoundError) Error 

func (err NoBackupsFoundError) Error() string

type NoBitmapFoundError 

type NoBitmapFoundError struct {
	// contains filtered or unexported fields
}

func (NoBitmapFoundError) Error 

func (err NoBitmapFoundError) Error() string

type NoFilesToExtractError 

type NoFilesToExtractError struct {
	// contains filtered or unexported fields
}

func (NoFilesToExtractError) Error 

func (err NoFilesToExtractError) Error() string

type NoPostgresVersionError 

type NoPostgresVersionError struct {
	// contains filtered or unexported fields
}

func (NoPostgresVersionError) Error 

func (err NoPostgresVersionError) Error() string

type NoSuchFileError 

type NoSuchFileError struct {
	// contains filtered or unexported fields
}

func NewNoSuchFileError 

func NewNoSuchFileError(filename string) NoSuchFileError

func (NoSuchFileError) Error 

func (err NoSuchFileError) Error() string

type NonEmptyDbDataDirectoryError 

type NonEmptyDbDataDirectoryError struct {
	// contains filtered or unexported fields
}

func (NonEmptyDbDataDirectoryError) Error 

func (err NonEmptyDbDataDirectoryError) Error() string

type NopASM added in v0.2.14 

type NopASM struct {
	// contains filtered or unexported fields
}

type NotWalFilenameError 

type NotWalFilenameError struct {
	// contains filtered or unexported fields
}

func (NotWalFilenameError) Error 

func (err NotWalFilenameError) Error() string

type PagedFileDeltaMap 

type PagedFileDeltaMap map[walparser.RelFileNode]*roaring.Bitmap

func NewPagedFileDeltaMap 

func NewPagedFileDeltaMap() PagedFileDeltaMap

func (*PagedFileDeltaMap) AddLocationToDelta added in v0.2.12 

func (deltaMap *PagedFileDeltaMap) AddLocationToDelta(location walparser.BlockLocation)

func (*PagedFileDeltaMap) AddLocationsToDelta added in v0.2.12 

func (deltaMap *PagedFileDeltaMap) AddLocationsToDelta(locations []walparser.BlockLocation)

func (*PagedFileDeltaMap) GetDeltaBitmapFor 

func (deltaMap *PagedFileDeltaMap) GetDeltaBitmapFor(filePath string) (*roaring.Bitmap, error)

TODO : unit test no bitmap found

type PgChecksummablePage added in v0.2.18 

type PgChecksummablePage [DatabasePageSize / int64(NSums*sizeofInt32)][NSums]uint32

PgChecksummablePage represents single database page divided by NSums blocks for checksum calculation

type PgControlNotFoundError 

type PgControlNotFoundError struct {
	// contains filtered or unexported fields
}

func (PgControlNotFoundError) Error 

func (err PgControlNotFoundError) Error() string

type PgDatabaseInfo added in v0.2.18 

type PgDatabaseInfo struct {
	// contains filtered or unexported fields
}

type PgDatabasePage added in v0.2.18 

type PgDatabasePage [DatabasePageSize]byte

PgDatabasePage represents single database page

type PgQueryRunner 

type PgQueryRunner struct {
	Version          int
	SystemIdentifier *uint64
	// contains filtered or unexported fields
}

PgQueryRunner is implementation for controlling PostgreSQL 9.0+

func (*PgQueryRunner) BuildGetDatabasesQuery added in v0.2.18 

func (queryRunner *PgQueryRunner) BuildGetDatabasesQuery() (string, error)

BuildGetDatabasesQuery formats a query to get all databases in cluster which are allowed to connect

func (*PgQueryRunner) BuildStartBackup 

func (queryRunner *PgQueryRunner) BuildStartBackup() (string, error)

BuildStartBackup formats a query that starts backup according to server features and version

func (*PgQueryRunner) BuildStatisticsQuery added in v0.2.18 

func (queryRunner *PgQueryRunner) BuildStatisticsQuery() (string, error)

BuildStatisticsQuery formats a query that fetch relations statistics from database

func (*PgQueryRunner) BuildStopBackup 

func (queryRunner *PgQueryRunner) BuildStopBackup() (string, error)

BuildStopBackup formats a query that stops backup according to server features and version

func (*PgQueryRunner) GetParameter added in v0.2.18 

func (queryRunner *PgQueryRunner) GetParameter(parameterName string) (string, error)

GetParameter reads a Postgres setting TODO: Unittest

func (*PgQueryRunner) GetPhysicalSlotInfo added in v0.2.18 

func (queryRunner *PgQueryRunner) GetPhysicalSlotInfo(slotName string) (PhysicalSlot, error)

GetPhysicalSlotInfo reads information on a physical replication slot TODO: Unittest

func (*PgQueryRunner) GetWalSegmentBytes added in v0.2.18 

func (queryRunner *PgQueryRunner) GetWalSegmentBytes() (segBlocks uint64, err error)

GetWalSegmentBytes reads the wals segment size (in bytes) and converts it to uint64 TODO: Unittest

type PgRelationStat added in v0.2.18 

type PgRelationStat struct {
	// contains filtered or unexported fields
}

type PhysicalSlot added in v0.2.18 

type PhysicalSlot struct {
	Name       string
	Exists     bool
	Active     bool
	RestartLSN pglogrepl.LSN
}

The PhysicalSlot represents a Physical Replication Slot.

func NewPhysicalSlot added in v0.2.18 

func NewPhysicalSlot(name string, exists bool, active bool, restartLSN string) (slot PhysicalSlot, err error)

NewPhysicalSlot is a helper function to declare a new PhysicalSlot object and set vaues from the parsed arguments

type PostgresPageHeader 

type PostgresPageHeader struct {
	// contains filtered or unexported fields
}

type ProcessMessageResult added in v0.2.18 

type ProcessMessageResult int

The ProcessMessageResult is an enum representing possible results from the methods processing the messages as received from Postgres into the wal segment. 

const (
	ProcessMessageOK ProcessMessageResult = iota
	ProcessMessageUnknown
	ProcessMessageCopyDone
	ProcessMessageReplyRequested
	ProcessMessageSegmentGap
	ProcessMessageMismatch
)

These are the multiple results that the methods can return

type QueryRunner 

type QueryRunner interface {
	// This call should inform the database that we are going to copy cluster's contents
	// Should fail if backup is currently impossible
	StartBackup(backup string) (string, string, bool, error)
	// Inform database that contents are copied, get information on backup
	StopBackup() (string, string, string, error)
}

The QueryRunner interface for controlling database during backup

type RatedComposeFileInfo added in v0.2.18 

type RatedComposeFileInfo struct {
	ComposeFileInfo
	// contains filtered or unexported fields
}

type RatingTarBallComposer added in v0.2.18 

type RatingTarBallComposer struct {
	// contains filtered or unexported fields
}

RatingTarBallComposer receives all files and tar headers that are going to be written to the backup, and composes the tarballs by placing the files with similar update rating in the same tarballs

func NewRatingTarBallComposer added in v0.2.18 

func NewRatingTarBallComposer(
	tarSizeThreshold uint64, updateRatingEvaluator ComposeRatingEvaluator,
	incrementBaseLsn *uint64, deltaMap PagedFileDeltaMap, tarBallQueue *TarBallQueue,
	crypter crypto.Crypter, fileStats RelFileStatistics, bundleFiles BundleFiles, packer *TarBallFilePacker) (*RatingTarBallComposer, error)

func (*RatingTarBallComposer) AddFile added in v0.2.18 

func (c *RatingTarBallComposer) AddFile(info *ComposeFileInfo)

func (*RatingTarBallComposer) AddHeader added in v0.2.18 

func (c *RatingTarBallComposer) AddHeader(fileInfoHeader *tar.Header, info os.FileInfo) error

func (*RatingTarBallComposer) GetFiles added in v0.2.18 

func (c *RatingTarBallComposer) GetFiles() BundleFiles

func (*RatingTarBallComposer) PackTarballs added in v0.2.18 

func (c *RatingTarBallComposer) PackTarballs() (TarFileSets, error)

func (*RatingTarBallComposer) SkipFile added in v0.2.18 

func (c *RatingTarBallComposer) SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)

type RatingTarBallComposerMaker added in v0.2.18 

type RatingTarBallComposerMaker struct {
	// contains filtered or unexported fields
}

func NewRatingTarBallComposerMaker added in v0.2.18 

func NewRatingTarBallComposerMaker(relFileStats RelFileStatistics,
	filePackerOptions TarBallFilePackerOptions) (*RatingTarBallComposerMaker, error)

func (*RatingTarBallComposerMaker) Make added in v0.2.18 

func (maker *RatingTarBallComposerMaker) Make(bundle *Bundle) (TarBallComposer, error)

type ReachedStopSegmentError added in v0.2.18 

type ReachedStopSegmentError struct {
	// contains filtered or unexported fields
}

func (ReachedStopSegmentError) Error added in v0.2.18 

func (err ReachedStopSegmentError) Error() string

type ReadWriterAt added in v0.2.17 

type ReadWriterAt interface {
	io.ReaderAt
	io.WriterAt
	Size() int64
	Name() string
}

func NewReadWriterAtFrom added in v0.2.17 

func NewReadWriterAtFrom(file *os.File) (ReadWriterAt, error)

type ReadWriterAtFileImpl added in v0.2.17 

type ReadWriterAtFileImpl struct {
	*os.File
	// contains filtered or unexported fields
}

func (*ReadWriterAtFileImpl) Size added in v0.2.17 

func (rw *ReadWriterAtFileImpl) Size() int64

type ReaderMaker 

type ReaderMaker interface {
	Reader() (io.ReadCloser, error)
	Path() string
}

ReaderMaker is the generic interface used by extract. It allows for ease of handling different file formats.

type RegularBundleFiles added in v0.2.18 

type RegularBundleFiles struct {
	sync.Map
}

func (*RegularBundleFiles) AddFile added in v0.2.18 

func (files *RegularBundleFiles) AddFile(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool)

func (*RegularBundleFiles) AddFileWithCorruptBlocks added in v0.2.18 

func (files *RegularBundleFiles) AddFileWithCorruptBlocks(tarHeader *tar.Header, fileInfo os.FileInfo,
	isIncremented bool, corruptedBlocks []uint32, storeAllBlocks bool)

func (*RegularBundleFiles) AddSkippedFile added in v0.2.18 

func (files *RegularBundleFiles) AddSkippedFile(tarHeader *tar.Header, fileInfo os.FileInfo)

func (*RegularBundleFiles) GetUnderlyingMap added in v0.2.18 

func (files *RegularBundleFiles) GetUnderlyingMap() *sync.Map

type RegularTarBallComposer added in v0.2.18 

type RegularTarBallComposer struct {
	// contains filtered or unexported fields
}

func NewRegularTarBallComposer added in v0.2.18 

func NewRegularTarBallComposer(
	tarBallQueue *TarBallQueue,
	tarBallFilePacker *TarBallFilePacker,
	files *RegularBundleFiles,
	crypter crypto.Crypter,
) *RegularTarBallComposer

func (*RegularTarBallComposer) AddFile added in v0.2.18 

func (c *RegularTarBallComposer) AddFile(info *ComposeFileInfo)

func (*RegularTarBallComposer) AddHeader added in v0.2.18 

func (c *RegularTarBallComposer) AddHeader(fileInfoHeader *tar.Header, info os.FileInfo) error

func (*RegularTarBallComposer) GetFiles added in v0.2.18 

func (c *RegularTarBallComposer) GetFiles() BundleFiles

func (*RegularTarBallComposer) PackTarballs added in v0.2.18 

func (c *RegularTarBallComposer) PackTarballs() (TarFileSets, error)

func (*RegularTarBallComposer) SkipFile added in v0.2.18 

func (c *RegularTarBallComposer) SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)

type RegularTarBallComposerMaker added in v0.2.18 

type RegularTarBallComposerMaker struct {
	// contains filtered or unexported fields
}

func NewRegularTarBallComposerMaker added in v0.2.18 

func NewRegularTarBallComposerMaker(filePackerOptions TarBallFilePackerOptions) *RegularTarBallComposerMaker

func (*RegularTarBallComposerMaker) Make added in v0.2.18 

func (maker *RegularTarBallComposerMaker) Make(bundle *Bundle) (TarBallComposer, error)

type RelFileStatistics added in v0.2.18 

type RelFileStatistics map[walparser.RelFileNode]PgRelationStat

type Saver 

type Saver interface {
	Save(writer io.Writer) error
}

type ScannedSegmentDescription added in v0.2.18 

type ScannedSegmentDescription struct {
	WalSegmentDescription
	// contains filtered or unexported fields
}

type ScannedSegmentStatus added in v0.2.18 

type ScannedSegmentStatus int
const (
	// Surely lost missing segment
	Lost ScannedSegmentStatus = iota + 1
	// Missing but probably still uploading segment
	ProbablyUploading
	// Missing but probably delayed segment
	ProbablyDelayed
	// Segment exists in storage
	Found
)

func (ScannedSegmentStatus) MarshalJSON added in v0.2.18 

func (status ScannedSegmentStatus) MarshalJSON() ([]byte, error)

MarshalJSON marshals the ScannedSegmentStatus enum as a quoted json string

func (ScannedSegmentStatus) String added in v0.2.18 

func (status ScannedSegmentStatus) String() string

type SegmentScanConfig added in v0.2.18 

type SegmentScanConfig struct {
	UnlimitedScan bool
	// ScanSegmentsLimit is used in case of UnlimitedScan is set to false
	ScanSegmentsLimit       int
	StopOnFirstFoundSegment bool
	// MissingSegmentStatus is set to all missing segments encountered during scan
	MissingSegmentStatus ScannedSegmentStatus
}

SegmentScanConfig is used to configure the single Scan() call of the WalSegmentScanner

type Sentinel 

type Sentinel struct {
	Info os.FileInfo
	// contains filtered or unexported fields
}

Sentinel is used to signal completion of a walked directory.

type SentinelMarshallingError added in v0.2.7 

type SentinelMarshallingError struct {
	// contains filtered or unexported fields
}

func (SentinelMarshallingError) Error added in v0.2.7 

func (err SentinelMarshallingError) Error() string

type SkippedFileError added in v0.2.18 

type SkippedFileError struct {
	// contains filtered or unexported fields
}

func (SkippedFileError) Error added in v0.2.18 

func (err SkippedFileError) Error() string

type StatBundleFiles added in v0.2.18 

type StatBundleFiles struct {
	sync.Map
	// contains filtered or unexported fields
}

StatBundleFiles contains the bundle files. Additionally, it calculates and stores the updates count for each added file

func (*StatBundleFiles) AddFile added in v0.2.18 

func (files *StatBundleFiles) AddFile(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool)

func (*StatBundleFiles) AddFileWithCorruptBlocks added in v0.2.18 

func (files *StatBundleFiles) AddFileWithCorruptBlocks(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool, corruptedBlocks []uint32, storeAllBlocks bool)

func (*StatBundleFiles) AddSkippedFile added in v0.2.18 

func (files *StatBundleFiles) AddSkippedFile(tarHeader *tar.Header, fileInfo os.FileInfo)

func (*StatBundleFiles) GetUnderlyingMap added in v0.2.18 

func (files *StatBundleFiles) GetUnderlyingMap() *sync.Map

type StorageAdapter added in v0.2.7 

type StorageAdapter struct {
	// contains filtered or unexported fields
}

type StorageReaderMaker 

type StorageReaderMaker struct {
	Folder       storage.Folder
	RelativePath string
}

StorageReaderMaker creates readers for downloading from storage

func (*StorageReaderMaker) Path 

func (readerMaker *StorageReaderMaker) Path() string

func (*StorageReaderMaker) Reader 

func (readerMaker *StorageReaderMaker) Reader() (io.ReadCloser, error)

type StorageTarBall 

type StorageTarBall struct {
	// contains filtered or unexported fields
}

StorageTarBall represents a tar file that is going to be uploaded to storage.

func (*StorageTarBall) AddSize 

func (tarBall *StorageTarBall) AddSize(i int64)

AddSize to total Size

func (*StorageTarBall) AwaitUploads 

func (tarBall *StorageTarBall) AwaitUploads()

func (*StorageTarBall) CloseTar 

func (tarBall *StorageTarBall) CloseTar() error

CloseTar closes the tar writer, flushing any unwritten data to the underlying writer before also closing the underlying writer.

func (*StorageTarBall) Name added in v0.2.18 

func (tarBall *StorageTarBall) Name() string

func (*StorageTarBall) SetUp 

func (tarBall *StorageTarBall) SetUp(crypter crypto.Crypter, names ...string)

SetUp creates a new tar writer and starts upload to storage. Upload will block until the tar file is finished writing. If a name for the file is not given, default name is of the form `part_....tar.[Compressor file extension]`.

func (*StorageTarBall) Size 

func (tarBall *StorageTarBall) Size() int64

Size accumulated in this tarball

func (*StorageTarBall) TarWriter 

func (tarBall *StorageTarBall) TarWriter() *tar.Writer

type StorageTarBallMaker 

type StorageTarBallMaker struct {
	// contains filtered or unexported fields
}

StorageTarBallMaker creates tarballs that are uploaded to storage.

func NewStorageTarBallMaker 

func NewStorageTarBallMaker(backupName string, uploader *Uploader) *StorageTarBallMaker

func (*StorageTarBallMaker) Make 

func (tarBallMaker *StorageTarBallMaker) Make(dedicatedUploader bool) TarBall

Make returns a tarball with required storage fields.

type TablespaceLocation added in v0.2.13 

type TablespaceLocation struct {
	Location string `json:"loc"`
	Symlink  string `json:"link"`
}

type TablespaceSpec added in v0.2.13 

type TablespaceSpec struct {
	// contains filtered or unexported fields
}

The mandatory keys for this map are "base_prefix" and "tablespaces". "base_prefix" contains Location of pg_data folder. "tablespaces" contains array of keys, which also happen to be names of tablespace folders. The rest keys should be these names of tablespace folders and values should be TablespaceLocation structs.

func NewTablespaceSpec added in v0.2.13 

func NewTablespaceSpec(basePrefix string) TablespaceSpec

func (*TablespaceSpec) BasePrefix added in v0.2.13 

func (spec *TablespaceSpec) BasePrefix() (string, bool)

func (*TablespaceSpec) MarshalJSON added in v0.2.13 

func (spec *TablespaceSpec) MarshalJSON() ([]byte, error)

func (*TablespaceSpec) TablespaceNames added in v0.2.13 

func (spec *TablespaceSpec) TablespaceNames() []string

func (*TablespaceSpec) UnmarshalJSON added in v0.2.13 

func (spec *TablespaceSpec) UnmarshalJSON(b []byte) error

type TarBall 

type TarBall interface {
	SetUp(crypter crypto.Crypter, args ...string)
	CloseTar() error
	Size() int64
	AddSize(int64)
	TarWriter() *tar.Writer
	AwaitUploads()
	Name() string
}

A TarBall represents one tar file.

type TarBallComposer added in v0.2.18 

type TarBallComposer interface {
	AddFile(info *ComposeFileInfo)
	AddHeader(header *tar.Header, fileInfo os.FileInfo) error
	SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)
	PackTarballs() (TarFileSets, error)
	GetFiles() BundleFiles
}

TarBallComposer is used to compose files into tarballs.

type TarBallComposerMaker added in v0.2.18 

type TarBallComposerMaker interface {
	Make(bundle *Bundle) (TarBallComposer, error)
}

TarBallComposerMaker is used to make an instance of TarBallComposer

func NewTarBallComposerMaker added in v0.2.18 

func NewTarBallComposerMaker(composerType TarBallComposerType, conn *pgx.Conn,
	filePackOptions TarBallFilePackerOptions) (TarBallComposerMaker, error)

type TarBallComposerType added in v0.2.18 

type TarBallComposerType int
const (
	RegularComposer TarBallComposerType = iota + 1
	RatingComposer
)

type TarBallFilePacker added in v0.2.18 

type TarBallFilePacker struct {
	// contains filtered or unexported fields
}

TarBallFilePacker is used to pack bundle file into tarball.

func (*TarBallFilePacker) PackFileIntoTar added in v0.2.18 

func (p *TarBallFilePacker) PackFileIntoTar(cfi *ComposeFileInfo, tarBall TarBall) error

TODO : unit tests

func (*TarBallFilePacker) UpdateDeltaMap added in v0.2.18 

func (p *TarBallFilePacker) UpdateDeltaMap(deltaMap PagedFileDeltaMap)

type TarBallFilePackerOptions added in v0.2.18 

type TarBallFilePackerOptions struct {
	// contains filtered or unexported fields
}

func NewTarBallFilePackerOptions added in v0.2.18 

func NewTarBallFilePackerOptions(verifyPageChecksums, storeAllCorruptBlocks bool) TarBallFilePackerOptions

type TarBallMaker 

type TarBallMaker interface {
	Make(dedicatedUploader bool) TarBall
}

TarBallMaker is used to allow for flexible creation of different TarBalls.

type TarBallQueue added in v0.2.18 

type TarBallQueue struct {
	TarSizeThreshold   int64
	AllTarballsSize    *int64
	TarBallMaker       TarBallMaker
	LastCreatedTarball TarBall
	// contains filtered or unexported fields
}

TarBallQueue is used to process multiple tarballs concurrently

func (*TarBallQueue) CheckSizeAndEnqueueBack added in v0.2.18 

func (tarQueue *TarBallQueue) CheckSizeAndEnqueueBack(tarBall TarBall) error

func (*TarBallQueue) CloseTarball added in v0.2.18 

func (tarQueue *TarBallQueue) CloseTarball(tarBall TarBall) error

func (*TarBallQueue) Deque added in v0.2.18 

func (tarQueue *TarBallQueue) Deque() TarBall

func (*TarBallQueue) EnqueueBack added in v0.2.18 

func (tarQueue *TarBallQueue) EnqueueBack(tarBall TarBall)

func (*TarBallQueue) FinishQueue added in v0.2.18 

func (tarQueue *TarBallQueue) FinishQueue() error

func (*TarBallQueue) FinishTarBall added in v0.2.18 

func (tarQueue *TarBallQueue) FinishTarBall(tarBall TarBall) error

func (*TarBallQueue) NewTarBall added in v0.2.18 

func (tarQueue *TarBallQueue) NewTarBall(dedicatedUploader bool) TarBall

NewTarBall starts writing new tarball

func (*TarBallQueue) StartQueue added in v0.2.18 

func (tarQueue *TarBallQueue) StartQueue() error

type TarFileSets added in v0.2.18 

type TarFileSets map[string][]string

type TarFilesCollection added in v0.2.18 

type TarFilesCollection struct {
	// contains filtered or unexported fields
}

TarFilesCollection stores the files which are going to be written to the same tarball

func (*TarFilesCollection) AddFile added in v0.2.18 

func (collection *TarFilesCollection) AddFile(file *RatedComposeFileInfo)

type TarInterpreter 

type TarInterpreter interface {
	Interpret(reader io.Reader, header *tar.Header) error
}

TarInterpreter behaves differently for different file types.

type TarSizeError 

type TarSizeError struct {
	// contains filtered or unexported fields
}

func (TarSizeError) Error 

func (err TarSizeError) Error() string

type TimeLineHistFile added in v0.2.18 

type TimeLineHistFile struct {
	TimeLineID uint32
	Filename   string
	// contains filtered or unexported fields
}

The TimeLineHistFile struct represents a TimeLineHistory file containing TimeLineHistFileRows. Since TimeLineHistFileRows are only parsed 0 or 1 rimes, the data is only preserved as []byte and parsed to TimeLineHistFileRows when required.

func NewTimeLineHistFile added in v0.2.18 

func NewTimeLineHistFile(timelineid uint32, filename string, body []byte) (TimeLineHistFile, error)

NewTimeLineHistFile is a helper function to define a new TimeLineHistFile

func (TimeLineHistFile) LSNToTimeLine added in v0.2.18 

func (tlh TimeLineHistFile) LSNToTimeLine(lsn pglogrepl.LSN) (uint32, error)

LSNToTimeLine uses rows() to get all TimeLineHistFileRows and from those rows get the timeline that a LS belongs too.

func (TimeLineHistFile) Name added in v0.2.18 

func (tlh TimeLineHistFile) Name() string

Name returns the filename of this wal segment. This is a convenience function used by the WalUploader.

func (TimeLineHistFile) Read added in v0.2.18 

func (tlh TimeLineHistFile) Read(p []byte) (n int, err error)

Read is what makes the WalSegment an io.Reader, which can be handled by WalUploader.UploadWalFile.

type TimeLineHistFileRow added in v0.2.18 

type TimeLineHistFileRow struct {
	TimeLineID uint32
	StartLSN   pglogrepl.LSN
	Comment    string
}

The TimeLineHistFileRow struct represents one line in the TimeLineHistory file

type TimelineHistoryRecord added in v0.2.18 

type TimelineHistoryRecord struct {
	// contains filtered or unexported fields
}

TimelineHistoryRecord represents entry in .history file

func NewTimelineHistoryRecord added in v0.2.18 

func NewTimelineHistoryRecord(timeline uint32, lsn uint64, comment string) *TimelineHistoryRecord

type TimelineInfo added in v0.2.18 

type TimelineInfo struct {
	Id               uint32          `json:"id"`
	ParentId         uint32          `json:"parent_id"`
	SwitchPointLsn   uint64          `json:"switch_point_lsn"`
	StartSegment     string          `json:"start_segment"`
	EndSegment       string          `json:"end_segment"`
	SegmentsCount    int             `json:"segments_count"`
	MissingSegments  []string        `json:"missing_segments"`
	Backups          []*BackupDetail `json:"backups,omitempty"`
	SegmentRangeSize uint64          `json:"segment_range_size"`
	Status           string          `json:"status"`
}

TimelineInfo contains information about some timeline in storage

func NewTimelineInfo added in v0.2.18 

func NewTimelineInfo(walSegments *WalSegmentsSequence, historyRecords []*TimelineHistoryRecord) (*TimelineInfo, error)

type UnconfiguredStorageError added in v0.2.8 

type UnconfiguredStorageError struct {
	// contains filtered or unexported fields
}

func (UnconfiguredStorageError) Error added in v0.2.8 

func (err UnconfiguredStorageError) Error() string

type UnexpectedTarDataError 

type UnexpectedTarDataError struct {
	// contains filtered or unexported fields
}

func (UnexpectedTarDataError) Error 

func (err UnexpectedTarDataError) Error() string

type UnknownCompressionMethodError 

type UnknownCompressionMethodError struct {
	// contains filtered or unexported fields
}

func (UnknownCompressionMethodError) Error 

func (err UnknownCompressionMethodError) Error() string

type UnknownIncrementFileHeaderError 

type UnknownIncrementFileHeaderError struct {
	// contains filtered or unexported fields
}

func (UnknownIncrementFileHeaderError) Error 

func (err UnknownIncrementFileHeaderError) Error() string

type UnknownTableSpaceError 

type UnknownTableSpaceError struct {
	// contains filtered or unexported fields
}

func (UnknownTableSpaceError) Error 

func (err UnknownTableSpaceError) Error() string

type UnmarshallingError added in v0.2.10 

type UnmarshallingError struct {
	// contains filtered or unexported fields
}

func (UnmarshallingError) Error added in v0.2.10 

func (err UnmarshallingError) Error() string

type UnsetRequiredSettingError added in v0.2.10 

type UnsetRequiredSettingError struct {
	// contains filtered or unexported fields
}

func NewUnsetRequiredSettingError added in v0.2.10 

func NewUnsetRequiredSettingError(settingName string) UnsetRequiredSettingError

func (UnsetRequiredSettingError) Error added in v0.2.10 

func (err UnsetRequiredSettingError) Error() string

type UnsupportedFileTypeError 

type UnsupportedFileTypeError struct {
	// contains filtered or unexported fields
}

UnsupportedFileTypeError is used to signal file types that are unsupported by WAL-G.

func (UnsupportedFileTypeError) Error added in v0.2.8 

func (err UnsupportedFileTypeError) Error() string

type UnsupportedPostgresVersionError 

type UnsupportedPostgresVersionError struct {
	// contains filtered or unexported fields
}

func (UnsupportedPostgresVersionError) Error 

func (err UnsupportedPostgresVersionError) Error() string

type UnwrapResult added in v0.2.17 

type UnwrapResult struct {
	// contains filtered or unexported fields
}

UnwrapResult stores information about the result of single backup unwrap operation

type UploadObject added in v0.2.10 

type UploadObject struct {
	Path    string
	Content io.Reader
}

UploadObject

func GetMarkedBackupMetadataToUpload added in v0.2.12 

func GetMarkedBackupMetadataToUpload(
	folder storage.Folder,
	backupName string,
	toPermanent bool) ([]UploadObject, error)

GetMarkedBackupMetadataToUpload retrieves all previous permanent or impermanent backup metas, including itself, any previous delta backups and initial full backup, in increasing order beginning from full backup, returning modified metadata ready to be uploaded

For example, when marking backups from impermanent to permanent, we retrieve all currently impermanent backup metadata, set them to permanent, and return the modified metadata as a slice of uploadable objects

type Uploader 

type Uploader struct {
	UploadingFolder storage.Folder
	Compressor      compression.Compressor

	ArchiveStatusManager ArchiveStatusManager
	Failed               atomic.Value
	// contains filtered or unexported fields
}

Uploader contains fields associated with uploading tarballs. Multiple tarballs can share one uploader.

func ConfigureUploader added in v0.2.8 

func ConfigureUploader() (uploader *Uploader, err error)

ConfigureUploader connects to storage and creates an uploader. It makes sure that a valid session has started; if invalid, returns AWS error and `<nil>` values.

func ConfigureUploaderWithoutCompressMethod added in v0.2.15 

func ConfigureUploaderWithoutCompressMethod() (uploader *Uploader, err error)

func NewUploader 

func NewUploader(
	compressor compression.Compressor,
	uploadingLocation storage.Folder,
) *Uploader

func (*Uploader) Compression added in v0.2.17 

func (uploader *Uploader) Compression() compression.Compressor

Compression returns configured compressor

func (*Uploader) DisableSizeTracking added in v0.2.17 

func (uploader *Uploader) DisableSizeTracking()

DisableSizeTracking stops bandwidth tracking

func (*Uploader) PushStream added in v0.2.10 

func (uploader *Uploader) PushStream(stream io.Reader) (string, error)

TODO : unit tests PushStream compresses a stream and push it

func (*Uploader) PushStreamToDestination added in v0.2.10 

func (uploader *Uploader) PushStreamToDestination(stream io.Reader, dstPath string) error

TODO : unit tests PushStreamToDestination compresses a stream and push it to specifyed destination

func (*Uploader) Upload added in v0.2.7 

func (uploader *Uploader) Upload(path string, content io.Reader) error

TODO : unit tests

func (*Uploader) UploadFile 

func (uploader *Uploader) UploadFile(file NamedReader) error

TODO : unit tests UploadFile compresses a file and uploads it.

type UploaderProvider added in v0.2.17 

type UploaderProvider interface {
	Upload(path string, content io.Reader) error
	UploadFile(file NamedReader) error
	PushStream(stream io.Reader) (string, error)
	PushStreamToDestination(stream io.Reader, dstPath string) error
	Compression() compression.Compressor
	DisableSizeTracking()
}

type WalDeltaRecorder 

type WalDeltaRecorder struct {
	// contains filtered or unexported fields
}

func NewWalDeltaRecorder 

func NewWalDeltaRecorder(blockLocationConsumer chan walparser.BlockLocation) *WalDeltaRecorder

type WalDeltaRecordingReader 

type WalDeltaRecordingReader struct {
	PageReader       walparser.WalPageReader
	WalParser        walparser.WalParser
	PageDataLeftover []byte
	Recorder         *WalDeltaRecorder
	// contains filtered or unexported fields
}

In case of recording error WalDeltaRecordingReader stops recording, but continues reading data correctly

func NewWalDeltaRecordingReader 

func NewWalDeltaRecordingReader(walFileReader io.Reader, walFilename string, manager *DeltaFileManager) (*WalDeltaRecordingReader, error)

func (*WalDeltaRecordingReader) Close 

func (reader *WalDeltaRecordingReader) Close() error

func (*WalDeltaRecordingReader) Read 

func (reader *WalDeltaRecordingReader) Read(p []byte) (n int, err error)

func (*WalDeltaRecordingReader) RecordBlockLocationsFromPage 

func (reader *WalDeltaRecordingReader) RecordBlockLocationsFromPage() error

type WalIntegrityScanSegmentSequence added in v0.2.18 

type WalIntegrityScanSegmentSequence struct {
	TimelineId    uint32               `json:"timeline_id"`
	StartSegment  string               `json:"start_segment"`
	EndSegment    string               `json:"end_segment"`
	SegmentsCount int                  `json:"segments_count"`
	Status        ScannedSegmentStatus `json:"status"`
}

WalIntegrityScanSegmentSequence is a continuous sequence of segments with the same timeline and status

type WalPart 

type WalPart struct {
	// contains filtered or unexported fields
}

func LoadWalPart 

func LoadWalPart(reader io.Reader) (*WalPart, error)

func NewWalPart 

func NewWalPart(dataType WalPartDataType, id uint8, data []byte) *WalPart

func (*WalPart) Save 

func (part *WalPart) Save(writer io.Writer) error

type WalPartDataType 

type WalPartDataType uint8
const (
	PreviousWalHeadType WalPartDataType = 0
	WalTailType         WalPartDataType = 1
	WalHeadType         WalPartDataType = 2
)

type WalPartFile 

type WalPartFile struct {
	WalTails        [][]byte
	PreviousWalHead []byte
	WalHeads        [][]byte
}

func LoadPartFile 

func LoadPartFile(reader io.Reader) (*WalPartFile, error)

func NewWalPartFile 

func NewWalPartFile() *WalPartFile

func (*WalPartFile) CombineRecords 

func (partFile *WalPartFile) CombineRecords() ([]walparser.XLogRecord, error)

func (*WalPartFile) IsComplete 

func (partFile *WalPartFile) IsComplete() bool

func (*WalPartFile) Save 

func (partFile *WalPartFile) Save(writer io.Writer) error

type WalPartRecorder 

type WalPartRecorder struct {
	// contains filtered or unexported fields
}

func NewWalPartRecorder 

func NewWalPartRecorder(walFilename string, manager *DeltaFileManager) (*WalPartRecorder, error)

func (*WalPartRecorder) SaveNextWalHead 

func (recorder *WalPartRecorder) SaveNextWalHead(head []byte) error

func (*WalPartRecorder) SavePreviousWalTail 

func (recorder *WalPartRecorder) SavePreviousWalTail(tailData []byte) error

type WalSegment added in v0.2.18 

type WalSegment struct {
	TimeLine uint32
	StartLSN pglogrepl.LSN
	// contains filtered or unexported fields
}

The WalSegment object represents a Postgres Wal Segment, holding all wal data for a wal file.

func NewWalSegment added in v0.2.18 

func NewWalSegment(timeline uint32, location pglogrepl.LSN, walSegmentBytes uint64) *WalSegment

NewWalSegment is a helper function to declare a new WalSegment.

func (*WalSegment) Name added in v0.2.18 

func (seg *WalSegment) Name() string

Name returns the filename of this wal segment. This is also used by the WalUploader to set the name of the destination file during upload of the wal segment.

func (*WalSegment) NextWalSegment added in v0.2.18 

func (seg *WalSegment) NextWalSegment() (*WalSegment, error)

NextWalSegment is a helper function to create the next wal segment which comes after this wal segment. Note that this will be on the same timeline. the convenience is that it also automatically processes a message that crosses the boundary between the two segments.

func (*WalSegment) Read added in v0.2.18 

func (seg *WalSegment) Read(p []byte) (n int, err error)

Read is what makes the WalSegment an io.Reader, which can be handled by WalUploader.UploadWalFile to write to a file.

func (*WalSegment) Stream added in v0.2.18 

func (seg *WalSegment) Stream(conn *pgconn.PgConn, standbyMessageTimeout time.Duration) (ProcessMessageResult, error)

Stream is a helper function to retrieve messages from Postgres and have them processed by processMessage().

type WalSegmentDescription added in v0.2.18 

type WalSegmentDescription struct {
	Number   WalSegmentNo
	Timeline uint32
}

func QueryCurrentWalSegment added in v0.2.18 

func QueryCurrentWalSegment() WalSegmentDescription

QueryCurrentWalSegment() gets start WAL segment from Postgres cluster

func (WalSegmentDescription) GetFileName added in v0.2.18 

func (desc WalSegmentDescription) GetFileName() string

type WalSegmentNo added in v0.2.12 

type WalSegmentNo uint64

type WalSegmentNotFoundError added in v0.2.18 

type WalSegmentNotFoundError struct {
	// contains filtered or unexported fields
}

func (WalSegmentNotFoundError) Error added in v0.2.18 

func (err WalSegmentNotFoundError) Error() string

type WalSegmentRunner added in v0.2.18 

type WalSegmentRunner struct {
	// contains filtered or unexported fields
}

WalSegmentRunner is used for sequential iteration over WAL segments in the storage

func NewWalSegmentRunner added in v0.2.18 

func NewWalSegmentRunner(
	startWalSegment WalSegmentDescription,
	segments map[WalSegmentDescription]bool,
	stopSegmentNo WalSegmentNo,
	timelineSwitchMap map[WalSegmentNo]*TimelineHistoryRecord,
) *WalSegmentRunner

func (*WalSegmentRunner) Current added in v0.2.18 

func (r *WalSegmentRunner) Current() WalSegmentDescription

func (*WalSegmentRunner) ForceMoveNext added in v0.2.18 

func (r *WalSegmentRunner) ForceMoveNext()

ForceMoveNext do a force-switch to the next segment without accessing storage

func (*WalSegmentRunner) Next added in v0.2.18 

func (r *WalSegmentRunner) Next() (WalSegmentDescription, error)

Next tries to get the next segment from storage

type WalSegmentScanner added in v0.2.18 

type WalSegmentScanner struct {
	ScannedSegments []ScannedSegmentDescription
	// contains filtered or unexported fields
}

WalSegmentScanner is used to scan the WAL segments storage

func NewWalSegmentScanner added in v0.2.18 

func NewWalSegmentScanner(walSegmentRunner *WalSegmentRunner) *WalSegmentScanner

func (*WalSegmentScanner) AddScannedSegment added in v0.2.18 

func (scanner *WalSegmentScanner) AddScannedSegment(description WalSegmentDescription, status ScannedSegmentStatus)

func (*WalSegmentScanner) GetMissingSegmentsDescriptions added in v0.2.18 

func (scanner *WalSegmentScanner) GetMissingSegmentsDescriptions() []WalSegmentDescription

GetMissingSegmentsDescriptions returns a slice containing WalSegmentDescription of each missing segment

func (*WalSegmentScanner) Scan added in v0.2.18 

func (scanner *WalSegmentScanner) Scan(config SegmentScanConfig) error

Scan traverse the WAL storage with WalSegmentRunner. Scan starts from the WalSegmentRunner's current position, so in case of subsequent Scan() call it will continue from the position where it stopped previously.

Scan always stops if: - Stop segment is reached OR - Unknown error encountered Also, it may be configured to stop after: - Scanning the ScanSegmentsLimit of segments - Finding the first segment which exists in WAL storage

type WalSegmentsSequence added in v0.2.18 

type WalSegmentsSequence struct {
	// contains filtered or unexported fields
}

WalSegmentsSequence represents some collection of wal segments with the same timeline

func NewSegmentsSequence added in v0.2.18 

func NewSegmentsSequence(id uint32, segmentNo WalSegmentNo) *WalSegmentsSequence

func (*WalSegmentsSequence) AddWalSegmentNo added in v0.2.18 

func (seq *WalSegmentsSequence) AddWalSegmentNo(number WalSegmentNo)

AddWalSegmentNo adds the provided segment number to collection

func (*WalSegmentsSequence) FindMissingSegments added in v0.2.18 

func (seq *WalSegmentsSequence) FindMissingSegments() ([]WalSegmentDescription, error)

FindMissingSegments finds missing segments in range [minSegmentNo, maxSegmentNo]

type WalShowJsonOutputWriter added in v0.2.18 

type WalShowJsonOutputWriter struct {
	// contains filtered or unexported fields
}

WalShowJsonOutputWriter writes the detailed JSON output

func (*WalShowJsonOutputWriter) Write added in v0.2.18 

func (writer *WalShowJsonOutputWriter) Write(timelineInfos []*TimelineInfo) error

type WalShowOutputType added in v0.2.18 

type WalShowOutputType int
const (
	TableOutput WalShowOutputType = iota + 1
	JsonOutput
)

type WalShowOutputWriter added in v0.2.18 

type WalShowOutputWriter interface {
	Write(timelineInfos []*TimelineInfo) error
}

WalShowOutputWriter writes the output of wal-show command execution result

func NewWalShowOutputWriter added in v0.2.18 

func NewWalShowOutputWriter(outputType WalShowOutputType, output io.Writer, includeBackups bool) WalShowOutputWriter

type WalShowTableOutputWriter added in v0.2.18 

type WalShowTableOutputWriter struct {
	// contains filtered or unexported fields
}

WalShowTableOutputWriter writes the output in compact pretty table

func (*WalShowTableOutputWriter) Write added in v0.2.18 

func (writer *WalShowTableOutputWriter) Write(timelineInfos []*TimelineInfo) error

type WalStorageStatus added in v0.2.18 

type WalStorageStatus int
const (
	// No missing segments in storage
	Ok WalStorageStatus = iota + 1
	// Storage contains some ProbablyUploading or ProbablyDelayed missing segments
	Warning
	// Storage contains lost missing segments
	Failure
)

func (WalStorageStatus) MarshalJSON added in v0.2.18 

func (storageStatus WalStorageStatus) MarshalJSON() ([]byte, error)

MarshalJSON marshals the WalStorageStatus enum as a quoted json string

func (WalStorageStatus) String added in v0.2.18 

func (storageStatus WalStorageStatus) String() string

type WalUploader added in v0.2.15 

type WalUploader struct {
	*Uploader
	*DeltaFileManager
}

WalUploader extends uploader with wal specific functionality.

func ConfigureWalUploader added in v0.2.15 

func ConfigureWalUploader() (uploader *WalUploader, err error)

ConfigureWalUploader connects to storage and creates an uploader. It makes sure that a valid session has started; if invalid, returns AWS error and `<nil>` values.

func ConfigureWalUploaderWithoutCompressMethod added in v0.2.15 

func ConfigureWalUploaderWithoutCompressMethod() (uploader *WalUploader, err error)

func NewWalUploader added in v0.2.15 

func NewWalUploader(
	compressor compression.Compressor,
	uploadingLocation storage.Folder,
	deltaFileManager *DeltaFileManager,
) *WalUploader

func (*WalUploader) FlushFiles added in v0.2.15 

func (walUploader *WalUploader) FlushFiles()

func (*WalUploader) UploadWalFile added in v0.2.15 

func (walUploader *WalUploader) UploadWalFile(file NamedReader) error

TODO : unit tests

type WalVerifyJsonOutputWriter added in v0.2.18 

type WalVerifyJsonOutputWriter struct {
	// contains filtered or unexported fields
}

WalVerifyJsonOutputWriter writes the detailed JSON output

func (*WalVerifyJsonOutputWriter) Write added in v0.2.18 

func (writer *WalVerifyJsonOutputWriter) Write(result WalVerifyResult) error

type WalVerifyOutputType added in v0.2.18 

type WalVerifyOutputType int
const (
	WalVerifyTableOutput WalVerifyOutputType = iota + 1
	WalVerifyJsonOutput
)

type WalVerifyOutputWriter added in v0.2.18 

type WalVerifyOutputWriter interface {
	Write(result WalVerifyResult) error
}

WalVerifyOutputWriter writes the output of wal-verify command execution result

func NewWalVerifyOutputWriter added in v0.2.18 

func NewWalVerifyOutputWriter(outputType WalVerifyOutputType, output io.Writer) WalVerifyOutputWriter

type WalVerifyResult added in v0.2.18 

type WalVerifyResult struct {
	StorageStatus       WalStorageStatus                   `json:"storage_status"`
	IntegrityScanResult []*WalIntegrityScanSegmentSequence `json:"integrity_scan_result"`
}

type WalVerifyTableOutputWriter added in v0.2.18 

type WalVerifyTableOutputWriter struct {
	// contains filtered or unexported fields
}

WalVerifyTableOutputWriter writes the output as pretty table

func (*WalVerifyTableOutputWriter) Write added in v0.2.18 

func (writer *WalVerifyTableOutputWriter) Write(result WalVerifyResult) error

type WithSizeReader added in v0.2.13 

type WithSizeReader struct {
	// contains filtered or unexported fields
}

func (*WithSizeReader) Read added in v0.2.13 

func (reader *WithSizeReader) Read(p []byte) (n int, err error)

type WrongTypeError 

type WrongTypeError struct {
	// contains filtered or unexported fields
}

func (WrongTypeError) Error 

func (err WrongTypeError) Error() string

Source Files

View all Source files

Directories

Path Synopsis
Package abool provides atomic Boolean type for cleaner code and better performance.
 Package abool provides atomic Boolean type for cleaner code and better performance.
computils
lz4
lzma
zstd
awskms
libsodium
openpgp
databases
fdb
mongo
mongo/archive
mongo/archive/mocks
mongo/client
mongo/client/mocks
mongo/discovery
mongo/models
mongo/oplog
mongo/stages
mongo/stages/mocks
mongo/stats
mysql
postgres
redis
sqlserver
sqlserver/blob
testtools
Package mock_internal is a generated GoMock package.
 Package mock_internal is a generated GoMock package.
parsingutil

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.