postgres

package
v1.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 12, 2021 License: Apache-2.0 Imports: 46 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	PgControlPath     = "/global/pg_control"
	FilesMetadataName = "files_metadata.json"
)
View Source 
const (
	PgControl             = "pg_control"
	BackupLabelFilename   = "backup_label"
	TablespaceMapFilename = "tablespace_map"
	TablespaceFolder      = "pg_tblspc"
)
View Source 
const (
	RelFileSizeBound               = 1 << 30
	BlocksInRelFile                = int(RelFileSizeBound / DatabasePageSize)
	DefaultSpcNode   walparser.Oid = 1663
)
View Source 
const (
	// number of checksums to calculate in parallel
	NSums int = 32
	// prime multiplier of FNV-1a hash
	FnvPrime uint32 = 16777619
	// page header checksum offset
	PdChecksumOffset = 8
	// page header checksum length (in bytes)
	PdChecksumLen = 2
)
View Source 
const (
	DatabasePageSize = int64(walparser.BlockSize)

	SignatureMagicNumber byte = 0x55

	DefaultTablespace    = "base"
	GlobalTablespace     = "global"
	NonDefaultTablespace = "pg_tblspc"
)
View Source 
const (
	BasePrefix  = "base_prefix"
	Tablespaces = "tablespaces"
)
View Source 
const (
	WalFileInDelta      uint64 = 16
	DeltaFilenameSuffix string = "_delta"
	PartFilenameSuffix  string = "_part"
)
View Source 
const (
	WalBulkMetadataLevel       = "BULK"
	WalIndividualMetadataLevel = "INDIVIDUAL"
	WalNoMetadataLevel         = "NOMETADATA"
)
View Source 
const (
	TimelineOkStatus          = "OK"
	TimelineLostSegmentStatus = "LOST_SEGMENTS"
)
View Source 
const (
	WalVerifyIntegrityCheck = iota + 1
	WalVerifyTimelineCheck
)
View Source 
const MetadataDatetimeFormat = "%Y-%m-%dT%H:%M:%S.%fZ"
View Source 
const PatternTimelineAndLogSegNo = "[0-9A-F]{24}"
View Source 
const (
	RecordPartFilename = "currentRecord.part"
)
View Source 
const (
	// Sets standbyMessageTimeout in Streaming Replication Protocol.
	StandbyMessageTimeout = time.Second * 10
)

Variables

View Source 
var ErrorBasePrefixMissing = fmt.Errorf("base prefix not set while working with tablespaces")
View Source 
var ExcludedFilenames = make(map[string]utility.Empty)

ExcludedFilenames is a list of excluded members from the bundled backup.

View Source 
var IncrementFileHeader = []byte{'w', 'i', '1', SignatureMagicNumber}

IncrementFileHeader contains "wi" at the head which stands for "wal-g increment" format version "1", signature magic number

View Source 
var UnwrapAll map[string]bool = nil
View Source 
var UtilityFilePaths = map[string]bool{
	PgControlPath:         true,
	BackupLabelFilename:   true,
	TablespaceMapFilename: true,
}
View Source 
var WalMetadataLevels = []string{WalBulkMetadataLevel, WalIndividualMetadataLevel, WalNoMetadataLevel}
View Source 
var (
	// WalSegmentSize is the size of one WAL file
	WalSegmentSize = uint64(16 * 1024 * 1024)
)

Functions

func ApplyFileIncrement added in v0.2.22 

func ApplyFileIncrement(fileName string, increment io.Reader, createNewIncrementalFiles bool) error

ApplyFileIncrement changes pages according to supplied change map file

func BackupCopyingInfo 

func BackupCopyingInfo(backup Backup, from storage.Folder, to storage.Folder) ([]copy.InfoProvider, error)

func CleanupPrefetchDirectories added in v0.2.22 

func CleanupPrefetchDirectories(walFileName string, location string, cleaner Cleaner)

func Connect added in v0.2.22 

func Connect(configOptions ...func(config *pgx.ConnConfig) error) (*pgx.Conn, error)

Connect establishes a connection to postgres using a UNIX socket. Must export PGHOST and run with `sudo -E -u postgres`. If PGHOST is not set or if the connection fails, an error is returned and the connection is `<nil>`.

Example: PGHOST=/var/run/postgresql or PGHOST=10.0.0.1

func CreateFileFromIncrement added in v0.2.22 

func CreateFileFromIncrement(increment io.Reader, target ReadWriterAt) (int64, error)

CreateFileFromIncrement writes the pages from the increment to local file and write empty blocks in place of pages which are not present in the increment

func FetchPgBackupName added in v0.2.22 

func FetchPgBackupName(object storage.Object) string

func GetBaseFilesToUnwrap added in v0.2.22 

func GetBaseFilesToUnwrap(backupFileStates internal.BackupFileList, currentFilesToUnwrap map[string]bool) (map[string]bool, error)

func GetDeltaFilenameFor added in v0.2.22 

func GetDeltaFilenameFor(walFilename string) (string, error)

func GetIncrementHeaderFields added in v0.2.22 

func GetIncrementHeaderFields(increment io.Reader) (uint64, uint32, []byte, error)

func GetLastWalFilename added in v0.2.22 

func GetLastWalFilename(backup Backup) (string, error)

func GetNextWalFilename added in v0.2.22 

func GetNextWalFilename(name string) (string, error)

GetNextWalFilename computes name of next WAL segment

func GetPermanentBackupsAndWals added in v0.2.22 

func GetPermanentBackupsAndWals(folder storage.Folder) (map[string]bool, map[string]bool)

func GetPgFetcherNew added in v0.2.22 

func GetPgFetcherNew(dbDataDirectory, fileMask, restoreSpecPath string, skipRedundantTars bool,
) func(folder storage.Folder, backup internal.Backup)

func GetPgFetcherOld added in v0.2.22 

func GetPgFetcherOld(dbDataDirectory, fileMask, restoreSpecPath string) func(rootFolder storage.Folder, backup internal.Backup)

func GetPositionInDelta added in v0.2.22 

func GetPositionInDelta(walFilename string) int

func GetRelFileIDFrom added in v0.2.22 

func GetRelFileIDFrom(filePath string) (int, error)

func GetRelFileNodeFrom added in v0.2.22 

func GetRelFileNodeFrom(filePath string) (*walparser.RelFileNode, error)

func HandleCatchupFetch added in v0.2.22 

func HandleCatchupFetch(folder storage.Folder, dbDirectory, backupName string, useNewUnwrap bool)

HandleCatchupFetch is invoked to perform wal-g catchup-fetch

func HandleCatchupPush added in v0.2.22 

func HandleCatchupPush(pgDataDirectory string, fromLSN uint64)

HandleCatchupPush is invoked to perform a wal-g catchup-push

func HandleCopy 

func HandleCopy(fromConfigFile string, toConfigFile string, backupName string, withoutHistory bool)

HandleCopy copy specific or all backups from one storage to another

func HandleDetailedBackupList added in v0.2.22 

func HandleDetailedBackupList(folder storage.Folder, pretty bool, json bool)

TODO : unit tests

func HandleWALFetch added in v0.2.22 

func HandleWALFetch(folder storage.Folder, walFileName string, location string, triggerPrefetch bool)

TODO : unit tests HandleWALFetch is invoked to performa wal-g wal-fetch

func HandleWALPrefetch added in v0.2.22 

func HandleWALPrefetch(uploader *WalUploader, walFileName string, location string)

TODO : unit tests HandleWALPrefetch is invoked by wal-fetch command to speed up database restoration

func HandleWALPush added in v0.2.22 

func HandleWALPush(uploader *WalUploader, walFilePath string)

TODO : unit tests HandleWALPush is invoked to perform wal-g wal-push

func HandleWALReceive added in v0.2.22 

func HandleWALReceive(uploader *WalUploader)

HandleWALReceive is invoked to receive wal with a replication connection and push

func HandleWalPurge added in v1.1.1 

func HandleWalPurge(folder storage.Folder, deleteHandler *internal.DeleteHandler, confirm bool) error

HandleWalPurge delete outdated WAL archives

func HandleWalShow added in v0.2.22 

func HandleWalShow(rootFolder storage.Folder, showBackups bool, outputWriter WalShowOutputWriter)

HandleWalShow gets the list of files inside WAL folder, detects the available WAL segments, groups WAL segments by the timeline and shows detailed info about each timeline stored in storage

func HandleWalVerify added in v0.2.22 

func HandleWalVerify(
	checkTypes []WalVerifyCheckType,
	rootFolder storage.Folder,
	currentWalSegment WalSegmentDescription,
	outputWriter WalVerifyOutputWriter,
)

HandleWalVerify builds a check runner for each check type and writes the check results to the provided output writer

func HistoryCopyingInfo 

func HistoryCopyingInfo(backup Backup, from storage.Folder, to storage.Folder) ([]copy.InfoProvider, error)

func IsPermanent added in v0.2.22 

func IsPermanent(objectName string, permanentBackups, permanentWals map[string]bool) bool

func IsPgControlRequired added in v0.2.22 

func IsPgControlRequired(backup Backup, sentinelDto BackupSentinelDto) bool

func NewDeleteHandler added in v1.1.1 

func NewDeleteHandler(folder storage.Folder, permanentBackups, permanentWals map[string]bool,
	useSentinelTime bool,
) (*internal.DeleteHandler, error)

func ParseTimelineFromBackupName added in v0.2.22 

func ParseTimelineFromBackupName(backupName string) (uint32, error)

func ParseTimelineFromString added in v0.2.22 

func ParseTimelineFromString(timelineString string) (uint32, error)

func ParseWALFilename added in v0.2.22 

func ParseWALFilename(name string) (timelineID uint32, logSegNo uint64, err error)

ParseWALFilename extracts numeric parts from WAL file name

func PrepareDirs added in v0.2.22 

func PrepareDirs(fileName string, targetPath string) error

PrepareDirs makes sure all dirs exist

func ReadIncrementFileHeader added in v0.2.22 

func ReadIncrementFileHeader(reader io.Reader) error

func ReadIncrementLocations added in v0.2.22 

func ReadIncrementLocations(filePath string, fileSize int64, lsn uint64) ([]walparser.BlockLocation, error)

func ReadIncrementalFile added in v0.2.22 

func ReadIncrementalFile(filePath string,
	fileSize int64,
	lsn uint64,
	deltaBitmap *roaring.Bitmap) (fileReader io.ReadCloser, size int64, err error)

func RestoreMissingPages added in v0.2.22 

func RestoreMissingPages(base io.Reader, target ReadWriterAt) error

RestoreMissingPages restores missing pages (zero blocks) of local file with their base backup version

func SelectRelFileBlocks added in v0.2.22 

func SelectRelFileBlocks(bitmap *roaring.Bitmap, relFileID int) *roaring.Bitmap

func SetWalSize added in v0.2.22 

func SetWalSize(sizeMb uint64)

func SortBackupDetails added in v1.1.1 

func SortBackupDetails(backupDetails []BackupDetail)

func ToPartFilename added in v0.2.22 

func ToPartFilename(deltaFilename string) string

func TryFetchTimelineAndLogSegNo added in v0.2.22 

func TryFetchTimelineAndLogSegNo(objectName string) (uint32, uint64, bool)

func ValidateSlotName added in v0.2.22 

func ValidateSlotName(pgSlotName string) (err error)

ValidateSlotName validates pgSlotName to be a valid slot name

func VerifyPagedFileBase added in v0.2.22 

func VerifyPagedFileBase(path string, fileInfo os.FileInfo, pagedFile io.Reader) ([]uint32, error)

VerifyPagedFileBase verifies pages of a standard paged file

func VerifyPagedFileIncrement added in v0.2.22 

func VerifyPagedFileIncrement(path string, fileInfo os.FileInfo, increment io.Reader) ([]uint32, error)

VerifyPagedFileIncrement verifies pages of an increment

func WildcardInfo 

func WildcardInfo(from storage.Folder, to storage.Folder) ([]copy.InfoProvider, error)

func WriteBackupListDetails added in v1.1.1 

func WriteBackupListDetails(backupDetails []BackupDetail, output io.Writer) error

TODO : unit tests

func WritePagesFromIncrement added in v0.2.22 

func WritePagesFromIncrement(increment io.Reader, target ReadWriterAt, overwriteExisting bool) (int64, error)

WritePagesFromIncrement writes pages from delta backup according to diffMap

func WritePrettyBackupListDetails added in v1.1.1 

func WritePrettyBackupListDetails(backupDetails []BackupDetail, output io.Writer)

TODO : unit tests

Types

type Backup added in v0.2.22 

type Backup struct {
	internal.Backup
	SentinelDto      *BackupSentinelDto // used for storage query caching
	FilesMetadataDto *FilesMetadataDto
}

Backup contains information about a valid Postgres backup generated and uploaded by WAL-G.

func NewBackup added in v0.2.22 

func NewBackup(baseBackupFolder storage.Folder, name string) Backup

func ToPgBackup added in v0.2.22 

func ToPgBackup(source internal.Backup) (output Backup)

func (*Backup) FetchMeta added in v0.2.22 

func (backup *Backup) FetchMeta() (ExtendedMetadataDto, error)

func (*Backup) GetFilesToUnwrap added in v0.2.22 

func (backup *Backup) GetFilesToUnwrap(fileMask string) (map[string]bool, error)

func (*Backup) GetSentinel added in v0.2.22 

func (backup *Backup) GetSentinel() (BackupSentinelDto, error)

func (*Backup) GetSentinelAndFilesMetadata added in v1.1.1 

func (backup *Backup) GetSentinelAndFilesMetadata() (BackupSentinelDto, FilesMetadataDto, error)

func (*Backup) GetTarNames added in v0.2.22 

func (backup *Backup) GetTarNames() ([]string, error)

type BackupArguments added in v0.2.22 

type BackupArguments struct {
	// contains filtered or unexported fields
}

BackupArguments holds all arguments parsed from cmd to this handler class

func NewBackupArguments added in v0.2.22 

func NewBackupArguments(pgDataDirectory string, backupsFolder string, isPermanent bool, verifyPageChecksums bool,
	isFullBackup bool, storeAllCorruptBlocks bool, tarBallComposerType TarBallComposerType,
	deltaBaseSelector internal.BackupSelector, userData interface{}) BackupArguments

NewBackupArguments creates a BackupArgument object to hold the arguments from the cmd

type BackupDetail added in v0.2.22 

type BackupDetail struct {
	internal.BackupTime
	ExtendedMetadataDto
}

BackupDetails is used to append ExtendedMetadataDto details to BackupTime struct

func GetBackupDetails added in v0.2.22 

func GetBackupDetails(folder storage.Folder, backupTime internal.BackupTime) (BackupDetail, error)

func GetBackupsDetails added in v0.2.22 

func GetBackupsDetails(folder storage.Folder, backups []internal.BackupTime) ([]BackupDetail, error)

type BackupFileOptions added in v0.2.22 

type BackupFileOptions struct {
	// contains filtered or unexported fields
}

type BackupFileUnwrapper added in v0.2.22 

type BackupFileUnwrapper struct {
	// contains filtered or unexported fields
}

type BackupHandler added in v0.2.22 

type BackupHandler struct {
	// contains filtered or unexported fields
}

BackupHandler is the main struct which is handling the backup process

func NewBackupHandler added in v0.2.22 

func NewBackupHandler(arguments BackupArguments) (bh *BackupHandler, err error)

NewBackupHandler returns a backup handler object, which can handle the backup

func (*BackupHandler) HandleBackupPush added in v0.2.22 

func (bh *BackupHandler) HandleBackupPush()

HandleBackupPush handles the backup being read from Postgres or filesystem and being pushed to the repository TODO : unit tests

type BackupObject added in v1.1.1 

type BackupObject struct {
	storage.Object
	BackupName string
	// contains filtered or unexported fields
}

func (BackupObject) GetBackupName added in v1.1.1 

func (o BackupObject) GetBackupName() string

func (BackupObject) GetBackupTime added in v1.1.1 

func (o BackupObject) GetBackupTime() time.Time

func (BackupObject) GetBaseBackupName added in v1.1.1 

func (o BackupObject) GetBaseBackupName() string

func (BackupObject) GetIncrementFromName added in v1.1.1 

func (o BackupObject) GetIncrementFromName() string

func (BackupObject) IsFullBackup added in v1.1.1 

func (o BackupObject) IsFullBackup() bool

type BackupPgInfo added in v0.2.22 

type BackupPgInfo struct {
	// contains filtered or unexported fields
}

BackupPgInfo holds the PostgreSQL info that the handler queries before running the backup

type BackupSentinelDto added in v0.2.22 

type BackupSentinelDto struct {
	BackupStartLSN    *uint64 `json:"LSN"`
	IncrementFromLSN  *uint64 `json:"DeltaLSN,omitempty"`
	IncrementFrom     *string `json:"DeltaFrom,omitempty"`
	IncrementFullName *string `json:"DeltaFullName,omitempty"`
	IncrementCount    *int    `json:"DeltaCount,omitempty"`

	PgVersion        int     `json:"PgVersion"`
	BackupFinishLSN  *uint64 `json:"FinishLSN"`
	SystemIdentifier *uint64 `json:"SystemIdentifier,omitempty"`

	UncompressedSize int64           `json:"UncompressedSize"`
	CompressedSize   int64           `json:"CompressedSize"`
	TablespaceSpec   *TablespaceSpec `json:"Spec"`

	UserData interface{} `json:"UserData,omitempty"`
}

BackupSentinelDto describes file structure of json sentinel

func NewBackupSentinelDto added in v0.2.22 

func NewBackupSentinelDto(bh *BackupHandler, tbsSpec *TablespaceSpec) BackupSentinelDto

func (*BackupSentinelDto) IsIncremental added in v0.2.22 

func (dto *BackupSentinelDto) IsIncremental() (isIncremental bool)

TODO : unit tests TODO : get rid of panic here IsIncremental checks that sentinel represents delta backup

type BackupSentinelDtoV2 added in v1.1.1 

type BackupSentinelDtoV2 struct {
	BackupSentinelDto
	Version        int       `json:"Version"`
	StartTime      time.Time `json:"StartTime"`
	FinishTime     time.Time `json:"FinishTime"`
	DatetimeFormat string    `json:"DateFmt"`
	Hostname       string    `json:"Hostname"`
	DataDir        string    `json:"DataDir"`
	IsPermanent    bool      `json:"IsPermanent"`
}

BackupSentinelDtoV2 is the future version of the backup sentinel. Basically, it is a union of BackupSentinelDto and ExtendedMetadataDto. Currently, WAL-G only uploads it, but use as the regular BackupSentinelDto. WAL-G will switch to the BackupSentinelDtoV2 in the next major release.

func NewBackupSentinelDtoV2 added in v1.1.1 

func NewBackupSentinelDtoV2(sentinel BackupSentinelDto, meta ExtendedMetadataDto) BackupSentinelDtoV2

type BackupTimeSlicesOrder added in v1.1.1 

type BackupTimeSlicesOrder int
const (
	ByCreationTime BackupTimeSlicesOrder = iota
	ByModificationTime
)

type BackupWorkers added in v0.2.22 

type BackupWorkers struct {
	// contains filtered or unexported fields
}

BackupWorkers holds the external objects that the handler uses to get the backup data / write the backup data

type BgUploader added in v0.2.22 

type BgUploader struct {
	// contains filtered or unexported fields
}

BgUploader represents the state of concurrent WAL upload

func NewBgUploader added in v0.2.22 

func NewBgUploader(walFilePath string,
	maxParallelWorkers int32,
	maxNumUploaded int32,
	uploader *WalUploader,
	preventWalOverwrite bool,
	readyRename bool) *BgUploader

NewBgUploader creates a new BgUploader which looks for WAL files adjacent to walFilePath. maxParallelWorkers and maxNumUploaded limits maximum concurrency and total work done by this BgUploader respectively.

func (*BgUploader) Start added in v0.2.22 

func (b *BgUploader) Start()

Start up checking what's inside archive_status

func (*BgUploader) Stop added in v0.2.22 

func (b *BgUploader) Stop() error

Stop pipeline. Stop can be safely called concurrently and repeatedly.

type Bundle added in v0.2.22 

type Bundle struct {
	Directory string
	Sentinel  *internal.Sentinel

	TarBallComposer TarBallComposer
	TarBallQueue    *internal.TarBallQueue

	Crypter            crypto.Crypter
	Timeline           uint32
	Replica            bool
	IncrementFromLsn   *uint64
	IncrementFromFiles internal.BackupFileList
	DeltaMap           PagedFileDeltaMap
	TablespaceSpec     TablespaceSpec

	TarSizeThreshold int64
	// contains filtered or unexported fields
}

A Bundle represents the directory to be walked. Contains at least one TarBall if walk has started. Each TarBall except for the last one will be at least TarSizeThreshold bytes. The Sentinel is used to ensure complete uploaded backups; in this case, pg_control is used as the sentinel.

func NewBundle added in v0.2.22 

func NewBundle(
	directory string, crypter crypto.Crypter,
	incrementFromLsn *uint64, incrementFromFiles internal.BackupFileList,
	forceIncremental bool, tarSizeThreshold int64,
) *Bundle

TODO: use DiskDataFolder

func (*Bundle) DownloadDeltaMap added in v0.2.22 

func (bundle *Bundle) DownloadDeltaMap(folder storage.Folder, backupStartLSN uint64) error

func (*Bundle) FinishQueue added in v0.2.22 

func (bundle *Bundle) FinishQueue() error

func (*Bundle) GetFiles added in v0.2.22 

func (bundle *Bundle) GetFiles() *sync.Map

func (*Bundle) HandleWalkedFSObject added in v0.2.22 

func (bundle *Bundle) HandleWalkedFSObject(path string, info os.FileInfo, err error) error

TODO : unit tests HandleWalkedFSObject walks files provided by the passed in directory and creates compressed tar members labeled as `part_00i.tar.*`, where '*' is compressor file extension.

To see which files and directories are Skipped, please consult ExcludedFilenames. Excluded directories will be created but their contents will not be included in the tar bundle.

func (*Bundle) NewTarBall added in v0.2.22 

func (bundle *Bundle) NewTarBall(dedicatedUploader bool) internal.TarBall

NewTarBall starts writing new tarball

func (*Bundle) PackTarballs added in v0.2.22 

func (bundle *Bundle) PackTarballs() (TarFileSets, error)

func (*Bundle) SetupComposer added in v0.2.22 

func (bundle *Bundle) SetupComposer(composerMaker TarBallComposerMaker) (err error)

func (*Bundle) StartBackup added in v0.2.22 

func (bundle *Bundle) StartBackup(conn *pgx.Conn,
	backup string) (backupName string, lsn uint64, err error)

TODO : unit tests StartBackup starts a non-exclusive base backup immediately. When finishing the backup, `backup_label` and `tablespace_map` contents are not immediately written to a file but returned instead. Returns empty string and an error if backup fails.

func (*Bundle) StartQueue added in v0.2.22 

func (bundle *Bundle) StartQueue(tarBallMaker internal.TarBallMaker) error

func (*Bundle) UploadPgControl added in v0.2.22 

func (bundle *Bundle) UploadPgControl(compressorFileExtension string) error

TODO : unit tests UploadPgControl should only be called after the rest of the backup is successfully uploaded to S3.

type BundleFiles added in v0.2.22 

type BundleFiles interface {
	AddSkippedFile(tarHeader *tar.Header, fileInfo os.FileInfo)
	AddFile(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool)
	AddFileWithCorruptBlocks(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool,
		corruptedBlocks []uint32, storeAllBlocks bool)
	GetUnderlyingMap() *sync.Map
}

BundleFiles represents the files in the backup that is going to be created

type BytesPerWalSegmentError added in v0.2.22 

type BytesPerWalSegmentError struct {
	// contains filtered or unexported fields
}

func (BytesPerWalSegmentError) Error added in v0.2.22 

func (err BytesPerWalSegmentError) Error() string

type CantDiscardWalDataError added in v0.2.22 

type CantDiscardWalDataError struct {
	// contains filtered or unexported fields
}

func (CantDiscardWalDataError) Error added in v0.2.22 

func (err CantDiscardWalDataError) Error() string

type CantOverwriteWalFileError added in v0.2.22 

type CantOverwriteWalFileError struct {
	// contains filtered or unexported fields
}

func (CantOverwriteWalFileError) Error added in v0.2.22 

func (err CantOverwriteWalFileError) Error() string

type CatchupFileUnwrapper added in v0.2.22 

type CatchupFileUnwrapper struct {
	BackupFileUnwrapper
}

CatchupFileUnwrapper is used for catchup (catchup-push) backups

func (*CatchupFileUnwrapper) UnwrapExistingFile added in v0.2.22 

func (u *CatchupFileUnwrapper) UnwrapExistingFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

func (*CatchupFileUnwrapper) UnwrapNewFile added in v0.2.22 

func (u *CatchupFileUnwrapper) UnwrapNewFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

type Cleaner added in v0.2.22 

type Cleaner interface {
	GetFiles(directory string) ([]string, error)
	Remove(file string)
}

Cleaner interface serves to separate file system logic from prefetch clean logic to make it testable

type ComposeFileInfo added in v0.2.22 

type ComposeFileInfo struct {
	// contains filtered or unexported fields
}

ComposeFileInfo holds data which is required to pack a file to some tarball

func NewComposeFileInfo added in v0.2.22 

func NewComposeFileInfo(path string, fileInfo os.FileInfo, wasInBase, isIncremented bool,
	header *tar.Header) *ComposeFileInfo

type CopyTarBallComposer added in v1.1.1 

type CopyTarBallComposer struct {
	// contains filtered or unexported fields
}

func NewCopyTarBallComposer added in v1.1.1 

func NewCopyTarBallComposer(
	tarBallQueue *internal.TarBallQueue,
	tarBallFilePacker *TarBallFilePacker,
	files *RegularBundleFiles,
	crypter crypto.Crypter,
	prevBackup Backup,
	newBackupName string,
	tarUnchangedFilesCount map[string]int,
	prevFileTar map[string]string,
	prevTarFileSets TarFileSets,
) (*CopyTarBallComposer, error)

func (*CopyTarBallComposer) AddFile added in v1.1.1 

func (c *CopyTarBallComposer) AddFile(info *ComposeFileInfo)

func (*CopyTarBallComposer) AddHeader added in v1.1.1 

func (c *CopyTarBallComposer) AddHeader(fileInfoHeader *tar.Header, info os.FileInfo) error

func (*CopyTarBallComposer) GetFiles added in v1.1.1 

func (c *CopyTarBallComposer) GetFiles() BundleFiles

func (*CopyTarBallComposer) PackTarballs added in v1.1.1 

func (c *CopyTarBallComposer) PackTarballs() (TarFileSets, error)

func (*CopyTarBallComposer) SkipFile added in v1.1.1 

func (c *CopyTarBallComposer) SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)

type CopyTarBallComposerMaker added in v1.1.1 

type CopyTarBallComposerMaker struct {
	// contains filtered or unexported fields
}

func NewCopyTarBallComposerMaker added in v1.1.1 

func NewCopyTarBallComposerMaker(previousBackup Backup, newBackupName string,
	filePackerOptions TarBallFilePackerOptions) *CopyTarBallComposerMaker

func (*CopyTarBallComposerMaker) Make added in v1.1.1 

func (maker *CopyTarBallComposerMaker) Make(bundle *Bundle) (TarBallComposer, error)

type CurBackupInfo added in v0.2.22 

type CurBackupInfo struct {
	// contains filtered or unexported fields
}

CurBackupInfo holds all information that is harvest during the backup process

type DefaultFileUnwrapper added in v0.2.22 

type DefaultFileUnwrapper struct {
	BackupFileUnwrapper
}

DefaultFileUnwrapper is used for default (backup-push) backups

func (*DefaultFileUnwrapper) UnwrapExistingFile added in v0.2.22 

func (u *DefaultFileUnwrapper) UnwrapExistingFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

func (*DefaultFileUnwrapper) UnwrapNewFile added in v0.2.22 

func (u *DefaultFileUnwrapper) UnwrapNewFile(reader io.Reader, header *tar.Header,
	file *os.File) (*FileUnwrapResult, error)

type DeltaFile added in v0.2.22 

type DeltaFile struct {
	Locations []walparser.BlockLocation
	WalParser *walparser.WalParser
}

func LoadDeltaFile added in v0.2.22 

func LoadDeltaFile(reader io.Reader) (*DeltaFile, error)

func NewDeltaFile added in v0.2.22 

func NewDeltaFile(walParser *walparser.WalParser) (*DeltaFile, error)

func (*DeltaFile) Save added in v0.2.22 

func (deltaFile *DeltaFile) Save(writer io.Writer) error

type DeltaFileChanWriter added in v0.2.22 

type DeltaFileChanWriter struct {
	DeltaFile             *DeltaFile
	BlockLocationConsumer chan walparser.BlockLocation
}

func NewDeltaFileChanWriter added in v0.2.22 

func NewDeltaFileChanWriter(deltaFile *DeltaFile) *DeltaFileChanWriter

func (*DeltaFileChanWriter) Consume added in v0.2.22 

func (writer *DeltaFileChanWriter) Consume(waitGroup *sync.WaitGroup)

type DeltaFileManager added in v0.2.22 

type DeltaFileManager struct {
	PartFiles        *internal.LazyCache
	DeltaFileWriters *internal.LazyCache

	CanceledDeltaFiles map[string]bool
	// contains filtered or unexported fields
}

func NewDeltaFileManager added in v0.2.22 

func NewDeltaFileManager(dataFolder fsutil.DataFolder) *DeltaFileManager

func (*DeltaFileManager) CancelRecording added in v0.2.22 

func (manager *DeltaFileManager) CancelRecording(walFilename string)

func (*DeltaFileManager) CombinePartFile added in v0.2.22 

func (manager *DeltaFileManager) CombinePartFile(deltaFilename string, partFile *WalPartFile) error

func (*DeltaFileManager) FlushDeltaFiles added in v0.2.22 

func (manager *DeltaFileManager) FlushDeltaFiles(uploader *internal.Uploader, completedPartFiles map[string]bool)

func (*DeltaFileManager) FlushFiles added in v0.2.22 

func (manager *DeltaFileManager) FlushFiles(uploader *internal.Uploader)

func (*DeltaFileManager) FlushPartFiles added in v0.2.22 

func (manager *DeltaFileManager) FlushPartFiles() (completedPartFiles map[string]bool)

func (*DeltaFileManager) GetBlockLocationConsumer added in v0.2.22 

func (manager *DeltaFileManager) GetBlockLocationConsumer(deltaFilename string) (chan walparser.BlockLocation, error)

func (*DeltaFileManager) GetPartFile added in v0.2.22 

func (manager *DeltaFileManager) GetPartFile(deltaFilename string) (*WalPartFile, error)

type DeltaFileWriterNotFoundError added in v0.2.22 

type DeltaFileWriterNotFoundError struct {
	// contains filtered or unexported fields
}

func (DeltaFileWriterNotFoundError) Error added in v0.2.22 

func (err DeltaFileWriterNotFoundError) Error() string

type DeltaNo added in v0.2.22 

type DeltaNo uint64

type DeprecatedSentinelFields added in v1.1.1 

type DeprecatedSentinelFields struct {
	FilesMetadataDto
	DeltaFromLSN *uint64 `json:"DeltaFromLSN,omitempty"`
}

type ExtendedMetadataDto added in v0.2.22 

type ExtendedMetadataDto struct {
	StartTime        time.Time `json:"start_time"`
	FinishTime       time.Time `json:"finish_time"`
	DatetimeFormat   string    `json:"date_fmt"`
	Hostname         string    `json:"hostname"`
	DataDir          string    `json:"data_dir"`
	PgVersion        int       `json:"pg_version"`
	StartLsn         uint64    `json:"start_lsn"`
	FinishLsn        uint64    `json:"finish_lsn"`
	IsPermanent      bool      `json:"is_permanent"`
	SystemIdentifier *uint64   `json:"system_identifier"`

	UncompressedSize int64 `json:"uncompressed_size"`
	CompressedSize   int64 `json:"compressed_size"`

	UserData interface{} `json:"user_data,omitempty"`
}

Extended metadata should describe backup in more details, but be small enough to be downloaded often

func NewExtendedMetadataDto added in v0.2.22 

func NewExtendedMetadataDto(isPermanent bool, dataDir string, startTime time.Time,
	sentinelDto BackupSentinelDto) (meta ExtendedMetadataDto)

type FetchConfig added in v0.2.22 

type FetchConfig struct {
	// contains filtered or unexported fields
}

func NewFetchConfig added in v0.2.22 

func NewFetchConfig(backupName, dbDataDirectory string, folder storage.Folder, spec *TablespaceSpec,
	filesToUnwrap map[string]bool, skipRedundantTars bool) *FetchConfig

func (*FetchConfig) SkipRedundantFiles added in v0.2.22 

func (fc *FetchConfig) SkipRedundantFiles(unwrapResult *UnwrapResult)

type FileNotExistError added in v0.2.22 

type FileNotExistError struct {
	// contains filtered or unexported fields
}

func (FileNotExistError) Error added in v0.2.22 

func (err FileNotExistError) Error() string

type FileTarInterpreter added in v0.2.22 

type FileTarInterpreter struct {
	DBDataDirectory string
	Sentinel        BackupSentinelDto
	FilesMetadata   FilesMetadataDto
	FilesToUnwrap   map[string]bool
	UnwrapResult    *UnwrapResult
	// contains filtered or unexported fields
}

FileTarInterpreter extracts input to disk.

func NewFileTarInterpreter added in v0.2.22 

func NewFileTarInterpreter(
	dbDataDirectory string, sentinel BackupSentinelDto, filesMetadata FilesMetadataDto,
	filesToUnwrap map[string]bool, createNewIncrementalFiles bool,
) *FileTarInterpreter

func (*FileTarInterpreter) AddFileUnwrapResult added in v0.2.22 

func (tarInterpreter *FileTarInterpreter) AddFileUnwrapResult(result *FileUnwrapResult, fileName string)

func (*FileTarInterpreter) Interpret added in v0.2.22 

func (tarInterpreter *FileTarInterpreter) Interpret(fileReader io.Reader, fileInfo *tar.Header) error

Interpret extracts a tar file to disk and creates needed directories. Returns the first error encountered. Calls fsync after each file is written successfully.

type FileUnwrapResult added in v0.2.22 

type FileUnwrapResult struct {
	FileUnwrapResultType
	// contains filtered or unexported fields
}

func NewCompletedResult added in v0.2.22 

func NewCompletedResult() *FileUnwrapResult

func NewCreatedFromIncrementResult added in v0.2.22 

func NewCreatedFromIncrementResult(missingBlockCount int64) *FileUnwrapResult

func NewSkippedResult added in v0.2.22 

func NewSkippedResult() *FileUnwrapResult

func NewWroteIncrementBlocksResult added in v0.2.22 

func NewWroteIncrementBlocksResult(restoredBlockCount int64) *FileUnwrapResult

type FileUnwrapResultType added in v0.2.22 

type FileUnwrapResultType int
const (
	Completed FileUnwrapResultType = iota + 1
	CreatedFromIncrement
	WroteIncrementBlocks
	Skipped
)

type FileUnwrapperType added in v0.2.22 

type FileUnwrapperType int
const (
	DefaultBackupFileUnwrapper FileUnwrapperType = iota + 1
	CatchupBackupFileUnwrapper
)

type FilesMetadataDto added in v1.1.1 

type FilesMetadataDto struct {
	Files       internal.BackupFileList `json:"Files,omitempty"`
	TarFileSets TarFileSets             `json:"TarFileSets,omitempty"`
}

FilesMetadataDto contains the information about the backup files. It can be pretty large on some databases, sometimes more than 1GB

func NewFilesMetadataDto added in v1.1.1 

func NewFilesMetadataDto(files internal.BackupFileList, tarFileSets TarFileSets) FilesMetadataDto

type GenericMetaFetcher added in v0.2.22 

type GenericMetaFetcher struct{}

func NewGenericMetaFetcher added in v0.2.22 

func NewGenericMetaFetcher() GenericMetaFetcher

func (GenericMetaFetcher) Fetch added in v0.2.22 

func (mf GenericMetaFetcher) Fetch(backupName string, backupFolder storage.Folder) (internal.GenericMetadata, error)

type GenericMetaInteractor added in v0.2.22 

type GenericMetaInteractor struct {
	GenericMetaFetcher
	GenericMetaSetter
}

func NewGenericMetaInteractor added in v0.2.22 

func NewGenericMetaInteractor() GenericMetaInteractor

type GenericMetaSetter added in v0.2.22 

type GenericMetaSetter struct{}

func NewGenericMetaSetter added in v0.2.22 

func NewGenericMetaSetter() GenericMetaSetter

func (GenericMetaSetter) SetIsPermanent added in v0.2.22 

func (ms GenericMetaSetter) SetIsPermanent(backupName string, backupFolder storage.Folder, isPermanent bool) error

func (GenericMetaSetter) SetUserData added in v0.2.22 

func (ms GenericMetaSetter) SetUserData(backupName string, backupFolder storage.Folder, userData interface{}) error

type HistoryFileNotFoundError added in v0.2.22 

type HistoryFileNotFoundError struct {
	// contains filtered or unexported fields
}

func (HistoryFileNotFoundError) Error added in v0.2.22 

func (err HistoryFileNotFoundError) Error() string

type IBackupFileUnwrapper added in v0.2.22 

type IBackupFileUnwrapper interface {
	UnwrapNewFile(reader io.Reader, header *tar.Header, file *os.File) (*FileUnwrapResult, error)
	UnwrapExistingFile(reader io.Reader, header *tar.Header, file *os.File) (*FileUnwrapResult, error)
}

func NewFileUnwrapper added in v0.2.22 

func NewFileUnwrapper(unwrapperType FileUnwrapperType, options *BackupFileOptions) IBackupFileUnwrapper

type IncorrectBackupNameError added in v0.2.22 

type IncorrectBackupNameError struct {
	// contains filtered or unexported fields
}

func (IncorrectBackupNameError) Error added in v0.2.22 

func (err IncorrectBackupNameError) Error() string

type IncorrectLogSegNoError added in v0.2.22 

type IncorrectLogSegNoError struct {
	// contains filtered or unexported fields
}

func (IncorrectLogSegNoError) Error added in v0.2.22 

func (err IncorrectLogSegNoError) Error() string

type IncrementDetailsFetcher added in v0.2.22 

type IncrementDetailsFetcher struct {
	// contains filtered or unexported fields
}

func NewIncrementDetailsFetcher added in v0.2.22 

func NewIncrementDetailsFetcher(backup Backup) *IncrementDetailsFetcher

func (*IncrementDetailsFetcher) Fetch added in v0.2.22 

func (idf *IncrementDetailsFetcher) Fetch() (bool, internal.IncrementDetails, error)

type IncrementalPageReader added in v0.2.22 

type IncrementalPageReader struct {
	PagedFile ioextensions.ReadSeekCloser
	FileSize  int64
	Lsn       uint64
	Next      []byte
	Blocks    []uint32
}

IncrementalPageReader constructs difference map during initialization and than re-read file Diff map may consist of 1Gb/PostgresBlockSize elements == 512Kb

func (*IncrementalPageReader) AdvanceFileReader added in v0.2.22 

func (pageReader *IncrementalPageReader) AdvanceFileReader() error

func (*IncrementalPageReader) Close added in v0.2.22 

func (pageReader *IncrementalPageReader) Close() error

Close IncrementalPageReader

func (*IncrementalPageReader) DeltaBitmapInitialize added in v0.2.22 

func (pageReader *IncrementalPageReader) DeltaBitmapInitialize(deltaBitmap *roaring.Bitmap)

func (*IncrementalPageReader) DrainMoreData added in v0.2.22 

func (pageReader *IncrementalPageReader) DrainMoreData() (succeed bool, err error)

func (*IncrementalPageReader) FullScanInitialize added in v0.2.22 

func (pageReader *IncrementalPageReader) FullScanInitialize() error

func (*IncrementalPageReader) Read added in v0.2.22 

func (pageReader *IncrementalPageReader) Read(p []byte) (n int, err error)

func (*IncrementalPageReader) SelectNewValidPage added in v0.2.22 

func (pageReader *IncrementalPageReader) SelectNewValidPage(pageBytes []byte, blockNo uint32) (valid bool)

SelectNewValidPage checks whether page is valid and if it so, then blockNo is appended to Blocks list

func (*IncrementalPageReader) WriteDiffMapToHeader added in v0.2.22 

func (pageReader *IncrementalPageReader) WriteDiffMapToHeader(headerWriter io.Writer)

WriteDiffMapToHeader is currently used only with buffers, so we don't handle any writing errors

type IntegrityCheckDetails added in v0.2.22 

type IntegrityCheckDetails []*IntegrityScanSegmentSequence

func (IntegrityCheckDetails) NewPlainTextReader added in v0.2.22 

func (sequences IntegrityCheckDetails) NewPlainTextReader() (io.Reader, error)

type IntegrityCheckRunner added in v0.2.22 

type IntegrityCheckRunner struct {
	// contains filtered or unexported fields
}

IntegrityCheckRunner queries the current cluster WAL segment and timeline and travels through WAL segments in storage in reversed chronological order (starting from that segment) to find any missing WAL segments that could potentially fail the PITR procedure

func NewIntegrityCheckRunner added in v0.2.22 

func NewIntegrityCheckRunner(
	rootFolder storage.Folder,
	walFolderFilenames []string,
	currentWalSegment WalSegmentDescription,
) (IntegrityCheckRunner, error)

func (IntegrityCheckRunner) Run added in v0.2.22 

func (check IntegrityCheckRunner) Run() (WalVerifyCheckResult, error)

func (IntegrityCheckRunner) Type added in v0.2.22 

func (check IntegrityCheckRunner) Type() WalVerifyCheckType

type IntegrityScanSegmentSequence added in v0.2.22 

type IntegrityScanSegmentSequence struct {
	TimelineID    uint32               `json:"timeline_id"`
	StartSegment  string               `json:"start_segment"`
	EndSegment    string               `json:"end_segment"`
	SegmentsCount int                  `json:"segments_count"`
	Status        ScannedSegmentStatus `json:"status"`
}

IntegrityScanSegmentSequence is a continuous sequence of segments with the same timeline and Status

type InvalidBlockError added in v0.2.22 

type InvalidBlockError struct {
	// contains filtered or unexported fields
}

InvalidBlockError indicates that file contain invalid page and cannot be archived incrementally

func (InvalidBlockError) Error added in v0.2.22 

func (err InvalidBlockError) Error() string

type InvalidIncrementFileHeaderError added in v0.2.22 

type InvalidIncrementFileHeaderError struct {
	// contains filtered or unexported fields
}

func (InvalidIncrementFileHeaderError) Error added in v0.2.22 

func (err InvalidIncrementFileHeaderError) Error() string

type InvalidWalFileMagicError added in v0.2.22 

type InvalidWalFileMagicError struct {
	// contains filtered or unexported fields
}

func (InvalidWalFileMagicError) Error added in v0.2.22 

func (err InvalidWalFileMagicError) Error() string

type NilWalParserError added in v0.2.22 

type NilWalParserError struct {
	// contains filtered or unexported fields
}

func (NilWalParserError) Error added in v0.2.22 

func (err NilWalParserError) Error() string

type NoBitmapFoundError added in v0.2.22 

type NoBitmapFoundError struct {
	// contains filtered or unexported fields
}

func (NoBitmapFoundError) Error added in v0.2.22 

func (err NoBitmapFoundError) Error() string

type NoCorrectBackupFoundError added in v0.2.22 

type NoCorrectBackupFoundError struct {
	// contains filtered or unexported fields
}

func (NoCorrectBackupFoundError) Error added in v0.2.22 

func (err NoCorrectBackupFoundError) Error() string

type NoPostgresVersionError added in v0.2.22 

type NoPostgresVersionError struct {
	// contains filtered or unexported fields
}

func (NoPostgresVersionError) Error added in v0.2.22 

func (err NoPostgresVersionError) Error() string

type NonEmptyDBDataDirectoryError added in v0.2.22 

type NonEmptyDBDataDirectoryError struct {
	// contains filtered or unexported fields
}

func NewNonEmptyDBDataDirectoryError added in v0.2.22 

func NewNonEmptyDBDataDirectoryError(dbDataDirectory string) NonEmptyDBDataDirectoryError

func (NonEmptyDBDataDirectoryError) Error added in v0.2.22 

func (err NonEmptyDBDataDirectoryError) Error() string

type NotWalFilenameError added in v0.2.22 

type NotWalFilenameError struct {
	// contains filtered or unexported fields
}

func (NotWalFilenameError) Error added in v0.2.22 

func (err NotWalFilenameError) Error() string
type PageHeader struct {
	// contains filtered or unexported fields
}

type PagedFileDeltaMap added in v0.2.22 

type PagedFileDeltaMap map[walparser.RelFileNode]*roaring.Bitmap

func NewPagedFileDeltaMap added in v0.2.22 

func NewPagedFileDeltaMap() PagedFileDeltaMap

func (*PagedFileDeltaMap) AddLocationToDelta added in v0.2.22 

func (deltaMap *PagedFileDeltaMap) AddLocationToDelta(location walparser.BlockLocation)

func (*PagedFileDeltaMap) AddLocationsToDelta added in v0.2.22 

func (deltaMap *PagedFileDeltaMap) AddLocationsToDelta(locations []walparser.BlockLocation)

func (*PagedFileDeltaMap) GetDeltaBitmapFor added in v0.2.22 

func (deltaMap *PagedFileDeltaMap) GetDeltaBitmapFor(filePath string) (*roaring.Bitmap, error)

TODO : unit test no bitmap found

type PgChecksummablePage added in v0.2.22 

type PgChecksummablePage [DatabasePageSize / int64(NSums*sizeofInt32)][NSums]uint32

PgChecksummablePage represents single database page divided by NSums blocks for checksum calculation

type PgControlNotFoundError added in v0.2.22 

type PgControlNotFoundError struct {
	// contains filtered or unexported fields
}

func (PgControlNotFoundError) Error added in v0.2.22 

func (err PgControlNotFoundError) Error() string

type PgDatabaseInfo added in v0.2.22 

type PgDatabaseInfo struct {
	// contains filtered or unexported fields
}

type PgDatabasePage added in v0.2.22 

type PgDatabasePage [DatabasePageSize]byte

PgDatabasePage represents single database page

type PgQueryRunner added in v0.2.22 

type PgQueryRunner struct {
	Connection       *pgx.Conn
	Version          int
	SystemIdentifier *uint64
}

PgQueryRunner is implementation for controlling PostgreSQL 9.0+

func NewPgQueryRunner added in v1.1.1 

func NewPgQueryRunner(conn *pgx.Conn) (*PgQueryRunner, error)

NewPgQueryRunner builds QueryRunner from available connection

func (*PgQueryRunner) BuildGetDatabasesQuery added in v0.2.22 

func (queryRunner *PgQueryRunner) BuildGetDatabasesQuery() (string, error)

BuildGetDatabasesQuery formats a query to get all databases in cluster which are allowed to connect

func (*PgQueryRunner) BuildStartBackup added in v0.2.22 

func (queryRunner *PgQueryRunner) BuildStartBackup() (string, error)

BuildStartBackup formats a query that starts backup according to server features and version

func (*PgQueryRunner) BuildStatisticsQuery added in v0.2.22 

func (queryRunner *PgQueryRunner) BuildStatisticsQuery() (string, error)

BuildStatisticsQuery formats a query that fetch relations statistics from database

func (*PgQueryRunner) BuildStopBackup added in v0.2.22 

func (queryRunner *PgQueryRunner) BuildStopBackup() (string, error)

BuildStopBackup formats a query that stops backup according to server features and version

func (*PgQueryRunner) GetDataDir added in v0.2.22 

func (queryRunner *PgQueryRunner) GetDataDir() (dataDir string, err error)

GetDataDir reads the wals segment size (in bytes) and converts it to uint64 TODO: Unittest

func (*PgQueryRunner) GetParameter added in v0.2.22 

func (queryRunner *PgQueryRunner) GetParameter(parameterName string) (string, error)

GetParameter reads a Postgres setting TODO: Unittest

func (*PgQueryRunner) GetPhysicalSlotInfo added in v0.2.22 

func (queryRunner *PgQueryRunner) GetPhysicalSlotInfo(slotName string) (PhysicalSlot, error)

GetPhysicalSlotInfo reads information on a physical replication slot TODO: Unittest

func (*PgQueryRunner) GetWalSegmentBytes added in v0.2.22 

func (queryRunner *PgQueryRunner) GetWalSegmentBytes() (segBlocks uint64, err error)

GetWalSegmentBytes reads the wals segment size (in bytes) and converts it to uint64 TODO: Unittest

func (*PgQueryRunner) IsTablespaceMapExists added in v0.2.22 

func (queryRunner *PgQueryRunner) IsTablespaceMapExists() bool

tablespace map does not exist in < 9.6 TODO: Unittest

type PgRelationStat added in v0.2.22 

type PgRelationStat struct {
	// contains filtered or unexported fields
}

type PhysicalSlot added in v0.2.22 

type PhysicalSlot struct {
	Name       string
	Exists     bool
	Active     bool
	RestartLSN pglogrepl.LSN
}

The PhysicalSlot represents a Physical Replication Slot.

func NewPhysicalSlot added in v0.2.22 

func NewPhysicalSlot(name string, exists bool, active bool, restartLSN string) (slot PhysicalSlot, err error)

NewPhysicalSlot is a helper function to declare a new PhysicalSlot object and set vaues from the parsed arguments

type PrevBackupInfo added in v0.2.22 

type PrevBackupInfo struct {
	// contains filtered or unexported fields
}

PrevBackupInfo holds all information that is harvest during the backup process

type ProcessMessageResult added in v0.2.22 

type ProcessMessageResult int

The ProcessMessageResult is an enum representing possible results from the methods processing the messages as received from Postgres into the wal segment. 

const (
	ProcessMessageOK ProcessMessageResult = iota
	ProcessMessageUnknown
	ProcessMessageCopyDone
	ProcessMessageReplyRequested
	ProcessMessageSegmentGap
	ProcessMessageMismatch
)

These are the multiple results that the methods can return

type QueryRunner added in v0.2.22 

type QueryRunner interface {
	// This call should inform the database that we are going to copy cluster's contents
	// Should fail if backup is currently impossible
	StartBackup(backup string) (string, string, bool, error)
	// Inform database that contents are copied, get information on backup
	StopBackup() (string, string, string, error)
}

The QueryRunner interface for controlling database during backup

type RatedComposeFileInfo added in v0.2.22 

type RatedComposeFileInfo struct {
	ComposeFileInfo
	// contains filtered or unexported fields
}

type RatingTarBallComposer added in v0.2.22 

type RatingTarBallComposer struct {
	// contains filtered or unexported fields
}

RatingTarBallComposer receives all files and tar headers that are going to be written to the backup, and composes the tarballs by placing the files with similar update rating in the same tarballs

func NewRatingTarBallComposer added in v0.2.22 

func NewRatingTarBallComposer(
	tarSizeThreshold uint64, updateRatingEvaluator internal.ComposeRatingEvaluator,
	incrementBaseLsn *uint64, deltaMap PagedFileDeltaMap, tarBallQueue *internal.TarBallQueue,
	crypter crypto.Crypter, fileStats RelFileStatistics, bundleFiles BundleFiles, packer *TarBallFilePacker,
) (*RatingTarBallComposer, error)

func (*RatingTarBallComposer) AddFile added in v0.2.22 

func (c *RatingTarBallComposer) AddFile(info *ComposeFileInfo)

func (*RatingTarBallComposer) AddHeader added in v0.2.22 

func (c *RatingTarBallComposer) AddHeader(fileInfoHeader *tar.Header, info os.FileInfo) error

func (*RatingTarBallComposer) GetFiles added in v0.2.22 

func (c *RatingTarBallComposer) GetFiles() BundleFiles

func (*RatingTarBallComposer) PackTarballs added in v0.2.22 

func (c *RatingTarBallComposer) PackTarballs() (TarFileSets, error)

func (*RatingTarBallComposer) SkipFile added in v0.2.22 

func (c *RatingTarBallComposer) SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)

type RatingTarBallComposerMaker added in v0.2.22 

type RatingTarBallComposerMaker struct {
	// contains filtered or unexported fields
}

func NewRatingTarBallComposerMaker added in v0.2.22 

func NewRatingTarBallComposerMaker(relFileStats RelFileStatistics,
	filePackerOptions TarBallFilePackerOptions) (*RatingTarBallComposerMaker, error)

func (*RatingTarBallComposerMaker) Make added in v0.2.22 

func (maker *RatingTarBallComposerMaker) Make(bundle *Bundle) (TarBallComposer, error)

type ReachedStopSegmentError added in v0.2.22 

type ReachedStopSegmentError struct {
	// contains filtered or unexported fields
}

func (ReachedStopSegmentError) Error added in v0.2.22 

func (err ReachedStopSegmentError) Error() string

type ReadWriterAt added in v0.2.22 

type ReadWriterAt interface {
	io.ReaderAt
	io.WriterAt
	Size() int64
	Name() string
}

func NewReadWriterAtFrom added in v0.2.22 

func NewReadWriterAtFrom(file *os.File) (ReadWriterAt, error)

type ReadWriterAtFileImpl added in v0.2.22 

type ReadWriterAtFileImpl struct {
	*os.File
	// contains filtered or unexported fields
}

func (*ReadWriterAtFileImpl) Size added in v0.2.22 

func (rw *ReadWriterAtFileImpl) Size() int64

type RegularBundleFiles added in v0.2.22 

type RegularBundleFiles struct {
	sync.Map
}

func (*RegularBundleFiles) AddFile added in v0.2.22 

func (files *RegularBundleFiles) AddFile(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool)

func (*RegularBundleFiles) AddFileWithCorruptBlocks added in v0.2.22 

func (files *RegularBundleFiles) AddFileWithCorruptBlocks(tarHeader *tar.Header, fileInfo os.FileInfo,
	isIncremented bool, corruptedBlocks []uint32, storeAllBlocks bool)

func (*RegularBundleFiles) AddSkippedFile added in v0.2.22 

func (files *RegularBundleFiles) AddSkippedFile(tarHeader *tar.Header, fileInfo os.FileInfo)

func (*RegularBundleFiles) GetUnderlyingMap added in v0.2.22 

func (files *RegularBundleFiles) GetUnderlyingMap() *sync.Map

type RegularTarBallComposer added in v0.2.22 

type RegularTarBallComposer struct {
	// contains filtered or unexported fields
}

func NewRegularTarBallComposer added in v0.2.22 

func NewRegularTarBallComposer(
	tarBallQueue *internal.TarBallQueue,
	tarBallFilePacker *TarBallFilePacker,
	files *RegularBundleFiles,
	crypter crypto.Crypter,
) *RegularTarBallComposer

func (*RegularTarBallComposer) AddFile added in v0.2.22 

func (c *RegularTarBallComposer) AddFile(info *ComposeFileInfo)

func (*RegularTarBallComposer) AddHeader added in v0.2.22 

func (c *RegularTarBallComposer) AddHeader(fileInfoHeader *tar.Header, info os.FileInfo) error

func (*RegularTarBallComposer) GetFiles added in v0.2.22 

func (c *RegularTarBallComposer) GetFiles() BundleFiles

func (*RegularTarBallComposer) PackTarballs added in v0.2.22 

func (c *RegularTarBallComposer) PackTarballs() (TarFileSets, error)

func (*RegularTarBallComposer) SkipFile added in v0.2.22 

func (c *RegularTarBallComposer) SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)

type RegularTarBallComposerMaker added in v0.2.22 

type RegularTarBallComposerMaker struct {
	// contains filtered or unexported fields
}

func NewRegularTarBallComposerMaker added in v0.2.22 

func NewRegularTarBallComposerMaker(filePackerOptions TarBallFilePackerOptions) *RegularTarBallComposerMaker

func (*RegularTarBallComposerMaker) Make added in v0.2.22 

func (maker *RegularTarBallComposerMaker) Make(bundle *Bundle) (TarBallComposer, error)

type RelFileStatistics added in v0.2.22 

type RelFileStatistics map[walparser.RelFileNode]PgRelationStat

type ScannedSegmentDescription added in v0.2.22 

type ScannedSegmentDescription struct {
	WalSegmentDescription
	// contains filtered or unexported fields
}

type ScannedSegmentStatus added in v0.2.22 

type ScannedSegmentStatus int
const (
	// Surely lost missing segment
	Lost ScannedSegmentStatus = iota + 1
	// Missing but probably still uploading segment
	ProbablyUploading
	// Missing but probably delayed segment
	ProbablyDelayed
	// Segment exists in storage
	Found
)

func (ScannedSegmentStatus) MarshalText added in v0.2.22 

func (status ScannedSegmentStatus) MarshalText() ([]byte, error)

MarshalText marshals the ScannedSegmentStatus enum as a string

func (ScannedSegmentStatus) String added in v0.2.22 

func (status ScannedSegmentStatus) String() string

type SegmentScanConfig added in v0.2.22 

type SegmentScanConfig struct {
	UnlimitedScan bool
	// ScanSegmentsLimit is used in case of UnlimitedScan is set to false
	ScanSegmentsLimit       int
	StopOnFirstFoundSegment bool
	// MissingSegmentStatus is set to all missing segments encountered during scan
	MissingSegmentStatus ScannedSegmentStatus
}

SegmentScanConfig is used to configure the single Scan() call of the WalSegmentScanner

type SkippedFileError added in v0.2.22 

type SkippedFileError struct {
	// contains filtered or unexported fields
}

func (SkippedFileError) Error added in v0.2.22 

func (err SkippedFileError) Error() string

type StatBundleFiles added in v0.2.22 

type StatBundleFiles struct {
	sync.Map
	// contains filtered or unexported fields
}

StatBundleFiles contains the bundle files. Additionally, it calculates and stores the updates count for each added file

func (*StatBundleFiles) AddFile added in v0.2.22 

func (files *StatBundleFiles) AddFile(tarHeader *tar.Header, fileInfo os.FileInfo, isIncremented bool)

func (*StatBundleFiles) AddFileWithCorruptBlocks added in v0.2.22 

func (files *StatBundleFiles) AddFileWithCorruptBlocks(tarHeader *tar.Header,
	fileInfo os.FileInfo,
	isIncremented bool,
	corruptedBlocks []uint32,
	storeAllBlocks bool)

func (*StatBundleFiles) AddSkippedFile added in v0.2.22 

func (files *StatBundleFiles) AddSkippedFile(tarHeader *tar.Header, fileInfo os.FileInfo)

func (*StatBundleFiles) GetUnderlyingMap added in v0.2.22 

func (files *StatBundleFiles) GetUnderlyingMap() *sync.Map

type StreamingBaseBackup added in v0.2.22 

type StreamingBaseBackup struct {
	TimeLine uint32
	StartLSN pglogrepl.LSN
	EndLSN   pglogrepl.LSN

	UncompressedSize int64

	Files internal.BackupFileList
	// contains filtered or unexported fields
}

The StreamingBaseBackup object represents a Postgres BASE_BACKUP, connecting to Postgres, and streaming backup data. For every tablespace, all files are combined in a tar format and streamed in a CopyData stream.

func NewStreamingBaseBackup added in v0.2.22 

func NewStreamingBaseBackup(pgDataDir string, maxTarSize int64, pgConn *pgconn.PgConn) (bb *StreamingBaseBackup)

NewStreamingBaseBackup will define a new StreamingBaseBackup object

func (*StreamingBaseBackup) BackupName added in v0.2.22 

func (bb *StreamingBaseBackup) BackupName() string

FolderName returns the name of the folder where the backup should be stored.

func (*StreamingBaseBackup) FileName added in v0.2.22 

func (bb *StreamingBaseBackup) FileName() string

Name returns the filename of a tablespace backup file. This is used by the WalUploader to set the name of the destination file during upload of the backup file.

func (*StreamingBaseBackup) Finish added in v0.2.22 

func (bb *StreamingBaseBackup) Finish() (err error)

Finish will wrap up a backup after finalizing upload.

func (*StreamingBaseBackup) GetTablespaceSpec added in v0.2.22 

func (bb *StreamingBaseBackup) GetTablespaceSpec() *TablespaceSpec

Name returns the filename of a tablespace backup file. This is used by the WalUploader to set the name of the destination file during upload of the backup file.

func (*StreamingBaseBackup) Path added in v0.2.22 

func (bb *StreamingBaseBackup) Path() string

FolderName returns the name of the folder where the backup should be stored.

func (*StreamingBaseBackup) Read added in v0.2.22 

func (bb *StreamingBaseBackup) Read(p []byte) (n int, err error)

Read makes the StreamingBaseBackup an io.Reader, to be handled by WalUploader.UploadWalFile written to a file.

func (*StreamingBaseBackup) Start added in v0.2.22 

func (bb *StreamingBaseBackup) Start(verifyChecksum bool, diskLimit int32) (err error)

Start will start a base_backup read the backup info, and prepare for uploading tar files

func (*StreamingBaseBackup) Upload added in v0.2.22 

func (bb *StreamingBaseBackup) Upload(uploader *WalUploader) (err error)

Upload will read all tar files from Postgres, and use the uploader to upload to the backup location

type TablespaceLocation added in v0.2.22 

type TablespaceLocation struct {
	Location string `json:"loc"`
	Symlink  string `json:"link"`
}

type TablespaceSpec added in v0.2.22 

type TablespaceSpec struct {
	// contains filtered or unexported fields
}

The mandatory keys for this map are "base_prefix" and "tablespaces". "base_prefix" contains Location of pg_data folder. "tablespaces" contains array of keys, which also happen to be names of tablespace folders. The rest keys should be these names of tablespace folders and values should be TablespaceLocation structs.

func NewTablespaceSpec added in v0.2.22 

func NewTablespaceSpec(basePrefix string) TablespaceSpec

func (*TablespaceSpec) BasePrefix added in v0.2.22 

func (spec *TablespaceSpec) BasePrefix() (string, bool)

func (*TablespaceSpec) MarshalJSON added in v0.2.22 

func (spec *TablespaceSpec) MarshalJSON() ([]byte, error)

func (*TablespaceSpec) TablespaceNames added in v0.2.22 

func (spec *TablespaceSpec) TablespaceNames() []string

func (*TablespaceSpec) UnmarshalJSON added in v0.2.22 

func (spec *TablespaceSpec) UnmarshalJSON(b []byte) error

type TarBallComposer added in v0.2.22 

type TarBallComposer interface {
	AddFile(info *ComposeFileInfo)
	AddHeader(header *tar.Header, fileInfo os.FileInfo) error
	SkipFile(tarHeader *tar.Header, fileInfo os.FileInfo)
	PackTarballs() (TarFileSets, error)
	GetFiles() BundleFiles
}

TarBallComposer is used to compose files into tarballs.

type TarBallComposerMaker added in v0.2.22 

type TarBallComposerMaker interface {
	Make(bundle *Bundle) (TarBallComposer, error)
}

TarBallComposerMaker is used to make an instance of TarBallComposer

func NewTarBallComposerMaker added in v0.2.22 

func NewTarBallComposerMaker(composerType TarBallComposerType, conn *pgx.Conn,
	folder storage.Folder, newBackupName string,
	filePackOptions TarBallFilePackerOptions) (TarBallComposerMaker, error)

type TarBallComposerType added in v0.2.22 

type TarBallComposerType int
const (
	RegularComposer TarBallComposerType = iota + 1
	RatingComposer
	CopyComposer
)

type TarBallFilePacker added in v0.2.22 

type TarBallFilePacker struct {
	// contains filtered or unexported fields
}

TarBallFilePacker is used to pack bundle file into tarball.

func (*TarBallFilePacker) PackFileIntoTar added in v0.2.22 

func (p *TarBallFilePacker) PackFileIntoTar(cfi *ComposeFileInfo, tarBall internal.TarBall) error

TODO : unit tests

func (*TarBallFilePacker) UpdateDeltaMap added in v0.2.22 

func (p *TarBallFilePacker) UpdateDeltaMap(deltaMap PagedFileDeltaMap)

type TarBallFilePackerOptions added in v0.2.22 

type TarBallFilePackerOptions struct {
	// contains filtered or unexported fields
}

func NewTarBallFilePackerOptions added in v0.2.22 

func NewTarBallFilePackerOptions(verifyPageChecksums, storeAllCorruptBlocks bool) TarBallFilePackerOptions

type TarFileSets added in v0.2.22 

type TarFileSets map[string][]string

type TarFilesCollection added in v0.2.22 

type TarFilesCollection struct {
	// contains filtered or unexported fields
}

TarFilesCollection stores the files which are going to be written to the same tarball

func (*TarFilesCollection) AddFile added in v0.2.22 

func (collection *TarFilesCollection) AddFile(file *RatedComposeFileInfo)

type TarSizeError added in v0.2.22 

type TarSizeError struct {
	// contains filtered or unexported fields
}

func (TarSizeError) Error added in v0.2.22 

func (err TarSizeError) Error() string

type TarballStreamer added in v0.2.22 

type TarballStreamer struct {

	// files to write to extra tar (teeTar)
	Tee []string

	// io buffer where writer writes data to
	TeeIo *bytes.Buffer

	// list of remaps, remapping input file names to output file names
	Remaps TarballStreamerRemaps
	// list of processed files
	Files internal.BackupFileList
	// contains filtered or unexported fields
}

TarballStreamer is used to modify tar files which are received streaming. Two modifications are: * remap: change (some of) the paths for files in the tar file, and * tee: copy some of the files to a second tar file In addition TarballStreamer maintains a list of files with their info

func NewTarballStreamer added in v0.2.22 

func NewTarballStreamer(input io.Reader, maxTarSize int64) (streamer *TarballStreamer)

func (*TarballStreamer) NextInputFile added in v0.2.22 

func (streamer *TarballStreamer) NextInputFile() (err error)

NextFile is what makes the TarballStreamer move to the next file.

func (*TarballStreamer) Read added in v0.2.22 

func (streamer *TarballStreamer) Read(p []byte) (n int, err error)

Read is what makes the TarballStreamer an io.Reader, which can be handled by WalUploader.UploadFile.

type TarballStreamerRemap added in v0.2.22 

type TarballStreamerRemap struct {
	// contains filtered or unexported fields
}

func NewTarballStreamerRemap added in v0.2.22 

func NewTarballStreamerRemap(from string, to string) (tsr *TarballStreamerRemap, err error)

type TarballStreamerRemaps added in v0.2.22 

type TarballStreamerRemaps []TarballStreamerRemap

type TimeLineHistFile added in v0.2.22 

type TimeLineHistFile struct {
	TimeLineID uint32
	Filename   string
	// contains filtered or unexported fields
}

The TimeLineHistFile struct represents a TimeLineHistory file containing TimeLineHistFileRows. Since TimeLineHistFileRows are only parsed 0 or 1 rimes, the data is only preserved as []byte and parsed to TimeLineHistFileRows when required.

func NewTimeLineHistFile added in v0.2.22 

func NewTimeLineHistFile(timelineid uint32, filename string, body []byte) (TimeLineHistFile, error)

NewTimeLineHistFile is a helper function to define a new TimeLineHistFile

func (TimeLineHistFile) LSNToTimeLine added in v0.2.22 

func (tlh TimeLineHistFile) LSNToTimeLine(lsn pglogrepl.LSN) (uint32, error)

LSNToTimeLine uses rows() to get all TimeLineHistFileRows and from those rows get the timeline that a LS belongs too.

func (TimeLineHistFile) Name added in v0.2.22 

func (tlh TimeLineHistFile) Name() string

Name returns the filename of this wal segment. This is a convenience function used by the WalUploader.

func (TimeLineHistFile) Read added in v0.2.22 

func (tlh TimeLineHistFile) Read(p []byte) (n int, err error)

Read is what makes the WalSegment an io.Reader, which can be handled by WalUploader.UploadWalFile.

type TimeLineHistFileRow added in v0.2.22 

type TimeLineHistFileRow struct {
	TimeLineID uint32
	StartLSN   pglogrepl.LSN
	Comment    string
}

The TimeLineHistFileRow struct represents one line in the TimeLineHistory file

type TimelineCheckDetails added in v0.2.22 

type TimelineCheckDetails struct {
	CurrentTimelineID        uint32 `json:"current_timeline_id"`
	HighestStorageTimelineID uint32 `json:"highest_storage_timeline_id"`
}

func (TimelineCheckDetails) NewPlainTextReader added in v0.2.22 

func (details TimelineCheckDetails) NewPlainTextReader() (io.Reader, error)

type TimelineCheckRunner added in v0.2.22 

type TimelineCheckRunner struct {
	// contains filtered or unexported fields
}

TimelineCheckRunner is used to verify that the current timeline is the highest among the storage timelines

func NewTimelineCheckRunner added in v0.2.22 

func NewTimelineCheckRunner(walFolderFilenames []string,
	currentSegment WalSegmentDescription) (TimelineCheckRunner, error)

func (TimelineCheckRunner) Name added in v0.2.22 

func (check TimelineCheckRunner) Name() string

func (TimelineCheckRunner) Run added in v0.2.22 

func (check TimelineCheckRunner) Run() (WalVerifyCheckResult, error)

func (TimelineCheckRunner) Type added in v0.2.22 

func (check TimelineCheckRunner) Type() WalVerifyCheckType

type TimelineHistoryRecord added in v0.2.22 

type TimelineHistoryRecord struct {
	// contains filtered or unexported fields
}

TimelineHistoryRecord represents entry in .history file

func NewTimelineHistoryRecord added in v0.2.22 

func NewTimelineHistoryRecord(timeline uint32, lsn uint64, comment string) *TimelineHistoryRecord

type TimelineInfo added in v0.2.22 

type TimelineInfo struct {
	ID               uint32          `json:"id"`
	ParentID         uint32          `json:"parent_id"`
	SwitchPointLsn   uint64          `json:"switch_point_lsn"`
	StartSegment     string          `json:"start_segment"`
	EndSegment       string          `json:"end_segment"`
	SegmentsCount    int             `json:"segments_count"`
	MissingSegments  []string        `json:"missing_segments"`
	Backups          []*BackupDetail `json:"backups,omitempty"`
	SegmentRangeSize uint64          `json:"segment_range_size"`
	Status           string          `json:"status"`
}

TimelineInfo contains information about some timeline in storage

func NewTimelineInfo added in v0.2.22 

func NewTimelineInfo(walSegments *WalSegmentsSequence, historyRecords []*TimelineHistoryRecord) (*TimelineInfo, error)

type UnexpectedTarDataError added in v0.2.22 

type UnexpectedTarDataError struct {
	// contains filtered or unexported fields
}

func (UnexpectedTarDataError) Error added in v0.2.22 

func (err UnexpectedTarDataError) Error() string

type UnknownIncrementFileHeaderError added in v0.2.22 

type UnknownIncrementFileHeaderError struct {
	// contains filtered or unexported fields
}

func (UnknownIncrementFileHeaderError) Error added in v0.2.22 

func (err UnknownIncrementFileHeaderError) Error() string

type UnknownTableSpaceError added in v0.2.22 

type UnknownTableSpaceError struct {
	// contains filtered or unexported fields
}

func (UnknownTableSpaceError) Error added in v0.2.22 

func (err UnknownTableSpaceError) Error() string

type UnknownWalVerifyCheckError added in v0.2.22 

type UnknownWalVerifyCheckError struct {
	// contains filtered or unexported fields
}

func NewUnknownWalVerifyCheckError added in v0.2.22 

func NewUnknownWalVerifyCheckError(checkType WalVerifyCheckType) UnknownWalVerifyCheckError

func (UnknownWalVerifyCheckError) Error added in v0.2.22 

func (err UnknownWalVerifyCheckError) Error() string

type UnsupportedPostgresVersionError added in v0.2.22 

type UnsupportedPostgresVersionError struct {
	// contains filtered or unexported fields
}

func (UnsupportedPostgresVersionError) Error added in v0.2.22 

func (err UnsupportedPostgresVersionError) Error() string

type UnwrapResult added in v0.2.22 

type UnwrapResult struct {
	// contains filtered or unexported fields
}

UnwrapResult stores information about the result of single backup unwrap operation

type WalDeltaRecorder added in v0.2.22 

type WalDeltaRecorder struct {
	// contains filtered or unexported fields
}

func NewWalDeltaRecorder added in v0.2.22 

func NewWalDeltaRecorder(blockLocationConsumer chan walparser.BlockLocation) *WalDeltaRecorder

type WalDeltaRecordingReader added in v0.2.22 

type WalDeltaRecordingReader struct {
	PageReader       walparser.WalPageReader
	WalParser        walparser.WalParser
	PageDataLeftover []byte
	Recorder         *WalDeltaRecorder
	// contains filtered or unexported fields
}

In case of recording error WalDeltaRecordingReader stops recording, but continues reading data correctly

func NewWalDeltaRecordingReader added in v0.2.22 

func NewWalDeltaRecordingReader(walFileReader io.Reader,
	walFilename string,
	manager *DeltaFileManager) (*WalDeltaRecordingReader, error)

func (*WalDeltaRecordingReader) Close added in v0.2.22 

func (reader *WalDeltaRecordingReader) Close() error

func (*WalDeltaRecordingReader) Read added in v0.2.22 

func (reader *WalDeltaRecordingReader) Read(p []byte) (n int, err error)

func (*WalDeltaRecordingReader) RecordBlockLocationsFromPage added in v0.2.22 

func (reader *WalDeltaRecordingReader) RecordBlockLocationsFromPage() error

type WalMetadataDescription added in v0.2.22 

type WalMetadataDescription struct {
	CreatedTime    time.Time `json:"created_time"`
	DatetimeFormat string    `json:"date_fmt"`
}

type WalMetadataUploader added in v0.2.22 

type WalMetadataUploader struct {
	// contains filtered or unexported fields
}

func NewWalMetadataUploader added in v0.2.22 

func NewWalMetadataUploader(walMetadataSetting string) (*WalMetadataUploader, error)

func (*WalMetadataUploader) UploadWalMetadata added in v0.2.22 

func (u *WalMetadataUploader) UploadWalMetadata(walFileName string, createdTime time.Time, uploader *internal.Uploader) error

type WalPart added in v0.2.22 

type WalPart struct {
	// contains filtered or unexported fields
}

func LoadWalPart added in v0.2.22 

func LoadWalPart(reader io.Reader) (*WalPart, error)

func NewWalPart added in v0.2.22 

func NewWalPart(dataType WalPartDataType, id uint8, data []byte) *WalPart

func (*WalPart) Save added in v0.2.22 

func (part *WalPart) Save(writer io.Writer) error

type WalPartDataType added in v0.2.22 

type WalPartDataType uint8
const (
	PreviousWalHeadType WalPartDataType = 0
	WalTailType         WalPartDataType = 1
	WalHeadType         WalPartDataType = 2
)

type WalPartFile added in v0.2.22 

type WalPartFile struct {
	WalTails        [][]byte
	PreviousWalHead []byte
	WalHeads        [][]byte
}

func LoadPartFile added in v0.2.22 

func LoadPartFile(reader io.Reader) (*WalPartFile, error)

func NewWalPartFile added in v0.2.22 

func NewWalPartFile() *WalPartFile

func (*WalPartFile) CombineRecords added in v0.2.22 

func (partFile *WalPartFile) CombineRecords() ([]walparser.XLogRecord, error)

func (*WalPartFile) IsComplete added in v0.2.22 

func (partFile *WalPartFile) IsComplete() bool

func (*WalPartFile) Save added in v0.2.22 

func (partFile *WalPartFile) Save(writer io.Writer) error

type WalPartRecorder added in v0.2.22 

type WalPartRecorder struct {
	// contains filtered or unexported fields
}

func NewWalPartRecorder added in v0.2.22 

func NewWalPartRecorder(walFilename string, manager *DeltaFileManager) (*WalPartRecorder, error)

func (*WalPartRecorder) SaveNextWalHead added in v0.2.22 

func (recorder *WalPartRecorder) SaveNextWalHead(head []byte) error

func (*WalPartRecorder) SavePreviousWalTail added in v0.2.22 

func (recorder *WalPartRecorder) SavePreviousWalTail(tailData []byte) error

type WalSegment added in v0.2.22 

type WalSegment struct {
	TimeLine uint32
	StartLSN pglogrepl.LSN
	// contains filtered or unexported fields
}

The WalSegment object represents a Postgres Wal Segment, holding all wal data for a wal file.

func NewWalSegment added in v0.2.22 

func NewWalSegment(timeline uint32, location pglogrepl.LSN, walSegmentBytes uint64) *WalSegment

NewWalSegment is a helper function to declare a new WalSegment.

func (*WalSegment) Name added in v0.2.22 

func (seg *WalSegment) Name() string

Name returns the filename of this wal segment. This is also used by the WalUploader to set the name of the destination file during upload of the wal segment.

func (*WalSegment) NextWalSegment added in v0.2.22 

func (seg *WalSegment) NextWalSegment() (*WalSegment, error)

NextWalSegment is a helper function to create the next wal segment which comes after this wal segment. Note that this will be on the same timeline. the convenience is that it also automatically processes a message that crosses the boundary between the two segments.

func (*WalSegment) Read added in v0.2.22 

func (seg *WalSegment) Read(p []byte) (n int, err error)

Read is what makes the WalSegment an io.Reader, which can be handled by WalUploader.UploadWalFile to write to a file.

func (*WalSegment) Stream added in v0.2.22 

func (seg *WalSegment) Stream(conn *pgconn.PgConn, standbyMessageTimeout time.Duration) (ProcessMessageResult, error)

Stream is a helper function to retrieve messages from Postgres and have them processed by processMessage().

type WalSegmentDescription added in v0.2.22 

type WalSegmentDescription struct {
	Number   WalSegmentNo
	Timeline uint32
}

func NewWalSegmentDescription added in v0.2.22 

func NewWalSegmentDescription(name string) (WalSegmentDescription, error)

func QueryCurrentWalSegment added in v0.2.22 

func QueryCurrentWalSegment() WalSegmentDescription

QueryCurrentWalSegment() gets start WAL segment from Postgres cluster

func (WalSegmentDescription) GetFileName added in v0.2.22 

func (desc WalSegmentDescription) GetFileName() string

type WalSegmentNo added in v0.2.22 

type WalSegmentNo uint64

type WalSegmentNotFoundError added in v0.2.22 

type WalSegmentNotFoundError struct {
	// contains filtered or unexported fields
}

func (WalSegmentNotFoundError) Error added in v0.2.22 

func (err WalSegmentNotFoundError) Error() string

type WalSegmentRunner added in v0.2.22 

type WalSegmentRunner struct {
	// contains filtered or unexported fields
}

WalSegmentRunner is used for sequential iteration over WAL segments in the storage

func NewWalSegmentRunner added in v0.2.22 

func NewWalSegmentRunner(
	startWalSegment WalSegmentDescription,
	segments map[WalSegmentDescription]bool,
	stopSegmentNo WalSegmentNo,
	timelineSwitchMap map[WalSegmentNo]*TimelineHistoryRecord,
) *WalSegmentRunner

func (*WalSegmentRunner) Current added in v0.2.22 

func (r *WalSegmentRunner) Current() WalSegmentDescription

func (*WalSegmentRunner) ForceMoveNext added in v0.2.22 

func (r *WalSegmentRunner) ForceMoveNext()

ForceMoveNext do a force-switch to the next segment without accessing storage

func (*WalSegmentRunner) Next added in v0.2.22 

func (r *WalSegmentRunner) Next() (WalSegmentDescription, error)

Next tries to get the next segment from storage

type WalSegmentScanner added in v0.2.22 

type WalSegmentScanner struct {
	ScannedSegments []ScannedSegmentDescription
	// contains filtered or unexported fields
}

WalSegmentScanner is used to scan the WAL segments storage

func NewWalSegmentScanner added in v0.2.22 

func NewWalSegmentScanner(walSegmentRunner *WalSegmentRunner) *WalSegmentScanner

func (*WalSegmentScanner) AddScannedSegment added in v0.2.22 

func (sc *WalSegmentScanner) AddScannedSegment(description WalSegmentDescription, status ScannedSegmentStatus)

func (*WalSegmentScanner) GetMissingSegmentsDescriptions added in v0.2.22 

func (sc *WalSegmentScanner) GetMissingSegmentsDescriptions() []WalSegmentDescription

GetMissingSegmentsDescriptions returns a slice containing WalSegmentDescription of each missing segment

func (*WalSegmentScanner) Scan added in v0.2.22 

func (sc *WalSegmentScanner) Scan(config SegmentScanConfig) error

Scan traverse the WAL storage with WalSegmentRunner. Scan starts from the WalSegmentRunner's current position, so in case of subsequent Scan() call it will continue from the position where it stopped previously.

Scan always stops if: - Stop segment is reached OR - Unknown error encountered Also, it may be configured to stop after: - Scanning the ScanSegmentsLimit of segments - Finding the first segment which exists in WAL storage

type WalSegmentsSequence added in v0.2.22 

type WalSegmentsSequence struct {
	// contains filtered or unexported fields
}

WalSegmentsSequence represents some collection of wal segments with the same timeline

func NewSegmentsSequence added in v0.2.22 

func NewSegmentsSequence(id uint32, segmentNo WalSegmentNo) *WalSegmentsSequence

func (*WalSegmentsSequence) AddWalSegmentNo added in v0.2.22 

func (seq *WalSegmentsSequence) AddWalSegmentNo(number WalSegmentNo)

AddWalSegmentNo adds the provided segment number to collection

func (*WalSegmentsSequence) FindMissingSegments added in v0.2.22 

func (seq *WalSegmentsSequence) FindMissingSegments() ([]WalSegmentDescription, error)

FindMissingSegments finds missing segments in range [minSegmentNo, maxSegmentNo]

type WalShowJSONOutputWriter added in v0.2.22 

type WalShowJSONOutputWriter struct {
	// contains filtered or unexported fields
}

WalShowJsonOutputWriter writes the detailed JSON output

func (*WalShowJSONOutputWriter) Write added in v0.2.22 

func (writer *WalShowJSONOutputWriter) Write(timelineInfos []*TimelineInfo) error

type WalShowOutputType added in v0.2.22 

type WalShowOutputType int
const (
	TableOutput WalShowOutputType = iota + 1
	JSONOutput
)

type WalShowOutputWriter added in v0.2.22 

type WalShowOutputWriter interface {
	Write(timelineInfos []*TimelineInfo) error
}

WalShowOutputWriter writes the output of wal-show command execution result

func NewWalShowOutputWriter added in v0.2.22 

func NewWalShowOutputWriter(outputType WalShowOutputType, output io.Writer, includeBackups bool) WalShowOutputWriter

type WalShowTableOutputWriter added in v0.2.22 

type WalShowTableOutputWriter struct {
	// contains filtered or unexported fields
}

WalShowTableOutputWriter writes the output in compact pretty table

func (*WalShowTableOutputWriter) Write added in v0.2.22 

func (writer *WalShowTableOutputWriter) Write(timelineInfos []*TimelineInfo) error

type WalUploader added in v0.2.22 

type WalUploader struct {
	*internal.Uploader
	*DeltaFileManager
}

WalUploader extends uploader with wal specific functionality.

func ConfigureWalUploader added in v0.2.22 

func ConfigureWalUploader() (uploader *WalUploader, err error)

ConfigureWalUploader connects to storage and creates an uploader. It makes sure that a valid session has started; if invalid, returns AWS error and `<nil>` values.

func ConfigureWalUploaderWithoutCompressMethod added in v0.2.22 

func ConfigureWalUploaderWithoutCompressMethod() (uploader *WalUploader, err error)

func NewWalUploader added in v0.2.22 

func NewWalUploader(
	compressor compression.Compressor,
	uploadingLocation storage.Folder,
	deltaFileManager *DeltaFileManager,
) *WalUploader

func (*WalUploader) FlushFiles added in v0.2.22 

func (walUploader *WalUploader) FlushFiles()

func (*WalUploader) UploadWalFile added in v0.2.22 

func (walUploader *WalUploader) UploadWalFile(file ioextensions.NamedReader) error

TODO : unit tests

type WalVerifyCheckDetails added in v0.2.22 

type WalVerifyCheckDetails interface {
	NewPlainTextReader() (io.Reader, error) // used in plaintext output
}

type WalVerifyCheckResult added in v0.2.22 

type WalVerifyCheckResult struct {
	Status  WalVerifyCheckStatus  `json:"status"`
	Details WalVerifyCheckDetails `json:"details"`
}

WalVerifyCheckResult contains the result of some WalVerifyCheckRunner run

type WalVerifyCheckRunner added in v0.2.22 

type WalVerifyCheckRunner interface {
	Type() WalVerifyCheckType
	Run() (WalVerifyCheckResult, error)
}

WalVerifyCheckRunner performs the check of WAL storage

func BuildWalVerifyCheckRunner added in v0.2.22 

func BuildWalVerifyCheckRunner(
	checkType WalVerifyCheckType,
	rootFolder storage.Folder,
	walFolderFilenames []string,
	currentWalSegment WalSegmentDescription,
) (WalVerifyCheckRunner, error)

type WalVerifyCheckStatus added in v0.2.22 

type WalVerifyCheckStatus int
const (
	StatusOk WalVerifyCheckStatus = iota + 1
	StatusWarning
	StatusFailure
)

func (WalVerifyCheckStatus) MarshalText added in v0.2.22 

func (status WalVerifyCheckStatus) MarshalText() ([]byte, error)

MarshalText marshals the WalVerifyCheckStatus enum as a string

func (WalVerifyCheckStatus) String added in v0.2.22 

func (status WalVerifyCheckStatus) String() string

type WalVerifyCheckType added in v0.2.22 

type WalVerifyCheckType int

func (WalVerifyCheckType) MarshalText added in v0.2.22 

func (checkType WalVerifyCheckType) MarshalText() (text []byte, err error)

func (WalVerifyCheckType) String added in v0.2.22 

func (checkType WalVerifyCheckType) String() string

type WalVerifyJSONOutputWriter added in v0.2.22 

type WalVerifyJSONOutputWriter struct {
	// contains filtered or unexported fields
}

WalVerifyJsonOutputWriter writes the detailed JSON output

func (*WalVerifyJSONOutputWriter) Write added in v0.2.22 

func (writer *WalVerifyJSONOutputWriter) Write(results map[WalVerifyCheckType]WalVerifyCheckResult) error

type WalVerifyOutputType added in v0.2.22 

type WalVerifyOutputType int
const (
	WalVerifyTableOutput WalVerifyOutputType = iota + 1
	WalVerifyJSONOutput
)

type WalVerifyOutputWriter added in v0.2.22 

type WalVerifyOutputWriter interface {
	Write(results map[WalVerifyCheckType]WalVerifyCheckResult) error
}

WalVerifyOutputWriter writes the output of wal-verify command execution result

func NewWalVerifyOutputWriter added in v0.2.22 

func NewWalVerifyOutputWriter(outputType WalVerifyOutputType, output io.Writer) WalVerifyOutputWriter

type WalVerifyTableOutputWriter added in v0.2.22 

type WalVerifyTableOutputWriter struct {
	// contains filtered or unexported fields
}

WalVerifyTableOutputWriter writes the output as pretty table

func (*WalVerifyTableOutputWriter) Write added in v0.2.22 

func (writer *WalVerifyTableOutputWriter) Write(result map[WalVerifyCheckType]WalVerifyCheckResult) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.