Documentation ¶
Index ¶
- Constants
- Variables
- func Checksum(bo binary.ByteOrder, s0, s1 uint32, b []byte) (uint32, uint32)
- func FormatWALPath(index int) string
- func FormatWALPathWithOffset(index int, offset int64) string
- func IsGenerationName(s string) bool
- func IsSnapshotPath(s string) bool
- func IsWALPath(s string) bool
- func ParseSnapshotPath(s string) (index int, ext string, err error)
- func ParseWALPath(s string) (index int, offset int64, ext string, err error)
- func RestoreReplica(ctx context.Context, r Replica, opt RestoreOptions) error
- func SnapshotIndexAt(ctx context.Context, r Replica, generation string, timestamp time.Time) (int, error)
- func ValidateReplica(ctx context.Context, r Replica) error
- func WALIndexAt(ctx context.Context, r Replica, generation string, maxIndex int, ...) (int, error)
- type DB
- func (db *DB) CRC64() (uint64, Pos, error)
- func (db *DB) CalcRestoreTarget(ctx context.Context, opt RestoreOptions) (Replica, string, error)
- func (db *DB) Checkpoint(mode string) (err error)
- func (db *DB) Close() (err error)
- func (db *DB) CurrentGeneration() (string, error)
- func (db *DB) CurrentShadowWALIndex(generation string) (index int, size int64, err error)
- func (db *DB) CurrentShadowWALPath(generation string) (string, error)
- func (db *DB) GenerationNamePath() string
- func (db *DB) GenerationPath(generation string) string
- func (db *DB) MetaPath() string
- func (db *DB) Notify() <-chan struct{}
- func (db *DB) Open() (err error)
- func (db *DB) PageSize() int
- func (db *DB) Path() string
- func (db *DB) Pos() (Pos, error)
- func (db *DB) Replica(name string) Replica
- func (db *DB) SQLDB() *sql.DB
- func (db *DB) ShadowWALDir(generation string) string
- func (db *DB) ShadowWALPath(generation string, index int) string
- func (db *DB) ShadowWALReader(pos Pos) (r *ShadowWALReader, err error)
- func (db *DB) Snapshots(ctx context.Context) ([]*SnapshotInfo, error)
- func (db *DB) SoftClose() (err error)
- func (db *DB) Sync() (err error)
- func (db *DB) UpdatedAt() (time.Time, error)
- func (db *DB) WALPath() string
- func (db *DB) WALs(ctx context.Context) ([]*WALInfo, error)
- type FileReplica
- func (r *FileReplica) CalcPos(ctx context.Context, generation string) (pos Pos, err error)
- func (r *FileReplica) DB() *DB
- func (r *FileReplica) EnforceRetention(ctx context.Context) (err error)
- func (r *FileReplica) GenerationDir(generation string) string
- func (r *FileReplica) GenerationStats(ctx context.Context, generation string) (stats GenerationStats, err error)
- func (r *FileReplica) Generations(ctx context.Context) ([]string, error)
- func (r *FileReplica) LastPos() Pos
- func (r *FileReplica) MaxSnapshotIndex(generation string) (int, error)
- func (r *FileReplica) Name() string
- func (r *FileReplica) Path() string
- func (r *FileReplica) SnapshotDir(generation string) string
- func (r *FileReplica) SnapshotPath(generation string, index int) string
- func (r *FileReplica) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
- func (r *FileReplica) Snapshots(ctx context.Context) ([]*SnapshotInfo, error)
- func (r *FileReplica) Start(ctx context.Context)
- func (r *FileReplica) Stop()
- func (r *FileReplica) Sync(ctx context.Context) (err error)
- func (r *FileReplica) Type() string
- func (r *FileReplica) WALDir(generation string) string
- func (r *FileReplica) WALPath(generation string, index int) string
- func (r *FileReplica) WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
- func (r *FileReplica) WALs(ctx context.Context) ([]*WALInfo, error)
- type GenerationStats
- type Pos
- type Replica
- type RestoreOptions
- type ShadowWALReader
- type SnapshotInfo
- type WALInfo
Constants ¶
const ( DefaultMonitorInterval = 1 * time.Second DefaultCheckpointInterval = 1 * time.Minute DefaultMinCheckpointPageN = 1000 DefaultMaxCheckpointPageN = 10000 )
Default DB settings.
const ( WALHeaderChecksumOffset = 24 WALFrameHeaderChecksumOffset = 16 )
SQLite WAL constants
const ( MetaDirSuffix = "-litestream" WALDirName = "wal" WALExt = ".wal" SnapshotExt = ".snapshot" GenerationNameLen = 16 )
Naming constants.
const ( CheckpointModePassive = "PASSIVE" CheckpointModeFull = "FULL" CheckpointModeRestart = "RESTART" CheckpointModeTruncate = "TRUNCATE" )
SQLite checkpoint modes.
const ( // WALHeaderSize is the size of the WAL header, in bytes. WALHeaderSize = 32 // WALFrameHeaderSize is the size of the WAL frame header, in bytes. WALFrameHeaderSize = 24 )
const ( DefaultRetention = 24 * time.Hour DefaultRetentionCheckInterval = 1 * time.Hour )
Default file replica settings.
const BusyTimeout = 1 * time.Second
BusyTimeout is the timeout to wait for EBUSY from SQLite.
Variables ¶
var ( ErrNoSnapshots = errors.New("no snapshots available") ErrChecksumMismatch = errors.New("invalid replica, checksum mismatch") )
Litestream errors.
var Tracef = func(format string, a ...interface{}) {}
Tracef is used for low-level tracing.
Functions ¶
func FormatWALPath ¶ added in v0.2.0
FormatWALPath formats a WAL filename with a given index.
func FormatWALPathWithOffset ¶ added in v0.2.0
FormatWALPathWithOffset formats a WAL filename with a given index & offset.
func IsGenerationName ¶
IsGenerationName returns true if s is the correct length and is only lowercase hex characters.
func IsSnapshotPath ¶
IsSnapshotPath returns true if s is a path to a snapshot file.
func ParseSnapshotPath ¶
ParseSnapshotPath returns the index for the snapshot. Returns an error if the path is not a valid snapshot path.
func ParseWALPath ¶
ParseWALPath returns the index & offset for the WAL file. Returns an error if the path is not a valid snapshot path.
func RestoreReplica ¶ added in v0.3.1
func RestoreReplica(ctx context.Context, r Replica, opt RestoreOptions) error
RestoreReplica restores the database from a replica based on the options given. This method will restore into opt.OutputPath, if specified, or into the DB's original database path. It can optionally restore from a specific replica or generation or it will automatically choose the best one. Finally, a timestamp can be specified to restore the database to a specific point-in-time.
func SnapshotIndexAt ¶ added in v0.3.0
func SnapshotIndexAt(ctx context.Context, r Replica, generation string, timestamp time.Time) (int, error)
SnapshotIndexAt returns the highest index for a snapshot within a generation that occurs before timestamp. If timestamp is zero, returns the latest snapshot.
func ValidateReplica ¶ added in v0.3.0
ValidateReplica restores the most recent data from a replica and validates that the resulting database matches the current database.
Types ¶
type DB ¶
type DB struct { // Minimum threshold of WAL size, in pages, before a passive checkpoint. // A passive checkpoint will attempt a checkpoint but fail if there are // active transactions occurring at the same time. MinCheckpointPageN int // Maximum threshold of WAL size, in pages, before a forced checkpoint. // A forced checkpoint will block new transactions and wait for existing // transactions to finish before issuing a checkpoint and resetting the WAL. // // If zero, no checkpoints are forced. This can cause the WAL to grow // unbounded if there are always read transactions occurring. MaxCheckpointPageN int // Time between automatic checkpoints in the WAL. This is done to allow // more fine-grained WAL files so that restores can be performed with // better precision. CheckpointInterval time.Duration // Frequency at which to perform db sync. MonitorInterval time.Duration // List of replicas for the database. // Must be set before calling Open(). Replicas []Replica // contains filtered or unexported fields }
DB represents a managed instance of a SQLite database in the file system.
func (*DB) CRC64 ¶ added in v0.3.0
CRC64 returns a CRC-64 ISO checksum of the database and its current position.
This function obtains a read lock so it prevents syncs from occurring until the operation is complete. The database will still be usable but it will be unable to checkpoint during this time.
func (*DB) CalcRestoreTarget ¶ added in v0.3.1
CalcRestoreTarget returns a replica & generation to restore from based on opt criteria.
func (*DB) Checkpoint ¶ added in v0.3.0
Checkpoint performs a checkpoint on the WAL file.
func (*DB) Close ¶
Close releases the read lock & closes the database. This method should only be called by tests as it causes the underlying database to be checkpointed.
func (*DB) CurrentGeneration ¶
CurrentGeneration returns the name of the generation saved to the "generation" file in the meta data directory. Returns empty string if none exists.
func (*DB) CurrentShadowWALIndex ¶
CurrentShadowWALIndex returns the current WAL index & total size.
func (*DB) CurrentShadowWALPath ¶
CurrentShadowWALPath returns the path to the last shadow WAL in a generation.
func (*DB) GenerationNamePath ¶
GenerationNamePath returns the path of the name of the current generation.
func (*DB) GenerationPath ¶
GenerationPath returns the path of a single generation. Panics if generation is blank.
func (*DB) Notify ¶
func (db *DB) Notify() <-chan struct{}
Notify returns a channel that closes when the shadow WAL changes.
func (*DB) PageSize ¶
PageSize returns the page size of the underlying database. Only valid after database exists & Init() has successfully run.
func (*DB) ShadowWALDir ¶ added in v0.2.0
ShadowWALDir returns the path of the shadow wal directory. Panics if generation is blank.
func (*DB) ShadowWALPath ¶
ShadowWALPath returns the path of a single shadow WAL file. Panics if generation is blank or index is negative.
func (*DB) ShadowWALReader ¶
func (db *DB) ShadowWALReader(pos Pos) (r *ShadowWALReader, err error)
ShadowWALReader opens a reader for a shadow WAL file at a given position. If the reader is at the end of the file, it attempts to return the next file.
The caller should check Pos() & Size() on the returned reader to check offset.
func (*DB) Snapshots ¶ added in v0.2.0
func (db *DB) Snapshots(ctx context.Context) ([]*SnapshotInfo, error)
Snapshots returns a list of all snapshots across all replicas.
func (*DB) SoftClose ¶
SoftClose closes everything but the underlying db connection. This method is available because the binary needs to avoid closing the database on exit to prevent autocheckpointing.
type FileReplica ¶
type FileReplica struct { // Time to keep snapshots and related WAL files. // Database is snapshotted after interval and older WAL files are discarded. Retention time.Duration // Time between checks for retention. RetentionCheckInterval time.Duration // Time between validation checks. ValidationInterval time.Duration // If true, replica monitors database for changes automatically. // Set to false if replica is being used synchronously (such as in tests). MonitorEnabled bool // contains filtered or unexported fields }
FileReplica is a replica that replicates a DB to a local file path.
func NewFileReplica ¶
func NewFileReplica(db *DB, name, dst string) *FileReplica
NewFileReplica returns a new instance of FileReplica.
func (*FileReplica) CalcPos ¶ added in v0.2.0
CalcPos returns the position for the replica for the current generation. Returns a zero value if there is no active generation.
func (*FileReplica) DB ¶ added in v0.3.0
func (r *FileReplica) DB() *DB
DB returns the parent database reference.
func (*FileReplica) EnforceRetention ¶ added in v0.3.0
func (r *FileReplica) EnforceRetention(ctx context.Context) (err error)
EnforceRetention forces a new snapshot once the retention interval has passed. Older snapshots and WAL files are then removed.
func (*FileReplica) GenerationDir ¶ added in v0.3.0
func (r *FileReplica) GenerationDir(generation string) string
GenerationDir returns the path to a generation's root directory.
func (*FileReplica) GenerationStats ¶
func (r *FileReplica) GenerationStats(ctx context.Context, generation string) (stats GenerationStats, err error)
GenerationStats returns stats for a generation.
func (*FileReplica) Generations ¶
func (r *FileReplica) Generations(ctx context.Context) ([]string, error)
Generations returns a list of available generation names.
func (*FileReplica) LastPos ¶ added in v0.2.0
func (r *FileReplica) LastPos() Pos
LastPos returns the last successfully replicated position.
func (*FileReplica) MaxSnapshotIndex ¶ added in v0.2.0
func (r *FileReplica) MaxSnapshotIndex(generation string) (int, error)
MaxSnapshotIndex returns the highest index for the snapshots.
func (*FileReplica) Name ¶
func (r *FileReplica) Name() string
Name returns the name of the replica. Returns the type if no name set.
func (*FileReplica) Path ¶ added in v0.3.0
func (r *FileReplica) Path() string
Path returns the path the replica was initialized with.
func (*FileReplica) SnapshotDir ¶
func (r *FileReplica) SnapshotDir(generation string) string
SnapshotDir returns the path to a generation's snapshot directory.
func (*FileReplica) SnapshotPath ¶
func (r *FileReplica) SnapshotPath(generation string, index int) string
SnapshotPath returns the path to a snapshot file.
func (*FileReplica) SnapshotReader ¶
func (r *FileReplica) SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
SnapshotReader returns a reader for snapshot data at the given generation/index. Returns os.ErrNotExist if no matching index is found.
func (*FileReplica) Snapshots ¶ added in v0.2.0
func (r *FileReplica) Snapshots(ctx context.Context) ([]*SnapshotInfo, error)
Snapshots returns a list of available snapshots in the replica.
func (*FileReplica) Start ¶
func (r *FileReplica) Start(ctx context.Context)
Start starts replication for a given generation.
func (*FileReplica) Stop ¶
func (r *FileReplica) Stop()
Stop cancels any outstanding replication and blocks until finished.
func (*FileReplica) Sync ¶ added in v0.2.0
func (r *FileReplica) Sync(ctx context.Context) (err error)
Sync replays data from the shadow WAL into the file replica.
func (*FileReplica) WALDir ¶
func (r *FileReplica) WALDir(generation string) string
WALDir returns the path to a generation's WAL directory
func (*FileReplica) WALPath ¶
func (r *FileReplica) WALPath(generation string, index int) string
WALPath returns the path to a WAL file.
func (*FileReplica) WALReader ¶
func (r *FileReplica) WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error)
WALReader returns a reader for WAL data at the given index. Returns os.ErrNotExist if no matching index is found.
type GenerationStats ¶
type GenerationStats struct { // Count of snapshot & WAL files. SnapshotN int WALN int // Time range for the earliest snapshot & latest WAL file update. CreatedAt time.Time UpdatedAt time.Time }
GenerationStats represents high level stats for a single generation.
func CalcReplicaRestoreTarget ¶ added in v0.3.1
func CalcReplicaRestoreTarget(ctx context.Context, r Replica, opt RestoreOptions) (generation string, stats GenerationStats, err error)
CalcReplicaRestoreTarget returns a generation to restore from.
type Pos ¶
type Pos struct { Generation string // generation name Index int // wal file index Offset int64 // offset within wal file }
Pos is a position in the WAL for a generation.
type Replica ¶
type Replica interface { // The name of the replica. Defaults to type if no name specified. Name() string // String identifier for the type of replica ("file", "s3", etc). Type() string // The parent database. DB() *DB // Starts replicating in a background goroutine. Start(ctx context.Context) // Stops all replication processing. Blocks until processing stopped. Stop() // Returns the last replication position. LastPos() Pos // Returns the computed position of the replica for a given generation. CalcPos(ctx context.Context, generation string) (Pos, error) // Returns a list of generation names for the replica. Generations(ctx context.Context) ([]string, error) // Returns basic information about a generation including the number of // snapshot & WAL files as well as the time range covered. GenerationStats(ctx context.Context, generation string) (GenerationStats, error) // Returns a list of available snapshots in the replica. Snapshots(ctx context.Context) ([]*SnapshotInfo, error) // Returns a list of available WAL files in the replica. WALs(ctx context.Context) ([]*WALInfo, error) // Returns a reader for snapshot data at the given generation/index. SnapshotReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) // Returns a reader for WAL data at the given position. WALReader(ctx context.Context, generation string, index int) (io.ReadCloser, error) }
Replica represents a remote destination to replicate the database & WAL.
type RestoreOptions ¶
type RestoreOptions struct { // Target path to restore into. // If blank, the original DB path is used. OutputPath string // Specific replica to restore from. // If blank, all replicas are considered. ReplicaName string // Specific generation to restore from. // If blank, all generations considered. Generation string // Specific index to restore from. // Set to math.MaxInt64 to ignore index. Index int // Point-in-time to restore database. // If zero, database restore to most recent state available. Timestamp time.Time // If true, no actual restore is performed. // Only equivalent log output for a regular restore. DryRun bool // Logger used to print status to. Logger *log.Logger }
RestoreOptions represents options for DB.Restore().
func NewRestoreOptions ¶ added in v0.2.0
func NewRestoreOptions() RestoreOptions
NewRestoreOptions returns a new instance of RestoreOptions with defaults.
type ShadowWALReader ¶
type ShadowWALReader struct {
// contains filtered or unexported fields
}
ShadowWALReader represents a reader for a shadow WAL file that tracks WAL position.
func (*ShadowWALReader) Close ¶
func (r *ShadowWALReader) Close() error
Close closes the underlying WAL file handle.
func (*ShadowWALReader) N ¶
func (r *ShadowWALReader) N() int64
N returns the remaining bytes in the reader.
func (*ShadowWALReader) Pos ¶
func (r *ShadowWALReader) Pos() Pos
Pos returns the current WAL position.
type SnapshotInfo ¶ added in v0.2.0
type SnapshotInfo struct { Name string Replica string Generation string Index int Size int64 CreatedAt time.Time }
SnapshotInfo represents file information about a snapshot.
func FilterSnapshotsAfter ¶ added in v0.3.0
func FilterSnapshotsAfter(a []*SnapshotInfo, t time.Time) []*SnapshotInfo
FilterSnapshotsAfter returns all snapshots that were created on or after t.
func FindMinSnapshotByGeneration ¶ added in v0.3.0
func FindMinSnapshotByGeneration(a []*SnapshotInfo, generation string) *SnapshotInfo
FindMinSnapshotByGeneration finds the snapshot with the lowest index in a generation.