log

Documentation

Index

• Variables
• func Exist(dir string) bool
• func RegisterRecord(rt RecordType, ent interface{})
• func Repair(lg *zap.Logger, dirpath string) bool
• func Verify(lg *zap.Logger, walDir string, snap Snapshot) error
• type HardState
  • func NewEmptyState() (s HardState)
• type LogEntry
  • func NewEmptyEntry() (e LogEntry)
• type RecordData
• type RecordType
• type Snapshot
  • func NewEmptySnapshot() (s Snapshot)
  • func ValidSnapshotEntries(lg *zap.Logger, walDir string) ([]Snapshot, error)
• type WAL
  • func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error)
  • func Open(lg *zap.Logger, dirpath string, snap Snapshot) (*WAL, error)
  • func OpenForRead(lg *zap.Logger, dirpath string, snap Snapshot) (*WAL, error)
  • func (w *WAL) Close() error
  • func (w *WAL) ReadAll() (metadata []byte, state HardState, ents []LogEntry, err error)
  • func (w *WAL) ReleaseLockTo(index uint64) error
  • func (w *WAL) Save(st HardState, ents []LogEntry) error
  • func (w *WAL) SaveEntry(ents []LogEntry) error
  • func (w *WAL) SaveSnapshot(e Snapshot) error
  • func (w *WAL) SaveState(st HardState) error
  • func (w *WAL) SetUnsafeNoFsync()
  • func (w *WAL) Sync() error
• type WALAPI

Variables

var (
	ErrMetadataConflict             = errors.New("wal: conflicting metadata found")
	ErrFileNotFound                 = errors.New("wal: file not found")
	ErrCRCMismatch                  = errors.New("wal: crc mismatch")
	ErrSnapshotNotFound             = errors.New("wal: snapshot not found")
	ErrSliceOutOfRange              = errors.New("wal: slice bounds out of range")
	ErrMaxWALEntrySizeLimitExceeded = errors.New("wal: max entry size limit exceeded")
	ErrDecoderNotFound              = errors.New("wal: decoder not found")

	// SegmentSizeBytes is the preallocated size of each wal segment file.
	// The actual size might be larger than this. In general, the default
	// value should be used, but this is defined as an exported variable
	// so that tests can set a different segment size.
	SegmentSizeBytes int64 = 64 * 1000 * 1000 // 64MB
)

Functions

func Exist

func Exist(dir string) bool

Exist returns true if there are any files in a given directory.

func RegisterRecord

func RegisterRecord(rt RecordType, ent interface{})

func Repair

func Repair(lg *zap.Logger, dirpath string) bool

Repair tries to repair ErrUnexpectedEOF in the last wal file by truncating.

func Verify

func Verify(lg *zap.Logger, walDir string, snap Snapshot) error

Verify reads through the given WAL and verifies that it is not corrupted. It creates a new decoder to read through the records of the given WAL. It does not conflict with any open WAL, but it is recommended not to call this function after opening the WAL for writing. If it cannot read out the expected snap, it will return ErrSnapshotNotFound. If the loaded snap doesn't match with the expected one, it will return error ErrSnapshotMismatch.

Types

type HardState

type HardState interface {
	RecordData
	// The latest index that has been committed
	GetCommitted() uint64
	// Reset set state initial default value
	Reset()
}

HardState implement custom state data struct for save the system latest state to wal

func NewEmptyState

func NewEmptyState() (s HardState)

type LogEntry

type LogEntry interface {
	RecordData
	// index of the entry saved to the wal
	GetIndex() uint64
	// size of entry alloc memory
	Size() (n int)
}

LogEntry implement custom entry data struct with entry index

func NewEmptyEntry

func NewEmptyEntry() (e LogEntry)

type RecordData

type RecordData interface {
	Marshal() (data []byte, err error)
	Unmarshal(data []byte) error
}

type RecordType

type RecordType int64

const (
	MetadataType RecordType = iota + 1
	EntryType
	StateType
	CrcType
	SnapshotType
)

type Snapshot

type Snapshot interface {
	RecordData
	// index of the entry saved to the wal
	GetIndex() uint64
}

Snapshot implement custom snapshot struct with entry index

func NewEmptySnapshot

func NewEmptySnapshot() (s Snapshot)

func ValidSnapshotEntries

func ValidSnapshotEntries(lg *zap.Logger, walDir string) ([]Snapshot, error)

ValidSnapshotEntries returns all the valid snapshot entries in the wal logs in the given directory. Snapshot entries are valid if their index is less than or equal to the most recent committed hardstate.

type WAL

type WAL struct {
	// contains filtered or unexported fields
}

WAL is a logical representation of the stable storage. WAL is either in read mode or append mode but not both. A newly created WAL is in append mode, and ready for appending records. A just opened WAL is in read mode, and ready for reading records. The WAL will be ready for appending after reading out all the previous records.

func Create

func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error)

Create creates a WAL ready for appending records. The given metadata is recorded at the head of each WAL file, and can be retrieved with ReadAll after the file is Open.

func Open

func Open(lg *zap.Logger, dirpath string, snap Snapshot) (*WAL, error)

Open opens the WAL at the given snap. The snap SHOULD have been previously saved to the WAL, or the following ReadAll will fail. The returned WAL is ready to read and the first record will be the one after the given snap. The WAL cannot be appended to before reading out all of its previous records.

func OpenForRead

func OpenForRead(lg *zap.Logger, dirpath string, snap Snapshot) (*WAL, error)

OpenForRead only opens the wal files for read. Write on a read only wal panics.

func (*WAL) Close

func (w *WAL) Close() error

Close closes the current WAL file and directory.

func (*WAL) ReadAll

func (w *WAL) ReadAll() (metadata []byte, state HardState, ents []LogEntry, err error)

ReadAll reads out records of the current WAL. If opened in write mode, it must read out all records until EOF. Or an error will be returned. If opened in read mode, it will try to read all records if possible. If it cannot read out the expected snap, it will return ErrSnapshotNotFound. If loaded snap doesn't match with the expected one, it will return all the records and error ErrSnapshotMismatch. TODO: detect not-last-snap error. TODO: maybe loose the checking of match. After ReadAll, the WAL will be ready for appending new records.

func (*WAL) ReleaseLockTo

func (w *WAL) ReleaseLockTo(index uint64) error

ReleaseLockTo releases the locks, which has smaller index than the given index except the largest one among them. For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release lock 1,2 but keep 3. ReleaseLockTo(5) will release 1,2,3 but keep 4.

func (*WAL) Save

func (w *WAL) Save(st HardState, ents []LogEntry) error

func (*WAL) SaveEntry

func (w *WAL) SaveEntry(ents []LogEntry) error

func (*WAL) SaveSnapshot

func (w *WAL) SaveSnapshot(e Snapshot) error

func (*WAL) SaveState

func (w *WAL) SaveState(st HardState) error

func (*WAL) SetUnsafeNoFsync

func (w *WAL) SetUnsafeNoFsync()

func (*WAL) Sync

func (w *WAL) Sync() error

type WALAPI

type WALAPI interface {
	// Save function saves ents and state to the underlying stable storage.
	// Save MUST block until st and ents are on stable storage.
	Save(st HardState, ents []LogEntry) error
	// SaveState function saves state to the underlying stable storage.
	SaveState(st HardState) error
	// SaveState function saves ents to the underlying stable storage.
	SaveEntry(ents []LogEntry) error
	// SaveSnapshot function saves snapshot to the underlying stable storage.
	SaveSnapshot(e Snapshot) error
	// ReleaseLockTo releases the locks, which has smaller index than the given index
	// except the largest one among them.
	ReleaseLockTo(index uint64) error
	// ReadAll reads out records of the current WAL.
	ReadAll() (metadata []byte, state HardState, ents []LogEntry, err error)
	// Sync WAL
	Sync() error

	SetUnsafeNoFsync()
	// Close closes the Storage and performs finalization.
	Close() error
}