wal

package
v3.3.1+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 13, 2019 License: LGPL-3.0 Imports: 18 Imported by: 0

Documentation

Overview

Package wal This Module is in so many aspects inspired by etcd's WAL.

Index

Constants

This section is empty.

Variables

View Source
var (
	// SegmentSizeBytes is the preallocated size of each wal segment file.
	// The actual size might be larger than this. In general, the default
	// value should be used, but this is defined as an exported variable
	// so that tests can set a different segment size.
	SegmentSizeBytes int64 = 8 * 1000 * 1000 // 8MB

	// ErrMetadataConflict metadata not consist
	ErrMetadataConflict = errors.New("wal: conflicting metadata found")
	// ErrFileNotFound file not found
	ErrFileNotFound = errors.New("wal: file not found")
	// ErrCRCMismatch crc miss match
	ErrCRCMismatch = errors.New("wal: crc mismatch")
)
View Source
var LogType_name = map[int32]string{
	0: "crcType",
	2: "metaDataType",
	3: "entryType",
}
View Source
var LogType_value = map[string]int32{
	"crcType":      0,
	"metaDataType": 2,
	"entryType":    3,
}

Functions

func BytesToUint64

func BytesToUint64(p []byte) uint64

BytesToUint64 convert byte array to uint64

func Exist

func Exist(dir string) bool

Exist returns true if there are any files in a given directory.

func OpenDir

func OpenDir(path string) (*os.File, error)

OpenDir open a dir

func Uint64ToBytes

func Uint64ToBytes(i uint64) []byte

Uint64ToBytes convert uint64 to byte array

func ZeroToEnd

func ZeroToEnd(f *os.File) error

ZeroToEnd write zero to file from current to end

Types

type Entry

type Entry struct {
	Data                 []byte   `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
	ExtraMeta            []byte   `protobuf:"bytes,2,opt,name=ExtraMeta,proto3" json:"ExtraMeta,omitempty"`
	Index                uint64   `protobuf:"varint,3,opt,name=index,proto3" json:"index,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*Entry) Descriptor

func (*Entry) Descriptor() ([]byte, []int)

func (*Entry) GetData

func (m *Entry) GetData() []byte

func (*Entry) GetExtraMeta

func (m *Entry) GetExtraMeta() []byte

func (*Entry) GetIndex

func (m *Entry) GetIndex() uint64

func (*Entry) ProtoMessage

func (*Entry) ProtoMessage()

func (*Entry) Reset

func (m *Entry) Reset()

func (*Entry) String

func (m *Entry) String() string

func (*Entry) XXX_DiscardUnknown

func (m *Entry) XXX_DiscardUnknown()

func (*Entry) XXX_Marshal

func (m *Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Entry) XXX_Merge

func (m *Entry) XXX_Merge(src proto.Message)

func (*Entry) XXX_Size

func (m *Entry) XXX_Size() int

func (*Entry) XXX_Unmarshal

func (m *Entry) XXX_Unmarshal(b []byte) error

type Log

type Log struct {
	Type                 LogType  `protobuf:"varint,1,opt,name=type,proto3,enum=wal.LogType" json:"type,omitempty"`
	Data                 []byte   `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
	Checksum             uint64   `protobuf:"varint,3,opt,name=checksum,proto3" json:"checksum,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*Log) Check

func (log *Log) Check(crc uint64) error

Check check whether the log.checksum is same as crc

func (*Log) Descriptor

func (*Log) Descriptor() ([]byte, []int)

func (*Log) GetChecksum

func (m *Log) GetChecksum() uint64

func (*Log) GetData

func (m *Log) GetData() []byte

func (*Log) GetType

func (m *Log) GetType() LogType

func (*Log) ProtoMessage

func (*Log) ProtoMessage()

func (*Log) Reset

func (m *Log) Reset()

func (*Log) String

func (m *Log) String() string

func (*Log) XXX_DiscardUnknown

func (m *Log) XXX_DiscardUnknown()

func (*Log) XXX_Marshal

func (m *Log) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Log) XXX_Merge

func (m *Log) XXX_Merge(src proto.Message)

func (*Log) XXX_Size

func (m *Log) XXX_Size() int

func (*Log) XXX_Unmarshal

func (m *Log) XXX_Unmarshal(b []byte) error

type LogType

type LogType int32
const (
	LogType_crcType      LogType = 0
	LogType_metaDataType LogType = 2
	LogType_entryType    LogType = 3
)

func (LogType) EnumDescriptor

func (LogType) EnumDescriptor() ([]byte, []int)

func (LogType) String

func (x LogType) String() string

type PageWriter

type PageWriter struct {
	// contains filtered or unexported fields
}

PageWriter implements the io.Writer interface so that writes will either be in page chunks or from flushing.

func NewPageWriter

func NewPageWriter(w io.Writer, pageBytes, pageOffset int) *PageWriter

NewPageWriter creates a new PageWriter. pageBytes is the number of bytes to write per page. pageOffset is the starting offset of io.Writer.

func (*PageWriter) Flush

func (pw *PageWriter) Flush() error

Flush flush the page writer

func (*PageWriter) Write

func (pw *PageWriter) Write(p []byte) (n int, err error)

type StreamFile

type StreamFile struct {
	// contains filtered or unexported fields
}

StreamFile generate files like a endless stream

func (*StreamFile) Close

func (st *StreamFile) Close() error

Close close the stream file

func (*StreamFile) GetNewFile

func (st *StreamFile) GetNewFile() (f *os.File, err error)

GetNewFile get a new generated file

type WAL

type WAL struct {
	// contains filtered or unexported fields
}

WAL is a logical representation of the stable storage. WAL is either in read mode or append mode but not both. A newly created WAL is in append mode, and ready for appending records. A just opened WAL is in read mode, and ready for reading records. The WAL will be ready for appending after reading out all the previous records.

func Create

func Create(dirpath string, metadata []byte) (*WAL, error)

Create creates a WAL ready for appending records. The given metadata is recorded at the head of each WAL file, and can be retrieved with ReadAll. If there already are some wal files, it will try to recover from them.

func Open

func Open(dirpath string) (*WAL, error)

Open opens the WAL at the given snap. The snap SHOULD have been previously saved to the WAL, or the following ReadAll will fail. The returned WAL is ready to read and the first record will be the one after the given snap. The WAL cannot be appended to before reading out all of its previous records.

func OpenForRead

func OpenForRead(dirpath string) (*WAL, error)

OpenForRead only opens the wal files for read. Write on a read only wal panics.

func (*WAL) CleanDir

func (w *WAL) CleanDir() error

CleanDir clean the wal dir

func (*WAL) Close

func (w *WAL) Close() error

Close closes the current WAL file and directory.

func (*WAL) HasDecoder

func (w *WAL) HasDecoder() bool

HasDecoder check whether wal has decoder, has decoder means there already are some wal files.

func (*WAL) ReadAll

func (w *WAL) ReadAll() (metadata []byte, ents []Entry, err error)

nolint ReadAll reads out records of the current WAL. After ReadAll, the WAL will be ready for appending new records.

func (*WAL) ReleaseLockTo

func (w *WAL) ReleaseLockTo(index uint64) error

ReleaseLockTo releases the files, which has smaller index than the given index except the largest one among them. For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release lock 1,2 but keep 3. ReleaseLockTo(5) will release 1,2,3 but keep 4.

func (*WAL) RemoveFiles

func (w *WAL) RemoveFiles(index uint64) error

RemoveFiles remove files less than or equal to index

func (*WAL) RemoveFilesBefore

func (w *WAL) RemoveFilesBefore(index uint64) error

RemoveFilesBefore remove files less than index

func (*WAL) Save

func (w *WAL) Save(ents []Entry) (uint64, error)

Save save entries

func (*WAL) SaveSingle

func (w *WAL) SaveSingle(ent Entry) (uint64, error)

SaveSingle save single entry, Return entry index and error

func (*WAL) Size

func (w *WAL) Size() uint64

Size return WAL used data size include current tmp file.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL