Help us shape the future of Go! Take the 2026 Go Developer Survey

kiroku

package module
v0.3.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 8, 2021 License: MIT Imports: 15 Imported by: 9

README

Kiroku GoDoc Status Go Report Card Go Test Coverage

Kiroku is a general purpose historical record system which utilizes data blocks. It was built to be used as the action persistence layer for Mojura.

Usage

New
func ExampleNew() {
	var err error
	if testKiroku, err = New("./test_data", "tester", nil); err != nil {
		log.Fatal(err)
		return
	}
}
New (with custom Processor)
func ExampleNew_with_custom_Processor() {
	var err error
	pfn := func(r *Reader) (err error) {
		fmt.Println("Hello chunk!", r.Meta())
		return
	}

	if testKiroku, err = New("./test_data", "tester", pfn); err != nil {
		log.Fatal(err)
		return
	}
}
Kiroku.Meta
func ExampleKiroku_Meta() {
	var (
		m   Meta
		err error
	)

	if m, err = testKiroku.Meta(); err != nil {
		log.Fatal(err)
	}

	fmt.Println("Meta", m)
}
Kiroku.Transaction
func ExampleKiroku_Transaction() {
	var err error
	if err = testKiroku.Transaction(func(t *Transaction) (err error) {
		if err = t.SetIndex(1337); err != nil {
			return
		}

		return t.AddBlock(TypeWriteAction, []byte("hello world!"))
	}); err != nil {
		log.Fatal(err)
		return
	}
}
Kiroku.Snapshot
func ExampleKiroku_Snapshot() {
	var err error
	if err = testKiroku.Snapshot(func(s *Snapshot) (err error) {
		return s.Write([]byte("testKey"), []byte("hello world!"))
	}); err != nil {
		log.Fatal(err)
		return
	}
}
NewWriter
func ExampleNewWriter() {
	var err error
	if testWriter, err = NewWriter("./test_data", "testie"); err != nil {
		log.Fatal(err)
		return
	}
}
Writer.GetIndex
func ExampleWriter_GetIndex() {
	var (
		index uint64
		err   error
	)

	if index, err = testWriter.GetIndex(); err != nil {
		log.Fatal(err)
		return
	}

	fmt.Println("Current index:", index)
}
Writer.SetIndex
func ExampleWriter_SetIndex() {
	var err error
	if err = testWriter.SetIndex(1337); err != nil {
		log.Fatal(err)
	}
}
Writer.NextIndex
func ExampleWriter_NextIndex() {
	var (
		index uint64
		err   error
	)

	if index, err = testWriter.NextIndex(); err != nil {
		log.Fatal(err)
		return
	}

	fmt.Println("Next index:", index)
}
Writer.AddBlock
func ExampleWriter_AddBlock() {
	var err error
	if err = testWriter.AddBlock(TypeWriteAction, []byte("Hello world!")); err != nil {
		log.Fatalf("error adding row: %v", err)
		return
	}
}
NewReader
func ExampleNewReader() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Open("filename.moj"); err != nil {
		log.Fatalf("error opening: %v", err)
		return
	}

	if testReader, err = NewReader(f); err != nil {
		log.Fatalf("error initializing reader: %v", err)
		return
	}
}
Reader.Meta
func ExampleReader_Meta() {
	var m Meta
	m = testReader.Meta()
	fmt.Println("Meta!", m)
}
Reader.ForEach
func ExampleReader_ForEach() {
	var err error
	if err = testReader.ForEach(0, func(b *Block) (err error) {
		fmt.Println("Block data:", string(b.Value))
		return
	}); err != nil {
		log.Fatalf("Error iterating through blocks: %v", err)
	}
}
Reader.Copy
func ExampleReader_Copy() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Create("chunk.copy.moj"); err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if _, err = testReader.Copy(f); err != nil {
		log.Fatalf("Error copying chunk: %v", err)
	}
}
Reader.CopyBlocks
func ExampleReader_CopyBlocks() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Create("chunk.blocksOnly.copy.moj"); err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if _, err = testReader.CopyBlocks(f); err != nil {
		log.Fatalf("Error copying chunk: %v", err)
	}
}
Read
func ExampleRead() {
	var err error
	if err = Read("filename.moj", func(r *Reader) (err error) {
		var m Meta
		m = testReader.Meta()
		fmt.Println("Meta!", m)

		if err = r.ForEach(0, func(b *Block) (err error) {
			fmt.Println("Block data:", string(b.Value))
			return
		}); err != nil {
			log.Fatalf("Error iterating through blocks: %v", err)
		}

		return
	}); err != nil {
		log.Fatal(err)
		return
	}
}

Documentation

Index

Examples

Constants

View Source 
const (
	// ErrEmptyDirectory is returned when a directory is empty
	ErrEmptyDirectory = errors.Error("invalid directory, cannot be empty")
	// ErrEmptyName is returned when a name is empty
	ErrEmptyName = errors.Error("invalid name, cannot be empty")
)
View Source 
const (
	// DefaultEndOfResultsDelay is the default value for EndOfResultsDelay
	DefaultEndOfResultsDelay = time.Second * 10
)

Variables

This section is empty.

Functions

func Read 

func Read(filename string, p Processor) (err error)

Read will read a filename and provide a temporary reader

Example 
var err error
if err = Read("filename.moj", func(r *Reader) (err error) {
	m := testReader.Meta()
	fmt.Println("Meta!", m)

	if _, err = r.ForEach(0, func(b *Block) (err error) {
		fmt.Println("Block value:", string(b.Value))
		return
	}); err != nil {
		log.Fatalf("Error iterating through blocks: %v", err)
	}

	return
}); err != nil {
	log.Fatal(err)
	return
}

Types

type Block 

type Block struct {
	// Type of block
	Type Type
	// Key of block
	Key []byte
	// Value of block
	Value []byte
}

Block represents a block of data stored within history

func (*Block) MarshalEnkodo 

func (b *Block) MarshalEnkodo(enc *enkodo.Encoder) (err error)

MarshalEnkodo is a enkodo encoding helper func

func (*Block) UnmarshalEnkodo 

func (b *Block) UnmarshalEnkodo(dec *enkodo.Decoder) (err error)

UnmarshalEnkodo is a enkodo decoding helper func

type Exporter added in v0.2.0 

type Exporter interface {
	Export(filename string, r io.Reader) error
}

Exporter is used for exporting

type Importer added in v0.2.0 

type Importer interface {
	GetNext(ctx context.Context, prefix, lastFilename string) (filename string, err error)
	Import(ctx context.Context, filename string, w io.Writer) (err error)
}

Importer is used for importing

type Kiroku 

type Kiroku struct {
	// contains filtered or unexported fields
}

Kiroku represents historical DB entries

func New 

func New(o Options, e Exporter) (kp *Kiroku, err error)

New will initialize a new Kiroku instance Note: Processor and Options are optional

Example 
var err error
opts := MakeOptions("./test_data", "tester")
if testKiroku, err = New(opts, nil); err != nil {
	log.Fatal(err)
	return
}
Example (With_custom_Exporter) 
var (
	e   Exporter
	err error
)

// Utilize any Exporter, see https://github.com/mojura/sync-s3 for an example

opts := MakeOptions("./test_data", "tester")
if testKiroku, err = New(opts, e); err != nil {
	log.Fatal(err)
	return
}

func NewWithContext 

func NewWithContext(ctx context.Context, o Options, e Exporter) (kp *Kiroku, err error)

NewWithContext will initialize a new Kiroku instance with a provided context.Context Note: Processor and Options are optional

func (*Kiroku) Close 

func (k *Kiroku) Close() (err error)

Close will close the selected instance of Kiroku

func (*Kiroku) Filename 

func (k *Kiroku) Filename() (filename string, err error)

Filename returns the filename of the primary chunk

func (*Kiroku) Meta 

func (k *Kiroku) Meta() (m Meta, err error)

Meta will return a copy of the current Meta

Example 
var (
	m   Meta
	err error
)

if m, err = testKiroku.Meta(); err != nil {
	log.Fatal(err)
}

fmt.Println("Meta", m)

func (*Kiroku) Snapshot 

func (k *Kiroku) Snapshot(fn func(*Snapshot) error) (err error)

Snapshot will engage a new history snapshot

Example 
var err error
if err = testKiroku.Snapshot(func(s *Snapshot) (err error) {
	return s.Write([]byte("testKey"), []byte("hello world!"))
}); err != nil {
	log.Fatal(err)
	return
}

func (*Kiroku) Transaction 

func (k *Kiroku) Transaction(fn func(*Transaction) error) (err error)

Transaction will engage a new history transaction

Example 
var err error
if err = testKiroku.Transaction(func(t *Transaction) (err error) {
	if err = t.SetIndex(1337); err != nil {
		return
	}

	return t.AddBlock(TypeWriteAction, []byte("testKey"), []byte("hello world!"))
}); err != nil {
	log.Fatal(err)
	return
}

type Meta 

type Meta struct {
	// CurrentIndex would be the current index count
	CurrentIndex uint64 `json:"currentIndex"`

	// BlockCount is the number of blocks contained within the Chunk
	BlockCount int64 `json:"blockCount"`
	// TotalBlockSize is the total block size (in bytes)
	TotalBlockSize int64 `json:"totalBlockSize"`

	// LastSnapshotAt is the timestamp of the last snapshot as Unix Nano
	LastSnapshotAt int64 `json:"lastSnapshotAt"`
	// CreatedAt is a UnixNano timestamp of when the last chunk was created
	CreatedAt int64 `json:"createdAt"`
}

Meta represents the historical meta data

type Mirror added in v0.2.0 

type Mirror struct {
	// contains filtered or unexported fields
}

Mirror represents a read-only instance of historical DB entries Note: The mirror is updated through it's Importer

func NewMirror added in v0.2.0 

func NewMirror(opts MirrorOptions, i Importer) (mp *Mirror, err error)

NewMirror will initialize a new Mirror instance

Example 
var err error
opts := MakeMirrorOptions("./test_data", "tester", nil)
if testMirror, err = NewMirror(opts, newErrorImporter(io.EOF)); err != nil {
	log.Fatal(err)
	return
}

func NewMirrorWithContext added in v0.2.0 

func NewMirrorWithContext(ctx context.Context, opts MirrorOptions, i Importer) (mp *Mirror, err error)

NewMirrorWithContext will initialize a new Mirror instance with a provided context.Context

func (*Mirror) Channel added in v0.3.0 

func (m *Mirror) Channel() <-chan struct{}

func (*Mirror) Close added in v0.2.0 

func (m *Mirror) Close() (err error)

Close will close the selected instance of Kiroku

func (*Mirror) Filename added in v0.2.0 

func (m *Mirror) Filename() (filename string, err error)

Filename returns the filename of the primary chunk

func (*Mirror) Meta added in v0.2.0 

func (m *Mirror) Meta() (meta Meta, err error)

Meta will return a copy of the current Meta

type MirrorOptions added in v0.2.0 

type MirrorOptions struct {
	Options

	// EndOfResultsDelay represents the amount of time to wait before pulling "Next" after receiving empty results
	// Note: Default is 10 seconds
	EndOfResultsDelay time.Duration
}

MirrorOptions represent Mirror options

func MakeMirrorOptions added in v0.2.0 

func MakeMirrorOptions(dir, name string, onImport Processor) (o MirrorOptions)

MakeMirrorOptions will create new Mirror Options

func (*MirrorOptions) Validate added in v0.2.0 

func (m *MirrorOptions) Validate() (err error)

Validate ensures that the MirrorOptions have all the required fields set

type Options 

type Options struct {
	Dir  string
	Name string

	AvoidMergeOnInit   bool
	AvoidMergeOnClose  bool
	AvoidExportOnClose bool
}

Options represent Kiroku options

func MakeOptions added in v0.2.0 

func MakeOptions(dir, name string) (o Options)

MakeOptions will create new Options

func (*Options) Validate added in v0.2.0 

func (o *Options) Validate() (err error)

Validate ensures that the Options have all the required fields set

type Processor 

type Processor func(*Reader) error

Processor will process chunks

type Reader 

type Reader struct {
	// contains filtered or unexported fields
}

Reader will parse and read a history chunk

func NewReader 

func NewReader(rs io.ReadSeeker) (rp *Reader, err error)

NewReader will initialize a new chunk reader

Example 
var (
	f   *os.File
	err error
)

if f, err = os.Open("filename.moj"); err != nil {
	log.Fatalf("error opening: %v", err)
	return
}

if testReader, err = NewReader(f); err != nil {
	log.Fatalf("error initializing reader: %v", err)
	return
}

func (*Reader) Copy 

func (r *Reader) Copy(destination io.Writer) (n int64, err error)

Copy will copy the entire chunk (meta + blocks)

Example 
var (
	f   *os.File
	err error
)

if f, err = os.Create("chunk.copy.moj"); err != nil {
	log.Fatal(err)
	return
}
defer f.Close()

if _, err = testReader.Copy(f); err != nil {
	log.Fatalf("Error copying chunk: %v", err)
}

func (*Reader) CopyBlocks 

func (r *Reader) CopyBlocks(destination io.Writer) (n int64, err error)

CopyBlocks will copy the blocks only (no meta)

Example 
var (
	f   *os.File
	err error
)

if f, err = os.Create("chunk.blocksOnly.copy.moj"); err != nil {
	log.Fatal(err)
	return
}
defer f.Close()

if _, err = testReader.CopyBlocks(f); err != nil {
	log.Fatalf("Error copying chunk: %v", err)
}

func (*Reader) ForEach 

func (r *Reader) ForEach(seek int64, fn func(*Block) error) (lastPosition int64, err error)

ForEach will iterate through all the blocks within the reader

Example 
var (
	lastPosition int64
	err          error
)

if lastPosition, err = testReader.ForEach(0, func(b *Block) (err error) {
	fmt.Println("Block value:", string(b.Value))
	return
}); err != nil {
	log.Fatalf("Error iterating through blocks: %v", err)
}

fmt.Println("Last read block at", lastPosition)

func (*Reader) Meta 

func (r *Reader) Meta() Meta

Meta will return the meta information for the chunk

Example 
m := testReader.Meta()
fmt.Println("Meta!", m)

func (*Reader) ReadSeeker added in v0.1.2 

func (r *Reader) ReadSeeker() io.ReadSeeker

ReadSeeker will return the Reader's underlying ReadSeeker

type Snapshot 

type Snapshot struct {
	// contains filtered or unexported fields
}

Snapshot manages a Kiroku transaction

func (*Snapshot) Write 

func (s *Snapshot) Write(key, value []byte) (err error)

Write will add a write block to a writer

type Transaction 

type Transaction struct {
	// contains filtered or unexported fields
}

Transaction manages a Kiroku transaction

func (*Transaction) AddBlock 

func (t *Transaction) AddBlock(typ Type, key, value []byte) (err error)

AddBlock will add a row

func (*Transaction) GetIndex 

func (t *Transaction) GetIndex() (index uint64, err error)

GetIndex will get the current index value

func (*Transaction) NextIndex 

func (t *Transaction) NextIndex() (index uint64, err error)

NextIndex will get the current index value then increment the internal value

func (*Transaction) SetIndex 

func (t *Transaction) SetIndex(index uint64) (err error)

SetIndex will set the index value Note: This can be used to manually set an index to a desired value

type Type 

type Type uint8

Type represents a block type 

const (
	// TypeUnset represents an unset block type
	TypeUnset Type = iota
	// TypeWriteAction represents a write action block
	TypeWriteAction
	// TypeDeleteAction represets a delete action block
	TypeDeleteAction
	// TypeComment represents a comment block
	TypeComment
)

func (Type) String 

func (t Type) String() string

Validate will ensure a type is valid

func (Type) Validate 

func (t Type) Validate() (err error)

Validate will ensure a type is valid

type Writer 

type Writer struct {
	// contains filtered or unexported fields
}

Writer will write a history chunk

func NewWriter 

func NewWriter(dir, name string) (wp *Writer, err error)

NewWriter will initialize a new Writer instance

Example 
var err error
if testWriter, err = NewWriter("./test_data", "testie"); err != nil {
	log.Fatal(err)
	return
}

func NewWriterWithFile 

func NewWriterWithFile(f *os.File) (wp *Writer, err error)

NewWriterWithFile will initialize a new Writer instance

func (*Writer) AddBlock 

func (w *Writer) AddBlock(t Type, key, value []byte) (err error)

AddBlock will add a row

Example 
var err error
if err = testWriter.AddBlock(TypeWriteAction, []byte("greeting"), []byte("Hello world!")); err != nil {
	log.Fatalf("error adding row: %v", err)
	return
}

func (*Writer) Close added in v0.1.4 

func (w *Writer) Close() (err error)

Close will close a writer

func (*Writer) Filename added in v0.1.3 

func (w *Writer) Filename() string

Filename will return the underlying filename of the Writer

func (*Writer) GetIndex 

func (w *Writer) GetIndex() (index uint64, err error)

GetIndex will get the current index value

Example 
var (
	index uint64
	err   error
)

if index, err = testWriter.GetIndex(); err != nil {
	log.Fatal(err)
	return
}

fmt.Println("Current index:", index)

func (*Writer) Merge 

func (w *Writer) Merge(r *Reader) (err error)

Merge will merge a child chunk into a primary chunk

func (*Writer) Meta added in v0.1.1 

func (w *Writer) Meta() Meta

Meta will return a copy of the Writer's underlying Meta

func (*Writer) NextIndex 

func (w *Writer) NextIndex() (index uint64, err error)

NextIndex will get the current index value then increment the internal value

Example 
var (
	index uint64
	err   error
)

if index, err = testWriter.NextIndex(); err != nil {
	log.Fatal(err)
	return
}

fmt.Println("Next index:", index)

func (*Writer) SetIndex 

func (w *Writer) SetIndex(index uint64) (err error)

SetIndex will set the index value Note: This can be used to manually set an index to a desired value

Example 
var err error
if err = testWriter.SetIndex(1337); err != nil {
	log.Fatal(err)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.