kiroku

README

Kiroku

Kiroku is a general purpose historical record system which utilizes data blocks. It was built to be used as the action persistence layer for Mojura.

Usage

New

func ExampleNew() {
	var err error
	if testKiroku, err = New("./test_data", "tester", nil); err != nil {
		log.Fatal(err)
		return
	}
}

New (with custom Processor)

func ExampleNew_with_custom_Processor() {
	var err error
	pfn := func(r *Reader) (err error) {
		fmt.Println("Hello chunk!", r.Meta())
		return
	}

	if testKiroku, err = New("./test_data", "tester", pfn); err != nil {
		log.Fatal(err)
		return
	}
}

Kiroku.Meta

func ExampleKiroku_Meta() {
	var (
		m   Meta
		err error
	)

	if m, err = testKiroku.Meta(); err != nil {
		log.Fatal(err)
	}

	fmt.Println("Meta", m)
}

Kiroku.Transaction

func ExampleKiroku_Transaction() {
	var err error
	if err = testKiroku.Transaction(func(t *Transaction) (err error) {
		if err = t.SetIndex(1337); err != nil {
			return
		}

		return t.AddBlock(TypeWriteAction, []byte("hello world!"))
	}); err != nil {
		log.Fatal(err)
		return
	}
}

Kiroku.Snapshot

func ExampleKiroku_Snapshot() {
	var err error
	if err = testKiroku.Snapshot(func(s *Snapshot) (err error) {
		return s.Write([]byte("testKey"), []byte("hello world!"))
	}); err != nil {
		log.Fatal(err)
		return
	}
}

NewWriter

func ExampleNewWriter() {
	var err error
	if testWriter, err = NewWriter("./test_data", "testie"); err != nil {
		log.Fatal(err)
		return
	}
}

Writer.GetIndex

func ExampleWriter_GetIndex() {
	var (
		index uint64
		err   error
	)

	if index, err = testWriter.GetIndex(); err != nil {
		log.Fatal(err)
		return
	}

	fmt.Println("Current index:", index)
}

Writer.SetIndex

func ExampleWriter_SetIndex() {
	var err error
	if err = testWriter.SetIndex(1337); err != nil {
		log.Fatal(err)
	}
}

Writer.NextIndex

func ExampleWriter_NextIndex() {
	var (
		index uint64
		err   error
	)

	if index, err = testWriter.NextIndex(); err != nil {
		log.Fatal(err)
		return
	}

	fmt.Println("Next index:", index)
}

Writer.AddBlock

func ExampleWriter_AddBlock() {
	var err error
	if err = testWriter.AddBlock(TypeWriteAction, []byte("Hello world!")); err != nil {
		log.Fatalf("error adding row: %v", err)
		return
	}
}

NewReader

func ExampleNewReader() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Open("filename.moj"); err != nil {
		log.Fatalf("error opening: %v", err)
		return
	}

	if testReader, err = NewReader(f); err != nil {
		log.Fatalf("error initializing reader: %v", err)
		return
	}
}

Reader.Meta

func ExampleReader_Meta() {
	var m Meta
	m = testReader.Meta()
	fmt.Println("Meta!", m)
}

Reader.ForEach

func ExampleReader_ForEach() {
	var err error
	if err = testReader.ForEach(0, func(b *Block) (err error) {
		fmt.Println("Block data:", string(b.Value))
		return
	}); err != nil {
		log.Fatalf("Error iterating through blocks: %v", err)
	}
}

Reader.Copy

func ExampleReader_Copy() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Create("chunk.copy.moj"); err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if _, err = testReader.Copy(f); err != nil {
		log.Fatalf("Error copying chunk: %v", err)
	}
}

Reader.CopyBlocks

func ExampleReader_CopyBlocks() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Create("chunk.blocksOnly.copy.moj"); err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if _, err = testReader.CopyBlocks(f); err != nil {
		log.Fatalf("Error copying chunk: %v", err)
	}
}

Read

func ExampleRead() {
	var err error
	if err = Read("filename.moj", func(r *Reader) (err error) {
		var m Meta
		m = testReader.Meta()
		fmt.Println("Meta!", m)

		if err = r.ForEach(0, func(b *Block) (err error) {
			fmt.Println("Block data:", string(b.Value))
			return
		}); err != nil {
			log.Fatalf("Error iterating through blocks: %v", err)
		}

		return
	}); err != nil {
		log.Fatal(err)
		return
	}
}

Documentation

Index

• func Read(filename string, p Processor) (err error)
• type Block
  • func (b *Block) MarshalEnkodo(enc *enkodo.Encoder) (err error)
  • func (b *Block) UnmarshalEnkodo(dec *enkodo.Decoder) (err error)
• type Kiroku
  • func New(dir, name string, p Processor, o *Options) (kp *Kiroku, err error)
  • func NewWithContext(ctx context.Context, dir, name string, p Processor, o *Options) (kp *Kiroku, err error)
  • func (k *Kiroku) Close() (err error)
  • func (k *Kiroku) Filename() (filename string, err error)
  • func (k *Kiroku) Meta() (m Meta, err error)
  • func (k *Kiroku) Snapshot(fn func(*Snapshot) error) (err error)
  • func (k *Kiroku) Transaction(fn func(*Transaction) error) (err error)
• type Meta
• type Options
• type Processor
• type Reader
  • func NewReader(rs io.ReadSeeker) (rp *Reader, err error)
  • func (r *Reader) Copy(destination io.Writer) (n int64, err error)
  • func (r *Reader) CopyBlocks(destination io.Writer) (n int64, err error)
  • func (r *Reader) ForEach(seek int64, fn func(*Block) error) (err error)
  • func (r *Reader) Meta() Meta
• type Snapshot
  • func (s *Snapshot) Write(key, value []byte) (err error)
• type Transaction
  • func (t *Transaction) AddBlock(typ Type, key, value []byte) (err error)
  • func (t *Transaction) GetIndex() (index uint64, err error)
  • func (t *Transaction) NextIndex() (index uint64, err error)
  • func (t *Transaction) SetIndex(index uint64) (err error)
• type Type
  • func (t Type) String() string
  • func (t Type) Validate() (err error)
• type Writer
  • func NewWriter(dir, name string) (wp *Writer, err error)
  • func NewWriterWithFile(f *os.File) (wp *Writer, err error)
  • func (w *Writer) AddBlock(t Type, key, value []byte) (err error)
  • func (w *Writer) GetIndex() (index uint64, err error)
  • func (w *Writer) Merge(r *Reader) (err error)
  • func (w *Writer) NextIndex() (index uint64, err error)
  • func (w *Writer) SetIndex(index uint64) (err error)

Examples

• Kiroku.Meta
• Kiroku.Snapshot
• Kiroku.Transaction
• New
• New (With_custom_Processor)
• NewReader
• NewWriter
• Read
• Reader.Copy
• Reader.CopyBlocks
• Reader.ForEach
• Reader.Meta
• Writer.AddBlock
• Writer.GetIndex
• Writer.NextIndex
• Writer.SetIndex

Functions

func Read

func Read(filename string, p Processor) (err error)

Read will read a filename and provide a temporary reader

Example

var err error
if err = Read("filename.moj", func(r *Reader) (err error) {
	m := testReader.Meta()
	fmt.Println("Meta!", m)

	if err = r.ForEach(0, func(b *Block) (err error) {
		fmt.Println("Block value:", string(b.Value))
		return
	}); err != nil {
		log.Fatalf("Error iterating through blocks: %v", err)
	}

	return
}); err != nil {
	log.Fatal(err)
	return
}

Types

type Block

type Block struct {
	// Type of block
	Type Type
	// Key of block
	Key []byte
	// Value of block
	Value []byte
}

Block represents a block of data stored within history

func (*Block) MarshalEnkodo

func (b *Block) MarshalEnkodo(enc *enkodo.Encoder) (err error)

MarshalEnkodo is a enkodo encoding helper func

func (*Block) UnmarshalEnkodo

func (b *Block) UnmarshalEnkodo(dec *enkodo.Decoder) (err error)

UnmarshalEnkodo is a enkodo decoding helper func

type Kiroku

type Kiroku struct {
	// contains filtered or unexported fields
}

Kiroku represents historical DB entries

func New

func New(dir, name string, p Processor, o *Options) (kp *Kiroku, err error)

New will initialize a new Kiroku instance Note: Processor and Options are optional

Example

var err error
if testKiroku, err = New("./test_data", "tester", nil, nil); err != nil {
	log.Fatal(err)
	return
}

Example (With_custom_Processor)

var err error
pfn := func(r *Reader) (err error) {
	fmt.Println("Hello chunk!", r.Meta())
	return
}

if testKiroku, err = New("./test_data", "tester", pfn, nil); err != nil {
	log.Fatal(err)
	return
}

func NewWithContext

func NewWithContext(ctx context.Context, dir, name string, p Processor, o *Options) (kp *Kiroku, err error)

NewWithContext will initialize a new Kiroku instance with a provided context.Context Note: Processor and Options are optional

func (*Kiroku) Close

func (k *Kiroku) Close() (err error)

Close will close the selected instance of Kiroku

func (*Kiroku) Filename

func (k *Kiroku) Filename() (filename string, err error)

Filename returns the filename of the primary chunk

func (*Kiroku) Meta

func (k *Kiroku) Meta() (m Meta, err error)

Meta will return a copy of the current Meta

Example

var (
	m   Meta
	err error
)

if m, err = testKiroku.Meta(); err != nil {
	log.Fatal(err)
}

fmt.Println("Meta", m)

func (*Kiroku) Snapshot

func (k *Kiroku) Snapshot(fn func(*Snapshot) error) (err error)

Snapshot will engage a new history snapshot

Example

var err error
if err = testKiroku.Snapshot(func(s *Snapshot) (err error) {
	return s.Write([]byte("testKey"), []byte("hello world!"))
}); err != nil {
	log.Fatal(err)
	return
}

func (*Kiroku) Transaction

func (k *Kiroku) Transaction(fn func(*Transaction) error) (err error)

Transaction will engage a new history transaction

Example

var err error
if err = testKiroku.Transaction(func(t *Transaction) (err error) {
	if err = t.SetIndex(1337); err != nil {
		return
	}

	return t.AddBlock(TypeWriteAction, []byte("testKey"), []byte("hello world!"))
}); err != nil {
	log.Fatal(err)
	return
}

type Meta

type Meta struct {
	// CurrentIndex would be the current index count
	CurrentIndex uint64

	// BlockCount is the number of blocks contained within the Chunk
	BlockCount int64
	// TotalBlockSize is the total block size (in bytes)
	TotalBlockSize int64

	// LastSnapshotAt is the timestamp of the last snapshot as Unix Nano
	LastSnapshotAt int64
	// CreatedAt is a UnixNano timestamp of when the last chunk was created
	CreatedAt int64
}

Meta represents the historical meta data

type Options

type Options struct {
	AvoidMergeOnInit    bool
	AvoidProcessOnInit  bool
	AvoidMergeOnClose   bool
	AvoidProcessOnClose bool
}

Options represent Kiroku options

type Processor

type Processor func(*Reader) error

Processor will process chunks

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader will parse and read a history chunk

func NewReader

func NewReader(rs io.ReadSeeker) (rp *Reader, err error)

NewReader will initialize a new chunk reader

Example

var (
	f   *os.File
	err error
)

if f, err = os.Open("filename.moj"); err != nil {
	log.Fatalf("error opening: %v", err)
	return
}

if testReader, err = NewReader(f); err != nil {
	log.Fatalf("error initializing reader: %v", err)
	return
}

func (*Reader) Copy

func (r *Reader) Copy(destination io.Writer) (n int64, err error)

Copy will copy the entire chunk (meta + blocks)

Example

var (
	f   *os.File
	err error
)

if f, err = os.Create("chunk.copy.moj"); err != nil {
	log.Fatal(err)
	return
}
defer f.Close()

if _, err = testReader.Copy(f); err != nil {
	log.Fatalf("Error copying chunk: %v", err)
}

func (*Reader) CopyBlocks

func (r *Reader) CopyBlocks(destination io.Writer) (n int64, err error)

CopyBlocks will copy the blocks only (no meta)

Example

var (
	f   *os.File
	err error
)

if f, err = os.Create("chunk.blocksOnly.copy.moj"); err != nil {
	log.Fatal(err)
	return
}
defer f.Close()

if _, err = testReader.CopyBlocks(f); err != nil {
	log.Fatalf("Error copying chunk: %v", err)
}

func (*Reader) ForEach

func (r *Reader) ForEach(seek int64, fn func(*Block) error) (err error)

ForEach will iterate through all the blocks within the reader

Example

var err error
if err = testReader.ForEach(0, func(b *Block) (err error) {
	fmt.Println("Block value:", string(b.Value))
	return
}); err != nil {
	log.Fatalf("Error iterating through blocks: %v", err)
}

func (*Reader) Meta

func (r *Reader) Meta() Meta

Meta will return the meta information for the chunk

Example

m := testReader.Meta()
fmt.Println("Meta!", m)

type Snapshot

type Snapshot struct {
	// contains filtered or unexported fields
}

Snapshot manages a Kiroku transaction

func (*Snapshot) Write

func (s *Snapshot) Write(key, value []byte) (err error)

Write will add a write block to a writer

type Transaction

type Transaction struct {
	// contains filtered or unexported fields
}

Transaction manages a Kiroku transaction

func (*Transaction) AddBlock

func (t *Transaction) AddBlock(typ Type, key, value []byte) (err error)

AddBlock will add a row

func (*Transaction) GetIndex

func (t *Transaction) GetIndex() (index uint64, err error)

GetIndex will get the current index value

func (*Transaction) NextIndex

func (t *Transaction) NextIndex() (index uint64, err error)

NextIndex will get the current index value then increment the internal value

func (*Transaction) SetIndex

func (t *Transaction) SetIndex(index uint64) (err error)

SetIndex will set the index value Note: This can be used to manually set an index to a desired value

type Type

type Type uint8

Type represents a block type

const (
	// TypeUnset represents an unset block type
	TypeUnset Type = iota
	// TypeWriteAction represents a write action block
	TypeWriteAction
	// TypeDeleteAction represets a delete action block
	TypeDeleteAction
	// TypeComment represents a comment block
	TypeComment
)

func (Type) String

func (t Type) String() string

Validate will ensure a type is valid

func (Type) Validate

func (t Type) Validate() (err error)

Validate will ensure a type is valid

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer will write a history chunk

func NewWriter

func NewWriter(dir, name string) (wp *Writer, err error)

NewWriter will initialize a new Writer instance

Example

var err error
if testWriter, err = NewWriter("./test_data", "testie"); err != nil {
	log.Fatal(err)
	return
}

func NewWriterWithFile

func NewWriterWithFile(f *os.File) (wp *Writer, err error)

NewWriterWithFile will initialize a new Writer instance

func (*Writer) AddBlock

func (w *Writer) AddBlock(t Type, key, value []byte) (err error)

AddBlock will add a row

Example

var err error
if err = testWriter.AddBlock(TypeWriteAction, []byte("greeting"), []byte("Hello world!")); err != nil {
	log.Fatalf("error adding row: %v", err)
	return
}

func (*Writer) GetIndex

func (w *Writer) GetIndex() (index uint64, err error)

GetIndex will get the current index value

Example

var (
	index uint64
	err   error
)

if index, err = testWriter.GetIndex(); err != nil {
	log.Fatal(err)
	return
}

fmt.Println("Current index:", index)

func (*Writer) Merge

func (w *Writer) Merge(r *Reader) (err error)

Merge will merge a child chunk into a primary chunk

func (*Writer) NextIndex

func (w *Writer) NextIndex() (index uint64, err error)

NextIndex will get the current index value then increment the internal value

Example

var (
	index uint64
	err   error
)

if index, err = testWriter.NextIndex(); err != nil {
	log.Fatal(err)
	return
}

fmt.Println("Next index:", index)

func (*Writer) SetIndex

func (w *Writer) SetIndex(index uint64) (err error)

SetIndex will set the index value Note: This can be used to manually set an index to a desired value

Example

var err error
if err = testWriter.SetIndex(1337); err != nil {
	log.Fatal(err)
}