Help us shape the future of Go! Take the 2026 Go Developer Survey

kiroku

package module
v0.10.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 26, 2023 License: MIT Imports: 18 Imported by: 9

README

Kiroku GoDoc Status Go Report Card Go Test Coverage

Kiroku is a general purpose historical record system which utilizes data blocks. It was built to be used as the action persistence layer for Mojura.

Usage

New
func ExampleNew() {
	var err error
	if testKiroku, err = New("./test_data", "tester", nil); err != nil {
		log.Fatal(err)
		return
	}
}
New (with custom Processor)
func ExampleNew_with_custom_Processor() {
	var err error
	pfn := func(r *Reader) (err error) {
		fmt.Println("Hello chunk!", r.Meta())
		return
	}

	if testKiroku, err = New("./test_data", "tester", pfn); err != nil {
		log.Fatal(err)
		return
	}
}
Kiroku.Meta
func ExampleKiroku_Meta() {
	var (
		m   Meta
		err error
	)

	if m, err = testKiroku.Meta(); err != nil {
		log.Fatal(err)
	}

	fmt.Println("Meta", m)
}
Kiroku.Transaction
func ExampleKiroku_Transaction() {
	var err error
	if err = testKiroku.Transaction(func(t *Transaction) (err error) {
		return t.AddBlock(TypeWriteAction, []byte("hello world!"))
	}); err != nil {
		log.Fatal(err)
		return
	}
}
Kiroku.Snapshot
func ExampleKiroku_Snapshot() {
	var err error
	if err = testKiroku.Snapshot(func(s *Snapshot) (err error) {
		return s.Write([]byte("testKey"), []byte("hello world!"))
	}); err != nil {
		log.Fatal(err)
		return
	}
}
NewWriter
func ExampleNewWriter() {
	var err error
	if testWriter, err = NewWriter("./test_data", "testie"); err != nil {
		log.Fatal(err)
		return
	}
}
Writer.AddBlock
func ExampleWriter_AddBlock() {
	var err error
	if err = testWriter.AddBlock(TypeWriteAction, []byte("Hello world!")); err != nil {
		log.Fatalf("error adding row: %v", err)
		return
	}
}
NewReader
func ExampleNewReader() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Open("filename.moj"); err != nil {
		log.Fatalf("error opening: %v", err)
		return
	}

	if testReader, err = NewReader(f); err != nil {
		log.Fatalf("error initializing reader: %v", err)
		return
	}
}
Reader.Meta
func ExampleReader_Meta() {
	var m Meta
	m = testReader.Meta()
	fmt.Println("Meta!", m)
}
Reader.ForEach
func ExampleReader_ForEach() {
	var err error
	if err = testReader.ForEach(0, func(b *Block) (err error) {
		fmt.Println("Block data:", string(b.Value))
		return
	}); err != nil {
		log.Fatalf("Error iterating through blocks: %v", err)
	}
}
Reader.Copy
func ExampleReader_Copy() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Create("chunk.copy.moj"); err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if _, err = testReader.Copy(f); err != nil {
		log.Fatalf("Error copying chunk: %v", err)
	}
}
Reader.CopyBlocks
func ExampleReader_CopyBlocks() {
	var (
		f   *os.File
		err error
	)

	if f, err = os.Create("chunk.blocksOnly.copy.moj"); err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if _, err = testReader.CopyBlocks(f); err != nil {
		log.Fatalf("Error copying chunk: %v", err)
	}
}
Read
func ExampleRead() {
	var err error
	if err = Read("filename.moj", func(r *Reader) (err error) {
		var m Meta
		m = testReader.Meta()
		fmt.Println("Meta!", m)

		if err = r.ForEach(0, func(b *Block) (err error) {
			fmt.Println("Block data:", string(b.Value))
			return
		}); err != nil {
			log.Fatalf("Error iterating through blocks: %v", err)
		}

		return
	}); err != nil {
		log.Fatal(err)
		return
	}
}

Documentation

Index

Examples

Constants

View Source 
const (
	// ErrMirrorNilSource is returned when a mirror is initialized with a nil source
	ErrMirrorNilSource = errors.Error("mirrors cannot have a nil source")
	// ErrMirrorTransaction is returned when a mirror attempts a transaction
	ErrMirrorTransaction = errors.Error("mirrors cannot perform transactions")
	// ErrMirrorSnapshot is returned when a mirror attempts to snapshot
	ErrMirrorSnapshot = errors.Error("mirrors cannot perform snapshots")
)
View Source 
const (
	// ErrEmptyDirectory is returned when a directory is empty
	ErrEmptyDirectory = errors.Error("invalid directory, cannot be empty")
	// ErrEmptyName is returned when a name is empty
	ErrEmptyName = errors.Error("invalid name, cannot be empty")
)
View Source 
const (
	// DefaultEndOfResultsDelay is the default value for EndOfResultsDelay
	DefaultEndOfResultsDelay = time.Second * 10
	// DefaultErrorDelay is the default value for ErrorDelay
	DefaultErrorDelay = time.Second * 30
	// DefaultBatchDuration is the default value for BatchDuration
	DefaultBatchDuration = time.Second * 10
)

Variables

This section is empty.

Functions

func GenerateFilename added in v0.9.9 

func GenerateFilename(name, kind string, timestamp int64) string

func Read 

func Read(filename string, p Processor) (err error)

Read will read a filename and provide a temporary reader

Example 
var err error
if err = Read("filename.moj", func(r *Reader) (err error) {
	m := testReader.Meta()
	fmt.Println("Meta!", m)

	if err = r.ForEach(0, func(b *Block) (err error) {
		fmt.Println("Block value:", string(b.Value))
		return
	}); err != nil {
		log.Fatalf("Error iterating through blocks: %v", err)
	}

	return
}); err != nil {
	log.Fatal(err)
	return
}

Types

type BatchFn added in v0.10.5 

type BatchFn func(*Transaction)

type Block 

type Block struct {
	// Type of block
	Type Type
	// Key of block
	Key []byte
	// Value of block
	Value []byte
}

Block represents a block of data stored within history

func (*Block) MarshalEnkodo 

func (b *Block) MarshalEnkodo(enc *enkodo.Encoder) (err error)

MarshalEnkodo is a enkodo encoding helper func

func (*Block) UnmarshalEnkodo 

func (b *Block) UnmarshalEnkodo(dec *enkodo.Decoder) (err error)

UnmarshalEnkodo is a enkodo decoding helper func

type File added in v0.5.0 

type File interface {
	io.Seeker
	io.Reader
	io.ReaderAt
}

type Kiroku 

type Kiroku struct {
	// contains filtered or unexported fields
}

Kiroku represents historical DB entries

func New 

func New(o Options, src Source) (kp *Kiroku, err error)

New will initialize a new Kiroku instance Note: Processor and Options are optional

Example 
var err error
opts := MakeOptions("./test_data", "tester")
if testKiroku, err = New(opts, nil); err != nil {
	log.Fatal(err)
	return
}
Example (With_custom_Exporter) 
var (
	src Source
	err error
)

// Utilize any Source, see https://github.com/mojura/sync-s3 for an example

opts := MakeOptions("./test_data", "tester")
if testKiroku, err = New(opts, src); err != nil {
	log.Fatal(err)
	return
}

func NewWithContext 

func NewWithContext(ctx context.Context, o Options, src Source) (kp *Kiroku, err error)

NewWithContext will initialize a new Kiroku instance with a provided context.Context Note: Processor and Options are optional

func (*Kiroku) Close 

func (k *Kiroku) Close() (err error)

Close will close the selected instance of Kiroku

func (*Kiroku) Filename 

func (k *Kiroku) Filename() (filename string, err error)

Filename returns the filename of the primary chunk

func (*Kiroku) Meta 

func (k *Kiroku) Meta() (m Meta, err error)

Meta will return a copy of the current Meta

Example 
var (
	m   Meta
	err error
)

if m, err = testKiroku.Meta(); err != nil {
	log.Fatal(err)
}

fmt.Println("Meta", m)

func (*Kiroku) Snapshot 

func (k *Kiroku) Snapshot(fn func(*Snapshot) error) (err error)

Snapshot will engage a new history snapshot

Example 
var err error
if err = testKiroku.Snapshot(func(s *Snapshot) (err error) {
	return s.Write([]byte("testKey"), []byte("hello world!"))
}); err != nil {
	log.Fatal(err)
	return
}

func (*Kiroku) Transaction 

func (k *Kiroku) Transaction(fn func(*Transaction) error) (err error)

Transaction will engage a new history transaction

Example 
var err error
if err = testKiroku.Transaction(func(t *Transaction) (err error) {
	return t.AddBlock(TypeWriteAction, []byte("testKey"), []byte("hello world!"))
}); err != nil {
	log.Fatal(err)
	return
}

type Ledger added in v0.4.0 

type Ledger interface {
	Meta() (m Meta, err error)
	Transaction(fn func(*Transaction) error) (err error)
	Snapshot(fn func(*Snapshot) error) (err error)
	Filename() (filename string, err error)
	Close() (err error)
}

type Meta 

type Meta struct {
	// BlockCount is the number of blocks contained within the Chunk
	BlockCount int64 `json:"blockCount"`
	// TotalBlockSize is the total block size (in bytes)
	TotalBlockSize int64 `json:"totalBlockSize"`

	// LastSnapshotAt is the timestamp of the last snapshot as Unix Nano
	LastSnapshotAt int64 `json:"lastSnapshotAt"`
	// CreatedAt is a UnixNano timestamp of when the last chunk was created
	CreatedAt int64 `json:"createdAt"`
}

Meta represents the historical meta data

type Mirror added in v0.2.0 

type Mirror struct {
	// contains filtered or unexported fields
}

Mirror represents a read-only instance of historical DB entries Note: The mirror is updated through it's Importer

func NewMirror added in v0.2.0 

func NewMirror(opts Options, src Source) (mp *Mirror, err error)

NewMirror will initialize a new Mirror instance

Example 
var err error
opts := MakeOptions("./test_data", "tester")
if testMirror, err = NewMirror(opts, newErrorSource(io.EOF)); err != nil {
	log.Fatal(err)
	return
}

func NewMirrorWithContext added in v0.2.0 

func NewMirrorWithContext(ctx context.Context, opts Options, src Source) (mp *Mirror, err error)

NewMirrorWithContext will initialize a new Mirror instance with a provided context.Context

func (*Mirror) Channel added in v0.3.0 

func (m *Mirror) Channel() <-chan struct{}

func (*Mirror) Close added in v0.2.0 

func (m *Mirror) Close() (err error)

Close will close the selected instance of Kiroku

func (*Mirror) Filename added in v0.2.0 

func (m *Mirror) Filename() (filename string, err error)

Filename returns the filename of the primary chunk

func (*Mirror) Meta added in v0.2.0 

func (m *Mirror) Meta() (meta Meta, err error)

Meta will return a copy of the current Meta

func (*Mirror) Snapshot added in v0.4.0 

func (m *Mirror) Snapshot(fn func(*Snapshot) error) (err error)

func (*Mirror) Transaction added in v0.4.0 

func (m *Mirror) Transaction(fn func(*Transaction) error) (err error)

type NOOP added in v0.9.6 

type NOOP struct {
}

func (*NOOP) Export added in v0.9.6 

func (n *NOOP) Export(ctx context.Context, filename string, r io.Reader) error

func (*NOOP) Get added in v0.9.6 

func (n *NOOP) Get(ctx context.Context, filename string, fn func(io.Reader) error) error

func (*NOOP) GetNext added in v0.9.6 

func (n *NOOP) GetNext(ctx context.Context, prefix, lastFilename string) (filename string, err error)

func (*NOOP) Import added in v0.9.6 

func (n *NOOP) Import(ctx context.Context, filename string, w io.Writer) error

type Options 

type Options struct {
	Dir       string `toml:"dir" json:"dir"`
	Name      string `toml:"name" json:"name"`
	Namespace string `toml:"namespace" json:"namespace"`

	AvoidImportOnInit  bool `toml:"avoid_import_on_init" json:"avoidImportOnInit"`
	AvoidMergeOnInit   bool `toml:"avoid_merge_on_init" json:"avoidMergeOnInit"`
	AvoidMergeOnClose  bool `toml:"avoid_merge_on_close" json:"avoidMergeOnClose"`
	AvoidExportOnClose bool `toml:"avoid_export_on_close" json:"avoidExportOnClose"`

	IsMirror bool `toml:"is_mirror" json:"isMirror"`

	// Event handlers
	OnUpdate func(*Reader)

	// EndOfResultsDelay represents the amount of time to wait before pulling "Next" after
	// receiving empty results (Default is 10 seconds).
	EndOfResultsDelay time.Duration `toml:"end_of_results_delay" json:"endOfResultsDelay"`
	// ErrorDelay represents the amount of time to wait before pulling "Next" after
	// receiving an error
	ErrorDelay time.Duration `toml:"error_delay" json:"errorDelay"`
	// BatchDuration represents the amount of time to keep a transaction open for a
	// Batch operation
	BatchDuration time.Duration `toml:"batch_duration" json:"batchDuration"`

	// RangeStart will determine the moment in time from which syncs will begin
	RangeStart time.Time `toml:"range_start" json:"rangeStart"`
	// RangeEnd will determine the moment in time from which syncs will end
	// Note: This feature is slated to be implemented within the following
	// release. As of now, this will act as a field placeholder
	RangeEnd time.Time `toml:"range_end" json:"rangeEnd"`
}

Options represent Kiroku options

func MakeOptions added in v0.2.0 

func MakeOptions(dir, name string) (o Options)

MakeOptions will create new Options

func (*Options) FullName added in v0.10.1 

func (o *Options) FullName() string

func (*Options) Validate added in v0.2.0 

func (o *Options) Validate() (err error)

Validate ensures that the Options have all the required fields set

type Processor 

type Processor func(*Reader) error

Processor will process chunks

type Producer added in v0.10.5 

type Producer struct {
	// contains filtered or unexported fields
}

Producer represents historical DB entries

func NewProducer added in v0.10.5 

func NewProducer(o Options, src Source) (kp *Producer, err error)

New will initialize a new Producer instance Note: Processor and Options are optional

func NewProducerWithContext added in v0.10.5 

func NewProducerWithContext(ctx context.Context, o Options, src Source) (pp *Producer, err error)

NewWithContext will initialize a new Producer instance with a provided context.Context Note: Processor and Options are optional

func (*Producer) Batch added in v0.10.5 

func (p *Producer) Batch(fn BatchFn) (err error)

Batch will engage a new history batch transaction

func (*Producer) BatchBlock added in v0.10.5 

func (p *Producer) BatchBlock(typ Type, key []byte, value []byte) (err error)

Batch will engage a new history batch transaction

func (*Producer) Close added in v0.10.5 

func (p *Producer) Close() (err error)

Close will close the selected instance of Producer

func (*Producer) Transaction added in v0.10.5 

func (p *Producer) Transaction(fn TransactionFn) (err error)

Transaction will engage a new history transaction

type Reader 

type Reader struct {
	// contains filtered or unexported fields
}

Reader will parse and read a history chunk

func NewReader 

func NewReader(f File) (rp *Reader, err error)

NewReader will initialize a new chunk reader

Example 
var (
	f   *os.File
	err error
)

if f, err = os.Open("filename.moj"); err != nil {
	log.Fatalf("error opening: %v", err)
	return
}

if testReader, err = NewReader(f); err != nil {
	log.Fatalf("error initializing reader: %v", err)
	return
}

func (*Reader) Copy 

func (r *Reader) Copy(destination io.Writer) (n int64, err error)

Copy will copy the entire chunk (meta + blocks)

Example 
var (
	f   *os.File
	err error
)

if f, err = os.Create("chunk.copy.moj"); err != nil {
	log.Fatal(err)
	return
}
defer f.Close()

if _, err = testReader.Copy(f); err != nil {
	log.Fatalf("Error copying chunk: %v", err)
}

func (*Reader) CopyBlocks 

func (r *Reader) CopyBlocks(destination io.Writer) (n int64, err error)

CopyBlocks will copy the blocks only (no meta)

Example 
var (
	f   *os.File
	err error
)

if f, err = os.Create("chunk.blocksOnly.copy.moj"); err != nil {
	log.Fatal(err)
	return
}
defer f.Close()

if _, err = testReader.CopyBlocks(f); err != nil {
	log.Fatalf("Error copying chunk: %v", err)
}

func (*Reader) ForEach 

func (r *Reader) ForEach(seek int64, fn func(*Block) error) (err error)

ForEach will iterate through all the blocks within the reader

Example 
var err error
if err = testReader.ForEach(0, func(b *Block) (err error) {
	fmt.Println("Block value:", string(b.Value))
	return
}); err != nil {
	log.Fatalf("Error iterating through blocks: %v", err)
}

func (*Reader) Meta 

func (r *Reader) Meta() Meta

Meta will return the meta information for the chunk

Example 
m := testReader.Meta()
fmt.Println("Meta!", m)

func (*Reader) ReadSeeker added in v0.1.2 

func (r *Reader) ReadSeeker() io.ReadSeeker

ReadSeeker will return the Reader's underlying ReadSeeker

type Snapshot 

type Snapshot struct {
	// contains filtered or unexported fields
}

Snapshot manages a Kiroku transaction

func (*Snapshot) Write 

func (s *Snapshot) Write(key, value []byte) (err error)

Write will add a write block to a writer

type Source added in v0.6.0 

type Source interface {
	Export(ctx context.Context, filename string, r io.Reader) error
	Import(ctx context.Context, filename string, w io.Writer) error
	Get(ctx context.Context, filename string, fn func(io.Reader) error) error
	GetNext(ctx context.Context, prefix, lastFilename string) (filename string, err error)
}

Source is used for importing

type Transaction 

type Transaction struct {
	// contains filtered or unexported fields
}

Transaction manages a Kiroku transaction

func (*Transaction) AddBlock 

func (t *Transaction) AddBlock(typ Type, key, value []byte) (err error)

AddBlock will add a row

func (*Transaction) Meta added in v0.9.1 

func (t *Transaction) Meta() (m Meta)

AddBlock will add a row

type TransactionFn added in v0.10.5 

type TransactionFn func(*Transaction) error

type Type 

type Type uint8

Type represents a block type 

const (
	// TypeUnset represents an unset block type
	TypeUnset Type = iota
	// TypeWriteAction represents a write action block
	TypeWriteAction
	// TypeDeleteAction represets a delete action block
	TypeDeleteAction
	// TypeComment represents a comment block
	TypeComment
)

func (Type) String 

func (t Type) String() string

Validate will ensure a type is valid

func (Type) Validate 

func (t Type) Validate() (err error)

Validate will ensure a type is valid

type Writer 

type Writer struct {
	// contains filtered or unexported fields
}

Writer will write a history chunk

func NewWriter 

func NewWriter(dir, name string) (wp *Writer, err error)

NewWriter will initialize a new Writer instance

Example 
var err error
if testWriter, err = NewWriter("./test_data", "testie"); err != nil {
	log.Fatal(err)
	return
}

func NewWriterWithFile 

func NewWriterWithFile(f *os.File) (wp *Writer, err error)

NewWriterWithFile will initialize a new Writer instance

func (*Writer) AddBlock 

func (w *Writer) AddBlock(t Type, key, value []byte) (err error)

AddBlock will add a row

Example 
var err error
if err = testWriter.AddBlock(TypeWriteAction, []byte("greeting"), []byte("Hello world!")); err != nil {
	log.Fatalf("error adding row: %v", err)
	return
}

func (*Writer) Close added in v0.1.4 

func (w *Writer) Close() (err error)

Close will close a writer

func (*Writer) Filename added in v0.1.3 

func (w *Writer) Filename() string

Filename will return the underlying filename of the Writer

func (*Writer) Merge 

func (w *Writer) Merge(r *Reader) (err error)

Merge will merge a child chunk into a primary chunk

func (*Writer) Meta added in v0.1.1 

func (w *Writer) Meta() Meta

Meta will return a copy of the Writer's underlying Meta

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.