ourdb

package
v1.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 23, 2025 License: Apache-2.0 Imports: 7 Imported by: 0

README

OurDB

OurDB is a simple key-value database implementation that provides:

  • Efficient key-value storage with history tracking
  • Data integrity verification using CRC32
  • Support for multiple backend files
  • Lookup table for fast data retrieval

Overview

The database consists of three main components:

  1. DB Interface - Provides the public API for database operations
  2. Lookup Table - Maps keys to data locations for efficient retrieval
  3. Backend Storage - Handles the actual data storage and file management

Features

  • Key-Value Storage: Store and retrieve binary data using numeric keys
  • History Tracking: Maintain a linked list of previous values for each key
  • Data Integrity: Verify data integrity using CRC32 checksums
  • Multiple Backends: Support for multiple storage files to handle large datasets
  • Incremental Mode: Automatically assign IDs for new records

Usage

Basic Usage
package main

import (
    "fmt"
    "log"
    
    "github.com/freeflowuniverse/herolauncher/pkg/ourdb"
)

func main() {
    // Create a new database
    config := ourdb.DefaultConfig()
    config.Path = "/path/to/database"
    
    db, err := ourdb.New(config)
    if err != nil {
        log.Fatalf("Failed to create database: %v", err)
    }
    defer db.Close()
    
    // Store data
    data := []byte("Hello, World!")
    id := uint32(1)
    _, err = db.Set(ourdb.OurDBSetArgs{
        ID:   &id,
        Data: data,
    })
    if err != nil {
        log.Fatalf("Failed to store data: %v", err)
    }
    
    // Retrieve data
    retrievedData, err := db.Get(id)
    if err != nil {
        log.Fatalf("Failed to retrieve data: %v", err)
    }
    
    fmt.Printf("Retrieved data: %s\n", string(retrievedData))
}
Using the Client
package main

import (
    "fmt"
    "log"
    
    "github.com/freeflowuniverse/herolauncher/pkg/ourdb"
)

func main() {
    // Create a new client
    client, err := ourdb.NewClient("/path/to/database")
    if err != nil {
        log.Fatalf("Failed to create client: %v", err)
    }
    defer client.Close()
    
    // Add data with auto-generated ID
    data := []byte("Hello, World!")
    id, err := client.Add(data)
    if err != nil {
        log.Fatalf("Failed to add data: %v", err)
    }
    
    fmt.Printf("Data stored with ID: %d\n", id)
    
    // Retrieve data
    retrievedData, err := client.Get(id)
    if err != nil {
        log.Fatalf("Failed to retrieve data: %v", err)
    }
    
    fmt.Printf("Retrieved data: %s\n", string(retrievedData))
    
    // Store data with specific ID
    err = client.Set(2, []byte("Another value"))
    if err != nil {
        log.Fatalf("Failed to set data: %v", err)
    }
    
    // Get history of a value
    history, err := client.GetHistory(id, 5)
    if err != nil {
        log.Fatalf("Failed to get history: %v", err)
    }
    
    fmt.Printf("History count: %d\n", len(history))
    
    // Delete data
    err = client.Delete(id)
    if err != nil {
        log.Fatalf("Failed to delete data: %v", err)
    }
}

Configuration Options

  • RecordNrMax: Maximum number of records (default: 16777215)
  • RecordSizeMax: Maximum size of a record in bytes (default: 4KB)
  • FileSize: Maximum size of a database file (default: 500MB)
  • IncrementalMode: Automatically assign IDs for new records (default: true)
  • Reset: Reset the database on initialization (default: false)

Notes

This is a Go port of the original V implementation from the herolib repository.

Documentation

Overview

Package ourdb provides a simple key-value database implementation with history tracking

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Client 

type Client struct {
	// contains filtered or unexported fields
}

Client provides a simplified interface to the OurDB database

func NewClient 

func NewClient(path string) (*Client, error)

NewClient creates a new client for the specified database path

func NewClientWithConfig 

func NewClientWithConfig(path string, baseConfig OurDBConfig) (*Client, error)

NewClientWithConfig creates a new client with a custom configuration

func (*Client) Add 

func (c *Client) Add(data []byte) (uint32, error)

Add stores data and returns the auto-generated ID

func (*Client) Close 

func (c *Client) Close() error

Close closes the database

func (*Client) Delete 

func (c *Client) Delete(id uint32) error

Delete removes data for the specified ID

func (*Client) Destroy 

func (c *Client) Destroy() error

Destroy closes and removes the database

func (*Client) Get 

func (c *Client) Get(id uint32) ([]byte, error)

Get retrieves data for the specified ID

func (*Client) GetHistory 

func (c *Client) GetHistory(id uint32, depth uint8) ([][]byte, error)

GetHistory retrieves historical values for the specified ID

func (*Client) Set 

func (c *Client) Set(id uint32, data []byte) error

Set stores data with the specified ID

type Location 

type Location struct {
	FileNr   uint16
	Position uint32
}

Location represents a position in a database file

func (Location) ToBytes 

func (loc Location) ToBytes() ([]byte, error)

ToBytes converts a Location to a 6-byte array

func (Location) ToLookupBytes 

func (loc Location) ToLookupBytes(keysize uint8) ([]byte, error)

ToLookupBytes converts a Location to bytes according to the keysize

func (Location) ToUint64 

func (loc Location) ToUint64() (uint64, error)

ToUint64 converts a Location to uint64, with file_nr as most significant (big endian)

type LookupConfig 

type LookupConfig struct {
	Size            uint32
	KeySize         uint8
	LookupPath      string
	IncrementalMode bool
}

LookupConfig contains configuration for the lookup table

type LookupTable 

type LookupTable struct {
	KeySize     uint8
	LookupPath  string
	Data        []byte
	Incremental *uint32
}

LookupTable manages the mapping between IDs and data locations

func NewLookup 

func NewLookup(config LookupConfig) (*LookupTable, error)

NewLookup creates a new lookup table

func (*LookupTable) Delete 

func (lut *LookupTable) Delete(x uint32) error

Delete removes an entry from the lookup table

func (*LookupTable) ExportSparse 

func (lut *LookupTable) ExportSparse(path string) error

ExportSparse exports the lookup table to a file in sparse format

func (*LookupTable) FindLastEntry 

func (lut *LookupTable) FindLastEntry() (uint32, error)

FindLastEntry scans the lookup table to find the highest ID with a non-zero entry

func (*LookupTable) Get 

func (lut *LookupTable) Get(x uint32) (Location, error)

Get retrieves a location from the lookup table

func (*LookupTable) GetDataFilePath 

func (lut *LookupTable) GetDataFilePath() (string, error)

GetDataFilePath returns the path to the data file

func (*LookupTable) GetIncFilePath 

func (lut *LookupTable) GetIncFilePath() (string, error)

GetIncFilePath returns the path to the incremental file

func (*LookupTable) GetNextID 

func (lut *LookupTable) GetNextID() (uint32, error)

GetNextID returns the next available ID for incremental mode

func (*LookupTable) ImportSparse 

func (lut *LookupTable) ImportSparse(path string) error

ImportSparse imports the lookup table from a file in sparse format

func (*LookupTable) IncrementIndex 

func (lut *LookupTable) IncrementIndex() error

IncrementIndex increments the index for the next insertion

func (*LookupTable) LocationNew 

func (lut *LookupTable) LocationNew(b_ []byte) (Location, error)

LocationNew creates a new Location from bytes

func (*LookupTable) Set 

func (lut *LookupTable) Set(x uint32, location Location) error

Set updates a location in the lookup table

type OurDB 

type OurDB struct {
	// contains filtered or unexported fields
}

OurDB represents a binary database with variable-length records

func New 

func New(config OurDBConfig) (*OurDB, error)

New creates a new database with the given configuration

func (*OurDB) Close 

func (db *OurDB) Close() error

Close closes the database file

func (*OurDB) Condense 

func (db *OurDB) Condense() error

Condense removes empty records and updates positions This is a complex operation that creates a new file without the deleted records

func (*OurDB) Delete 

func (db *OurDB) Delete(x uint32) error

Delete removes the data at the specified key position This operation zeros out the record but maintains the space in the file Use condense() to reclaim space from deleted records (happens in step after)

func (*OurDB) Destroy 

func (db *OurDB) Destroy() error

Destroy closes and removes the database

func (*OurDB) Get 

func (db *OurDB) Get(x uint32) ([]byte, error)

Get retrieves data stored at the specified key position Returns error if the key doesn't exist or data is corrupted

func (*OurDB) GetHistory 

func (db *OurDB) GetHistory(x uint32, depth uint8) ([][]byte, error)

GetHistory retrieves a list of previous values for the specified key depth parameter controls how many historical values to retrieve (max) Returns error if key doesn't exist or if there's an issue accessing the data

func (*OurDB) GetNextID 

func (db *OurDB) GetNextID() (uint32, error)

GetNextID returns the next id which will be used when storing

func (*OurDB) Load 

func (db *OurDB) Load() error

Load metadata if exists

func (*OurDB) Save 

func (db *OurDB) Save() error

Save ensures we have the metadata stored on disk

func (*OurDB) Set 

func (db *OurDB) Set(args OurDBSetArgs) (uint32, error)

Set stores data at the specified key position The data is stored with a CRC32 checksum for integrity verification and maintains a linked list of previous values for history tracking Returns the ID used (either x if specified, or auto-incremented if x=0)

type OurDBConfig 

type OurDBConfig struct {
	RecordNrMax     uint32
	RecordSizeMax   uint32
	FileSize        uint32
	Path            string
	IncrementalMode bool
	Reset           bool
}

OurDBConfig contains configuration options for creating a new database

func DefaultConfig 

func DefaultConfig() OurDBConfig

DefaultConfig returns a default configuration

type OurDBSetArgs 

type OurDBSetArgs struct {
	ID   *uint32
	Data []byte
}

OurDBSetArgs contains the parameters for the Set method

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.