snapshotkv

README

SnapshotKV

A lightweight, snapshot-based key-value storage engine with optional large blob support.

Features

• In-memory operations with snapshot persistence
• Memory-only mode - use without any file system (empty path)
• Flexible value types - store any Go type (strings, ints, maps, slices, etc.)
• Debounced saves with configurable interval (default: 10s)
• TTL support with automatic cleanup
• Blob storage for large binary data (loaded on demand)
• Thread-safe for concurrent access from multiple goroutines
• Pluggable serialization (MessagePack default, JSON available)
• HLC-style naming for snapshots (65,535 snapshots per second)
• Atomic writes with temp file + rename pattern
• Configurable snapshot history (rollback capability)

Installation

go get github.com/paularlott/snapshotkv

Quick Start

Persistent Storage

package main

import (
    "fmt"
    "time"

    "github.com/paularlott/snapshotkv"
)

func main() {
    // Open database with persistence
    db, err := snapshotkv.Open("./data", nil)
    if err != nil {
        panic(err)
    }
    defer db.Close()

    // Store a map
    db.Set("user:123", map[string]any{
        "name":  "Alice",
        "email": "alice@example.com",
    })

    // Store a simple string
    db.Set("config:api_key", "secret123")

    // Store an integer
    db.Set("counter", 42)

    // With TTL (30 days)
    db.SetEx("session:abc", map[string]any{
        "user_id": "123",
    }, 30*24*time.Hour)

    // Get by key (returns any, type assert as needed)
    user, err := db.Get("user:123")
    if err != nil {
        panic(err)
    }
    if m, ok := user.(map[string]any); ok {
        fmt.Println("User:", m["name"])
    }

    // Get simple value
    apiKey, _ := db.Get("config:api_key")
    fmt.Println("API Key:", apiKey.(string))

    // Check if key exists
    if db.Exists("user:123") {
        fmt.Println("User exists")
    }

    // Get remaining TTL
    ttl := db.TTL("session:abc")
    fmt.Println("Session expires in:", ttl)

    // Find keys by prefix
    keys := db.FindKeysByPrefix("user:")
    fmt.Println("Users:", keys)

    // Delete
    db.Delete("user:123")
}

Memory-Only Mode

For in-memory use without any file system operations:

// Empty path = memory-only mode
db, err := snapshotkv.Open("", nil)
if err != nil {
    panic(err)
}
defer db.Close()

// All operations work except blob storage
db.Set("cache:key", "data")
val, _ := db.Get("cache:key")

// Blob operations return ErrMemoryOnly in this mode
_, err = db.GetBlob("some:key")
if err == snapshotkv.ErrMemoryOnly {
    fmt.Println("Blobs not available in memory-only mode")
}

Blob Storage

For large data like conversation history or embeddings:

// Store with blob (requires persistent mode)
db.SetWithBlobEx("conversation:conv-001", map[string]any{
    "status":     "active",
    "created_at": time.Now(),
}, conversationData, 30*24*time.Hour)

// Load blob on demand
blob, err := db.GetBlob("conversation:conv-001")
if err != nil {
    panic(err)
}
fmt.Println("Conversation data:", len(blob), "bytes")

// Update blob
db.UpdateBlob("conversation:conv-001", newConversationData)

Transaction Operations

For bulk writes, use transaction mode to defer saves:

db.BeginTransaction()
for i := 0; i < 1000; i++ {
    key := fmt.Sprintf("item:%d", i)
    db.Set(key, map[string]any{"index": i})
}
db.Commit() // Single save at the end

Or rollback without saving:

db.BeginTransaction()
db.Set("temp:key", data)
db.Rollback() // Changes discarded, no save triggered

Configuration

cfg := &snapshotkv.Config{
    Path:               "./data",           // Storage directory (empty = memory-only)
    MaxSnapshots:       10,                 // Snapshot history (default: 5)
    SaveDebounce:       10 * time.Second,   // Debounce interval (default: 10s)
    DisableCompression: false,              // Set true to disable gzip (default: false)
    TTLCleanupInterval: 5 * time.Minute,    // TTL cleanup interval (default: 5m, 0 to disable)
    Codec:              snapshotkv.MsgpackCodec{}, // Serialization (default: MsgpackCodec)
}

db, err := snapshotkv.Open("", cfg)  // Path can also be set in Config

Pluggable Codecs

// MessagePack (default, more compact)
db, _ := snapshotkv.Open("./data", &snapshotkv.Config{
    Codec: snapshotkv.MsgpackCodec{},
})

// JSON (human-readable snapshots)
db, _ := snapshotkv.Open("./data", &snapshotkv.Config{
    Codec:              snapshotkv.JSONCodec{},
    DisableCompression: true, // Disable gzip for readable files
})

Storage Layout

data/
  snapshots/
    data-20260301-143022-0001.msgpack.gz
    data-20260301-143022-0000.msgpack.gz
    data-20260301-120000-0000.msgpack.gz
  blobs/
    0ab/
      0abc12345678-1a2b3c4d5e6f.bin
    ff1/
      ff1234567890-2b3c4d5e6f7a.bin

Snapshot Files

• Format: MessagePack (or JSON) with optional gzip compression
• Naming: data-YYYYMMDD-HHMMSS-CCCC.<ext>.gz
• HLC counter: 16-bit hex counter for multiple snapshots per second
• Atomic writes: Write to temp file, then rename

Blob Files

• Naming: UUID v7 (time-sortable) with .bin extension
• Sharding: 3 hex chars = 4,096 directories

API Reference

Database Operations

// Open/close
db, err := snapshotkv.Open(path string, cfg *Config) (*DB, error)
db.Close() error
db.Save() error  // Force immediate snapshot save (no-op in memory-only mode)
db.IsMemoryOnly() bool  // Check if running in memory-only mode

Key-Value Operations

// Get value by key (returns any - type assert as needed)
value, err := db.Get(key string) (any, error)

// Set without expiry (value can be any type)
db.Set(key string, value any) error

// Set with TTL
db.SetEx(key string, value any, ttl time.Duration) error

// Delete key
db.Delete(key string) error

// Check if key exists and is not expired
exists := db.Exists(key string) bool

// Get remaining TTL (-1 if no expiration, -2 if key doesn't exist)
ttl := db.TTL(key string) time.Duration

// Find keys by prefix
keys := db.FindKeysByPrefix(prefix string) []string

Blob Operations

// Set with blob (no expiry) - returns ErrMemoryOnly in memory-only mode
db.SetWithBlob(key string, value any, blob []byte) error

// Set with blob and TTL
db.SetWithBlobEx(key string, value any, blob []byte, ttl time.Duration) error

// Get blob
blob, err := db.GetBlob(key string) ([]byte, error)

// Update blob
db.UpdateBlob(key string, blob []byte) error

Error Handling

value, err := db.Get("nonexistent")
if err == snapshotkv.ErrNotFound {
    // Key doesn't exist
}

blob, err := db.GetBlob("key")
if err == snapshotkv.ErrNoBlob {
    // Key exists but has no blob
}

err = db.SetWithBlob("key", data, blob)
if err == snapshotkv.ErrMemoryOnly {
    // Can't use blobs in memory-only mode
}

Thread Safety

SnapshotKV is fully safe for concurrent use:

• All operations protected by RWMutex
• Read operations use read lock
• Write operations use write lock
• TTL cleanup runs in background goroutine with proper locking

When to Use SnapshotKV

Good fit:

• Configuration storage
• Session management
• Conversation/message storage
• MCP server configurations
• LLM memory systems
• Small to medium datasets (KB to low MB)
• Workloads with infrequent writes

Not ideal for:

• High-throughput write workloads (writes are debounced but still trigger full snapshots)
• Very large datasets (entire dataset loaded into memory)
• Real-time analytics queries

License

MIT License

Documentation

Index

• Constants
• Variables
• type Codec
• type Config
• type DB
  • func Open(path string, cfg *Config) (*DB, error)
  • func (db *DB) BeginTransaction() error
  • func (db *DB) Close() error
  • func (db *DB) Commit() error
  • func (db *DB) Count(prefix string) int
  • func (db *DB) Delete(key string) error
  • func (db *DB) Exists(key string) bool
  • func (db *DB) FindKeysByPrefix(prefix string) []string
  • func (db *DB) Get(key string) (any, error)
  • func (db *DB) GetBlob(key string) ([]byte, error)
  • func (db *DB) IsMemoryOnly() bool
  • func (db *DB) Rollback() error
  • func (db *DB) Save() error
  • func (db *DB) Scan(prefix string, fn func(key string, value any) bool)
  • func (db *DB) Set(key string, value any) error
  • func (db *DB) SetEx(key string, value any, ttl time.Duration) error
  • func (db *DB) SetWithBlob(key string, value any, blob []byte) error
  • func (db *DB) SetWithBlobEx(key string, value any, blob []byte, ttl time.Duration) error
  • func (db *DB) TTL(key string) time.Duration
  • func (db *DB) UpdateBlob(key string, blob []byte) error
• type JSONCodec
  • func (c JSONCodec) Decode(data []byte, v any) error
  • func (c JSONCodec) Encode(v any) ([]byte, error)
  • func (c JSONCodec) Extension() string
• type MsgpackCodec
  • func (c MsgpackCodec) Decode(data []byte, v any) error
  • func (c MsgpackCodec) Encode(v any) ([]byte, error)
  • func (c MsgpackCodec) Extension() string
• type Snapshot

Constants

const SnapshotVersion = 2

SnapshotVersion is the current snapshot format version

Variables

var (
	// ErrNotFound is returned when a key does not exist
	ErrNotFound = errors.New("key not found")

	// ErrBlobNotFound is returned when a blob file does not exist
	ErrBlobNotFound = errors.New("blob not found")

	// ErrNoBlob is returned when trying to get a blob from a key without one
	ErrNoBlob = errors.New("key has no associated blob")

	// ErrDatabaseClosed is returned when operations are called after Close()
	ErrDatabaseClosed = errors.New("database is closed")

	// ErrNotInTransaction is returned when Commit/Rollback is called outside transaction mode
	ErrNotInTransaction = errors.New("not in transaction mode")

	// ErrAlreadyInTransaction is returned when BeginTransaction is called while already in transaction mode
	ErrAlreadyInTransaction = errors.New("already in transaction mode")

	// ErrNoSnapshots is returned when no valid snapshots can be loaded
	ErrNoSnapshots = errors.New("no valid snapshots found")

	// ErrMemoryOnly is returned when trying to use blob operations in memory-only mode
	ErrMemoryOnly = errors.New("blob operations not supported in memory-only mode")
)

var (
	ErrNotInBatch     = ErrNotInTransaction
	ErrAlreadyInBatch = ErrAlreadyInTransaction
)

Backward compatibility aliases

Types

type Codec

type Codec interface {
	Encode(v any) ([]byte, error)
	Decode(data []byte, v any) error
	Extension() string // File extension without dot, e.g., "msgpack" or "json"
}

Codec interface for pluggable serialization

type Config

type Config struct {
	// Path is the base directory for storage (empty = memory-only mode)
	Path string
	// MaxSnapshots is the maximum number of snapshot files to keep (default: 5)
	MaxSnapshots int
	// SaveDebounce is the debounce interval for saves (default: 10s)
	SaveDebounce time.Duration
	// DisableCompression disables gzip compression for snapshots (default: false, compression enabled)
	DisableCompression bool
	// TTLCleanupInterval is the interval for TTL cleanup (default: 5m, 0 to disable)
	TTLCleanupInterval time.Duration
	// Codec is the encoder/decoder (default: MsgpackCodec)
	Codec Codec
}

Config holds database configuration

type DB

type DB struct {
	// contains filtered or unexported fields
}

DB is the main database structure

func Open

func Open(path string, cfg *Config) (*DB, error)

Open opens or creates a database at the given path. If path is empty, operates in memory-only mode with no persistence or blob support.

func (*DB) BeginTransaction

func (db *DB) BeginTransaction() error

BeginTransaction starts transaction mode (defers save until Commit).

func (*DB) Close

func (db *DB) Close() error

Close closes the database and stops background goroutines

func (*DB) Commit

func (db *DB) Commit() error

Commit exits transaction mode and saves immediately.

func (*DB) Count

func (db *DB) Count(prefix string) int

Count returns the number of non-expired keys matching a prefix.

func (*DB) Delete

func (db *DB) Delete(key string) error

Delete removes a key and its associated blob.

func (*DB) Exists

func (db *DB) Exists(key string) bool

Exists checks if a key exists and is not expired.

func (*DB) FindKeysByPrefix

func (db *DB) FindKeysByPrefix(prefix string) []string

FindKeysByPrefix returns all non-expired keys matching a prefix.

func (*DB) Get

func (db *DB) Get(key string) (any, error)

Get retrieves a value by key. Returns ErrNotFound if the key doesn't exist or has expired.

func (*DB) GetBlob

func (db *DB) GetBlob(key string) ([]byte, error)

GetBlob retrieves the blob data for a key. Returns ErrMemoryOnly in memory-only mode. Returns ErrNoBlob if the key has no associated blob.

func (*DB) IsMemoryOnly

func (db *DB) IsMemoryOnly() bool

IsMemoryOnly returns true if the database is in memory-only mode

func (*DB) Rollback

func (db *DB) Rollback() error

Rollback exits transaction mode without saving and deletes any blobs created.

func (*DB) Save

func (db *DB) Save() error

Save forces an immediate snapshot save (no-op in memory-only mode)

func (*DB) Scan

func (db *DB) Scan(prefix string, fn func(key string, value any) bool)

Scan iterates over all non-expired keys matching prefix, calling fn for each. Stops early if fn returns false. The lock is held for the duration of the scan so fn must not call any DB methods.

func (*DB) Set

func (db *DB) Set(key string, value any) error

Set serialises value using the DB codec and stores it without expiry.

func (*DB) SetEx

func (db *DB) SetEx(key string, value any, ttl time.Duration) error

SetEx serialises value using the DB codec and stores it with TTL. Use ttl=0 for no expiration.

func (*DB) SetWithBlob

func (db *DB) SetWithBlob(key string, value any, blob []byte) error

SetWithBlob stores a value with blob data (no expiry). Returns ErrMemoryOnly in memory-only mode.

func (*DB) SetWithBlobEx

func (db *DB) SetWithBlobEx(key string, value any, blob []byte, ttl time.Duration) error

SetWithBlobEx stores a value with blob data and TTL. Returns ErrMemoryOnly in memory-only mode.

func (*DB) TTL

func (db *DB) TTL(key string) time.Duration

TTL returns the remaining time-to-live for a key. Returns -1 if the key has no expiration, -2 if the key doesn't exist.

func (*DB) UpdateBlob

func (db *DB) UpdateBlob(key string, blob []byte) error

UpdateBlob updates the blob data for an existing key. Returns ErrMemoryOnly in memory-only mode.

type JSONCodec

type JSONCodec struct{}

JSONCodec uses encoding/json for serialization

func (JSONCodec) Decode

func (c JSONCodec) Decode(data []byte, v any) error

func (JSONCodec) Encode

func (c JSONCodec) Encode(v any) ([]byte, error)

func (JSONCodec) Extension

func (c JSONCodec) Extension() string

type MsgpackCodec

type MsgpackCodec struct{}

MsgpackCodec uses github.com/vmihailenco/msgpack/v5 for serialization

func (MsgpackCodec) Decode

func (c MsgpackCodec) Decode(data []byte, v any) error

func (MsgpackCodec) Encode

func (c MsgpackCodec) Encode(v any) ([]byte, error)

func (MsgpackCodec) Extension

func (c MsgpackCodec) Extension() string

type Snapshot

type Snapshot struct {
	Version   int               `msgpack:"version" json:"version"`
	CreatedAt time.Time         `msgpack:"created_at" json:"created_at"`
	Data      map[string]*entry `msgpack:"data" json:"data"`
}

Snapshot represents the on-disk snapshot format