ebolt

README

ebolt

A secure, encrypted wrapper over bbolt providing hierarchical key-value storage with transparent encryption capabilities.

Features

• Hierarchical Path Structure: Use intuitive paths like "users/profiles/john" with automatic bucket creation
• Transparent Encryption: All values are automatically encrypted and decrypted
• Key Obfuscation: Leaf keys are obfuscated while preserving bucket structure for efficient navigation
• Transaction Support: Full atomic operations with commit/abort capabilities
• Backup Support: Live, encrypted database backups without interrupting service
• Cross-Platform: Works on Linux, macOS, and Windows

Installation

go get github.com/opencoff/ebolt

Overview

Ebolt enhances the popular bbolt key-value store by adding:

1. Encryption Layer: All stored values are encrypted before writing and decrypted when read
2. Path-Based Access: Keys are specified as paths (e.g., "users/settings/theme") where intermediate components become buckets
3. Auto-Vivification: Intermediate buckets are automatically created when setting values
4. Key Obfuscation: Only leaf keys are obfuscated, keeping intermediate bucket names readable for easier debugging

This library is ideal for applications that need to store sensitive data while maintaining the performance and simplicity of bbolt.

Interface Documentation

Types and Interfaces

// KV represents a key-value pair for storage operations
type KV struct {
    Key string
    Val []byte
}

// Ops interface defines the core operations for the encrypted database
type Ops interface {
    // Get retrieves and decrypts the value stored at the specified path.
    // The path format "a/b/name" is interpreted where intermediate components
    // are buckets and the final component is the key.
    Get(p string) ([]byte, error)
    
    // Set encrypts and stores a value at the specified path, automatically
    // creating any intermediate buckets as needed. The leaf component of the
    // path is obfuscated while bucket names remain in plaintext.
    Set(p string, v []byte) error

    // SetMany encrypts and stores multiple key-value pairs. Each key follows
    // the path format with automatic bucket creation.
    SetMany(v []KV) error

    // Del removes the encrypted value at the specified path.
    Del(p string) error

    // DelMany deletes multiple keys in a single transaction.
    // Each path is processed according to the hierarchical bucket structure.
    DelMany(v []string) error

    // All retrieves all entries within a given bucket path, returning a map
    // of decrypted key-value pairs. The keys in the map are the original 
    // unobfuscated keys.
    All(p string) (map[string][]byte, error)
    
    // AllKeys returns all keys within a given bucket path without
    // retrieving their values. The returned keys are the original
    // unobfuscated keys.
    AllKeys(p string) ([]string, error)
    
    // Dir returns all sub-buckets under the specified path without
    // retrieving individual key-value pairs. In boltdb terminology,
    // this returns all sub-buckets of a bucket.
    Dir(p string) ([][]byte, error)
}

// DB interface extends Ops with database management functionality
type DB interface {
    // Ops embeds all operations from the Ops interface
    Ops
    
    // Close finalizes all transactions and releases database resources.
    Close() error
    
    // BeginTransaction starts a new transaction that can be either read-only
    // or read-write. Multiple read-only transactions can run concurrently,
    // but write transactions are exclusive.
    BeginTransaction(writable bool) (Tx, error)
    
    // Backup performs a live backup of the encrypted database to the provided
    // io.Writer, returning the number of bytes written. The database remains
    // usable during the backup process.
    Backup(wr io.Writer) (int64, error)
}

// Tx interface represents an active transaction
type Tx interface {
    // Ops embeds all operations from the Ops interface
    Ops
    
    // Commit persists all changes made within this transaction to the database.
    // After calling Commit, the transaction is no longer usable.
    Commit() error
    
    // Rollback discards all changes made within this transaction.
    // After calling Rollback, the transaction is no longer usable.
    Rollback() error
}

Usage Examples

Basic Operations

package main

import (
    "fmt"
    "log"

    "github.com/opencoff/ebolt"
    "github.com/opencoff/go-utils"
)

func main() {
    pw, err := utils.Askpass("Enter DB Password", true)
    if err != nil {
        log.Fatal(err)
    }

    // This is just an example. For production uses, you
    // must use a strong KDF like Argon2i to derive a
    // key from the user's passphrase.

    // Open or create an encrypted database
    db, err := ebolt.Open("users.db", []byte(pw), nil)
    if err != nil {
        log.Fatalf("Failed to open database: %v", err)
    }
    defer db.Close()

    // Store values with hierarchical paths (buckets auto-created)
    if err := db.Set("app/settings/theme", []byte("dark")); err != nil {
        log.Fatalf("Failed to set theme: %v", err)
    }
    
    if err := db.Set("app/settings/language", []byte("en-US")); err != nil {
        log.Fatalf("Failed to set language: %v", err)
    }

    // Retrieve a specific value
    theme, err := db.Get("app/settings/theme")
    if err != nil {
        log.Fatalf("Failed to get theme: %v", err)
    }
    fmt.Printf("Theme: %s\n", theme)

    // Get all settings
    settings, err := db.All("app/settings")
    if err != nil {
        log.Fatalf("Failed to get settings: %v", err)
    }

    fmt.Println("All settings:")
    for k, v := range settings {
        fmt.Printf("  %s: %s\n", k, v)
    }
}

Using Transactions

package main

import (
    "fmt"
    "log"

    "github.com/opencoff/ebolt"
    "github.com/opencoff/go-utils"
)

func main() {
    pw, err := utils.Askpass("Enter DB Password", true)
    if err != nil {
        log.Fatal(err)
    }

    // This is just an example. For production uses, you
    // must use a strong KDF like Argon2i to derive a
    // key from the user's passphrase.

    db, err := ebolt.Open("users.db", []byte(pw), nil)
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    // Start a writable transaction
    tx, err := db.BeginTransaction(true)
    if err != nil {
        log.Fatal(err)
    }

    // Set multiple values atomically
    err = tx.SetMany([]ebolt.KV{
        {Key: "users/1001/name", Val: []byte("Alice Smith")},
        {Key: "users/1001/email", Val: []byte("alice@example.com")},
        {Key: "users/1001/role", Val: []byte("admin")},
    })

    if err != nil {
        tx.Rollback()
        log.Fatalf("Transaction failed: %v", err)
    }

    // Commit changes
    if err = tx.Commit(); err != nil {
        log.Fatalf("Commit failed: %v", err)
    }

    // Read the data back
    name, _ := db.Get("users/1001/name")
    fmt.Printf("User name: %s\n", name)

    // Get all keys in a bucket
    keys, err := db.AllKeys("users/1001")
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("User profile fields:")
    for _, key := range keys {
        fmt.Printf("  - %s\n", key)
    }
}

Database Backup

package main

import (
    "log"
    "os"
    "time"

    "github.com/opencoff/ebolt"
    "github.com/opencoff/go-utils"
)

func main() {
    pw, err := utils.Askpass("Enter DB Password", true)
    if err != nil {
        log.Fatal(err)
    }

    // This is just an example. For production uses, you
    // must use a strong KDF like Argon2i to derive a
    // key from the user's passphrase.

    // Open the database
    db, err := ebolt.Open("production.db", []byte(pw), nil)
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    // Create a backup file
    backupFile, err := os.Create("backup-" + time.Now().Format("20060102") + ".db")
    if err != nil {
        log.Fatal(err)
    }
    defer backupFile.Close()

    // Perform live backup
    bytes, err := db.Backup(backupFile)
    if err != nil {
        log.Fatal(err)
    }

    log.Printf("Backup completed successfully: %d bytes written", bytes)
}

Database Encryption Keys

If your db encryption key is already part of some KMS regime or a previous HKDF-like key expansion, then it's safe to use with ebolt.Open().

Please DO NOT use string passwords as the input to "ebolt.Open()". This is a terrible idea. Consult your favorite cryptographer to safely convert a string passphrase into usable key material. I tend to use the following construct to generate a 32-byte key.

    salt = randombytes(32)
    key  = argon2id(32, passphrase, salt, Time, Mem, Par)

Of course, one has to store "salt" safely in some place. And choose "Time", "Mem", "Par" to account for your security needs.

Implementation Notes

• The encryption is applied only to the values stored in the database, not to the database file itself.
• Only leaf keys are obfuscated, keeping bucket names readable for easier debugging and navigation
• Performance impact of encryption is expected to be minimal for most use cases

Cryptography

cipher.go implements the necessary cryptography. The user provided key is expanded with domain separation into a MAC key and an AEAD key. The MAC key is used for obfuscating the leaf names of a key-path while the AEAD key is used to encrypt the values.

The encryption of the values uses AES-256-GCM with a random nonce. The actual key is stored along with the value as plaintext. This is necessary to implement the AllKeys() operation.

Related Projects

• go-logger - Simple logging library
• go-fio - Cross-platform file I/O utilities with support for concurrent file tree walking and directory tree comparison

License

MIT License

Documentation

Overview

ebolt is an encrypted wrapper over etcd/bbolt. It also implements a convenient notion of "path" for the key - where each intermediate "dir" is a boltdb bucket. And the last part of the key-path is the a "key" in the last bucket.

All values stored in the db are encrypted with AES-256-GCM. The last part of the key-path is obfuscated with its MAC.

Index

• type DB
  • func Open(fn string, key []byte, opt *bolt.Options) (DB, error)
• type KV
• type Ops
• type Options
• type StorageError
  • func (e *StorageError) Error() string
  • func (e *StorageError) Unwrap() error
• type Tx

Types

type DB

type DB interface {
	// Ops embeds all operations from the Ops interface
	Ops

	// Close finalizes all transactions and releases database resources.
	Close() error

	// BeginTransaction starts a new transaction that can be either read-only
	// or read-write. Multiple read-only transactions can run concurrently,
	// but write transactions are exclusive.
	BeginTransaction(writable bool) (Tx, error)

	// Backup performs a live backup of the encrypted database to the provided
	// io.Writer, returning the number of bytes written. The database remains
	// usable during the backup process.
	Backup(wr io.Writer) (int64, error)
}

DB interface extends Ops with database management functionality

func Open

func Open(fn string, key []byte, opt *bolt.Options) (DB, error)

Create or open a new encrypted bolt db. The supplied 'key' will be expanded and used to encrypt the values stored in the db. The leaf of a key-path is obfuscated while preserving the intermediate paths in plaintext. This compromise gives us better performance without sacrificing too much privacy.

type KV

type KV struct {
	Key string
	Val []byte
}

KV represents a "key, value" pair for storage operations

type Ops

type Ops interface {
	// Get retrieves and decrypts the value stored at the specified path.
	// The path format "a/b/name" is interpreted where intermediate components
	// are buckets and the final component is the key.
	Get(p string) ([]byte, error)

	// Set encrypts and stores a value at the specified path, automatically
	// creating any intermediate buckets as needed. The leaf component of the
	// path is obfuscated while bucket names remain in plaintext.
	Set(p string, v []byte) error

	// SetMany encrypts and stores multiple key-value pairs. Each key follows
	// the path format with automatic bucket creation.
	SetMany(v []KV) error

	// Del removes the encrypted value at the specified path.
	Del(p string) error

	// DelMany deletes multiple keys in a single transaction.
	// Each path is processed according to the hierarchical bucket structure.
	DelMany(v []string) error

	// All retrieves all entries within a given bucket path, returning a map
	// of decrypted key-value pairs. The keys in the map are the original
	// unobfuscated keys.
	All(p string) (map[string][]byte, error)

	// AllKeys returns all keys within a given bucket path without
	// retrieving their values. The returned keys are the original
	// unobfuscated keys.
	AllKeys(p string) ([]string, error)

	// Dir returns all sub-buckets under the specified path without
	// retrieving individual key-value pairs. In boltdb terminology,
	// this returns all sub-buckets of a bucket.
	Dir(p string) ([][]byte, error)
}

Ops interface defines the core operations for the encrypted database.

type Options

type Options = bolt.Options

type StorageError

type StorageError struct {
	Op  string
	Key string
	Err error
}

func (*StorageError) Error

func (e *StorageError) Error() string

func (*StorageError) Unwrap

func (e *StorageError) Unwrap() error

type Tx

type Tx interface {
	// Ops embeds all operations from the Ops interface
	Ops

	// Commit persists all changes made within this transaction to the database.
	// After calling Commit, the transaction is no longer usable.
	Commit() error

	// Rollback discards all changes made within this transaction.
	// After calling Rollback, the transaction is no longer usable.
	Rollback() error
}

Tx interface represents an active transaction. This enables callers to perform multiple operations in 'Ops' and commit in the end or abort.