loomlog

README

loomlog-go

Go library for reading, writing, and querying Loomlog (.lml) files.

Loomlog is a fast, line-oriented log format: data up front, metadata in a tiny trailer. Append-friendly, human-diffable, and storage-agnostic.

---

Installation

go get github.com/popchatlabs/loomlog-go@latest

---

Quick Start

package main

import (
    "fmt"
    "github.com/popchatlabs/loomlog-go"
)

func main() {
    // Create a new loomlog file
    data, _ := loomlog.Create()

    // Insert a record
    data, _ = loomlog.Insert(data, map[string]string{
        "id":   "msg-001",
        "data": "hello-world",
    }, map[string]string{
        "user":  "alice",
        "topic": "greetings",
    })

    // Insert another record
    data, _ = loomlog.Insert(data, map[string]string{
        "id":   "msg-002",
        "data": "goodbye",
    }, map[string]string{
        "user": "bob",
        "topic": "farewells",
    })

    // Query records
    results, _ := loomlog.Query(data, loomlog.QuerySpec{
        Where: []loomlog.Pred{
            {Key: "user", Op: "eq", Val: "alice"},
        },
    })

    fmt.Printf("Found %d records\n", len(results))
    // Output: Found 1 records
}

---

Core Operations

Creating and Parsing

// Create empty file
data, err := loomlog.Create()

// Check file structure
err := loomlog.CheckStructure(data)

// Parse file to index
index, err := loomlog.Parse(data, nil)

// Get file statistics
fileSize, recordCount, trailerSize, err := loomlog.Stats(data)

Mutations

// Insert a new record
data, err := loomlog.Insert(data, map[string]string{
    "id":   "rec-123",
    "data": "payload-token",
}, map[string]string{
    "priority": "high",
    "owner":    "alice",
})

// Update record data
data, err := loomlog.Update(data, "rec-123", map[string]string{
    "data": "new-payload",
})

// Update only metadata (fast path)
data, err := loomlog.UpdateExtras(data, "rec-123", map[string]string{
    "priority": "urgent",
    "status":   "processed",
})

// Update with same-size validation
data, err := loomlog.UpdateSameSize(data, "rec-123", map[string]string{
    "data": "same-length!",
})

// Delete a record
data, err := loomlog.Delete(data, "rec-123")

// Get record metadata
extras, err := loomlog.GetExtras(data, "rec-123")

Querying

// Simple query
results, err := loomlog.Query(data, loomlog.QuerySpec{
    Where: []loomlog.Pred{
        {Key: "status", Op: "eq", Val: "active"},
    },
    OrderBy: []loomlog.Order{
        {Key: "ts", Dir: "desc"},
    },
    Limit: 10,
})

// Get IDs only (faster)
ids, err := loomlog.FindIDs(data, loomlog.QuerySpec{
    Where: []loomlog.Pred{
        {Key: "user", Op: "eq", Val: "alice"},
    },
})

// Streaming query (memory-efficient)
err := loomlog.QueryIter(data, loomlog.QuerySpec{
    Select: loomlog.Select{
        Fields:      []string{"id", "ts", "user"},
        IncludeData: false,
    },
}, func(row map[string]any) bool {
    fmt.Printf("ID: %s, User: %s\n", row["id"], row["user"])
    return true // continue iteration
})

Query Operators

Supported predicate operators:

• eq, neq - Equality
• in, nin - Set membership (comma-separated values)
• gt, gte, lt, lte - Numeric comparisons
• prefix, suffix, contains, not_contains - String matching
• exists, not_exists - Key presence

Rendering

// Render to various formats
opts := &loomlog.RenderOpts{
    Fields:        []string{"id", "ts", "user", "topic"},
    IncludeHeader: true,
    Pretty:        true,
}

// Table format
err := loomlog.RenderRows(results, loomlog.FormatTable, opts, os.Stdout)

// JSON/JSONL
err := loomlog.RenderRows(results, loomlog.FormatJSON, opts, os.Stdout)
err := loomlog.RenderRows(results, loomlog.FormatJSONL, opts, os.Stdout)

// CSV
err := loomlog.RenderRows(results, loomlog.FormatCSV, opts, os.Stdout)

// Streaming render
iter := func(emit func(map[string]any) bool) *loomlog.ErrDetail {
    return loomlog.QueryIter(data, spec, emit)
}
err := loomlog.RenderRowsStream(iter, loomlog.FormatJSONL, opts, os.Stdout)

---

File Format

Loomlog files have a simple structure:

# LOOMLOG1
id: a1, len: 11, data: hello-world
id: a2, len: 7,  data: goodbye
+++
rec: ts: 1738960000000000000, id: a1, line: 2, off: 12, len: 34, user: 42, topic: intro
rec: ts: 1738960001000000000, id: a2, line: 3, off: 49, len: 28, user: 99, topic: outro

Key features:

• Data records up front (append-optimized)
• Metadata in trailer (query-optimized)
• Partial trailer rebuild on edits
• Human-readable and diffable

See the full specification for details.

---

Error Handling

All mutation and query functions return *ErrDetail for structured error information:

type ErrDetail struct {
    Code string  // Error code (e.g., "ERR_DUP_ID")
    Hint string  // Human-readable explanation
    Line uint32  // Line number (if applicable)
    Off  uint64  // Byte offset (if applicable)
}

// Example usage
data, err := loomlog.Insert(data, record, extras)
if err != nil {
    fmt.Printf("Error %s at line %d: %s\n", err.Code, err.Line, err.Hint)
}

---

Performance

• Appends: Add one record line + one trailer line (minimal rewrite)
• Queries: Read only trailer to filter; fetch payload for matches
• Updates: Partial trailer rebuild (only affected offsets updated)
• Same-size updates: Fast path with no offset recalculation

For large-scale production use, see the Profiles & Ops Guide for:

• Bloom filters for existence checks
• Segment tables for range queries
• .tlx sidecar files for metadata caching

---

API Documentation

Full API documentation is available at pkg.go.dev.

---

Command-Line Tool

For command-line usage, see loomlog-cli:

# View records
loomlog view file.lml

# Export to JSON
loomlog export file.lml --format jsonl

# Query
loomlog query file.lml --spec '{"where":[{"key":"user","op":"eq","val":"alice"}]}'

---

Other Implementations

• Ruby: loomlog-rb
• Python: loomlog-py

---

Testing

go test ./... -v

The library includes comprehensive test coverage:

• Parsing and validation tests
• Mutation lifecycle tests
• Query and filtering tests
• Rendering tests
• Error handling tests

---

Contributing

Contributions welcome! Please:

1. Run tests: go test ./...
2. Follow Go conventions
3. Maintain backward compatibility
4. Update tests for new features

See CONTRIBUTING.md for guidelines.

---

License

Apache 2.0

Copyright (c) 2025 Popchat, Inc. Popchat® is the registered trademark of Popchat, Inc.

See LICENSE for details.

Documentation

Overview

Package loomlog provides a Go library for reading, writing, and querying Loomlog (.lml) files.

Loomlog is a fast, line-oriented log format with data up front and metadata in a tiny trailer. It is append-friendly, human-diffable, and storage-agnostic.

File Format

A loomlog file consists of:

• Magic line: # LOOMLOG1
• Record lines: id: <id>, len: <uint>, data: <token>
• Trailer separator: +++
• Trailer entries: rec: ts:<i64>, id:<id>, line:<u32>, off:<u64>, len:<u32>[, <key>:<value> ...]

Basic Usage

Create a new file:

data, _ := loomlog.Create()

Insert records:

data, _ = loomlog.Insert(data, map[string]string{
    "id":   "msg-001",
    "data": "hello-world",
}, map[string]string{
    "user": "alice",
})

Query records:

results, _ := loomlog.Query(data, loomlog.QuerySpec{
    Where: []loomlog.Pred{
        {Key: "user", Op: "eq", Val: "alice"},
    },
})

Update records:

data, _ = loomlog.Update(data, "msg-001", map[string]string{
    "data": "new-payload",
})

Delete records:

data, _ = loomlog.Delete(data, "msg-001")

Error Handling

All mutation and query functions return *ErrDetail for structured errors:

data, err := loomlog.Insert(data, record, extras)
if err != nil {
    fmt.Printf("Error %s: %s\n", err.Code, err.Hint)
}

Performance

Loomlog is optimized for append-heavy workloads:

• Appends add one record line and one trailer entry
• Queries read only the trailer to filter results
• Updates trigger partial trailer rebuild
• Same-size updates use a fast path

For more details, see https://github.com/popchatlabs/loomlog/blob/main/SPEC.md

Index

• Variables
• func GetExtras(b []byte, id string) (Extras, *ErrDetail)
• type ErrDetail
  • func CheckStructure(b []byte) *ErrDetail
  • func Create() ([]byte, *ErrDetail)
  • func Delete(b []byte, id string) ([]byte, *ErrDetail)
  • func EncodeRecLine(ts int64, id string, line uint32, off uint64, length uint32, extras Extras) ([]byte, *ErrDetail)
  • func EncodeRecordData(kv map[string]string) ([]byte, *ErrDetail)
  • func FindIDs(b []byte, q QuerySpec) ([]string, *ErrDetail)
  • func Insert(b []byte, lineKV map[string]string, extras Extras) ([]byte, *ErrDetail)
  • func Parse(b []byte, opts *ParseOpts) (map[string]RecIndex, *ErrDetail)
  • func Plan(b []byte, in PlanIn) (int64, int64, *ErrDetail)
  • func Query(b []byte, q QuerySpec) ([]map[string]any, *ErrDetail)
  • func QueryIter(b []byte, q QuerySpec, emit func(map[string]any) bool) *ErrDetail
  • func RenderRows(rows []map[string]any, format ViewFormat, opts *RenderOpts, w io.Writer) *ErrDetail
  • func RenderRowsStream(iter RowIterator, format ViewFormat, opts *RenderOpts, w io.Writer) *ErrDetail
  • func Stats(b []byte) (int64, int, int64, *ErrDetail)
  • func Update(b []byte, id string, newKV map[string]string) ([]byte, *ErrDetail)
  • func UpdateExtras(b []byte, id string, extras Extras) ([]byte, *ErrDetail)
  • func UpdateSameSize(b []byte, id string, newKV map[string]string) ([]byte, *ErrDetail)
• type Extras
• type Op
• type Order
• type ParseOpts
• type PlanIn
• type Pred
• type QuerySpec
• type RecIndex
• type RenderOpts
• type RowIterator
• type Select
• type ViewFormat

Variables

var (
	ErrNoMagic              = errors.New("E_NO_MAGIC")
	ErrUnterminatedLastLine = errors.New("E_UNTERMINATED_LAST_LINE")
	ErrNoTrailer            = errors.New("E_NO_TRAILER")
	ErrMultipleTrailers     = errors.New("E_MULTIPLE_TRAILERS")
	ErrNoID                 = errors.New("E_NO_ID")
	ErrNoLen                = errors.New("E_NO_LEN")
	ErrNoData               = errors.New("E_NO_DATA")
	ErrDataNotLast          = errors.New("E_DATA_NOT_LAST")
	ErrDataWS               = errors.New("E_DATA_WS")
	ErrLenMismatch          = errors.New("E_LEN_MISMATCH")
	ErrDupKey               = errors.New("E_DUP_KEY")
	ErrDupID                = errors.New("E_DUP_ID")
	ErrIDNotFound           = errors.New("E_ID_NOT_FOUND")
	ErrTrailingComma        = errors.New("E_TRAILING_COMMA")
	ErrRenderFormat         = errors.New("E_RENDER_FORMAT")
	ErrRenderTarget         = errors.New("E_RENDER_TARGET")
)

Functions

func GetExtras

func GetExtras(b []byte, id string) (Extras, *ErrDetail)

Types

type ErrDetail

type ErrDetail struct {
	Code string
	Line uint
	Off  uint64
	Hint string
}

func CheckStructure

func CheckStructure(b []byte) *ErrDetail

func Create

func Create() ([]byte, *ErrDetail)

func Delete

func Delete(b []byte, id string) ([]byte, *ErrDetail)

func EncodeRecLine

func EncodeRecLine(ts int64, id string, line uint32, off uint64, length uint32, extras Extras) ([]byte, *ErrDetail)

func EncodeRecordData

func EncodeRecordData(kv map[string]string) ([]byte, *ErrDetail)

func FindIDs

func FindIDs(b []byte, q QuerySpec) ([]string, *ErrDetail)

func Insert

func Insert(b []byte, lineKV map[string]string, extras Extras) ([]byte, *ErrDetail)

func Parse

func Parse(b []byte, opts *ParseOpts) (map[string]RecIndex, *ErrDetail)

func Plan

func Plan(b []byte, in PlanIn) (int64, int64, *ErrDetail)

func Query

func Query(b []byte, q QuerySpec) ([]map[string]any, *ErrDetail)

func QueryIter

func QueryIter(b []byte, q QuerySpec, emit func(map[string]any) bool) *ErrDetail

func RenderRows

func RenderRows(rows []map[string]any, format ViewFormat, opts *RenderOpts, w io.Writer) *ErrDetail

func RenderRowsStream

func RenderRowsStream(iter RowIterator, format ViewFormat, opts *RenderOpts, w io.Writer) *ErrDetail

func Stats

func Stats(b []byte) (int64, int, int64, *ErrDetail)

func Update

func Update(b []byte, id string, newKV map[string]string) ([]byte, *ErrDetail)

func UpdateExtras

func UpdateExtras(b []byte, id string, extras Extras) ([]byte, *ErrDetail)

func UpdateSameSize

func UpdateSameSize(b []byte, id string, newKV map[string]string) ([]byte, *ErrDetail)

type Extras

type Extras = map[string]string

type Op

type Op string

const (
	OpEq          Op = "eq"
	OpNeq         Op = "neq"
	OpIn          Op = "in"
	OpNin         Op = "nin"
	OpPrefix      Op = "prefix"
	OpSuffix      Op = "suffix"
	OpContains    Op = "contains"
	OpNotContains Op = "not_contains"
	OpExists      Op = "exists"
	OpNotExists   Op = "not_exists"
	OpGt          Op = "gt"
	OpGte         Op = "gte"
	OpLt          Op = "lt"
	OpLte         Op = "lte"
)

type Order

type Order struct {
	Key string
	Dir string
}

type ParseOpts

type ParseOpts struct {
	MaxLineBytes    int64
	MaxTrailerBytes int64
	StrictParse     bool
}

type PlanIn

type PlanIn struct {
	Kind   string
	ID     string
	LineKV map[string]string
	Extras Extras
}

type Pred

type Pred struct {
	Key string
	Op  Op
	Val any
}

type QuerySpec

type QuerySpec struct {
	Select  Select
	Where   []Pred
	Or      [][]Pred
	Not     []Pred
	OrderBy []Order
	Limit   int
	Offset  int
	Min     int
}

type RecIndex

type RecIndex struct {
	Ts   int64
	Line uint32
	Off  uint64
	Len  uint32
}

type RenderOpts

type RenderOpts struct {
	Fields        []string
	IncludeHeader bool
	Pretty        bool
}

type RowIterator

type RowIterator func(func(map[string]any) bool) *ErrDetail

type Select

type Select struct {
	Fields      []string
	IncludeData bool
}

type ViewFormat

type ViewFormat string

const (
	FormatTable ViewFormat = "table"
	FormatBox   ViewFormat = "box"
	FormatJSON  ViewFormat = "json"
	FormatJSONL ViewFormat = "jsonl"
	FormatCSV   ViewFormat = "csv"
	FormatHTML  ViewFormat = "html"
)