anystore

README

Any Store

A document‑oriented database with a MongoDB‑like query language, running on top of a single SQLite file. Any Store brings schema‑less flexibility, rich indexes and ACID transactions to embedded Go applications.

⚠️ Status: pre‑1.0 – APIs may change. We actively dog‑food the library in production and welcome early adopters & contributors.

Features

• Mongo‑style queries – $in, $inc, comparison & logical operators out of the box.
• Automatic indexes – create, ensure or drop compound & unique indexes at runtime.
• ACID transactions – explicit read / write transactions plus convenience helpers.
• Streaming iterators – low‑memory scans with cursor API.
• CLI – quick inspection, import/export and interactive shell.
• Cross‑platform – pure Go, no CGO, runs anywhere Go runs.

Quick start

Install library

go get github.com/anyproto/any-store

Install CLI (optional)

go install github.com/anyproto/any-store/cmd/any-store-cli@latest

Hello, Any Store

package main

import (
    "context"
    "fmt"
    "log"

    anystore "github.com/anyproto/any-store"
    "github.com/anyproto/any-store/anyenc"
)

func main() {
    ctx := context.Background()

    db, err := anystore.Open(ctx, "/tmp/demo.db", nil)
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()

    users, _ := db.Collection(ctx, "users")

    _ = users.Insert(ctx,
        anyenc.MustParseJson(`{"id": 1, "name": "John"}`),
        anyenc.MustParseJson(`{"id": 2, "name": "Jane"}`),
    )

    res, _ := users.Find(`{"id": {"$in": [1,2]}}`).Sort("-name").Iter(ctx)
    for res.Next() {
        doc, _ := res.Doc()
        fmt.Println(doc.Value().String())
    }
}

The full end‑to‑end example lives in example/ and in the API docs.

Documentation

• API reference – https://pkg.go.dev/github.com/anyproto/any-store
• CLI manual – any-store-cli --help

Design highlights

Layer	Responsibility
Query builder	Parses Mongo‑like JSON filters and modifiers
Index engine	Generates composite SQLite indexes, picks optimal index via cost estimator
Encoding arena	Efficient AnyEnc value arena to minimise GC churn
Connection pool	Separate read / write SQLite connections for concurrent workloads

Contributing

1. Fork & clone
2. make test – run unit tests
3. Create your feature branch
4. Open a PR and sign the CLA

Please read our Code of Conduct before contributing.

⚖️ License

Any Store is released under the MIT License – see LICENSE for details.

Documentation

Index

• Variables
• type Collection
• type Config
• type DB
  • func Open(ctx context.Context, path string, config *Config) (DB, error)
• type DBStats
• type Doc
• type Explain
• type Index
• type IndexExplain
• type IndexHint
• type IndexInfo
• type Iterator
• type ModifyResult
• type Query
• type ReadTx
• type WriteTx

Variables

var (
	// ErrDocExists is returned when attempting to insert a document that already exists.
	ErrDocExists = errors.New("any-store: document already exists")

	// ErrDocNotFound is returned when a document cannot be found by its ID.
	ErrDocNotFound = errors.New("any-store: document not found")

	// ErrDocWithoutId is returned when a document is provided without a required ID.
	ErrDocWithoutId = errors.New("any-store: document missing ID")

	// ErrCollectionExists is returned when attempting to create a collection that already exists.
	ErrCollectionExists = errors.New("any-store: collection already exists")

	// ErrCollectionNotFound is returned when a collection cannot be found.
	ErrCollectionNotFound = errors.New("any-store: collection not found")

	// ErrIndexExists is returned when attempting to create an index that already exists.
	ErrIndexExists = errors.New("any-store: index already exists")

	// ErrIndexNotFound is returned when an index cannot be found.
	ErrIndexNotFound = errors.New("any-store: index not found")

	// ErrTxIsReadOnly is returned when a write operation is attempted in a read-only transaction.
	ErrTxIsReadOnly = errors.New("any-store: transaction is read-only")

	// ErrTxIsUsed is returned when an operation is attempted on a transaction that has already been committed or rolled back.
	ErrTxIsUsed = errors.New("any-store: transaction has already been used")

	// ErrTxOtherInstance is returned when an operation is attempted using a transaction from a different database instance.
	ErrTxOtherInstance = errors.New("any-store: transaction belongs to another database instance")

	// ErrUniqueConstraint is returned when a unique constraint violation occurs.
	ErrUniqueConstraint = errors.New("any-store: unique constraint violation")

	// ErrIterClosed is returned when operations are attempted on a closed iterator.
	ErrIterClosed = errors.New("any-store: iterator is closed")

	ErrDBIsClosed          = driver.ErrDBIsClosed
	ErrDBIsNotOpened       = driver.ErrDBIsNotOpened
	ErrIncompatibleVersion = driver.ErrIncompatibleVersion
)

Types

type Collection

type Collection interface {
	// Name returns the name of the collection.
	Name() string

	// FindId finds a document by its ID.
	// Returns the document or an error if the document is not found.
	FindId(ctx context.Context, id any) (Doc, error)

	// FindIdWithParser finds a document by its ID. Uses provided anyenc parser.
	// Returns the document or an error if the document is not found.
	FindIdWithParser(ctx context.Context, p *anyenc.Parser, id any) (Doc, error)

	// Find returns a new Query object with given filter
	Find(filter any) Query

	// Insert inserts multiple documents into the collection.
	// Returns an error if the insertion fails.
	Insert(ctx context.Context, docs ...*anyenc.Value) (err error)

	// UpdateOne updates a single document in the collection.
	// Provided document must contain an id field
	// Returns an error if the update fails.
	UpdateOne(ctx context.Context, doc *anyenc.Value) (err error)

	// UpdateId updates a single document in the collection with provided modifier
	// Returns a modify result or error.
	UpdateId(ctx context.Context, id any, mod query.Modifier) (res ModifyResult, err error)

	// UpsertOne inserts a document if it does not exist, or updates it if it does.
	// Returns the ID of the upserted document or an error if the operation fails.
	UpsertOne(ctx context.Context, doc *anyenc.Value) (err error)

	// UpsertId updates a single document or creates new one
	// Returns a modify result or error.
	UpsertId(ctx context.Context, id any, mod query.Modifier) (res ModifyResult, err error)

	// DeleteId deletes a single document by its ID.
	// Returns an error if the deletion fails.
	DeleteId(ctx context.Context, id any) (err error)

	// Count returns the number of documents in the collection.
	// Returns the count of documents or an error if the operation fails.
	Count(ctx context.Context) (count int, err error)

	// CreateIndex creates a new index.
	// Returns an error if index exists or the operation fails.
	CreateIndex(ctx context.Context, info ...IndexInfo) (err error)

	// EnsureIndex ensures an index exists on the specified fields.
	// Returns an error if the operation fails.
	EnsureIndex(ctx context.Context, info ...IndexInfo) (err error)

	// DropIndex drops an index by its name.
	// Returns an error if the operation fails.
	DropIndex(ctx context.Context, indexName string) (err error)

	// GetIndexes returns a list of indexes on the collection.
	GetIndexes() (indexes []Index)

	// Rename renames the collection.
	// Returns an error if the operation fails.
	Rename(ctx context.Context, newName string) (err error)

	// Drop drops the collection.
	// Returns an error if the operation fails.
	Drop(ctx context.Context) (err error)

	// ReadTx starts a new read-only transaction. It's just a proxy to db object.
	// Returns a ReadTx or an error if there is an issue starting the transaction.
	ReadTx(ctx context.Context) (ReadTx, error)

	// WriteTx starts a new read-write transaction. It's just a proxy to db object.
	// Returns a WriteTx or an error if there is an issue starting the transaction.
	WriteTx(ctx context.Context) (WriteTx, error)

	// Close closes the collection.
	// Returns an error if the operation fails.
	Close() error
}

Collection represents a collection of documents.

type Config

type Config struct {
	// Namespace is a prefix for all created tables and indexes by any-store,
	// helping to isolate tables and indexes within the same database file.
	Namespace string

	// ReadConnections specifies the number of read connections to the database,
	// optimizing read operations by allowing multiple concurrent read connections.
	ReadConnections int

	// SQLiteConnectionOptions provides additional options for SQLite connections,
	// corresponding to SQLite pragmas or other connection settings.
	SQLiteConnectionOptions map[string]string

	// SQLiteGlobalPageCachePreallocateSizeBytes is the size of the global page cache to preallocate.
	// Initialised on the first call to NewConnManager and shared by all connections.
	// default value is 10M
	// set negative to disable preallocation
	SQLiteGlobalPageCachePreallocateSizeBytes int

	// SyncPoolElementMaxSize defines maximum size of buffer that can be returned to the syncpool
	// default value id 2MiB
	SyncPoolElementMaxSize int

	// StalledConnectionsDetectorEnabled enables the collection of stack traces and duration of acquired connections
	// You can then use StalledConnections method of the ConnManager
	StalledConnectionsDetectorEnabled bool
	// StalledConnectionsPanicOnClose enables panic on Close in case of any connection is not released after this timeout
	StalledConnectionsPanicOnClose time.Duration
}

Config provides the configuration options for the database.

type DB

type DB interface {
	// CreateCollection creates a new collection with the specified name.
	// Returns the created Collection or an error if the collection already exists.
	// Possible errors:
	// - ErrCollectionExists: if the collection already exists.
	CreateCollection(ctx context.Context, collectionName string) (Collection, error)

	// OpenCollection opens an existing collection with the specified name.
	// Returns the opened Collection or an error if the collection does not exist.
	// Possible errors:
	// - ErrCollectionNotFound: if the collection does not exist.
	OpenCollection(ctx context.Context, collectionName string) (Collection, error)

	// Collection is a convenience method to get or create a collection.
	// It first attempts to open the collection, and if it does not exist, it creates the collection.
	// Returns the Collection or an error if there is an issue creating or opening the collection.
	Collection(ctx context.Context, collectionName string) (Collection, error)

	// GetCollectionNames returns a list of all collection names in the database.
	// Returns a slice of collection names or an error if there is an issue retrieving the names.
	GetCollectionNames(ctx context.Context) ([]string, error)

	// Stats returns the statistics of the database.
	// Returns a DBStats struct containing the database statistics or an error if there is an issue retrieving the stats.
	Stats(ctx context.Context) (DBStats, error)

	// QuickCheck performs PRAGMA quick_check to sqlite. If result not ok returns error.
	QuickCheck(ctx context.Context) (err error)

	// Checkpoint performs PRAGMA wal_checkpoint to sqlite. isFull=true - wal_checkpoint(FULL), isFull=false - wal_checkpoint(PASSIVE);
	Checkpoint(ctx context.Context, isFull bool) (err error)

	// Backup creates a backup of the database at the specified file path.
	// Returns an error if the operation fails.
	Backup(ctx context.Context, path string) (err error)

	// ReadTx starts a new read-only transaction.
	// Returns a ReadTx or an error if there is an issue starting the transaction.
	ReadTx(ctx context.Context) (ReadTx, error)

	// WriteTx starts a new read-write transaction.
	// Returns a WriteTx or an error if there is an issue starting the transaction.
	WriteTx(ctx context.Context) (WriteTx, error)

	// Close closes the database connection.
	// Returns an error if there is an issue closing the connection.
	Close() error
}

DB represents a document-oriented database.

func Open

func Open(ctx context.Context, path string, config *Config) (DB, error)

Open opens a database at the specified path with the given configuration. The config parameter can be nil for default settings. Returns a DB instance or an error.

type DBStats

type DBStats struct {
	// CollectionsCount is the total number of collections in the database.
	CollectionsCount int

	// IndexesCount is the total number of indexes across all collections in the database.
	IndexesCount int

	// TotalSizeBytes is the total size of the database in bytes.
	TotalSizeBytes int

	// DataSizeBytes is the total size of the data stored in the database in bytes, excluding free space.
	DataSizeBytes int
}

DBStats represents the statistics of the database.

type Doc

type Doc interface {
	// Value returns the document as a *anyenc.Value.
	// Important: When used in an iterator, the returned value is valid only until the next call to Next.
	Value() *anyenc.Value
}

Doc represents a document in the collection.

type Explain

type Explain struct {
	Sql           string
	SqliteExplain []string
	Indexes       []IndexExplain
}

type Index

type Index interface {
	// Info returns the IndexInfo for this index.
	Info() IndexInfo

	// Len returns the length of the index.
	Len(ctx context.Context) (int, error)
}

Index represents an index on a collection.

type IndexExplain

type IndexExplain struct {
	Name   string
	Weight int
	Used   bool
}

type IndexHint

type IndexHint struct {
	IndexName string
	Boost     int
}

type IndexInfo

type IndexInfo struct {
	// Name is the name of the index. If empty, it will be generated
	// based on the fields (e.g., "name,-createdDate").
	Name string

	// Fields are the fields included in the index. Each field can specify
	// ascending (e.g., "name") or descending (e.g., "-createdDate") order.
	Fields []string

	// Unique indicates whether the index enforces a unique constraint.
	Unique bool

	// Sparse indicates whether the index is sparse, indexing only documents
	// with the specified fields.
	Sparse bool
}

IndexInfo provides information about an index.

type Iterator

type Iterator interface {
	// Next advances the iterator to the next document.
	// Returns false if there are no more documents.
	Next() bool

	// Doc returns the current document.
	// Returns an error if there is an issue retrieving the document.
	Doc() (Doc, error)

	// Err returns any error encountered during the lifetime of the iterator,
	Err() error

	// Close closes the iterator and releases any associated resources.
	// Returns an error if there is an issue closing the iterator or any other error encountered during its lifetime.
	Close() error
}

Iterator represents an iterator over query results.

Example of usage:

for iter.Next() {
    doc, err := iter.Doc()
    if err != nil {
        log.Fatalf("error retrieving document: %v", err)
    }
    fmt.Println("Document:", doc.Value().String())
}
if err := iter.Close(); err != nil {
    log.Fatalf("iteration error: %v", err)
}

type ModifyResult

type ModifyResult struct {
	// Matched is the number of documents matched by the query.
	Matched int

	// Modified is the number of documents that were actually modified.
	Modified int
}

ModifyResult represents the result of a modification operation.

type Query

type Query interface {

	// Limit sets the maximum number of documents to return.
	Limit(limit uint) Query

	// Offset sets the number of documents to skip before starting to return results.
	Offset(offset uint) Query

	// Sort sets the sort order for the query results.
	Sort(sort ...any) Query

	// IndexHint adds or removes boost for some indexes
	IndexHint(hints ...IndexHint) Query

	// Iter executes the query and returns an Iterator for the results.
	Iter(ctx context.Context) (Iterator, error)

	// Count returns the number of documents matching the query.
	Count(ctx context.Context) (count int, err error)

	// Update modifies documents matching the query.
	Update(ctx context.Context, modifier any) (res ModifyResult, err error)

	// Delete removes documents matching the query.
	Delete(ctx context.Context) (res ModifyResult, err error)

	// Explain provides the query execution plan.
	Explain(ctx context.Context) (explain Explain, err error)
}

Query represents a query on a collection.

type ReadTx

type ReadTx interface {
	// Context returns the context associated with the transaction.
	Context() context.Context

	// Commit commits the transaction.
	// Returns an error if the commit fails.
	Commit() error

	// Done returns true if the transaction is completed (committed or rolled back).
	Done() bool
	// contains filtered or unexported methods
}

ReadTx represents a read-only transaction.

type WriteTx

type WriteTx interface {
	// ReadTx is embedded to provide read-only transaction methods.
	ReadTx

	// Rollback rolls back the transaction.
	// Returns an error if the rollback fails.
	Rollback() error
}

WriteTx represents a read-write transaction.