scio

README

scio

URI-based data catalog with atomic operations for Go.

scio is the system's authoritative map of data sources — it knows where data lives and provides type-agnostic access via atoms.

The Map

scio registers storage resources and routes operations through URIs:

s := scio.New()

// Register resources (user code provides typed grub wrappers)
s.RegisterDatabase("db://users", usersDB.Atomic())
s.RegisterStore("kv://sessions", sessionsStore.Atomic())
s.RegisterBucket("bcs://documents", docsBucket.Atomic())

// Operations via URI — scio routes to the right provider
atom, _ := s.Get(ctx, "db://users/123")
s.Set(ctx, "kv://sessions/abc", sessionAtom)

// Query databases
results, _ := s.Query(ctx, "db://users", stmt, params)

// Introspect the topology
s.Sources()                    // all registered resources
s.FindBySpec(spec)             // resources sharing a type
s.Related("db://users")        // other resources with same spec

Install

go get github.com/zoobzio/scio

Requires Go 1.24 or higher.

Quick Start

package main

import (
    "context"
    "fmt"

    "github.com/zoobzio/grub"
    "github.com/zoobzio/scio"
)

func main() {
    // Create scio instance
    s := scio.New()

    // Register a database (assumes usersDB is a grub.Database[User])
    err := s.RegisterDatabase("db://users", usersDB.Atomic(),
        scio.WithDescription("User accounts"),
        scio.WithTag("owner", "auth-team"),
    )
    if err != nil {
        panic(err)
    }

    ctx := context.Background()

    // Get a record
    atom, err := s.Get(ctx, "db://users/123")
    if err != nil {
        panic(err)
    }

    // Access fields via typed maps
    fmt.Println(atom.Strings["email"])
    fmt.Println(atom.Ints["age"])

    // Find related resources
    for _, r := range s.Related("db://users") {
        fmt.Printf("Related: %s (%s)\n", r.URI, r.Variant)
    }
}

URI Semantics

Variant	Pattern	Example
db://	table/key	db://users/123
kv://	store/key	kv://sessions/abc
bcs://	bucket/path	bcs://docs/reports/q4.pdf

Why scio?

• URI-based routing — logical addressing decoupled from physical storage
• Type-agnostic — operates on atoms, never needs to know your types
• Topology awareness — knows what resources exist and their relationships
• Spec tracking — auto-detects when multiple resources share the same type
• Metadata support — annotate resources for system use (LLM context, ownership, versioning)

Documentation

Learn

• Quick Start
• Core Concepts
• Architecture

Reference

• API Reference

Contributing

See CONTRIBUTING.md for development setup and guidelines.

License

MIT License - see LICENSE for details.

Documentation

Overview

Package scio provides a URI-based data catalog with atomic operations. It serves as the authoritative map of data sources, providing topology intelligence and type-agnostic access via atoms.

Index

• Variables
• type Catalog
• type Metadata
• type Operations
• type RegistrationOption
  • func WithDescription(desc string) RegistrationOption
  • func WithTag(key, value string) RegistrationOption
  • func WithVersion(ver string) RegistrationOption
• type Registry
• type Resource
• type Scio
  • func New() *Scio
  • func (s *Scio) Buckets() []Resource
  • func (s *Scio) Databases() []Resource
  • func (s *Scio) Delete(ctx context.Context, uri string) error
  • func (s *Scio) Exists(ctx context.Context, uri string) (bool, error)
  • func (s *Scio) FindByField(field string) []Resource
  • func (s *Scio) FindBySpec(spec atom.Spec) []Resource
  • func (s *Scio) Get(ctx context.Context, uri string) (*atom.Atom, error)
  • func (s *Scio) Put(ctx context.Context, uri string, obj *grub.AtomicObject) error
  • func (s *Scio) Query(ctx context.Context, uri string, stmt edamame.QueryStatement, ...) ([]*atom.Atom, error)
  • func (s *Scio) RegisterBucket(uri string, bucket grub.AtomicBucket, opts ...RegistrationOption) error
  • func (s *Scio) RegisterDatabase(uri string, db grub.AtomicDatabase, opts ...RegistrationOption) error
  • func (s *Scio) RegisterStore(uri string, store grub.AtomicStore, opts ...RegistrationOption) error
  • func (s *Scio) Related(uri string) []Resource
  • func (s *Scio) Resource(uri string) *Resource
  • func (s *Scio) Select(ctx context.Context, uri string, stmt edamame.SelectStatement, ...) (*atom.Atom, error)
  • func (s *Scio) Set(ctx context.Context, uri string, data *atom.Atom) error
  • func (s *Scio) SetWithTTL(ctx context.Context, uri string, data *atom.Atom, ttl time.Duration) error
  • func (s *Scio) Sources() []Resource
  • func (s *Scio) Spec(uri string) (atom.Spec, error)
  • func (s *Scio) Stores() []Resource
• type URI
  • func ParseURI(raw string) (*URI, error)
  • func (u *URI) ResourceURI() string
  • func (u *URI) String() string
• type Variant

Variables

var (
	// ErrInvalidURI indicates the URI could not be parsed.
	ErrInvalidURI = errors.New("invalid URI")

	// ErrUnknownVariant indicates an unrecognized URI scheme.
	ErrUnknownVariant = errors.New("unknown variant")

	// ErrResourceNotFound indicates the requested resource is not registered.
	ErrResourceNotFound = errors.New("resource not found")

	// ErrResourceExists indicates a resource is already registered at the URI.
	ErrResourceExists = errors.New("resource already exists")

	// ErrVariantMismatch indicates an operation was attempted on the wrong variant.
	ErrVariantMismatch = errors.New("variant mismatch")

	// ErrKeyRequired indicates a key is required but was not provided.
	ErrKeyRequired = errors.New("key required")

	// ErrKeyNotExpected indicates a key was provided but is not used by the operation.
	ErrKeyNotExpected = errors.New("key not expected")
)

Semantic errors for scio operations.

var (
	ErrNotFound        = grub.ErrNotFound
	ErrDuplicate       = grub.ErrDuplicate
	ErrConflict        = grub.ErrConflict
	ErrConstraint      = grub.ErrConstraint
	ErrInvalidKey      = grub.ErrInvalidKey
	ErrReadOnly        = grub.ErrReadOnly
	ErrTTLNotSupported = grub.ErrTTLNotSupported
)

Re-export grub errors for convenience.

Types

type Catalog

type Catalog interface {
	// Sources returns all registered resources.
	Sources() []Resource

	// Databases returns all db:// resources.
	Databases() []Resource

	// Stores returns all kv:// resources.
	Stores() []Resource

	// Buckets returns all bcs:// resources.
	Buckets() []Resource

	// Spec returns the atom spec for a specific resource.
	Spec(uri string) (atom.Spec, error)

	// FindBySpec returns all resources sharing the given spec (by FQDN).
	FindBySpec(spec atom.Spec) []Resource

	// FindByField returns all resources containing the given field.
	FindByField(field string) []Resource

	// Related returns other resources with the same spec as the given URI.
	Related(uri string) []Resource
}

Catalog defines topology introspection operations.

type Metadata

type Metadata struct {
	Description string
	Version     string
	Tags        map[string]string
}

Metadata holds resource annotations for system use.

type Operations

type Operations interface {
	// Get retrieves an atom at the given URI.
	// Returns ErrNotFound if the key does not exist.
	Get(ctx context.Context, uri string) (*atom.Atom, error)

	// Set stores an atom at the given URI.
	Set(ctx context.Context, uri string, data *atom.Atom) error

	// Delete removes the record at the given URI.
	// Returns ErrNotFound if the key does not exist.
	Delete(ctx context.Context, uri string) error

	// Exists checks whether a record exists at the given URI.
	Exists(ctx context.Context, uri string) (bool, error)

	// Query executes a query statement against a database resource.
	// The URI should reference a db:// resource without a key.
	Query(ctx context.Context, uri string, stmt edamame.QueryStatement, params map[string]any) ([]*atom.Atom, error)

	// Select executes a select statement against a database resource.
	// The URI should reference a db:// resource without a key.
	Select(ctx context.Context, uri string, stmt edamame.SelectStatement, params map[string]any) (*atom.Atom, error)

	// SetWithTTL stores an atom at the given kv:// URI with a TTL.
	// TTL of 0 means no expiration.
	SetWithTTL(ctx context.Context, uri string, data *atom.Atom, ttl time.Duration) error

	// Put stores a blob object at the given bcs:// URI.
	Put(ctx context.Context, uri string, obj *grub.AtomicObject) error
}

Operations defines data access operations.

type RegistrationOption

type RegistrationOption func(*registrationConfig)

RegistrationOption configures resource registration.

func WithDescription

func WithDescription(desc string) RegistrationOption

WithDescription sets the resource description.

func WithTag

func WithTag(key, value string) RegistrationOption

WithTag adds a tag to the resource metadata.

func WithVersion

func WithVersion(ver string) RegistrationOption

WithVersion sets the resource version.

type Registry

type Registry interface {
	// RegisterDatabase registers an atomic database at the given URI.
	// The URI should be in the form db://table.
	RegisterDatabase(uri string, db grub.AtomicDatabase, opts ...RegistrationOption) error

	// RegisterStore registers an atomic store at the given URI.
	// The URI should be in the form kv://store.
	RegisterStore(uri string, store grub.AtomicStore, opts ...RegistrationOption) error

	// RegisterBucket registers an atomic bucket at the given URI.
	// The URI should be in the form bcs://bucket.
	RegisterBucket(uri string, bucket grub.AtomicBucket, opts ...RegistrationOption) error
}

Registry defines resource registration operations.

type Resource

type Resource struct {
	URI      string
	Variant  Variant
	Name     string
	Spec     atom.Spec
	Metadata Metadata
}

Resource represents a registered data source.

type Scio

type Scio struct {
	// contains filtered or unexported fields
}

Scio is the central data catalog and access point.

func New

func New() *Scio

New creates a new Scio instance.

func (*Scio) Buckets

func (s *Scio) Buckets() []Resource

Buckets returns all bcs:// resources.

func (*Scio) Databases

func (s *Scio) Databases() []Resource

Databases returns all db:// resources.

func (*Scio) Delete

func (s *Scio) Delete(ctx context.Context, uri string) error

Delete removes the record at the given URI. Returns ErrNotFound if the key does not exist. Returns ErrResourceNotFound if the resource is not registered.

func (*Scio) Exists

func (s *Scio) Exists(ctx context.Context, uri string) (bool, error)

Exists checks whether a record exists at the given URI. Returns ErrResourceNotFound if the resource is not registered.

func (*Scio) FindByField

func (s *Scio) FindByField(field string) []Resource

FindByField returns all resources containing the given field.

func (*Scio) FindBySpec

func (s *Scio) FindBySpec(spec atom.Spec) []Resource

FindBySpec returns all resources sharing the given spec (by FQDN).

func (*Scio) Get

func (s *Scio) Get(ctx context.Context, uri string) (*atom.Atom, error)

Get retrieves an atom at the given URI. Returns ErrNotFound if the key does not exist. Returns ErrResourceNotFound if the resource is not registered. Returns ErrKeyRequired if no key is provided.

func (*Scio) Put

func (s *Scio) Put(ctx context.Context, uri string, obj *grub.AtomicObject) error

Put stores a blob object at the given bcs:// URI. Returns ErrVariantMismatch if the URI is not a bcs:// resource.

func (*Scio) Query

func (s *Scio) Query(ctx context.Context, uri string, stmt edamame.QueryStatement, params map[string]any) ([]*atom.Atom, error)

Query executes a query statement against a database resource. The URI should reference a db:// resource without a key component. Returns ErrVariantMismatch if the URI is not a db:// resource. Returns ErrKeyNotExpected if a key component is provided.

func (*Scio) RegisterBucket

func (s *Scio) RegisterBucket(uri string, bucket grub.AtomicBucket, opts ...RegistrationOption) error

RegisterBucket registers an atomic bucket at the given URI. The URI should be in the form bcs://bucket.

func (*Scio) RegisterDatabase

func (s *Scio) RegisterDatabase(uri string, db grub.AtomicDatabase, opts ...RegistrationOption) error

RegisterDatabase registers an atomic database at the given URI. The URI should be in the form db://table.

func (*Scio) RegisterStore

func (s *Scio) RegisterStore(uri string, store grub.AtomicStore, opts ...RegistrationOption) error

RegisterStore registers an atomic store at the given URI. The URI should be in the form kv://store.

func (*Scio) Related

func (s *Scio) Related(uri string) []Resource

Related returns other resources with the same spec as the given URI. The resource at the given URI is excluded from the results.

func (*Scio) Resource

func (s *Scio) Resource(uri string) *Resource

Resource returns the resource metadata for a specific URI. Returns nil if the resource is not registered.

func (*Scio) Select

func (s *Scio) Select(ctx context.Context, uri string, stmt edamame.SelectStatement, params map[string]any) (*atom.Atom, error)

Select executes a select statement against a database resource. The URI should reference a db:// resource without a key component. Returns ErrVariantMismatch if the URI is not a db:// resource. Returns ErrKeyNotExpected if a key component is provided.

func (*Scio) Set

func (s *Scio) Set(ctx context.Context, uri string, data *atom.Atom) error

Set stores an atom at the given URI. Returns ErrResourceNotFound if the resource is not registered. Returns ErrKeyRequired if no key is provided. For kv:// resources, use SetWithTTL to specify expiration.

func (*Scio) SetWithTTL

func (s *Scio) SetWithTTL(ctx context.Context, uri string, data *atom.Atom, ttl time.Duration) error

SetWithTTL stores an atom at the given kv:// URI with a TTL. TTL of 0 means no expiration. Returns ErrVariantMismatch if the URI is not a kv:// resource.

func (*Scio) Sources

func (s *Scio) Sources() []Resource

Sources returns all registered resources.

func (*Scio) Spec

func (s *Scio) Spec(uri string) (atom.Spec, error)

Spec returns the atom spec for a specific resource. Returns ErrResourceNotFound if the resource is not registered.

func (*Scio) Stores

func (s *Scio) Stores() []Resource

Stores returns all kv:// resources.

type URI

type URI struct {
	Variant  Variant
	Resource string // table, store, or bucket name
	Key      string // record key or blob path (may be empty for queries)
}

URI represents a parsed data address.

func ParseURI

func ParseURI(raw string) (*URI, error)

ParseURI parses a raw URI string into its components. Supported formats:

• db://table/key -> Variant: db, Resource: table, Key: key
• kv://store/key -> Variant: kv, Resource: store, Key: key
• bcs://bucket/path -> Variant: bcs, Resource: bucket, Key: path (full remainder)

func (*URI) ResourceURI

func (u *URI) ResourceURI() string

ResourceURI returns the URI without the key component.

func (*URI) String

func (u *URI) String() string

String returns the URI as a string.

type Variant

type Variant string

Variant represents a storage type.

const (
	// VariantDatabase represents SQL database storage (db://).
	VariantDatabase Variant = "db"

	// VariantStore represents key-value storage (kv://).
	VariantStore Variant = "kv"

	// VariantBucket represents blob/object storage (bcs://).
	VariantBucket Variant = "bcs"
)