db

Documentation

Index

• Constants
• Variables
• type BatchWriteError
  • func (e *BatchWriteError) Error() string
  • func (e *BatchWriteError) Unwrap() []error
• type CollectionOptions
• type CollectionStats
• type CollectionStore
  • func CreateCollection(ctx context.Context, dir string, schema json.RawMessage, ...) (*CollectionStore, error)
  • func OpenCollection(ctx context.Context, dir string, options CollectionOptions) (*CollectionStore, error)
  • func (c *CollectionStore) Close() error
  • func (c *CollectionStore) Delete(ctx context.Context, primaryKeys []string) ([]WriteResult, error)
  • func (c *CollectionStore) DocumentCount() uint64
  • func (c *CollectionStore) Fetch(ctx context.Context, primaryKeys []string) ([]segmentstore.FetchResult, error)
  • func (c *CollectionStore) Flush(ctx context.Context) error
  • func (c *CollectionStore) Insert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)
  • func (c *CollectionStore) LiveDocuments(ctx context.Context) ([]segmentstore.StoredDocument, error)
  • func (c *CollectionStore) Manifest() common.Manifest
  • func (c *CollectionStore) OptimizationNeeded(ctx context.Context) (bool, error)
  • func (c *CollectionStore) PruneObsoleteArtifacts(ctx context.Context) error
  • func (c *CollectionStore) PublishSchema(ctx context.Context, schema json.RawMessage) (committed bool, err error)
  • func (c *CollectionStore) PublishSegmentIndexSnapshots(ctx context.Context, snapshots []common.SegmentIndexSnapshotMetadata) (committed bool, err error)
  • func (c *CollectionStore) ReadOnly() bool
  • func (c *CollectionStore) RewriteDocuments(ctx context.Context, schema json.RawMessage, ...) (committed bool, err error)
  • func (c *CollectionStore) SegmentSnapshots(ctx context.Context) ([]SegmentSnapshot, error)
  • func (c *CollectionStore) Stats() CollectionStats
  • func (c *CollectionStore) Update(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)
  • func (c *CollectionStore) Upsert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)
• type SegmentSnapshot
• type WriteEngine
  • func NewWriteEngine(manager *segment.SegmentManager, wal *wal.WAL) (*WriteEngine, error)
  • func (e *WriteEngine) Delete(ctx context.Context, primaryKeys []string) ([]WriteResult, error)
  • func (e *WriteEngine) Err() error
  • func (e *WriteEngine) Insert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)
  • func (e *WriteEngine) Update(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)
  • func (e *WriteEngine) Upsert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)
• type WriteInput
• type WriteResult

Constants

const (
	DefaultSegmentMaxDocuments = uint64(65_536)
)

Variables

var (
	ErrCollectionClosed  = errors.New("db: collection is closed")
	ErrCollectionCorrupt = errors.New("db: corrupt collection")
	ErrReadOnly          = errors.New("db: collection is read-only")
)

var (
	ErrPrimaryKeyExists    = errors.New("db: primary key already exists")
	ErrPrimaryKeyNotFound  = errors.New("db: primary key not found")
	ErrWriteEnginePoisoned = errors.New("db: write engine requires reopen")
)

Types

type BatchWriteError

type BatchWriteError struct {
	Failed int
	// contains filtered or unexported fields
}

BatchWriteError summarizes per-document failures while preserving errors.Is.

func (*BatchWriteError) Error

func (e *BatchWriteError) Error() string

func (*BatchWriteError) Unwrap

func (e *BatchWriteError) Unwrap() []error

type CollectionOptions

type CollectionOptions struct {
	ReadOnly            bool
	EnableMmap          bool
	SegmentMaxDocuments uint64
	WAL                 walstore.WALOptions
}

CollectionOptions controls the native storage lifecycle. SegmentMaxDocuments is persisted at creation; zero selects DefaultSegmentMaxDocuments. ReadOnly is an open-handle property and is never persisted.

type CollectionStats

type CollectionStats struct {
	DocumentCount         uint64
	ImmutableSegmentCount uint64
	MutableDocumentCount  uint64
	DeletedDocumentCount  uint64
	MemoryUsageBytes      uint64
}

CollectionStats describes current live keys and retained segment resources.

type CollectionStore

type CollectionStore struct {
	// contains filtered or unexported fields
}

CollectionStore owns one consistent manifest, WAL, and segment view. A writable handle holds the exclusive collection lock; any number of read-only handles can hold the shared lock together.

func CreateCollection

func CreateCollection(ctx context.Context, dir string, schema json.RawMessage, options CollectionOptions) (*CollectionStore, error)

CreateCollection creates a native Go collection and returns its sole writer.

func OpenCollection

func OpenCollection(ctx context.Context, dir string, options CollectionOptions) (*CollectionStore, error)

OpenCollection opens the exact version named by CURRENT and replays the complete WAL prefix. Read-only recovery never modifies an incomplete tail.

func (*CollectionStore) Close

func (c *CollectionStore) Close() error

Close releases the outer WAL and IDMap before the collection lock. Working IDMap state is disposable because the outer WAL can recreate it.

func (*CollectionStore) Delete

func (c *CollectionStore) Delete(ctx context.Context, primaryKeys []string) ([]WriteResult, error)

Delete delegates a primary-key batch delete to the current WAL writer.

func (*CollectionStore) DocumentCount

func (c *CollectionStore) DocumentCount() uint64

DocumentCount returns the number of live primary keys in memory.

func (*CollectionStore) Fetch

func (c *CollectionStore) Fetch(ctx context.Context, primaryKeys []string) ([]segmentstore.FetchResult, error)

Fetch resolves primary keys against the stable in-memory version.

func (*CollectionStore) Flush

func (c *CollectionStore) Flush(ctx context.Context) error

Flush atomically turns the non-empty write segment into an immutable segment, checkpoints IDMap/deletion state, publishes a new manifest, and rotates the outer WAL and disposable IDMap working copy.

func (*CollectionStore) Insert

func (c *CollectionStore) Insert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Insert delegates a batch insert to the current WAL writer.

func (*CollectionStore) LiveDocuments

func (c *CollectionStore) LiveDocuments(ctx context.Context) ([]segmentstore.StoredDocument, error)

LiveDocuments returns a stable collection-level view while excluding a concurrent flush. Public query orchestration additionally serializes writes so multi-step upserts cannot be observed halfway through application.

func (*CollectionStore) Manifest

func (c *CollectionStore) Manifest() common.Manifest

Manifest returns an independent copy of the current published metadata.

func (*CollectionStore) OptimizationNeeded

func (c *CollectionStore) OptimizationNeeded(ctx context.Context) (bool, error)

OptimizationNeeded reports whether rewriting would flush mutable documents, remove deleted versions, or reduce the immutable layout to the canonical contiguous-ID runs bounded by SegmentMaxDocuments.

func (*CollectionStore) PruneObsoleteArtifacts

func (c *CollectionStore) PruneObsoleteArtifacts(ctx context.Context) error

PruneObsoleteArtifacts removes only storage artifacts owned by this package that are no longer referenced by the current manifest. CURRENT is already the commit point, so an interrupted prune is harmless and can be retried. Unknown files and manifest generations are deliberately left untouched.

func (*CollectionStore) PublishSchema

func (c *CollectionStore) PublishSchema(ctx context.Context, schema json.RawMessage) (committed bool, err error)

PublishSchema atomically installs a validated schema payload in a new manifest generation. committed is true once CURRENT names that generation, including the rare case where a post-commit directory sync reports an error. Callers must update their in-memory schema whenever committed is true.

func (*CollectionStore) PublishSegmentIndexSnapshots

func (c *CollectionStore) PublishSegmentIndexSnapshots(ctx context.Context, snapshots []common.SegmentIndexSnapshotMetadata) (committed bool, err error)

PublishSegmentIndexSnapshots atomically installs immutable per-segment index metadata. Vector artifacts must already exist as regular files and FTS/INVERT artifacts as Pebble directories below the collection directory.

func (*CollectionStore) ReadOnly

func (c *CollectionStore) ReadOnly() bool

ReadOnly reports whether this handle rejects mutations.

func (*CollectionStore) RewriteDocuments

func (c *CollectionStore) RewriteDocuments(ctx context.Context, schema json.RawMessage, documents []segmentstore.StoredDocument) (committed bool, err error)

RewriteDocuments atomically replaces every live document payload together with the collection schema. Document IDs and primary keys must exactly match the current live snapshot in ascending document-ID order. Superseded and deleted versions are reclaimed by the rewrite, while the next document ID remains monotonic. committed reports whether CURRENT reached the new generation even if a post-commit sync or old WAL close reports an error.

func (*CollectionStore) SegmentSnapshots

func (c *CollectionStore) SegmentSnapshots(ctx context.Context) ([]SegmentSnapshot, error)

SegmentSnapshots returns retained documents grouped by physical segment. Logical deletions and superseded versions remain present so immutable index artifacts never need rewriting; query-time live masks exclude them.

func (*CollectionStore) Stats

func (c *CollectionStore) Stats() CollectionStats

Stats returns a point-in-time storage snapshot without cloning documents.

func (*CollectionStore) Update

func (c *CollectionStore) Update(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Update delegates a batch update to the current WAL writer.

func (*CollectionStore) Upsert

func (c *CollectionStore) Upsert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Upsert delegates a batch upsert to the current WAL writer.

type SegmentSnapshot

type SegmentSnapshot struct {
	Metadata  common.SegmentMetadata
	Documents []segmentstore.StoredDocument
	Mutable   bool
}

SegmentSnapshot is an owned, stable view used by the collection index layer. Immutable snapshots correspond to PersistedSegments; the final mutable snapshot corresponds to the current WAL-backed writing segment.

type WriteEngine

type WriteEngine struct {
	// contains filtered or unexported fields
}

WriteEngine serializes WAL-backed mutations over a SegmentManager. When an automatic synchronization fails after a complete append, the engine still applies that record, returns the synchronization error, and relies on the poisoned WAL to reject later mutations until reopen.

func NewWriteEngine

func NewWriteEngine(manager *segment.SegmentManager, wal *wal.WAL) (*WriteEngine, error)

NewWriteEngine validates the dependencies required for WAL-backed mutations.

func (*WriteEngine) Delete

func (e *WriteEngine) Delete(ctx context.Context, primaryKeys []string) ([]WriteResult, error)

Delete appends removals to the WAL before updating primary-key mappings and logically deleting current document IDs. Immutable segment bytes are never rewritten.

func (*WriteEngine) Err

func (e *WriteEngine) Err() error

Err reports whether a record reached the outer WAL but failed to apply completely. Such a handle must be reopened and replayed before more writes or publication operations are safe.

func (*WriteEngine) Insert

func (e *WriteEngine) Insert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Insert appends documents whose primary keys do not exist to the WAL before applying them. WALOptions controls when appended records are synchronized. Validation and duplicate errors are reported per input; every failure is also included in the returned BatchWriteError.

func (*WriteEngine) Update

func (e *WriteEngine) Update(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Update appends replacement document versions for existing keys to the WAL.

func (*WriteEngine) Upsert

func (e *WriteEngine) Upsert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Upsert appends a new document version to the WAL. If the key already exists, its prior document ID is logically deleted after the WAL append succeeds.

type WriteInput

type WriteInput struct {
	PrimaryKey string
	Payload    []byte
}

WriteInput is one schema-encoded document requested by a write API.

type WriteResult

type WriteResult struct {
	PrimaryKey string
	DocID      uint64
	Err        error
}

WriteResult reports the outcome for one input in the same position.