db

package
v0.0.0-...-6e4a073 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 10, 2026 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultSegmentMaxDocuments = uint64(65_536)
)
View Source 
const (
	// DiskFormatVersion is the Pebble-backed native Go collection format. It is
	// not compatible with v1 or the C++ collection format.
	DiskFormatVersion uint32 = 3
)
View Source 
const (
	MaxDocumentPayloadSize = 64 << 20
)
View Source 
const (

	// MaxWALRecordSize matches the pinned baseline's per-record safety limit.
	MaxWALRecordSize = 4 << 20
)

Variables

View Source 
var (
	ErrCollectionClosed  = errors.New("db: collection is closed")
	ErrCollectionCorrupt = errors.New("db: corrupt collection")
	ErrReadOnly          = errors.New("db: collection is read-only")
)
View Source 
var (
	ErrManifestNotFound         = errors.New("db: manifest not found")
	ErrManifestCorrupt          = errors.New("db: corrupt manifest")
	ErrManifestConflict         = errors.New("db: manifest version conflict")
	ErrManifestExists           = errors.New("db: collection manifest already exists")
	ErrUnsupportedFormatVersion = errors.New("db: unsupported disk format version")
)
View Source 
var (
	ErrSegmentCorrupt   = errors.New("db: corrupt segment")
	ErrSegmentSealed    = errors.New("db: write segment is sealed")
	ErrSegmentFull      = errors.New("db: write segment is full")
	ErrSegmentNotFound  = errors.New("db: segment not found")
	ErrDocumentNotFound = errors.New("db: document not found")
)
View Source 
var (
	ErrWALNotFound       = errors.New("db: WAL not found")
	ErrWALExists         = errors.New("db: WAL already exists")
	ErrWALCorrupt        = errors.New("db: corrupt WAL")
	ErrWALClosed         = errors.New("db: WAL is closed")
	ErrWALReadOnly       = errors.New("db: WAL is read-only")
	ErrWALPoisoned       = errors.New("db: WAL append state is poisoned")
	ErrWALRecordTooLarge = errors.New("db: WAL record is too large")
)
View Source 
var (
	ErrPrimaryKeyExists    = errors.New("db: primary key already exists")
	ErrPrimaryKeyNotFound  = errors.New("db: primary key not found")
	ErrWriteEnginePoisoned = errors.New("db: write engine requires reopen")
)
View Source 
var ErrIDMapCorrupt = errors.New("db: corrupt IDMap")
View Source 
var ErrSnapshotCorrupt = errors.New("db: corrupt snapshot")

Functions

func MarshalManifest 

func MarshalManifest(m Manifest) ([]byte, error)

MarshalManifest encodes m with a magic value, format version, payload length, generation, and CRC32C checksum.

Types

type BatchWriteError 

type BatchWriteError struct {
	Failed int
	// contains filtered or unexported fields
}

BatchWriteError summarizes per-document failures while preserving errors.Is.

func (*BatchWriteError) Error 

func (e *BatchWriteError) Error() string

func (*BatchWriteError) Unwrap 

func (e *BatchWriteError) Unwrap() []error

type CollectionOptions 

type CollectionOptions struct {
	ReadOnly            bool
	EnableMmap          bool
	SegmentMaxDocuments uint64
	WAL                 WALOptions
}

CollectionOptions controls the native storage lifecycle. SegmentMaxDocuments is persisted at creation; zero selects DefaultSegmentMaxDocuments. ReadOnly is an open-handle property and is never persisted.

type CollectionStats 

type CollectionStats struct {
	DocumentCount         uint64
	ImmutableSegmentCount uint64
	MutableDocumentCount  uint64
	DeletedDocumentCount  uint64
	MemoryUsageBytes      uint64
}

CollectionStats describes current live keys and retained segment resources.

type CollectionStore 

type CollectionStore struct {
	// contains filtered or unexported fields
}

CollectionStore owns one consistent manifest, WAL, and segment view. A writable handle holds the exclusive collection lock; any number of read-only handles can hold the shared lock together.

func CreateCollection 

func CreateCollection(ctx context.Context, dir string, schema json.RawMessage, options CollectionOptions) (*CollectionStore, error)

CreateCollection creates a native Go collection and returns its sole writer.

func OpenCollection 

func OpenCollection(ctx context.Context, dir string, options CollectionOptions) (*CollectionStore, error)

OpenCollection opens the exact version named by CURRENT and replays the complete WAL prefix. Read-only recovery never modifies an incomplete tail.

func (*CollectionStore) Close 

func (c *CollectionStore) Close() error

Close releases the outer WAL and IDMap before the collection lock. Working IDMap state is disposable because the outer WAL can recreate it.

func (*CollectionStore) Delete 

func (c *CollectionStore) Delete(ctx context.Context, primaryKeys []string) ([]WriteResult, error)

Delete delegates a primary-key batch delete to the current WAL writer.

func (*CollectionStore) DocumentCount 

func (c *CollectionStore) DocumentCount() uint64

DocumentCount returns the number of live primary keys in memory.

func (*CollectionStore) Fetch 

func (c *CollectionStore) Fetch(ctx context.Context, primaryKeys []string) ([]FetchResult, error)

Fetch resolves primary keys against the stable in-memory version.

func (*CollectionStore) Flush 

func (c *CollectionStore) Flush(ctx context.Context) error

Flush atomically turns the non-empty write segment into an immutable segment, checkpoints IDMap/deletion state, publishes a new manifest, and rotates the outer WAL and disposable IDMap working copy.

func (*CollectionStore) Insert 

func (c *CollectionStore) Insert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Insert delegates a batch insert to the current WAL writer.

func (*CollectionStore) LiveDocuments 

func (c *CollectionStore) LiveDocuments(ctx context.Context) ([]StoredDocument, error)

LiveDocuments returns a stable collection-level view while excluding a concurrent flush. Public query orchestration additionally serializes writes so multi-step upserts cannot be observed halfway through application.

func (*CollectionStore) Manifest 

func (c *CollectionStore) Manifest() Manifest

Manifest returns an independent copy of the current published metadata.

func (*CollectionStore) OptimizationNeeded 

func (c *CollectionStore) OptimizationNeeded(ctx context.Context) (bool, error)

OptimizationNeeded reports whether rewriting would flush mutable documents, remove deleted versions, or reduce the immutable layout to the canonical contiguous-ID runs bounded by SegmentMaxDocuments.

func (*CollectionStore) PruneObsoleteArtifacts 

func (c *CollectionStore) PruneObsoleteArtifacts(ctx context.Context) error

PruneObsoleteArtifacts removes only storage artifacts owned by this package that are no longer referenced by the current manifest. CURRENT is already the commit point, so an interrupted prune is harmless and can be retried. Unknown files and manifest generations are deliberately left untouched.

func (*CollectionStore) PublishSchema 

func (c *CollectionStore) PublishSchema(ctx context.Context, schema json.RawMessage) (committed bool, err error)

PublishSchema atomically installs a validated schema payload in a new manifest generation. committed is true once CURRENT names that generation, including the rare case where a post-commit directory sync reports an error. Callers must update their in-memory schema whenever committed is true.

func (*CollectionStore) PublishSegmentIndexSnapshots 

func (c *CollectionStore) PublishSegmentIndexSnapshots(ctx context.Context, snapshots []SegmentIndexSnapshotMetadata) (committed bool, err error)

PublishSegmentIndexSnapshots atomically installs immutable per-segment index metadata. Vector artifacts must already exist as regular files and FTS/INVERT artifacts as Pebble directories below the collection directory.

func (*CollectionStore) ReadOnly 

func (c *CollectionStore) ReadOnly() bool

ReadOnly reports whether this handle rejects mutations.

func (*CollectionStore) RewriteDocuments 

func (c *CollectionStore) RewriteDocuments(ctx context.Context, schema json.RawMessage, documents []StoredDocument) (committed bool, err error)

RewriteDocuments atomically replaces every live document payload together with the collection schema. Document IDs and primary keys must exactly match the current live snapshot in ascending document-ID order. Superseded and deleted versions are reclaimed by the rewrite, while the next document ID remains monotonic. committed reports whether CURRENT reached the new generation even if a post-commit sync or old-WAL close reports an error.

func (*CollectionStore) SegmentSnapshots 

func (c *CollectionStore) SegmentSnapshots(ctx context.Context) ([]SegmentSnapshot, error)

SegmentSnapshots returns retained documents grouped by physical segment. Logical deletions and superseded versions remain present so immutable index artifacts never need rewriting; query-time live masks exclude them.

func (*CollectionStore) Stats 

func (c *CollectionStore) Stats() CollectionStats

Stats returns a point-in-time storage snapshot without cloning documents.

func (*CollectionStore) Update 

func (c *CollectionStore) Update(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Update delegates a batch update to the current WAL writer.

func (*CollectionStore) Upsert 

func (c *CollectionStore) Upsert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Upsert delegates a batch upsert to the current WAL writer.

type DeleteStore 

type DeleteStore struct {
	// contains filtered or unexported fields
}

DeleteStore tracks global document IDs hidden by logical deletion.

func LoadDeleteStore 

func LoadDeleteStore(ctx context.Context, name string) (*DeleteStore, error)

LoadDeleteStore reads and validates a logical-deletion snapshot.

func NewDeleteStore 

func NewDeleteStore() *DeleteStore

NewDeleteStore returns an empty logical deletion set.

func (*DeleteStore) Clone 

func (s *DeleteStore) Clone() *DeleteStore

Clone returns an independent delete set.

func (*DeleteStore) Count 

func (s *DeleteStore) Count() uint64

Count returns the number of deleted document IDs.

func (*DeleteStore) IsDeleted 

func (s *DeleteStore) IsDeleted(docID uint64) bool

IsDeleted reports whether docID is logically deleted.

func (*DeleteStore) MarkDeleted 

func (s *DeleteStore) MarkDeleted(ctx context.Context, docID uint64) (bool, error)

MarkDeleted marks docID and reports whether the set changed.

func (*DeleteStore) RangeCount 

func (s *DeleteStore) RangeCount(minDocID, maxDocID uint64) uint64

RangeCount returns deletions in the inclusive document-ID interval.

func (*DeleteStore) Restore 

func (s *DeleteStore) Restore(ctx context.Context, docID uint64) (bool, error)

Restore clears a logical deletion and reports whether the set changed.

func (*DeleteStore) WriteSnapshot 

func (s *DeleteStore) WriteSnapshot(ctx context.Context, name string) error

WriteSnapshot writes sorted fixed-width document IDs atomically.

type FetchResult 

type FetchResult struct {
	PrimaryKey string
	Document   *StoredDocument
	Err        error
}

FetchResult preserves request order. A nil Document with nil Err means the primary key is absent or logically deleted, matching the pinned baseline.

type ImmutableSegment 

type ImmutableSegment struct {
	// contains filtered or unexported fields
}

ImmutableSegment is a verified read-only segment snapshot.

func OpenImmutableSegment 

func OpenImmutableSegment(ctx context.Context, collectionDir string, metadata SegmentMetadata) (*ImmutableSegment, error)

OpenImmutableSegment loads and verifies the first data file in metadata.

func (*ImmutableSegment) Document 

func (s *ImmutableSegment) Document(docID uint64) (StoredDocument, bool)

Document returns an independent document copy.

func (*ImmutableSegment) Documents 

func (s *ImmutableSegment) Documents() []StoredDocument

Documents returns all immutable documents in ascending ID order.

func (*ImmutableSegment) ID 

func (s *ImmutableSegment) ID() uint64

ID returns the immutable segment ID.

func (*ImmutableSegment) MemoryUsageBytes 

func (s *ImmutableSegment) MemoryUsageBytes() uint64

MemoryUsageBytes returns the encoded record bytes retained by the segment.

func (*ImmutableSegment) Metadata 

func (s *ImmutableSegment) Metadata() SegmentMetadata

Metadata returns a deep copy.

type IndexArtifactMetadata 

type IndexArtifactMetadata struct {
	Field string `json:"field"`
	Kind  string `json:"kind"`
	File  string `json:"file"`
}

IndexArtifactMetadata identifies one collection index file. Kind is interpreted by the public collection layer; the storage layer owns only its portable path and publication lifecycle.

type Manifest 

type Manifest struct {
	FormatVersion            uint32                         `json:"format_version"`
	Generation               uint64                         `json:"generation"`
	Schema                   json.RawMessage                `json:"schema"`
	EnableMmap               bool                           `json:"enable_mmap"`
	SegmentMaxDocuments      uint64                         `json:"segment_max_documents"`
	PersistedSegments        []SegmentMetadata              `json:"persisted_segments,omitempty"`
	WritingSegment           *SegmentMetadata               `json:"writing_segment,omitempty"`
	WritingSegmentStartDocID uint64                         `json:"writing_segment_start_doc_id"`
	IDMap                    string                         `json:"id_map"`
	DeleteSnapshotGeneration uint64                         `json:"delete_snapshot_generation"`
	NextSegmentID            uint64                         `json:"next_segment_id"`
	SegmentIndexSnapshots    []SegmentIndexSnapshotMetadata `json:"segment_index_snapshots,omitempty"`
}

Manifest is one immutable collection metadata snapshot. Schema contains the versioned JSON representation owned by the schema codec; the manifest layer preserves it without importing the public package.

func UnmarshalManifest 

func UnmarshalManifest(encoded []byte) (Manifest, error)

UnmarshalManifest verifies and decodes one complete manifest file.

func (Manifest) Clone 

func (m Manifest) Clone() Manifest

Clone returns a deep copy of m.

func (Manifest) Validate 

func (m Manifest) Validate() error

Validate checks all format-level invariants. Schema semantics are checked by the schema codec; this layer requires a non-null JSON object.

type PrimaryKeyMap 

type PrimaryKeyMap struct {
	// contains filtered or unexported fields
}

PrimaryKeyMap is the collection IDMap: a Pebble point map from a primary key directly to a collection-global document ID. Writable maps are disposable working state whose durability comes from the outer WAL. Read-only maps use an immutable Pebble checkpoint plus an in-memory replay overlay.

func CreatePrimaryKeyMap 

func CreatePrimaryKeyMap(ctx context.Context, name string) (*PrimaryKeyMap, error)

CreatePrimaryKeyMap creates a disposable writable Pebble IDMap at path.

func NewPrimaryKeyMap 

func NewPrimaryKeyMap() *PrimaryKeyMap

NewPrimaryKeyMap creates an in-memory writable IDMap for unit-level users. Collection lifecycle paths use the filesystem constructors below.

func OpenPrimaryKeyMap 

func OpenPrimaryKeyMap(ctx context.Context, checkpoint, working string) (*PrimaryKeyMap, error)

OpenPrimaryKeyMap copies checkpoint into a new disposable working directory and opens the copy for replay and mutations. Existing working directories are never reused, so the outer WAL remains the sole recovery authority.

func OpenPrimaryKeyMapReadOnly 

func OpenPrimaryKeyMapReadOnly(ctx context.Context, checkpoint string) (*PrimaryKeyMap, error)

OpenPrimaryKeyMapReadOnly opens checkpoint without creating files and keeps WAL replay changes in memory.

func (*PrimaryKeyMap) Checkpoint 

func (m *PrimaryKeyMap) Checkpoint(ctx context.Context, target string) error

Checkpoint flushes disposable working state and creates an immutable Pebble directory. The checkpoint is not visible until an outer manifest naming it is published through CURRENT.

func (*PrimaryKeyMap) Close 

func (m *PrimaryKeyMap) Close() error

Close releases Pebble resources. It is idempotent.

func (*PrimaryKeyMap) Count 

func (m *PrimaryKeyMap) Count() int

Count returns the exact logical key count maintained across mutations and a read-only overlay. Corrupt checkpoint values are rejected while opening.

func (*PrimaryKeyMap) Delete 

func (m *PrimaryKeyMap) Delete(ctx context.Context, key string) (uint64, bool, error)

Delete removes key and returns its prior global document ID, if any.

func (*PrimaryKeyMap) Get 

func (m *PrimaryKeyMap) Get(key string) (uint64, bool, error)

Get performs a point lookup and never converts a Pebble error into absence.

func (*PrimaryKeyMap) MultiGet 

func (m *PrimaryKeyMap) MultiGet(keys []string) ([]uint64, []bool, error)

MultiGet returns document IDs and found flags in input order.

func (*PrimaryKeyMap) Put 

func (m *PrimaryKeyMap) Put(ctx context.Context, key string, docID uint64) (uint64, bool, error)

Put adds or replaces key and returns its prior global document ID, if any.

type SegmentIndexSnapshotMetadata 

type SegmentIndexSnapshotMetadata struct {
	SegmentID     uint64                  `json:"segment_id"`
	SchemaSHA256  string                  `json:"schema_sha256"`
	DocumentCount uint64                  `json:"document_count"`
	MinDocumentID uint64                  `json:"min_document_id"`
	MaxDocumentID uint64                  `json:"max_document_id"`
	Artifacts     []IndexArtifactMetadata `json:"artifacts,omitempty"`
}

SegmentIndexSnapshotMetadata identifies indexes owned by one immutable data segment. Document bounds bind the artifacts to the exact segment payload; SchemaSHA256 binds their interpretation to one collection schema.

type SegmentManager 

type SegmentManager struct {
	// contains filtered or unexported fields
}

SegmentManager owns sorted immutable segments, one optional write segment, the primary-key map, and the logical deletion snapshot.

func NewSegmentManager 

func NewSegmentManager(primaryKey *PrimaryKeyMap, deletes *DeleteStore) *SegmentManager

NewSegmentManager constructs an empty manager. Nil stores are replaced with empty instances.

func (*SegmentManager) AddImmutable 

func (m *SegmentManager) AddImmutable(segment *ImmutableSegment) error

AddImmutable adds a verified immutable segment.

func (*SegmentManager) ClearWriting 

func (m *SegmentManager) ClearWriting() *WriteSegment

ClearWriting removes and returns the current write segment.

func (*SegmentManager) Deletes 

func (m *SegmentManager) Deletes() *DeleteStore

Deletes returns the manager's shared logical deletion set.

func (*SegmentManager) Document 

func (m *SegmentManager) Document(docID uint64) (StoredDocument, bool)

Document returns a live document by global ID.

func (*SegmentManager) DocumentByPrimaryKey 

func (m *SegmentManager) DocumentByPrimaryKey(key string) (StoredDocument, bool, error)

DocumentByPrimaryKey resolves the IDMap and verifies the target document's primary key so a corrupt mapping cannot return another document.

func (*SegmentManager) Fetch 

func (m *SegmentManager) Fetch(ctx context.Context, primaryKeys []string) ([]FetchResult, error)

Fetch resolves primary keys in input order. Missing keys are successful nil results; context cancellation is returned at batch level and attached to all unprocessed entries.

func (*SegmentManager) ImmutableMetadata 

func (m *SegmentManager) ImmutableMetadata() []SegmentMetadata

ImmutableMetadata returns independent metadata sorted like segments.

func (*SegmentManager) ImmutableSegments 

func (m *SegmentManager) ImmutableSegments() []*ImmutableSegment

ImmutableSegments returns segments sorted by minimum document ID and ID.

func (*SegmentManager) LiveDocuments 

func (m *SegmentManager) LiveDocuments(ctx context.Context) ([]StoredDocument, error)

LiveDocuments returns independent copies of every current document in ascending global document-ID order. Superseded and deleted versions are omitted. The caller is responsible for excluding concurrent multi-step writes when it needs a transactionally stable query snapshot.

func (*SegmentManager) PrimaryKeys 

func (m *SegmentManager) PrimaryKeys() *PrimaryKeyMap

PrimaryKeys returns the manager's shared primary-key map.

func (*SegmentManager) RemoveImmutable 

func (m *SegmentManager) RemoveImmutable(segmentID uint64) (*ImmutableSegment, error)

RemoveImmutable removes and returns a segment without deleting its files.

func (*SegmentManager) ReplacePrimaryKeys 

func (m *SegmentManager) ReplacePrimaryKeys(primaryKey *PrimaryKeyMap) error

ReplacePrimaryKeys swaps the logical IDMap while the collection-level lock excludes readers and writers. It is used only after CURRENT commits a new immutable checkpoint.

func (*SegmentManager) RotateWriting 

func (m *SegmentManager) RotateWriting(currentID uint64, immutable *ImmutableSegment, next *WriteSegment) error

RotateWriting atomically replaces the current write segment with its immutable snapshot and installs the next empty write segment. The caller must have already durably published matching manifest metadata.

func (*SegmentManager) SetWriting 

func (m *SegmentManager) SetWriting(segment *WriteSegment) error

SetWriting installs a write segment after checking ID and document-range conflicts.

func (*SegmentManager) StorageStats 

func (m *SegmentManager) StorageStats() StorageStats

StorageStats returns a stable snapshot of retained segment resources.

func (*SegmentManager) Writing 

func (m *SegmentManager) Writing() *WriteSegment

Writing returns the current write segment.

type SegmentMetadata 

type SegmentMetadata struct {
	ID       uint64   `json:"id"`
	MinDocID uint64   `json:"min_doc_id"`
	MaxDocID uint64   `json:"max_doc_id"`
	DocCount uint64   `json:"doc_count"`
	Files    []string `json:"files,omitempty"`
}

SegmentMetadata identifies immutable files owned by one segment. File names use slash-separated paths relative to the collection directory so manifests remain portable across operating systems.

type SegmentSnapshot 

type SegmentSnapshot struct {
	Metadata  SegmentMetadata
	Documents []StoredDocument
	Mutable   bool
}

SegmentSnapshot is an owned, stable view used by the collection index layer. Immutable snapshots correspond to PersistedSegments; the final mutable snapshot corresponds to the current WAL-backed writing segment.

type StorageStats 

type StorageStats struct {
	ImmutableSegmentCount uint64
	MutableDocumentCount  uint64
	DeletedDocumentCount  uint64
	MemoryUsageBytes      uint64
}

StorageStats describes retained segment data without allocating document copies. MemoryUsageBytes counts encoded record headers, keys, payloads, and logical-deletion IDs.

type StoredDocument 

type StoredDocument struct {
	DocID      uint64
	PrimaryKey string
	Payload    []byte
}

StoredDocument is the schema-independent representation held by segments. Payload is produced and interpreted by the collection's schema codec.

func (StoredDocument) Clone 

func (d StoredDocument) Clone() StoredDocument

Clone returns a deep copy of d.

type VersionManager 

type VersionManager struct {
	// contains filtered or unexported fields
}

VersionManager owns the current immutable manifest snapshot. Publishing is serialized across goroutines and processes and never mutates an existing manifest file.

func CreateVersionManager 

func CreateVersionManager(ctx context.Context, dir string, initial Manifest) (*VersionManager, error)

CreateVersionManager creates and atomically publishes an initial manifest. Existing unreferenced manifests from a failed create are skipped safely.

func OpenVersionManager 

func OpenVersionManager(ctx context.Context, dir string) (*VersionManager, error)

OpenVersionManager loads the manifest named by CURRENT. It never selects an unreferenced manifest, even when that file has a larger generation.

func (*VersionManager) Current 

func (m *VersionManager) Current() Manifest

Current returns an independent copy of the published version.

func (*VersionManager) Publish 

func (m *VersionManager) Publish(ctx context.Context, next Manifest) (Manifest, error)

Publish atomically replaces CURRENT with an immutable copy of next. The manager assigns its generation. If another manager has published since this manager was opened, Publish returns ErrManifestConflict.

func (*VersionManager) Update 

func (m *VersionManager) Update(ctx context.Context, mutate func(*Manifest) error) (Manifest, error)

Update clones the current manifest, invokes mutate, and publishes the clone. A mutate error leaves memory and disk unchanged.

type WAL 

type WAL struct {
	// contains filtered or unexported fields
}

WAL is a single-writer, append-only write-ahead log. A sidecar advisory lock prevents separate handles or processes from appending concurrently.

func CreateWAL 

func CreateWAL(ctx context.Context, name string, options WALOptions) (*WAL, error)

CreateWAL creates a new log, writes and syncs its file header, and keeps an exclusive writer lock until Close.

func OpenWAL 

func OpenWAL(ctx context.Context, name string, options WALOptions) (*WAL, error)

OpenWAL validates an existing log. A partial final header or payload is truncated and reported; all other structural or checksum damage is fatal.

func OpenWALReadOnly 

func OpenWALReadOnly(ctx context.Context, name string) (*WAL, error)

OpenWALReadOnly validates an existing log under a shared lock. An incomplete crash tail is excluded from replay but is not modified; a later writer will repair it while holding the exclusive lock.

func (*WAL) Append 

func (w *WAL) Append(ctx context.Context, payload []byte) (uint64, error)

Append writes one opaque payload and returns its monotonically increasing LSN. Once a write or automatic synchronization fails, the handle is poisoned and must be closed and reopened so recovery can establish a safe state.

func (*WAL) Close 

func (w *WAL) Close() error

Close syncs complete records, closes the file, and releases the writer lock. It is idempotent. A poisoned log is closed without another synchronization.

func (*WAL) HasRecords 

func (w *WAL) HasRecords() bool

HasRecords reports whether at least one complete record is present.

func (*WAL) NewReader 

func (w *WAL) NewReader() (*WALReader, error)

NewReader opens an independent reader over the complete-record prefix that existed at the time of this call.

func (*WAL) Recovery 

func (w *WAL) Recovery() WALRecovery

Recovery returns the immutable result from opening the log.

func (*WAL) Replay 

func (w *WAL) Replay(ctx context.Context, apply func(WALRecord) error) error

Replay invokes apply in LSN order over a stable WAL snapshot.

func (*WAL) Sync 

func (w *WAL) Sync(ctx context.Context) error

Sync makes every successfully appended record durable before returning. A synchronization failure poisons the handle because durability is uncertain.

type WALOptions 

type WALOptions struct {
	SyncEvery uint64
}

WALOptions controls explicit durability batching. SyncEvery zero disables automatic record-count-based syncing; callers can use Sync directly.

type WALReader 

type WALReader struct {
	// contains filtered or unexported fields
}

WALReader iterates an immutable valid-prefix snapshot. It owns an independent file handle and is safe for sequential use.

func (*WALReader) Close 

func (r *WALReader) Close() error

Close releases the reader file handle and is idempotent.

func (*WALReader) Next 

func (r *WALReader) Next(ctx context.Context) (WALRecord, error)

Next returns the next verified record or io.EOF.

type WALRecord 

type WALRecord struct {
	LSN     uint64
	Payload []byte
}

WALRecord is one replayed opaque operation payload.

type WALRecovery 

type WALRecovery struct {
	Records        uint64
	LastLSN        uint64
	ValidBytes     int64
	TruncatedBytes int64
}

WALRecovery describes the valid prefix found while opening a WAL.

type WriteEngine 

type WriteEngine struct {
	// contains filtered or unexported fields
}

WriteEngine serializes WAL-backed mutations over a SegmentManager. When an automatic synchronization fails after a complete append, the engine still applies that record, returns the synchronization error, and relies on the poisoned WAL to reject later mutations until reopen.

func NewWriteEngine 

func NewWriteEngine(manager *SegmentManager, wal *WAL) (*WriteEngine, error)

NewWriteEngine validates the dependencies required for WAL-backed mutations.

func (*WriteEngine) Delete 

func (e *WriteEngine) Delete(ctx context.Context, primaryKeys []string) ([]WriteResult, error)

Delete appends removals to the WAL before updating primary-key mappings and logically deleting current document IDs. Immutable segment bytes are never rewritten.

func (*WriteEngine) Err 

func (e *WriteEngine) Err() error

Err reports whether a record reached the outer WAL but failed to apply completely. Such a handle must be reopened and replayed before more writes or publication operations are safe.

func (*WriteEngine) Insert 

func (e *WriteEngine) Insert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Insert appends documents whose primary keys do not exist to the WAL before applying them. WALOptions controls when appended records are synchronized. Validation and duplicate errors are reported per input; every failure is also included in the returned BatchWriteError.

func (*WriteEngine) Update 

func (e *WriteEngine) Update(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Update appends replacement document versions for existing keys to the WAL.

func (*WriteEngine) Upsert 

func (e *WriteEngine) Upsert(ctx context.Context, inputs []WriteInput) ([]WriteResult, error)

Upsert appends a new document version to the WAL. If the key already exists, its prior document ID is logically deleted after the WAL append succeeds.

type WriteInput 

type WriteInput struct {
	PrimaryKey string
	Payload    []byte
}

WriteInput is one schema-encoded document requested by a write API.

type WriteResult 

type WriteResult struct {
	PrimaryKey string
	DocID      uint64
	Err        error
}

WriteResult reports the outcome for one input in the same position.

type WriteSegment 

type WriteSegment struct {
	// contains filtered or unexported fields
}

WriteSegment accepts sequential documents until it is sealed.

func NewWriteSegment 

func NewWriteSegment(id, minDocID, maxDocs uint64) (*WriteSegment, error)

NewWriteSegment constructs an empty segment with a fixed global ID range start and capacity.

func (*WriteSegment) Append 

func (s *WriteSegment) Append(ctx context.Context, primaryKey string, payload []byte) (StoredDocument, error)

Append stores a cloned payload and assigns the next contiguous document ID.

func (*WriteSegment) AppendExpected 

func (s *WriteSegment) AppendExpected(ctx context.Context, expectedDocID uint64, primaryKey string, payload []byte) (StoredDocument, error)

AppendExpected appends only if expectedDocID is still next. It lets the WAL record and in-memory application agree on the assigned global ID.

func (*WriteSegment) Document 

func (s *WriteSegment) Document(docID uint64) (StoredDocument, bool)

Document returns an independent document copy.

func (*WriteSegment) Documents 

func (s *WriteSegment) Documents() []StoredDocument

Documents returns all documents in ascending ID order.

func (*WriteSegment) ID 

func (s *WriteSegment) ID() uint64

ID returns the segment ID.

func (*WriteSegment) MemoryUsageBytes 

func (s *WriteSegment) MemoryUsageBytes() uint64

MemoryUsageBytes returns the encoded record bytes retained by the segment.

func (*WriteSegment) Metadata 

func (s *WriteSegment) Metadata() SegmentMetadata

Metadata returns the current in-memory range without file references.

func (*WriteSegment) NextDocumentID 

func (s *WriteSegment) NextDocumentID() (uint64, error)

NextDocumentID returns the ID that the next append will receive.

func (*WriteSegment) ReservedRange 

func (s *WriteSegment) ReservedRange() (uint64, uint64)

ReservedRange returns the inclusive document-ID range owned by the segment.

func (*WriteSegment) Seal 

func (s *WriteSegment) Seal(ctx context.Context, collectionDir, relativeName string) (*ImmutableSegment, error)

Seal writes one immutable segment file relative to collectionDir and makes this write segment reject further appends.

func (*WriteSegment) Snapshot 

func (s *WriteSegment) Snapshot(ctx context.Context, collectionDir, relativeName string) (*ImmutableSegment, error)

Snapshot writes the current non-empty contents as an immutable segment without sealing the write segment. Collection flush uses this before the manifest commit point so a failed publication can safely keep accepting WAL backed writes and retry with fresh immutable artifacts.

Source Files

View all Source files

Directories

Path Synopsis
Package sql parses and evaluates the SQL-style scalar filter language used by zvec.
 Package sql parses and evaluates the SQL-style scalar filter language used by zvec.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.