Documentation
¶
Index ¶
- Variables
- type Branch
- type Collection
- type Document
- type DocumentBlob
- type Section
- type SnapshottedCollection
- type SnapshottedDocument
- type SnapshottedSection
- type Store
- func (s *Store) Close() error
- func (s *Store) CreateCollection(ctx context.Context, label string) (model.PersistedCollection, error)
- func (s *Store) DB() *gorm.DB
- func (s *Store) DeleteCollection(ctx context.Context, id model.CollectionID) error
- func (s *Store) DeleteDocumentByID(ctx context.Context, ids ...model.DocumentID) error
- func (s *Store) DeleteDocumentBySource(ctx context.Context, source *url.URL) error
- func (s *Store) GenerateSnapshot(ctx context.Context) (io.ReadCloser, error)
- func (s *Store) GetCollectionByID(ctx context.Context, id model.CollectionID, full bool) (model.PersistedCollection, error)
- func (s *Store) GetCollectionStats(ctx context.Context, id model.CollectionID) (*model.CollectionStats, error)
- func (s *Store) GetDocumentByID(ctx context.Context, id model.DocumentID) (model.PersistedDocument, error)
- func (s *Store) GetDocumentsMetadataBySources(ctx context.Context, sources []string) (map[string]map[string]any, error)
- func (s *Store) GetSectionByID(ctx context.Context, id model.SectionID) (model.Section, error)
- func (s *Store) GetSectionsByIDs(ctx context.Context, ids []model.SectionID) (map[model.SectionID]model.Section, error)
- func (s *Store) ListCollectionLanguages(ctx context.Context, ids []model.CollectionID) ([]string, error)
- func (s *Store) ListCollections(ctx context.Context, ids []model.CollectionID) ([]model.Collection, error)
- func (s *Store) ListDocumentDigests(ctx context.Context, sourcePrefix string, page int, pageSize int) ([]ingest.DocumentDigest, error)
- func (s *Store) ListReferencedBlobs(ctx context.Context, fn func(blob.Hash) error) error
- func (s *Store) QueryCollections(ctx context.Context, opts ingest.QueryCollectionsOptions) ([]model.PersistedCollection, error)
- func (s *Store) QueryDocuments(ctx context.Context, opts ingest.QueryDocumentsOptions) ([]model.PersistedDocument, int64, error)
- func (s *Store) QueryDocumentsByCollectionID(ctx context.Context, collectionID model.CollectionID, ...) ([]model.PersistedDocument, int64, error)
- func (s *Store) RestoreSnapshot(ctx context.Context, r io.Reader) error
- func (s *Store) SaveDocuments(ctx context.Context, documents ...model.Document) error
- func (s *Store) SectionExists(ctx context.Context, id model.SectionID) (bool, error)
- func (s *Store) SectionsExist(ctx context.Context, ids []model.SectionID) (map[model.SectionID]bool, error)
- func (s *Store) UpdateCollection(ctx context.Context, id model.CollectionID, updates ingest.CollectionUpdates) (model.PersistedCollection, error)
Constants ¶
This section is empty.
Variables ¶
var (
ErrMissingSource = errors.New("missing source")
)
Functions ¶
This section is empty.
Types ¶
type Collection ¶
type Document ¶
type Document struct {
ID string `gorm:"primaryKey;autoIncrement:false"`
ETag string `gorm:"index"`
CreatedAt time.Time
UpdatedAt time.Time
Source string `gorm:"unique;not null;index"`
Sections []*Section `gorm:"constraint:OnDelete:CASCADE"`
Collections []*Collection `gorm:"many2many:documents_collections;"`
Content []byte
// Metadata holds arbitrary document metadata serialized as JSON, used for
// metadata filtering at search time.
Metadata []byte
}
type DocumentBlob ¶ added in v0.9.0
type DocumentBlob struct {
DocumentID string `gorm:"primaryKey;size:64"`
Hash string `gorm:"primaryKey;size:64;index"`
// The foreign key deletes the references along with their document, so a
// document removed by any path — including one that bypasses
// DeleteDocumentByID — cannot leave stale references behind.
Document *Document `gorm:"foreignKey:DocumentID;constraint:OnDelete:CASCADE"`
}
DocumentBlob records that a document references a blob. It is a derived index over the content of the documents, maintained in the same transaction as the write that produced it — which is what keeps it from drifting away from its source of truth.
It exists so the cleanup task can compute the set of live blobs with a single indexed query instead of reading every document. Without it the sweep costs a full pass over the corpus; the trade is one small row per image reference.
func (DocumentBlob) TableName ¶ added in v0.9.0
func (DocumentBlob) TableName() string
TableName pins the table name so it does not depend on gorm's pluralization.
type Section ¶
type Section struct {
ID string `gorm:"primaryKey;autoIncrement:false"`
CreatedAt time.Time
UpdatedAt time.Time
Document *Document
DocumentID string
Parent *Section
ParentID *string
// Fixed: Self-referencing relationship should only use ParentID as foreign key
// ParentID references the ID field of the same Section table
Sections []*Section `gorm:"foreignKey:ParentID;references:ID;constraint:OnDelete:CASCADE"`
Branch *Branch
Level uint
Start int
End int
}
type SnapshottedCollection ¶
type SnapshottedDocument ¶
type SnapshottedDocument struct {
ID string
Source string
ETag string
Content []byte
Collections []SnapshottedCollection
Sections []SnapshottedSection
}
type SnapshottedSection ¶
type Store ¶
type Store struct {
// contains filtered or unexported fields
}
func NewPostgresStore ¶
NewPostgresStore opens a PostgreSQL-backed ingest.Store from the given DSN (e.g. "postgres://user:pass@host:5432/db?sslmode=disable"). The schema is migrated lazily on first use. The context is used to verify connectivity eagerly so an unreachable database fails fast.
The caller does not need any extra replace directive; the PostgreSQL driver is pulled through this subpackage only. Wire the resulting store into the facade with amoxtli.WithStore.
func NewSQLiteStore ¶
NewSQLiteStore opens a SQLite-backed ingest.Store at the given DSN (a file path, or ":memory:"). The connection is tuned for the library's usage (WAL journal, foreign keys, single writer) and the schema is migrated lazily on first use.
Wire the resulting store into the facade with amoxtli.WithStore; the caller owns the store and must Close it.
func (*Store) Close ¶
Close releases the underlying database connection. The store owns the connection only when it was created through NewSQLiteStore/NewPostgresStore; when the *gorm.DB was provided to NewStore by the caller, closing it here still closes that shared connection, so use it accordingly.
func (*Store) CreateCollection ¶
func (s *Store) CreateCollection(ctx context.Context, label string) (model.PersistedCollection, error)
CreateCollection implements ingest.Store.
func (*Store) DB ¶ added in v0.0.3
DB returns the underlying *gorm.DB. It is intended for advanced usage such as sharing the connection with a persistent task runner (task/gorm).
func (*Store) DeleteCollection ¶
DeleteCollection implements ingest.Store.
func (*Store) DeleteDocumentByID ¶
DeleteDocumentByID implements ingest.Store.
func (*Store) DeleteDocumentBySource ¶
DeleteDocumentBySource implements ingest.Store.
func (*Store) GenerateSnapshot ¶
GenerateSnapshot implements backup.Snapshotable.
func (*Store) GetCollectionByID ¶
func (s *Store) GetCollectionByID(ctx context.Context, id model.CollectionID, full bool) (model.PersistedCollection, error)
GetCollectionByID implements ingest.Store.
func (*Store) GetCollectionStats ¶
func (s *Store) GetCollectionStats(ctx context.Context, id model.CollectionID) (*model.CollectionStats, error)
GetCollectionStats implements ingest.Store.
func (*Store) GetDocumentByID ¶
func (s *Store) GetDocumentByID(ctx context.Context, id model.DocumentID) (model.PersistedDocument, error)
GetDocumentByID implements ingest.Store.
func (*Store) GetDocumentsMetadataBySources ¶ added in v0.0.3
func (s *Store) GetDocumentsMetadataBySources(ctx context.Context, sources []string) (map[string]map[string]any, error)
GetDocumentsMetadataBySources implements ingest.MetadataProvider.
func (*Store) GetSectionByID ¶
GetSectionByID implements ingest.Store.
func (*Store) GetSectionsByIDs ¶
func (s *Store) GetSectionsByIDs(ctx context.Context, ids []model.SectionID) (map[model.SectionID]model.Section, error)
GetSectionsByIDs implements ingest.Store.
func (*Store) ListCollectionLanguages ¶ added in v0.10.0
func (s *Store) ListCollectionLanguages(ctx context.Context, ids []model.CollectionID) ([]string, error)
ListCollectionLanguages implements pipeline.CollectionLanguageLister: the languages of the given collections (all of them when ids is empty), ordered by decreasing number of documents.
func (*Store) ListCollections ¶
func (s *Store) ListCollections(ctx context.Context, ids []model.CollectionID) ([]model.Collection, error)
ListCollections implements pipeline.CollectionLister.
func (*Store) ListDocumentDigests ¶
func (s *Store) ListDocumentDigests(ctx context.Context, sourcePrefix string, page int, pageSize int) ([]ingest.DocumentDigest, error)
ListDocumentDigests implements ingest.Store.
func (*Store) ListReferencedBlobs ¶ added in v0.9.0
ListReferencedBlobs implements ingest.BlobReferenceLister: it walks the distinct blob hashes referenced by at least one stored document — the live set of the blob garbage collector.
func (*Store) QueryCollections ¶
func (s *Store) QueryCollections(ctx context.Context, opts ingest.QueryCollectionsOptions) ([]model.PersistedCollection, error)
QueryCollections implements ingest.Store.
func (*Store) QueryDocuments ¶
func (s *Store) QueryDocuments(ctx context.Context, opts ingest.QueryDocumentsOptions) ([]model.PersistedDocument, int64, error)
QueryDocuments implements ingest.Store.
func (*Store) QueryDocumentsByCollectionID ¶
func (s *Store) QueryDocumentsByCollectionID(ctx context.Context, collectionID model.CollectionID, opts ingest.QueryDocumentsOptions) ([]model.PersistedDocument, int64, error)
QueryDocumentsByCollectionID implements ingest.Store.
func (*Store) RestoreSnapshot ¶
RestoreSnapshot implements backup.Snapshotable.
func (*Store) SaveDocuments ¶
SaveDocuments implements ingest.Store. The whole batch is written in a single transaction: one commit — and, on SQLite, one WAL sync — instead of one per document.
func (*Store) SectionExists ¶
SectionExists implements ingest.Store.
func (*Store) SectionsExist ¶
func (s *Store) SectionsExist(ctx context.Context, ids []model.SectionID) (map[model.SectionID]bool, error)
SectionsExist implements ingest.Store.
func (*Store) UpdateCollection ¶
func (s *Store) UpdateCollection(ctx context.Context, id model.CollectionID, updates ingest.CollectionUpdates) (model.PersistedCollection, error)
UpdateCollection implements ingest.Store.