library

package
v0.64.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 19, 2026 License: AGPL-3.0 Imports: 50 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MediaTypeMarkdown = "text/markdown"
	MediaTypeText     = "text/plain"
	MediaTypePDF      = "application/pdf"
	MediaTypeDOC      = "application/msword"
	MediaTypeDOCX     = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
	MediaTypeODT      = "application/vnd.oasis.opendocument.text"
	MediaTypeRTF      = "application/rtf"
	MediaTypeXLS      = "application/vnd.ms-excel"
	MediaTypeXLSX     = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
	MediaTypeODS      = "application/vnd.oasis.opendocument.spreadsheet"
	MediaTypeCSV      = "text/csv"
	MediaTypeTSV      = "text/tab-separated-values"
	MediaTypePPT      = "application/vnd.ms-powerpoint"
	MediaTypePPTX     = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
	MediaTypeODP      = "application/vnd.oasis.opendocument.presentation"
	MediaTypeHTML     = "text/html"
	MediaTypeXHTML    = "application/xhtml+xml"
	MediaTypeEPUB     = "application/epub+zip"
	MediaTypeFB2      = "application/x-fictionbook+xml"
	MediaTypeMDX      = "text/mdx"
	MediaTypeRST      = "text/x-rst"
	MediaTypeORG      = "text/org"
	MediaTypeJSON     = "application/json"
	MediaTypeYAML     = "application/yaml"
	MediaTypeTOML     = "application/toml"
	MediaTypeXML      = "application/xml"
)
View Source 
const (
	RawPrefix          = "library/files"
	MaxRawListPageSize = 500
	// DefaultFSMinFreeBytes keeps a local deployment from consuming the final
	// disk reserve with immutable library snapshots.
	DefaultFSMinFreeBytes int64 = 5 << 30
)
View Source 
const (
	MaxSearchQueryRunes = 500
	DefaultSearchLimit  = 5
	MaxSearchLimit      = 10
)
View Source 
const (
	// TextParserProfile is persisted with every generation. Any setting that can
	// change chunk bytes or locators must produce a new profile value.
	TextParserProfile = "" /* 139-byte string literal not displayed */

	TextChunkRunes        = 1_000
	TextChunkOverlapRunes = 200

	MaxParsedChunks            = 32_768
	MaxParsedChunkContentBytes = 40 << 20
	MaxStagedChunksPerTx       = 500
	MaxStagedContentBytesPerTx = 2 << 20
)
View Source 
const (
	MaxFileBytes = 25 << 20

	SystemMaxFiles      int64 = 4_000
	SystemMaxBytes      int64 = 20 << 30
	SystemAgentMaxFiles int64 = 1_000
	SystemAgentMaxBytes int64 = 5 << 30
	PersonalMaxFiles    int64 = 2_000
	PersonalMaxBytes    int64 = 10 << 30
)
View Source 
const LibraryQueue = "stella_library"
View Source 
const ToolName = "library_search"

ToolName is the provider-facing function name. Keep it compatible with the common OpenAI function-name contract; product documentation may still refer to the conceptual operation as library.search.

Variables

View Source 
var (
	ErrRawAlreadyExists    = errors.New("library raw object already exists")
	ErrRawStorageDegraded  = errors.New("library raw storage is degraded")
	ErrInvalidRawStorePage = errors.New("invalid library raw store page")
)
View Source 
var (
	ErrNoExtractedText   = errors.New("document contains no extractable text")
	ErrInvalidParserData = errors.New("parser returned invalid chunk data")
	ErrParseResultLimit  = errors.New("parser result exceeds the configured limit")
)
View Source 
var (
	ErrInvalidOwner        = errors.New("invalid library owner")
	ErrFileTooLarge        = errors.New("library file is too large")
	ErrUnsupportedFileType = errors.New("unsupported library file type")
	ErrInvalidFile         = errors.New("invalid library file")
	ErrQuotaExceeded       = errors.New("library quota exceeded")
	ErrNotFound            = errors.New("library file not found")
	ErrForbidden           = errors.New("library access forbidden")
	ErrServiceUnavailable  = errors.New("library service is unavailable")
	ErrSpoolCapacity       = errors.New("library upload spool is at capacity")
	ErrGenerationConflict  = errors.New("library chunk generation identity conflicts with durable state")
	ErrGenerationChanged   = errors.New("library chunk generation state changed")
	ErrRawIntegrity        = errors.New("library raw snapshot failed integrity validation")
	ErrInvalidSearch       = errors.New("invalid library search")
)

Functions

func FileIDFromRawKey 

func FileIDFromRawKey(key string) (string, error)

FileIDFromRawKey validates a canonical key before reconciliation uses it as a database identity. Malformed objects are retained, never guessed at.

func RawKey 

func RawKey(fileID string) (string, error)

RawKey derives the only canonical object key for a LibraryFile.

func SupportedExtensions added in v0.64.1 

func SupportedExtensions() []string

SupportedExtensions returns the canonical, sorted upload allowlist for transport errors and other non-parser callers.

func SupportedMediaTypes added in v0.64.1 

func SupportedMediaTypes() []string

SupportedMediaTypes returns a stable copy used by reconciliation so every admitted format participates in parser-profile upgrades and recovery.

func XbergMediaTypes added in v0.64.1 

func XbergMediaTypes() []string

XbergMediaTypes returns a stable copy used by the composition root to bind the single Xberg adapter to every format assigned to that runtime.

Types

type ChunkLocator 

type ChunkLocator struct {
	FirstPage   *uint32  `json:"first_page,omitempty"`
	LastPage    *uint32  `json:"last_page,omitempty"`
	HeadingPath []string `json:"heading_path,omitempty"`
	ByteStart   int      `json:"byte_start"`
	ByteEnd     int      `json:"byte_end"`
}

ChunkLocator records stable source positioning. Byte offsets are retained for internal diagnostics but are removed before a chunk is returned to a model.

type ChunkSetStatus 

type ChunkSetStatus string

ChunkSetStatus is the publication state of one deterministic generation. Only a ready set referenced by LibraryFile.ActiveChunkSetID may be retrieved. 

const (
	ChunkSetStatusBuilding ChunkSetStatus = "building"
	ChunkSetStatusReady    ChunkSetStatus = "ready"
	ChunkSetStatusFailed   ChunkSetStatus = "failed"
)

type FSRawStore 

type FSRawStore struct {
	// contains filtered or unexported fields
}

func NewFSRawStore 

func NewFSRawStore(root string, minFreeBytes int64) (*FSRawStore, error)

func (*FSRawStore) Create 

func (s *FSRawStore) Create(ctx context.Context, key string, reader io.Reader) error

func (*FSRawStore) Delete 

func (s *FSRawStore) Delete(ctx context.Context, key string) error

func (*FSRawStore) ListPage 

func (s *FSRawStore) ListPage(
	ctx context.Context,
	prefix, cursor string,
	limit int,
) (RawPage, error)

func (*FSRawStore) Open 

func (s *FSRawStore) Open(ctx context.Context, key string) (io.ReadCloser, error)

func (*FSRawStore) SupportsOrphanCollection 

func (*FSRawStore) SupportsOrphanCollection() bool

SupportsOrphanCollection reports that the local Library root belongs to this Stella deployment rather than a shared external namespace.

type FileStatus 

type FileStatus string

FileStatus is the durable processing state of one immutable snapshot. 

const (
	FileStatusProcessing FileStatus = "processing"
	FileStatusReady      FileStatus = "ready"
	FileStatusFailed     FileStatus = "failed"
)

type LibraryFile 

type LibraryFile struct {
	ID               string
	Owner            Owner
	FileName         string
	MediaType        string
	SizeBytes        int64
	RawSHA256        []byte
	Status           FileStatus
	ErrorMessage     string
	ActiveChunkSetID string
	DeletedAt        *time.Time
	CreatedAt        time.Time
	UpdatedAt        time.Time
}

LibraryFile is internal metadata for one canonical immutable source snapshot. Raw bytes remain in RawStore and are never included in this value.

type ListCursor added in v0.64.0 

type ListCursor struct {
	CreatedAt time.Time
	ID        string
}

ListCursor is the stable position after one management-list item. HTTP keeps its serialized form opaque and binds it to the authorized query.

type Owner 

type Owner struct {
	Scope   Scope
	UserID  string
	AgentID string
}

Owner is the normalized four-part scope tuple. Empty IDs are database NULLs.

func (Owner) Validate 

func (o Owner) Validate() error

Validate enforces the same four legal combinations as the database CHECK.

type ParsedChunk 

type ParsedChunk struct {
	Content string
	Locator ChunkLocator
}

ParsedChunk is the normalized parser result persisted as a Library chunk.

type Parser 

type Parser interface {
	// Profile must be a pure, stable lookup: lifecycle code calls it while
	// holding database locks and uses the value as durable generation identity.
	Profile(mediaType string) (string, error)
	Parse(ctx context.Context, path, mediaType string) ([]ParsedChunk, error)
}

Parser is the bounded document parser used by the asynchronous chunk worker.

type Quota 

type Quota struct {
	UsedFiles int64
	MaxFiles  int64
	UsedBytes int64
	MaxBytes  int64
}

Quota describes current usage and the fixed limit of one logical quota pool.

func (Quota) CanAdd 

func (q Quota) CanAdd(sizeBytes int64) bool

CanAdd reports whether adding one immutable file stays inside both limits.

type QuotaExceededError 

type QuotaExceededError struct {
	Quota Quota
}

QuotaExceededError carries the authoritative pool state at rejection time.

func (*QuotaExceededError) Error 

func (e *QuotaExceededError) Error() string

func (*QuotaExceededError) Unwrap 

func (e *QuotaExceededError) Unwrap() error

type RawObject 

type RawObject struct {
	Key          string
	Size         int64
	LastModified time.Time
}

RawObject is the minimum storage metadata required by bounded orphan GC.

type RawPage 

type RawPage struct {
	Objects    []RawObject
	NextCursor string
}

RawPage is one bounded canonical-key-ordered enumeration page. NextCursor is the last returned key and is an exclusive lower bound for the following page.

type RawStore 

type RawStore interface {
	Create(ctx context.Context, key string, reader io.Reader) error
	Open(ctx context.Context, key string) (io.ReadCloser, error)
	Delete(ctx context.Context, key string) error
	ListPage(ctx context.Context, prefix, cursor string, limit int) (RawPage, error)
	// SupportsOrphanCollection is true only when RawPrefix is owned exclusively
	// by this Stella deployment, so unknown objects can be deleted safely.
	SupportsOrphanCollection() bool
}

RawStore owns canonical immutable raw snapshots. Create must never replace an existing key. ListPage must order by canonical key and bound both returned objects and adapter memory; it does not provide snapshot isolation across concurrent pages.

func NewRawStoreFromConfig 

func NewRawStoreFromConfig(
	root string,
	deploymentS3 config.BlobS3Config,
	options RawStoreOptions,
) (RawStore, error)

NewRawStoreFromConfig selects local FS or deployment S3 using the existing STELLA_BLOB_S3_* configuration group.

type RawStoreOptions 

type RawStoreOptions struct {
	TempDir        string
	FSMinFreeBytes int64
	S3Admission    func(context.Context) error
}

RawStoreOptions holds backend admission knobs without exposing a second Library-specific storage configuration group.

type RoutingParser added in v0.64.0 

type RoutingParser struct {
	// contains filtered or unexported fields
}

RoutingParser is an immutable media-type dispatch table. Lifecycle code only depends on Parser and remains unaware of concrete document processors.

func NewRoutingParser added in v0.64.0 

func NewRoutingParser(routes map[string]Parser) (*RoutingParser, error)

func (*RoutingParser) Parse added in v0.64.0 

func (p *RoutingParser) Parse(ctx context.Context, path, mediaType string) ([]ParsedChunk, error)

func (*RoutingParser) Profile added in v0.64.0 

func (p *RoutingParser) Profile(mediaType string) (string, error)

type S3RawStore 

type S3RawStore struct {
	// contains filtered or unexported fields
}

func NewS3RawStore 

func NewS3RawStore(
	config blob.S3Config,
	tempDir string,
	admission func(context.Context) error,
) (*S3RawStore, error)

func (*S3RawStore) Create 

func (s *S3RawStore) Create(ctx context.Context, key string, reader io.Reader) error

func (*S3RawStore) Delete 

func (s *S3RawStore) Delete(ctx context.Context, key string) error

func (*S3RawStore) ListPage 

func (s *S3RawStore) ListPage(
	ctx context.Context,
	prefix, cursor string,
	limit int,
) (RawPage, error)

func (*S3RawStore) Open 

func (s *S3RawStore) Open(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3RawStore) SupportsOrphanCollection 

func (*S3RawStore) SupportsOrphanCollection() bool

SupportsOrphanCollection remains false until S3 keys carry a deployment namespace or another ownership marker. Exact-key tombstone cleanup is still safe and continues to use Delete directly.

type Scope 

type Scope string

Scope is the owner scope shared by Library management and retrieval. 

const (
	ScopeSystem      Scope = "system"
	ScopeSystemAgent Scope = "system_agent"
	ScopeUser        Scope = "user"
	ScopeUserAgent   Scope = "user_agent"
)

type SearchHit added in v0.64.0 

type SearchHit struct {
	FileName string         `json:"file_name"`
	Locator  *SearchLocator `json:"locator,omitempty"`
	Content  string         `json:"content"`
}

SearchHit is the complete published chunk returned to an Agent. It contains only source-facing citation metadata; internal IDs, owner scope, score, raw bytes, and byte offsets never cross this boundary.

type SearchLocator added in v0.64.0 

type SearchLocator struct {
	FirstPage   *uint32  `json:"first_page,omitempty"`
	LastPage    *uint32  `json:"last_page,omitempty"`
	HeadingPath []string `json:"heading_path,omitempty"`
}

SearchLocator is the safe, human-readable part of a parser locator. Page ranges and structural paths may be absent when a source has no such coordinate.

type Service 

type Service struct {
	// contains filtered or unexported fields
}

Service owns authorization, bounded acquisition, immutable raw publication, and the short metadata-plus-job transaction.

func NewService 

func NewService(config ServiceConfig) (*Service, error)

func (*Service) BindRiverClient 

func (s *Service) BindRiverClient(client *river.Client[pgx.Tx]) error

BindRiverClient injects the single shared working River client. Tests may still supply an insert-only client in ServiceConfig; production binds once before the client starts.

func (*Service) CreateManagedUpload 

func (s *Service) CreateManagedUpload(
	ctx context.Context,
	authority authz.Authority,
	scope Scope,
	agentID string,
	fileName string,
	source io.Reader,
) (LibraryFile, error)

CreateManagedUpload is the upload acquisition boundary. Scope and Agent authorization finish before prepareUpload consumes a single source byte.

func (*Service) DeleteManaged 

func (s *Service) DeleteManaged(
	ctx context.Context,
	authority authz.Authority,
	id string,
) error

DeleteManaged commits the tombstone before attempting cancellation or raw cleanup. The durable tombstone is the visibility guarantee; cancellation is only a best-effort resource optimization.

func (*Service) Get 

func (s *Service) Get(ctx context.Context, id string) (LibraryFile, error)

Get returns live internal metadata only; raw bytes are never exposed.

func (*Service) GetManaged 

func (s *Service) GetManaged(
	ctx context.Context,
	authority authz.Authority,
	id string,
) (LibraryFile, error)

GetManaged loads a file and authorizes the management view against its durable owner tuple. Every denial is opaque so knowing a foreign UUID cannot reveal whether that file exists.

func (*Service) ListManaged added in v0.64.0 

func (s *Service) ListManaged(
	ctx context.Context,
	authority authz.Authority,
	scope Scope,
	agentID string,
	query string,
	limit int32,
	cursor *ListCursor,
) ([]LibraryFile, Quota, error)

ListManaged returns files owned by the exact authorized scope tuple together with its authoritative logical quota. Callers fetch limit+1 rows to determine whether another page exists. Personal scopes intentionally share one quota.

func (*Service) QueueConfig 

func (s *Service) QueueConfig() (string, river.QueueConfig)

QueueConfig returns the dedicated per-node parser/maintenance concurrency.

func (*Service) RegisterRiverWorkers 

func (s *Service) RegisterRiverWorkers(workers *river.Workers)

RegisterRiverWorkers contributes all internal Library workers to the one process-wide River client. No management or Agent surface is registered here.

func (*Service) ResolveManageOwner 

func (s *Service) ResolveManageOwner(
	ctx context.Context,
	authority authz.Authority,
	scope Scope,
	agentID string,
) (Owner, error)

ResolveManageOwner authorizes a scope-keyed management operation and returns the exact owner tuple that may be passed to List or Create. HTTP parameters select a scope; they never supply the owning user, which always comes from the trusted Authority.

func (*Service) Search added in v0.64.0 

func (s *Service) Search(
	ctx context.Context,
	authority authz.Authority,
	query string,
	limit int,
) ([]SearchHit, error)

Search performs one permission-aware BM25 query for a trusted delegated Agent. The exact four-scope union is enforced in SQL, before any candidate can cross into Go.

func (*Service) StartReconciliation 

func (s *Service) StartReconciliation() (rivertype.PeriodicJobHandle, error)

StartReconciliation registers one leader-elected bounded repair chain.

func (*Service) StopReconciliation 

func (s *Service) StopReconciliation(handle rivertype.PeriodicJobHandle)

type ServiceConfig 

type ServiceConfig struct {
	DB                       *pgxpool.Pool
	RawStore                 RawStore
	Parser                   Parser
	River                    *river.Client[pgx.Tx]
	Logger                   *slog.Logger
	TempDir                  string
	MaxConcurrentUploads     int
	MaxSpoolBytes            int64
	AgentAccess              *agentaccess.Service
	SnapshotCommitTimeout    time.Duration
	DatabaseStatementTimeout time.Duration
	DatabaseLockTimeout      time.Duration
	ReconciliationInterval   time.Duration
	StaleDerivationAfter     time.Duration
	OrphanMinAge             time.Duration
	MaxClockSkew             time.Duration
	OrphanSafetyMargin       time.Duration
	MaxWorkers               int
}

ServiceConfig contains the internal Library ingestion and lifecycle dependencies. Public management and retrieval surfaces are composed later.

type TextParser 

type TextParser struct{}

TextParser deterministically chunks UTF-8 text and Markdown without an external runtime. Markdown is intentionally treated as plain source text in V1; richer format-specific parsing can introduce a separate profile later.

func NewTextParser 

func NewTextParser() *TextParser

func (*TextParser) Parse 

func (*TextParser) Parse(ctx context.Context, filePath, mediaType string) ([]ParsedChunk, error)

func (*TextParser) Profile added in v0.64.0 

func (*TextParser) Profile(mediaType string) (string, error)

type Tool added in v0.64.0 

type Tool struct {
	// contains filtered or unexported fields
}

Tool exposes the single read-only Library retrieval operation. Identity and scope are deliberately absent from its arguments and come only from runtime.

func NewTool added in v0.64.0 

func NewTool(service *Service) *Tool

func (*Tool) Definition added in v0.64.0 

func (*Tool) Definition() tools.Definition

func (*Tool) Execute added in v0.64.0 

func (t *Tool) Execute(ctx context.Context, args map[string]any) (string, error)

type XbergCLIParser added in v0.64.0 

type XbergCLIParser struct {
	// contains filtered or unexported fields
}

func NewXbergCLIParser added in v0.64.0 

func NewXbergCLIParser(ctx context.Context, binary string) (*XbergCLIParser, error)

func (*XbergCLIParser) Parse added in v0.64.0 

func (p *XbergCLIParser) Parse(ctx context.Context, path, mediaType string) ([]ParsedChunk, error)

func (*XbergCLIParser) Profile added in v0.64.0 

func (p *XbergCLIParser) Profile(mediaType string) (string, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.