core

Documentation

Index

• Constants
• func CountWords(text string) int
• func EnsureRuntime() error
• func L2Normalize(values []float32) []float32
• func NormalizeText(text string) string
• func RuntimeLibPath() (string, error)
• func ShouldSummarizeQuery(text string) bool
• func SupportsDocumentPath(path string) bool
• type Document
  • func LoadDocument(path string) (Document, error)
• type DocumentChunk
  • func ChunkDocument(text string, targetWords, overlapWords int) []DocumentChunk
• type EmbeddingModelInfo
  • func EnsureEmbeddingModel(db *sql.DB, modelName string) (EmbeddingModelInfo, error)
• type EncodingAdapter
• type RuntimeOptions
• type Summarizer
  • func NewSummarizer(info SummaryModelInfo, options RuntimeOptions) (*Summarizer, error)
  • func (s *Summarizer) Close() error
  • func (s *Summarizer) Summarize(text string) (string, error)
• type SummaryModelInfo
  • func EnsureSummaryModel(db *sql.DB, modelName string) (SummaryModelInfo, error)
• type TextEmbedder
  • func NewTextEmbedder(info EmbeddingModelInfo, options RuntimeOptions) (*TextEmbedder, error)
  • func (e *TextEmbedder) Close() error
  • func (e *TextEmbedder) EmbedDocument(text string) ([]float32, error)
  • func (e *TextEmbedder) EmbedQuery(text string) ([]float32, error)

Constants

const (
	DefaultChunkTargetWords  = 220
	DefaultChunkOverlapWords = 40
	LongQueryWordLimit       = 220
)

Functions

func CountWords

func CountWords(text string) int

func EnsureRuntime

func EnsureRuntime() error

func L2Normalize

func L2Normalize(values []float32) []float32

func NormalizeText

func NormalizeText(text string) string

func RuntimeLibPath

func RuntimeLibPath() (string, error)

func ShouldSummarizeQuery

func ShouldSummarizeQuery(text string) bool

func SupportsDocumentPath

func SupportsDocumentPath(path string) bool

Types

type Document

type Document struct {
	Path           string
	RawText        string
	NormalizedText string
	ContentHash    string
}

func LoadDocument

func LoadDocument(path string) (Document, error)

type DocumentChunk

type DocumentChunk struct {
	Index     int
	StartLine int
	EndLine   int
	Content   string
}

func ChunkDocument

func ChunkDocument(text string, targetWords, overlapWords int) []DocumentChunk

type EmbeddingModelInfo

type EmbeddingModelInfo struct {
	ID            string
	ModelPath     string
	TokenizerPath string
	Definition    embeddingModelDefinition
}

func EnsureEmbeddingModel

func EnsureEmbeddingModel(db *sql.DB, modelName string) (EmbeddingModelInfo, error)

type EncodingAdapter

type EncodingAdapter struct {
	IDs           []int
	AttentionMask []int
	TypeIDs       []int
}

type RuntimeOptions

type RuntimeOptions struct {
	Verbose bool
}

type Summarizer

type Summarizer struct {
	// contains filtered or unexported fields
}

func NewSummarizer

func NewSummarizer(info SummaryModelInfo, options RuntimeOptions) (*Summarizer, error)

func (*Summarizer) Close

func (s *Summarizer) Close() error

func (*Summarizer) Summarize

func (s *Summarizer) Summarize(text string) (string, error)

type SummaryModelInfo

type SummaryModelInfo struct {
	ID            string
	EncoderPath   string
	DecoderPath   string
	TokenizerPath string
	Definition    summaryModelDefinition
}

func EnsureSummaryModel

func EnsureSummaryModel(db *sql.DB, modelName string) (SummaryModelInfo, error)

type TextEmbedder

type TextEmbedder struct {
	// contains filtered or unexported fields
}

func NewTextEmbedder

func NewTextEmbedder(info EmbeddingModelInfo, options RuntimeOptions) (*TextEmbedder, error)

func (*TextEmbedder) Close

func (e *TextEmbedder) Close() error

func (*TextEmbedder) EmbedDocument

func (e *TextEmbedder) EmbedDocument(text string) ([]float32, error)

func (*TextEmbedder) EmbedQuery

func (e *TextEmbedder) EmbedQuery(text string) ([]float32, error)