embedding

Documentation

Index

• func ChunkText(text string, chunkSize int, overlap int) []string
• func CosineSimilarity(a, b []float32) float32
• func ExtractLinks(content string, baseURL string) []string
• func NormalizeEmbedding(embedding []float32) []float32
• type EmbeddingDocument
• type EmbeddingProvider
• type EmbeddingStore
  • func NewEmbeddingStore(filePath string, modelPath string, onnxLibPath string, ...) (*EmbeddingStore, error)
  • func (es *EmbeddingStore) AddDocument(doc EmbeddingDocument)
  • func (es *EmbeddingStore) Clear()
  • func (es *EmbeddingStore) Destroy()
  • func (es *EmbeddingStore) GetEmbedding(ctx context.Context, text string) ([]float32, error)
  • func (es *EmbeddingStore) GetEmbeddingsBatch(ctx context.Context, texts []string) ([][]float32, error)
  • func (es *EmbeddingStore) IsStale(days int) bool
  • func (es *EmbeddingStore) Load() error
  • func (es *EmbeddingStore) Save() error
  • func (es *EmbeddingStore) Search(ctx context.Context, query string, topK int, threshold float32) ([]SearchResult, error)
  • func (es *EmbeddingStore) SetProvider(provider EmbeddingProvider)
• type LocalEmbeddingProvider
  • func (lep *LocalEmbeddingProvider) GetEmbedding(text string) ([]float32, error)
  • func (lep *LocalEmbeddingProvider) GetEmbeddingsBatch(texts []string) ([][]float32, error)
• type OpenAIEmbeddingProvider
• type SearchResult

Functions

func ChunkText

func ChunkText(text string, chunkSize int, overlap int) []string

ChunkText splits text into smaller chunks for embedding This helps with token limits and more precise retrieval

func CosineSimilarity

func CosineSimilarity(a, b []float32) float32

CosineSimilarity calculates the cosine similarity between two vectors Note: Assumes vectors are already normalized (unit length)

func ExtractLinks

func ExtractLinks(content string, baseURL string) []string

ExtractLinks extracts page URLs from content based on the base URL pattern baseURL should be the domain pattern to match (e.g., "https://www.notion.so/")

func NormalizeEmbedding

func NormalizeEmbedding(embedding []float32) []float32

normalizeEmbedding performs L2 normalization on an embedding vector

Types

type EmbeddingDocument

type EmbeddingDocument struct {
	ID        string    `json:"id"`
	Content   string    `json:"content"`
	URL       string    `json:"url"`
	Title     string    `json:"title"`
	Embedding []float32 `json:"embedding"`
	Depth     int       `json:"depth"`
}

EmbeddingDocument represents a chunk of text with its embedding

type EmbeddingProvider

type EmbeddingProvider interface {
	GetEmbedding(text string) ([]float32, error)
	GetEmbeddingsBatch(texts []string) ([][]float32, error)
}

EmbeddingProvider is an interface for getting embeddings from any provider (local or remote)

type EmbeddingStore

type EmbeddingStore struct {
	Documents   []EmbeddingDocument `json:"documents"`
	LastUpdated string              `json:"last_updated"` // RFC3339 timestamp
	// contains filtered or unexported fields
}

EmbeddingStore manages embeddings in memory and persists to file

func NewEmbeddingStore

func NewEmbeddingStore(filePath string, modelPath string, onnxLibPath string, provider EmbeddingProvider) (*EmbeddingStore, error)

NewEmbeddingStore creates a new embedding store

func (*EmbeddingStore) AddDocument

func (es *EmbeddingStore) AddDocument(doc EmbeddingDocument)

AddDocument adds a document with its embedding

func (*EmbeddingStore) Clear

func (es *EmbeddingStore) Clear()

Clear removes all documents

func (*EmbeddingStore) Destroy

func (es *EmbeddingStore) Destroy()

Destroy cleans up the embedding store runtime resources (hugot session/pipeline) Note: This does NOT delete the persisted embeddings file - only frees memory

func (*EmbeddingStore) GetEmbedding

func (es *EmbeddingStore) GetEmbedding(ctx context.Context, text string) ([]float32, error)

GetEmbedding generates an embedding for the given text using the configured provider

func (*EmbeddingStore) GetEmbeddingsBatch

func (es *EmbeddingStore) GetEmbeddingsBatch(ctx context.Context, texts []string) ([][]float32, error)

GetEmbeddingsBatch generates embeddings for multiple texts at once using the configured provider

func (*EmbeddingStore) IsStale

func (es *EmbeddingStore) IsStale(days int) bool

IsStale checks if embeddings are older than the specified number of days

func (*EmbeddingStore) Load

func (es *EmbeddingStore) Load() error

Load loads embeddings from file

func (*EmbeddingStore) Save

func (es *EmbeddingStore) Save() error

Save persists embeddings to file

func (*EmbeddingStore) Search

func (es *EmbeddingStore) Search(ctx context.Context, query string, topK int, threshold float32) ([]SearchResult, error)

Search finds the most similar documents to the query

func (*EmbeddingStore) SetProvider

func (es *EmbeddingStore) SetProvider(provider EmbeddingProvider)

SetProvider sets a custom embedding provider (e.g., GPT-based)

type LocalEmbeddingProvider

type LocalEmbeddingProvider struct {
	// contains filtered or unexported fields
}

LocalEmbeddingProvider implements EmbeddingProvider using a local ONNX model

func (*LocalEmbeddingProvider) GetEmbedding

func (lep *LocalEmbeddingProvider) GetEmbedding(text string) ([]float32, error)

GetEmbedding generates an embedding using the local ONNX model

func (*LocalEmbeddingProvider) GetEmbeddingsBatch

func (lep *LocalEmbeddingProvider) GetEmbeddingsBatch(texts []string) ([][]float32, error)

GetEmbeddingsBatch generates embeddings for multiple texts using the local ONNX model

type OpenAIEmbeddingProvider

type OpenAIEmbeddingProvider struct {
	*gpt.OpenAI
}

OpenAIEmbeddingProvider implements EmbeddingProvider using OpenAI's API

type SearchResult

type SearchResult struct {
	Document   EmbeddingDocument
	Similarity float32
}

SearchResult represents a search result with similarity score