Documentation
¶
Index ¶
- Constants
- Variables
- type BM25Scorer
- func (s *BM25Scorer) CalculateTermScore(term string, docID int32) TermScore
- func (s *BM25Scorer) ExplainScore(queryTerms []string, docID int32) ScoreExplanation
- func (s *BM25Scorer) GetParams() (k1, b float64)
- func (s *BM25Scorer) Score(queryTerms []string, docID int32) float64
- func (s *BM25Scorer) ScoreAll(queryTerms []string) []ScoredDocument
- func (s *BM25Scorer) ScoreAllWithLimit(queryTerms []string, limit int) []ScoredDocument
- func (s *BM25Scorer) SetParams(k1, b float64)
- func (s *BM25Scorer) UpdateAvgDocLength()
- type Document
- type DocumentStore
- func (ds *DocumentStore) Add(doc *Document) int32
- func (ds *DocumentStore) AddWithID(docID int32, doc *Document)
- func (ds *DocumentStore) Clear()
- func (ds *DocumentStore) Count() int
- func (ds *DocumentStore) Get(docID int32) *Document
- func (ds *DocumentStore) GetAllSessionIDs() []string
- func (ds *DocumentStore) GetBySession(sessionID string) []*Document
- func (ds *DocumentStore) GetDocIDsBySession(sessionID string) []int32
- func (ds *DocumentStore) GetStats() DocumentStoreStats
- func (ds *DocumentStore) HasSession(sessionID string) bool
- func (ds *DocumentStore) Remove(docID int32)
- func (ds *DocumentStore) RemoveBySession(sessionID string)
- func (ds *DocumentStore) SessionCount() int
- type DocumentStoreStats
- type HighlightRange
- type IndexStore
- func (s *IndexStore) Delete() error
- func (s *IndexStore) DeleteSyncMetadata() error
- func (s *IndexStore) Exists() bool
- func (s *IndexStore) GetIndexDir() string
- func (s *IndexStore) GetVersion() (*IndexVersion, error)
- func (s *IndexStore) Load() (*InvertedIndex, *DocumentStore, error)
- func (s *IndexStore) LoadSyncMetadata() (*IndexSyncMetadata, error)
- func (s *IndexStore) Save(index *InvertedIndex, docStore *DocumentStore) error
- func (s *IndexStore) SaveSyncMetadata(meta *IndexSyncMetadata) error
- func (s *IndexStore) SyncMetadataExists() bool
- type IndexSyncMetadata
- type IndexVersion
- type InvertedIndex
- func (idx *InvertedIndex) AddDocument(docID int32, tokens []string, positions map[string][]int32)
- func (idx *InvertedIndex) AddDocumentSimple(docID int32, tokens []string)
- func (idx *InvertedIndex) Clear()
- func (idx *InvertedIndex) GetAllTerms() []string
- func (idx *InvertedIndex) GetAvgDocLength() float64
- func (idx *InvertedIndex) GetDocLength(docID int32) int
- func (idx *InvertedIndex) GetDocumentFrequency(term string) int
- func (idx *InvertedIndex) GetStats() InvertedIndexStats
- func (idx *InvertedIndex) GetTermCount() int
- func (idx *InvertedIndex) GetTotalDocs() int
- func (idx *InvertedIndex) HasDocument(docID int32) bool
- func (idx *InvertedIndex) RemoveDocument(docID int32)
- func (idx *InvertedIndex) Search(term string) *PostingsList
- func (idx *InvertedIndex) SearchMultiple(terms []string) map[string]*PostingsList
- type InvertedIndexStats
- type PostingsList
- type ScoreExplanation
- type ScoredDocument
- type SearchEngine
- func (e *SearchEngine) BuildIndex(history *session.ClaudeSessionHistory) error
- func (e *SearchEngine) Clear()
- func (e *SearchEngine) GetDocStore() *DocumentStore
- func (e *SearchEngine) GetDocument(docID int32) *Document
- func (e *SearchEngine) GetIndex() *InvertedIndex
- func (e *SearchEngine) GetStats() SearchEngineStats
- func (e *SearchEngine) GetSyncMetadata() *IndexSyncMetadata
- func (e *SearchEngine) GetTokenizer() *Tokenizer
- func (e *SearchEngine) HasSession(sessionID string) bool
- func (e *SearchEngine) IncrementalSync(history *session.ClaudeSessionHistory) (*SyncResult, error)
- func (e *SearchEngine) IndexMessage(sessionID string, msgIdx int, role, content string, timestamp time.Time) error
- func (e *SearchEngine) LoadIndex() error
- func (e *SearchEngine) LoadSyncMetadata() error
- func (e *SearchEngine) RemoveSession(sessionID string)
- func (e *SearchEngine) SaveIndex() error
- func (e *SearchEngine) SaveSyncMetadata() error
- func (e *SearchEngine) Search(query string, opts SearchOptions) (*SearchResults, error)
- func (e *SearchEngine) ShouldRebuild() bool
- func (e *SearchEngine) ShouldRebuildLocked() bool
- type SearchEngineStats
- type SearchOptions
- type SearchResult
- type SearchResults
- type SessionIndexMetadata
- type Snippet
- type SnippetGenerator
- type SyncResult
- type TermScore
- type TokenPosition
- type Tokenizer
Constants ¶
const ( // K1 controls term frequency saturation. Higher values increase the impact // of term frequency. Standard value is 1.2-2.0. K1 = 1.5 // B controls document length normalization. B=1.0 means full normalization, // B=0 means no normalization. Standard value is 0.75. B = 0.75 )
BM25 scoring parameters
const (
// CurrentIndexVersion is the schema version for the index format
CurrentIndexVersion = 1
)
const CurrentSyncMetadataVersion = 1
CurrentSyncMetadataVersion is the schema version for sync metadata. Increment this when making breaking changes to the metadata format.
Variables ¶
var ( // ErrMetadataNotFound is returned when sync metadata is not found. ErrMetadataNotFound = fmt.Errorf("sync metadata not found") )
Functions ¶
This section is empty.
Types ¶
type BM25Scorer ¶
type BM25Scorer struct {
// contains filtered or unexported fields
}
BM25Scorer implements the Okapi BM25 relevance scoring algorithm. BM25 is a probabilistic ranking function that improves upon TF-IDF by adding term frequency saturation and document length normalization.
func NewBM25Scorer ¶
func NewBM25Scorer(index *InvertedIndex) *BM25Scorer
NewBM25Scorer creates a new BM25 scorer for the given inverted index.
func NewBM25ScorerWithParams ¶
func NewBM25ScorerWithParams(index *InvertedIndex, k1, b float64) *BM25Scorer
NewBM25ScorerWithParams creates a BM25 scorer with custom parameters.
func (*BM25Scorer) CalculateTermScore ¶
func (s *BM25Scorer) CalculateTermScore(term string, docID int32) TermScore
CalculateTermScore calculates the BM25 score contribution from a single term. Useful for debugging or understanding score breakdown.
func (*BM25Scorer) ExplainScore ¶
func (s *BM25Scorer) ExplainScore(queryTerms []string, docID int32) ScoreExplanation
ExplainScore provides a detailed breakdown of the BM25 score for debugging.
func (*BM25Scorer) GetParams ¶
func (s *BM25Scorer) GetParams() (k1, b float64)
GetParams returns the current BM25 parameters (k1, b).
func (*BM25Scorer) Score ¶
func (s *BM25Scorer) Score(queryTerms []string, docID int32) float64
Score calculates the BM25 relevance score for a document given a query. The query is provided as a list of terms (already tokenized).
BM25 formula: score(D, Q) = Σ IDF(qi) * (f(qi, D) * (k1 + 1)) / (f(qi, D) + k1 * (1 - b + b * |D| / avgdl))
Where: - D = document - Q = query - qi = query term i - f(qi, D) = frequency of term qi in document D - |D| = document length (number of tokens) - avgdl = average document length - k1 = term frequency saturation parameter - b = length normalization parameter
func (*BM25Scorer) ScoreAll ¶
func (s *BM25Scorer) ScoreAll(queryTerms []string) []ScoredDocument
ScoreAll calculates BM25 scores for all documents containing any query term. Returns a slice of ScoredDocument sorted by score (descending).
func (*BM25Scorer) ScoreAllWithLimit ¶
func (s *BM25Scorer) ScoreAllWithLimit(queryTerms []string, limit int) []ScoredDocument
ScoreAllWithLimit is like ScoreAll but returns at most limit results.
func (*BM25Scorer) SetParams ¶
func (s *BM25Scorer) SetParams(k1, b float64)
SetParams updates the BM25 parameters.
func (*BM25Scorer) UpdateAvgDocLength ¶
func (s *BM25Scorer) UpdateAvgDocLength()
UpdateAvgDocLength updates the average document length. Call this after adding/removing documents if using cached scorer.
type Document ¶
type Document struct {
// SessionID is the ID of the conversation containing this message
SessionID string
// MessageIndex is the index of the message within the conversation
MessageIndex int
// MessageRole is the role of the message sender (user, assistant, system)
MessageRole string
// Content is the full text content of the message
Content string
// WordCount is the number of tokens in the message
WordCount int
// Timestamp is when the message was created
Timestamp time.Time
}
Document represents a single indexed document (message) with its metadata.
type DocumentStore ¶
type DocumentStore struct {
// Docs maps document IDs to document metadata
Docs map[int32]*Document
// SessionIndex maps session IDs to their document IDs for quick lookup
SessionIndex map[string][]int32
// NextDocID is the next available document ID
NextDocID int32
// contains filtered or unexported fields
}
DocumentStore is a thread-safe store mapping document IDs to their metadata. It's used alongside the InvertedIndex to retrieve document details after search.
func NewDocumentStore ¶
func NewDocumentStore() *DocumentStore
NewDocumentStore creates a new empty document store.
func (*DocumentStore) Add ¶
func (ds *DocumentStore) Add(doc *Document) int32
Add stores a new document and returns its assigned document ID.
func (*DocumentStore) AddWithID ¶
func (ds *DocumentStore) AddWithID(docID int32, doc *Document)
AddWithID stores a document with a specific document ID. Used when loading from persistence or when ID is known.
func (*DocumentStore) Clear ¶
func (ds *DocumentStore) Clear()
Clear removes all documents from the store.
func (*DocumentStore) Count ¶
func (ds *DocumentStore) Count() int
Count returns the total number of documents.
func (*DocumentStore) Get ¶
func (ds *DocumentStore) Get(docID int32) *Document
Get retrieves a document by its ID. Returns nil if not found.
func (*DocumentStore) GetAllSessionIDs ¶
func (ds *DocumentStore) GetAllSessionIDs() []string
GetAllSessionIDs returns all session IDs in the store.
func (*DocumentStore) GetBySession ¶
func (ds *DocumentStore) GetBySession(sessionID string) []*Document
GetBySession returns all documents belonging to a session.
func (*DocumentStore) GetDocIDsBySession ¶
func (ds *DocumentStore) GetDocIDsBySession(sessionID string) []int32
GetDocIDsBySession returns all document IDs belonging to a session.
func (*DocumentStore) GetStats ¶
func (ds *DocumentStore) GetStats() DocumentStoreStats
GetStats returns statistics about the document store.
func (*DocumentStore) HasSession ¶
func (ds *DocumentStore) HasSession(sessionID string) bool
HasSession returns true if the session exists in the store.
func (*DocumentStore) Remove ¶
func (ds *DocumentStore) Remove(docID int32)
Remove deletes a document from the store.
func (*DocumentStore) RemoveBySession ¶
func (ds *DocumentStore) RemoveBySession(sessionID string)
RemoveBySession removes all documents belonging to a session.
func (*DocumentStore) SessionCount ¶
func (ds *DocumentStore) SessionCount() int
SessionCount returns the number of unique sessions.
type DocumentStoreStats ¶
DocumentStoreStats contains statistics about the document store.
type HighlightRange ¶
HighlightRange represents a range to highlight in the snippet text.
type IndexStore ¶
type IndexStore struct {
// contains filtered or unexported fields
}
IndexStore handles persistence of the search index to disk. It uses Gob encoding for efficient storage and atomic writes for safety.
func NewIndexStore ¶
func NewIndexStore() (*IndexStore, error)
NewIndexStore creates a new IndexStore that persists to ~/.claude/search_index/
func NewIndexStoreWithDir ¶
func NewIndexStoreWithDir(indexDir string) (*IndexStore, error)
NewIndexStoreWithDir creates an IndexStore with a custom directory. Useful for testing.
func (*IndexStore) Delete ¶
func (s *IndexStore) Delete() error
Delete removes all persisted index files including sync metadata.
func (*IndexStore) DeleteSyncMetadata ¶
func (s *IndexStore) DeleteSyncMetadata() error
DeleteSyncMetadata removes the sync metadata file.
func (*IndexStore) Exists ¶
func (s *IndexStore) Exists() bool
Exists returns true if a persisted index exists.
func (*IndexStore) GetIndexDir ¶
func (s *IndexStore) GetIndexDir() string
GetIndexDir returns the directory where index files are stored.
func (*IndexStore) GetVersion ¶
func (s *IndexStore) GetVersion() (*IndexVersion, error)
GetVersion returns the version metadata of the persisted index.
func (*IndexStore) Load ¶
func (s *IndexStore) Load() (*InvertedIndex, *DocumentStore, error)
Load reads the inverted index and document store from disk. Returns error if files don't exist or are corrupted.
func (*IndexStore) LoadSyncMetadata ¶
func (s *IndexStore) LoadSyncMetadata() (*IndexSyncMetadata, error)
LoadSyncMetadata reads the sync metadata from disk. Returns nil, ErrMetadataNotFound if metadata doesn't exist (fresh index).
func (*IndexStore) Save ¶
func (s *IndexStore) Save(index *InvertedIndex, docStore *DocumentStore) error
Save persists the inverted index and document store to disk. Uses atomic writes (write to temp file, then rename) to prevent corruption.
func (*IndexStore) SaveSyncMetadata ¶
func (s *IndexStore) SaveSyncMetadata(meta *IndexSyncMetadata) error
SaveSyncMetadata persists the sync metadata to disk using atomic write.
func (*IndexStore) SyncMetadataExists ¶
func (s *IndexStore) SyncMetadataExists() bool
SyncMetadataExists returns true if sync metadata exists on disk.
type IndexSyncMetadata ¶
type IndexSyncMetadata struct {
// Version is the schema version of the sync metadata
Version int `json:"version"`
// LastFullSync is when the index was last fully rebuilt
LastFullSync time.Time `json:"last_full_sync"`
// LastIncrementalSync is when the index was last incrementally updated
LastIncrementalSync time.Time `json:"last_incremental_sync"`
// Sessions maps session IDs to their index metadata
Sessions map[string]*SessionIndexMetadata `json:"sessions"`
// TotalSessions is the count of indexed sessions
TotalSessions int `json:"total_sessions"`
// TotalDocuments is the total indexed document count
TotalDocuments int `json:"total_documents"`
}
IndexSyncMetadata tracks the overall index state for incremental updates. This metadata is persisted alongside the inverted index and document store.
func NewIndexSyncMetadata ¶
func NewIndexSyncMetadata() *IndexSyncMetadata
NewIndexSyncMetadata creates a new IndexSyncMetadata with initialized maps.
type IndexVersion ¶
type IndexVersion struct {
Version int `json:"version"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DocumentCount int `json:"document_count"`
TermCount int `json:"term_count"`
}
IndexVersion tracks metadata about the persisted index.
type InvertedIndex ¶
type InvertedIndex struct {
// Index maps terms to their posting lists
Index map[string]*PostingsList
// DocFrequency maps terms to the number of documents containing them (for IDF calculation)
DocFrequency map[string]int
// TotalDocs is the total number of documents indexed
TotalDocs int
// DocLengths maps document IDs to their lengths (number of tokens) for BM25
DocLengths map[int32]int
// AvgDocLength is the average document length for BM25 scoring
AvgDocLength float64
// contains filtered or unexported fields
}
InvertedIndex is a thread-safe inverted index for full-text search. It maps terms to posting lists containing document IDs, positions, and frequencies.
func NewInvertedIndex ¶
func NewInvertedIndex() *InvertedIndex
NewInvertedIndex creates a new empty inverted index.
func (*InvertedIndex) AddDocument ¶
func (idx *InvertedIndex) AddDocument(docID int32, tokens []string, positions map[string][]int32)
AddDocument adds a document to the inverted index. docID is a unique identifier for the document. tokens is the list of tokens (already processed by the tokenizer). positions maps each token to its positions in the original document.
func (*InvertedIndex) AddDocumentSimple ¶
func (idx *InvertedIndex) AddDocumentSimple(docID int32, tokens []string)
AddDocumentSimple is a simplified version of AddDocument that takes just tokens without position information. Useful when positions aren't needed.
func (*InvertedIndex) Clear ¶
func (idx *InvertedIndex) Clear()
Clear removes all documents and terms from the index.
func (*InvertedIndex) GetAllTerms ¶
func (idx *InvertedIndex) GetAllTerms() []string
GetAllTerms returns all terms in the index.
func (*InvertedIndex) GetAvgDocLength ¶
func (idx *InvertedIndex) GetAvgDocLength() float64
GetAvgDocLength returns the average document length.
func (*InvertedIndex) GetDocLength ¶
func (idx *InvertedIndex) GetDocLength(docID int32) int
GetDocLength returns the length of a specific document.
func (*InvertedIndex) GetDocumentFrequency ¶
func (idx *InvertedIndex) GetDocumentFrequency(term string) int
GetDocumentFrequency returns the number of documents containing the term.
func (*InvertedIndex) GetStats ¶
func (idx *InvertedIndex) GetStats() InvertedIndexStats
GetStats returns statistics about the index.
func (*InvertedIndex) GetTermCount ¶
func (idx *InvertedIndex) GetTermCount() int
GetTermCount returns the total number of unique terms in the index.
func (*InvertedIndex) GetTotalDocs ¶
func (idx *InvertedIndex) GetTotalDocs() int
GetTotalDocs returns the total number of indexed documents.
func (*InvertedIndex) HasDocument ¶
func (idx *InvertedIndex) HasDocument(docID int32) bool
HasDocument checks if a document ID exists in the index.
func (*InvertedIndex) RemoveDocument ¶
func (idx *InvertedIndex) RemoveDocument(docID int32)
RemoveDocument removes a document from the index. This is an expensive operation as it requires updating all posting lists.
func (*InvertedIndex) Search ¶
func (idx *InvertedIndex) Search(term string) *PostingsList
Search returns the posting list for a term. Returns nil if the term is not in the index.
func (*InvertedIndex) SearchMultiple ¶
func (idx *InvertedIndex) SearchMultiple(terms []string) map[string]*PostingsList
SearchMultiple returns posting lists for multiple terms. Useful for multi-term queries.
type InvertedIndexStats ¶
type InvertedIndexStats struct {
TotalDocs int
TotalTerms int
TotalPostings int
AvgDocLength float64
AvgTermsPerDoc float64
}
InvertedIndexStats contains statistics about the inverted index.
type PostingsList ¶
type PostingsList struct {
// DocIDs contains the list of document IDs containing this term
DocIDs []int32
// Positions contains the positions of the term within each document
// Positions[i] corresponds to DocIDs[i]
Positions [][]int32
// Frequency contains the term frequency in each document
// Frequency[i] corresponds to DocIDs[i]
Frequency []int32
}
PostingsList represents the posting list for a term in the inverted index. It contains document IDs, positions within each document, and term frequencies.
type ScoreExplanation ¶
type ScoreExplanation struct {
DocID int32
TotalScore float64
TermScores []TermScore
K1 float64
B float64
}
ScoreExplanation provides detailed information about how a score was calculated.
type ScoredDocument ¶
ScoredDocument represents a document with its BM25 relevance score.
type SearchEngine ¶
type SearchEngine struct {
// contains filtered or unexported fields
}
SearchEngine is the main interface for full-text search over Claude history. It combines tokenization, indexing, scoring, and snippet generation.
func NewSearchEngine ¶
func NewSearchEngine() *SearchEngine
NewSearchEngine creates a new search engine instance.
func NewSearchEngineWithPersistence ¶
func NewSearchEngineWithPersistence(indexStore *IndexStore) *SearchEngine
NewSearchEngineWithPersistence creates a search engine with index persistence.
func (*SearchEngine) BuildIndex ¶
func (e *SearchEngine) BuildIndex(history *session.ClaudeSessionHistory) error
BuildIndex indexes all messages from the provided history. This replaces any existing index.
func (*SearchEngine) GetDocStore ¶
func (e *SearchEngine) GetDocStore() *DocumentStore
GetDocStore returns the underlying document store (for advanced usage).
func (*SearchEngine) GetDocument ¶
func (e *SearchEngine) GetDocument(docID int32) *Document
GetDocument returns a document by its ID.
func (*SearchEngine) GetIndex ¶
func (e *SearchEngine) GetIndex() *InvertedIndex
GetIndex returns the underlying inverted index (for advanced usage).
func (*SearchEngine) GetStats ¶
func (e *SearchEngine) GetStats() SearchEngineStats
GetStats returns statistics about the search engine.
func (*SearchEngine) GetSyncMetadata ¶
func (e *SearchEngine) GetSyncMetadata() *IndexSyncMetadata
GetSyncMetadata returns the current sync metadata (for inspection).
func (*SearchEngine) GetTokenizer ¶
func (e *SearchEngine) GetTokenizer() *Tokenizer
GetTokenizer returns the tokenizer (for query highlighting).
func (*SearchEngine) HasSession ¶
func (e *SearchEngine) HasSession(sessionID string) bool
HasSession returns true if the session is indexed.
func (*SearchEngine) IncrementalSync ¶
func (e *SearchEngine) IncrementalSync(history *session.ClaudeSessionHistory) (*SyncResult, error)
IncrementalSync synchronizes the index with current history state. It only indexes new/modified sessions and removes deleted ones. On first run or when metadata is missing, falls back to full rebuild.
func (*SearchEngine) IndexMessage ¶
func (e *SearchEngine) IndexMessage(sessionID string, msgIdx int, role, content string, timestamp time.Time) error
IndexMessage adds a single message to the index. Use for incremental updates when new messages arrive.
func (*SearchEngine) LoadIndex ¶
func (e *SearchEngine) LoadIndex() error
LoadIndex loads a previously persisted index from disk. Also loads sync metadata if available.
func (*SearchEngine) LoadSyncMetadata ¶
func (e *SearchEngine) LoadSyncMetadata() error
LoadSyncMetadata loads persisted sync metadata from the index store.
func (*SearchEngine) RemoveSession ¶
func (e *SearchEngine) RemoveSession(sessionID string)
RemoveSession removes all documents from a session.
func (*SearchEngine) SaveIndex ¶
func (e *SearchEngine) SaveIndex() error
SaveIndex persists the current index to disk.
func (*SearchEngine) SaveSyncMetadata ¶
func (e *SearchEngine) SaveSyncMetadata() error
SaveSyncMetadata persists current sync metadata to disk.
func (*SearchEngine) Search ¶
func (e *SearchEngine) Search(query string, opts SearchOptions) (*SearchResults, error)
Search performs a full-text search on the indexed messages.
func (*SearchEngine) ShouldRebuild ¶
func (e *SearchEngine) ShouldRebuild() bool
ShouldRebuild returns true if a full rebuild is recommended. This happens when: no sync metadata, version mismatch, or corruption.
func (*SearchEngine) ShouldRebuildLocked ¶
func (e *SearchEngine) ShouldRebuildLocked() bool
ShouldRebuildLocked is the lock-free version of ShouldRebuild. Must be called with at least read lock held.
type SearchEngineStats ¶
type SearchEngineStats struct {
TotalDocuments int
TotalTerms int
TotalPostings int
TotalSessions int
AvgDocLength float64
AvgTermsPerDoc float64
}
SearchEngineStats contains statistics about the search engine.
type SearchOptions ¶
type SearchOptions struct {
// Limit is the maximum number of results to return (0 = no limit)
Limit int
// Offset is the number of results to skip for pagination
Offset int
// SessionID filters results to a specific session (empty = all sessions)
SessionID string
}
SearchOptions configures search behavior.
type SearchResult ¶
type SearchResult struct {
// DocID is the internal document ID
DocID int32
// SessionID is the conversation ID containing this message
SessionID string
// MessageIndex is the index of the message within the conversation
MessageIndex int
// MessageRole is the role (user, assistant, system)
MessageRole string
// Score is the BM25 relevance score
Score float64
// Content is the full message content
Content string
// Timestamp is when the message was created
Timestamp time.Time
}
SearchResult represents a single search result.
type SearchResults ¶
type SearchResults struct {
// Results is the list of search results
Results []SearchResult
// TotalMatches is the total number of matching documents (before pagination)
TotalMatches int
// QueryTime is the duration of the search operation
QueryTime time.Duration
}
SearchResults contains the results of a search query.
type SessionIndexMetadata ¶
type SessionIndexMetadata struct {
// SessionID is the unique conversation identifier
SessionID string `json:"session_id"`
// UpdatedAt is the last known update time of the conversation
UpdatedAt time.Time `json:"updated_at"`
// MessageCount is the number of messages in the conversation when indexed
MessageCount int `json:"message_count"`
// LastIndexedAt is when we last indexed this session
LastIndexedAt time.Time `json:"last_indexed_at"`
// DocCount is the number of documents we created for this session
DocCount int `json:"doc_count"`
}
SessionIndexMetadata tracks the indexing state for a single conversation session. Used to detect changes since last index build.
type Snippet ¶
type Snippet struct {
// Text is the snippet text with surrounding context
Text string
// HighlightRanges contains all highlight positions in the snippet
HighlightRanges []HighlightRange
// MessageRole is the role of the message sender (user, assistant, system)
MessageRole string
// MessageTime is when the message was created
MessageTime time.Time
}
Snippet represents a highlighted text snippet showing where a search term appears.
type SnippetGenerator ¶
type SnippetGenerator struct {
// contains filtered or unexported fields
}
SnippetGenerator generates contextual snippets from search results.
func NewSnippetGenerator ¶
func NewSnippetGenerator() *SnippetGenerator
NewSnippetGenerator creates a new snippet generator with default settings.
func NewSnippetGeneratorWithOptions ¶
func NewSnippetGeneratorWithOptions(contextWords, maxSnippets, maxSnippetLength int) *SnippetGenerator
NewSnippetGeneratorWithOptions creates a snippet generator with custom settings.
func (*SnippetGenerator) Generate ¶
func (g *SnippetGenerator) Generate(message string, query string, role string, timestamp time.Time) []Snippet
Generate creates snippets from a message containing the query terms. query should be the original query terms (will be tokenized for matching).
func (*SnippetGenerator) GenerateFromSearchResult ¶
func (g *SnippetGenerator) GenerateFromSearchResult(doc *Document, queryTokens []string) []Snippet
GenerateFromSearchResult generates snippets for a search result. This is a convenience method that uses the document content and query tokens.
type SyncResult ¶
type SyncResult struct {
// SessionsAdded is the count of newly indexed sessions
SessionsAdded int
// SessionsUpdated is the count of sessions with new messages re-indexed
SessionsUpdated int
// SessionsRemoved is the count of deleted sessions removed from index
SessionsRemoved int
// DocumentsAdded is the count of new documents indexed
DocumentsAdded int
// DocumentsRemoved is the count of documents removed
DocumentsRemoved int
// SyncDuration is how long the sync took
SyncDuration time.Duration
// WasFullRebuild is true if a full rebuild was performed instead of incremental
WasFullRebuild bool
// Errors contains any non-fatal errors encountered during sync
Errors []error
}
SyncResult contains statistics about an incremental sync operation.
func (*SyncResult) HasChanges ¶
func (r *SyncResult) HasChanges() bool
HasChanges returns true if any sessions were added, updated, or removed.
func (*SyncResult) String ¶
func (r *SyncResult) String() string
String returns a human-readable summary of the sync result.
type TermScore ¶
type TermScore struct {
Term string
TermFrequency float64
DocumentFrequency int
IDF float64
Score float64
DocumentLength int
AvgDocLength float64
}
TermScore provides detailed scoring breakdown for a single term.
type TokenPosition ¶
type TokenPosition struct {
Token string // The stemmed token
Start int // Start character position in original text
End int // End character position in original text
}
TokenPosition represents a token with its position in the original text.
type Tokenizer ¶
type Tokenizer struct {
// contains filtered or unexported fields
}
Tokenizer handles text tokenization for full-text search. It provides lowercasing, stop word removal, and Porter stemming.
func NewTokenizer ¶
func NewTokenizer() *Tokenizer
NewTokenizer creates a new Tokenizer with default English stop words.
func (*Tokenizer) IsStopWord ¶
IsStopWord returns true if the word is a stop word.
func (*Tokenizer) StemWord ¶
StemWord applies Porter stemming to a single word. The word should be lowercase.
func (*Tokenizer) Tokenize ¶
Tokenize splits text into normalized tokens suitable for indexing. It performs: lowercase, word splitting, stop word removal, and stemming.
func (*Tokenizer) TokenizeWithPositions ¶
func (t *Tokenizer) TokenizeWithPositions(text string) []TokenPosition
TokenizeWithPositions returns tokens along with their character positions. This is useful for highlighting search results.