cache

Documentation

Index

• type CacheStats
• type Embedding
  • func NewEmbedding(ttl, cleanup time.Duration) *Embedding
  • func (c *Embedding) Get(text string) ([]float32, bool)
  • func (c *Embedding) Set(text string, embedding []float32)
  • func (c *Embedding) Stats() CacheStats
• type ResponseCache
  • func NewResponseCache(ttl, cleanup time.Duration, maxItems int) *ResponseCache
  • func (c *ResponseCache) Get(text, historyKey string) (*models.ChatResponse, bool)
  • func (c *ResponseCache) Invalidate(pattern string)
  • func (c *ResponseCache) Set(text string, historyKey string, resp *models.ChatResponse, ...)
  • func (c *ResponseCache) Stats() CacheStats

Types

type CacheStats

type CacheStats struct {
	Hits    int64   `json:"hits"`
	Misses  int64   `json:"misses"`
	Size    int     `json:"size"`
	HitRate float64 `json:"hit_rate"`
}

CacheStats represents the basic metrics of cache

type Embedding

type Embedding struct {
	// contains filtered or unexported fields
}

Embedding prevents the recomputation for repeated texts

func NewEmbedding

func NewEmbedding(ttl, cleanup time.Duration) *Embedding

NewEmbedding creates a new instance of Embedding

func (*Embedding) Get

func (c *Embedding) Get(text string) ([]float32, bool)

Get retrieves the embedding for the given text from the cache. It returns the embedding and a boolean indicating if the key was found.

func (*Embedding) Set

func (c *Embedding) Set(text string, embedding []float32)

Set stores the embedding for the given text in the cache.

func (*Embedding) Stats

func (c *Embedding) Stats() CacheStats

type ResponseCache

type ResponseCache struct {
	// contains filtered or unexported fields
}

ResponseCache stores the complete responses by normalized hash

func NewResponseCache

func NewResponseCache(ttl, cleanup time.Duration, maxItems int) *ResponseCache

NewResponseCache creates a new instance of ResponseCache

func (*ResponseCache) Get

func (c *ResponseCache) Get(text, historyKey string) (*models.ChatResponse, bool)

Get retrieves the cached response for a given text and history key. It returns the cloned response and a boolean indicating if the key was found.

func (*ResponseCache) Invalidate

func (c *ResponseCache) Invalidate(pattern string)

Invalidate removes items from the cache that match a specific pattern. If the pattern is empty, it flushes the entire cache.

func (*ResponseCache) Set

func (c *ResponseCache) Set(
	text string,
	historyKey string,
	resp *models.ChatResponse,
	shortTools,
	mediumTools map[string]struct{},
)

Set stores the response for the given text and history key in the cache. It handles eviction if the cache size exceeds maxItems and determines TTL based on tool usage.

func (*ResponseCache) Stats

func (c *ResponseCache) Stats() CacheStats

Stats returns the current metrics of the response cache.