cache

Documentation

Overview

Package cache implements ephemeral prompt caching for improved performance.

This package provides caching mechanisms for AI prompts to reduce costs and improve response times through Anthropic's prompt caching feature.

Features:

• Cache control headers for system prompts
• Cache control headers for large context
• Automatic cache key generation
• Cache hit/miss metrics tracking
• Configurable cache behavior
• TTL-based cache expiration

Prompt caching works by marking portions of prompts as cacheable, allowing the AI provider to reuse processed prompts across requests.

Example usage:

import "github.com/AINative-studio/ainative-code/internal/cache"

// Create cache manager
manager := cache.NewManager(cache.Config{
    Enabled:           true,
    MinPromptLength:   1024,
    SystemPromptCache: true,
})

// Mark content as cacheable
cacheControl := manager.ShouldCache(content)
if cacheControl != nil {
    // Add cache control to API request
}

// Track cache metrics
manager.RecordCacheHit("system_prompt")
stats := manager.GetStats()

Index

• type CacheControl
• type CacheKeyMetrics
• type CacheMetrics
• type CacheStatus
• type CacheableContent
• type Config
  • func DefaultConfig() Config
• type Manager
  • func NewManager(config Config) *Manager
  • func (m *Manager) ClearCache()
  • func (m *Manager) FormatStats() string
  • func (m *Manager) GetCacheKeys() []string
  • func (m *Manager) GetCacheSize() int64
  • func (m *Manager) GetCacheStatus(cacheKey string) *CacheStatus
  • func (m *Manager) GetConfig() Config
  • func (m *Manager) GetStats() *CacheMetrics
  • func (m *Manager) InvalidateCache(cacheKey string)
  • func (m *Manager) IsCached(cacheKey string) bool
  • func (m *Manager) RecordBytesSaved(bytes int64)
  • func (m *Manager) RecordCacheHit(cacheKey string)
  • func (m *Manager) RecordCacheMiss(cacheKey string)
  • func (m *Manager) RecordCached(cacheKey string, bytesSize int64)
  • func (m *Manager) ResetMetrics()
  • func (m *Manager) ShouldCache(content *CacheableContent) *CacheControl
  • func (m *Manager) Stop()
  • func (m *Manager) UpdateConfig(config Config)

Types

type CacheControl

type CacheControl struct {
	Type       string `json:"type"`       // "ephemeral" for Anthropic prompt caching
	Enabled    bool   `json:"enabled"`    // Whether caching is enabled for this segment
	CacheKey   string `json:"cache_key"`  // Unique identifier for this cached content
	TTL        int    `json:"ttl"`        // Time-to-live in seconds (optional)
	Breakpoint bool   `json:"breakpoint"` // Whether this is a cache breakpoint
}

CacheControl represents cache control directives for a prompt segment

type CacheKeyMetrics

type CacheKeyMetrics struct {
	Key         string    `json:"key"`
	Hits        int64     `json:"hits"`
	Misses      int64     `json:"misses"`
	BytesCached int64     `json:"bytes_cached"`
	LastHit     time.Time `json:"last_hit"`
	Created     time.Time `json:"created"`
}

CacheKeyMetrics tracks metrics for a specific cache key

type CacheMetrics

type CacheMetrics struct {
	TotalRequests  int64                       `json:"total_requests"`
	CacheHits      int64                       `json:"cache_hits"`
	CacheMisses    int64                       `json:"cache_misses"`
	BytesCached    int64                       `json:"bytes_cached"`
	BytesSaved     int64                       `json:"bytes_saved"`
	AverageHitRate float64                     `json:"average_hit_rate"`
	LastReset      time.Time                   `json:"last_reset"`
	CacheByKey     map[string]*CacheKeyMetrics `json:"cache_by_key"`
}

CacheMetrics tracks cache performance metrics

type CacheStatus

type CacheStatus struct {
	Key        string    `json:"key"`
	Cached     bool      `json:"cached"`
	ExpiresAt  time.Time `json:"expires_at"`
	BytesSize  int64     `json:"bytes_size"`
	HitCount   int64     `json:"hit_count"`
	LastAccess time.Time `json:"last_access"`
}

CacheStatus represents the current status of a cache entry

type CacheableContent

type CacheableContent struct {
	Content    string `json:"content"`
	Type       string `json:"type"` // "system", "context", "tools"
	Length     int    `json:"length"`
	CacheKey   string `json:"cache_key"`
	Priority   int    `json:"priority"`   // Higher priority cached first
	Breakpoint bool   `json:"breakpoint"` // Mark as cache breakpoint
}

CacheableContent represents content that can be cached

type Config

type Config struct {
	Enabled           bool          `json:"enabled"`
	MinPromptLength   int           `json:"min_prompt_length"`   // Minimum length to cache (default: 1024)
	MaxPromptLength   int           `json:"max_prompt_length"`   // Maximum length to cache (default: 0 = unlimited)
	SystemPromptCache bool          `json:"system_prompt_cache"` // Cache system prompts
	ContextCache      bool          `json:"context_cache"`       // Cache conversation context
	TTL               time.Duration `json:"ttl"`                 // Default TTL (0 = use provider default)
	AutoCleanup       bool          `json:"auto_cleanup"`        // Automatically cleanup expired cache
	CleanupInterval   time.Duration `json:"cleanup_interval"`    // How often to run cleanup
}

Config represents caching configuration

func DefaultConfig

func DefaultConfig() Config

DefaultConfig returns the default caching configuration

type Manager

type Manager struct {
	// contains filtered or unexported fields
}

Manager manages prompt caching

func NewManager

func NewManager(config Config) *Manager

NewManager creates a new cache manager

func (*Manager) ClearCache

func (m *Manager) ClearCache()

ClearCache clears all cached entries

func (*Manager) FormatStats

func (m *Manager) FormatStats() string

FormatStats formats cache statistics as a human-readable string

func (*Manager) GetCacheKeys

func (m *Manager) GetCacheKeys() []string

GetCacheKeys returns all currently cached keys

func (*Manager) GetCacheSize

func (m *Manager) GetCacheSize() int64

GetCacheSize returns the total size of cached content in bytes

func (*Manager) GetCacheStatus

func (m *Manager) GetCacheStatus(cacheKey string) *CacheStatus

GetCacheStatus returns the status of a specific cache key

func (*Manager) GetConfig

func (m *Manager) GetConfig() Config

GetConfig returns the current configuration

func (*Manager) GetStats

func (m *Manager) GetStats() *CacheMetrics

GetStats returns current cache statistics

func (*Manager) InvalidateCache

func (m *Manager) InvalidateCache(cacheKey string)

InvalidateCache invalidates a specific cache key

func (*Manager) IsCached

func (m *Manager) IsCached(cacheKey string) bool

IsCached checks if content is currently cached

func (*Manager) RecordBytesSaved

func (m *Manager) RecordBytesSaved(bytes int64)

RecordBytesSaved records bytes saved due to caching

func (*Manager) RecordCacheHit

func (m *Manager) RecordCacheHit(cacheKey string)

RecordCacheHit records a cache hit for metrics

func (*Manager) RecordCacheMiss

func (m *Manager) RecordCacheMiss(cacheKey string)

RecordCacheMiss records a cache miss for metrics

func (*Manager) RecordCached

func (m *Manager) RecordCached(cacheKey string, bytesSize int64)

RecordCached records that content was cached

func (*Manager) ResetMetrics

func (m *Manager) ResetMetrics()

ResetMetrics resets all cache metrics

func (*Manager) ShouldCache

func (m *Manager) ShouldCache(content *CacheableContent) *CacheControl

ShouldCache determines if content should be cached and returns cache control

func (*Manager) Stop

func (m *Manager) Stop()

Stop stops the cache manager and cleanup processes

func (*Manager) UpdateConfig

func (m *Manager) UpdateConfig(config Config)

UpdateConfig updates the cache configuration