Documentation
¶
Overview ¶
Package store provides cross-request state for rate limiting, circuit breaking, sticky sessions, and latency stats. The StateStore interface lives in core (core.StateStore).
Index ¶
- Constants
- Variables
- func RedisKeyAlias(alias string) string
- func RedisKeyApiKeyHash(keyHash string) string
- func RedisKeyConfigEndpoints(modelCode string) string
- func RedisKeyModelAliases(modelCode string) string
- func RedisKeyTenantEndpoints(tenantCode, modelCode string) string
- func RedisKeyUserModels(userID string) string
- type ExpirableCache
- type MemoryStateStore
- func (s *MemoryStateStore) Close() error
- func (s *MemoryStateStore) GetAvgLatency(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
- func (s *MemoryStateStore) GetAvgTTFT(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
- func (s *MemoryStateStore) GetEMA(ctx context.Context, key string) (float64, error)
- func (s *MemoryStateStore) RateLimitIncr(ctx context.Context, key string, tokens int64, window time.Duration) (int64, error)
- func (s *MemoryStateStore) RateLimitRefund(ctx context.Context, key string, tokens int64) error
- func (s *MemoryStateStore) RateLimitTake(ctx context.Context, key string, tokens int64, rate int64, capacity int64, ...) (bool, int64, error)
- func (s *MemoryStateStore) RecordLatency(ctx context.Context, endpointID string, latency time.Duration) error
- func (s *MemoryStateStore) RecordTTFT(ctx context.Context, endpointID string, ttft time.Duration) error
- func (s *MemoryStateStore) StickyGet(ctx context.Context, sessionKey string) (string, error)
- func (s *MemoryStateStore) StickySet(ctx context.Context, sessionKey string, endpointID string, ttl time.Duration) error
- func (s *MemoryStateStore) UpdateEMA(ctx context.Context, key string, actual int64, alpha float64) (float64, error)
- type RedisStateStore
- func (s *RedisStateStore) Close() error
- func (s *RedisStateStore) GetAvgLatency(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
- func (s *RedisStateStore) GetAvgTTFT(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
- func (s *RedisStateStore) GetEMA(ctx context.Context, key string) (float64, error)
- func (s *RedisStateStore) RateLimitIncr(ctx context.Context, key string, tokens int64, window time.Duration) (int64, error)
- func (s *RedisStateStore) RateLimitRefund(ctx context.Context, key string, tokens int64) error
- func (s *RedisStateStore) RateLimitTake(ctx context.Context, key string, tokens int64, rate int64, capacity int64, ...) (bool, int64, error)
- func (s *RedisStateStore) RecordLatency(ctx context.Context, endpointID string, latency time.Duration) error
- func (s *RedisStateStore) RecordTTFT(ctx context.Context, endpointID string, ttft time.Duration) error
- func (s *RedisStateStore) StickyGet(ctx context.Context, sessionKey string) (string, error)
- func (s *RedisStateStore) StickySet(ctx context.Context, sessionKey string, endpointID string, ttl time.Duration) error
- func (s *RedisStateStore) UpdateEMA(ctx context.Context, key string, actual int64, alpha float64) (float64, error)
- type RedisStateStoreConfig
Constants ¶
const (
RedisKeyConfigModelVersions = "aigw:config:model_versions"
)
Variables ¶
var ( // ErrKeyNotFound is returned when a key does not exist. ErrKeyNotFound = errors.New("key not found") )
Functions ¶
func RedisKeyAlias ¶
RedisKeyAlias returns the Redis key for an alias mapping.
func RedisKeyApiKeyHash ¶
func RedisKeyConfigEndpoints ¶
RedisKeyConfigEndpoints returns the endpoints config key for a model_code.
func RedisKeyModelAliases ¶
RedisKeyModelAliases returns the Redis key for a model's alias set (reverse index).
func RedisKeyTenantEndpoints ¶
func RedisKeyUserModels ¶
Types ¶
type ExpirableCache ¶
type ExpirableCache[K comparable, V any] struct { // contains filtered or unexported fields }
func NewExpirableCache ¶
func NewExpirableCache[K comparable, V any](validCap int, validTTL time.Duration, invalidCap int, invalidTTL time.Duration) *ExpirableCache[K, V]
func (*ExpirableCache[K, V]) AddInvalid ¶
func (c *ExpirableCache[K, V]) AddInvalid(key K, errMsg string)
func (*ExpirableCache[K, V]) AddValid ¶
func (c *ExpirableCache[K, V]) AddValid(key K, val V)
func (*ExpirableCache[K, V]) Get ¶
func (c *ExpirableCache[K, V]) Get(key K) (V, string, bool)
Get queries valid and invalid caches. Returns (value, errorMsg, exists).
func (*ExpirableCache[K, V]) Remove ¶
func (c *ExpirableCache[K, V]) Remove(key K)
type MemoryStateStore ¶
type MemoryStateStore struct {
// contains filtered or unexported fields
}
MemoryStateStore is an in-memory StateStore for single-instance and tests.
func NewMemoryStateStore ¶
func NewMemoryStateStore() *MemoryStateStore
NewMemoryStateStore creates a MemoryStateStore.
func (*MemoryStateStore) Close ¶
func (s *MemoryStateStore) Close() error
Close is a no-op for MemoryStateStore.
func (*MemoryStateStore) GetAvgLatency ¶
func (s *MemoryStateStore) GetAvgLatency(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
GetAvgLatency returns average latency for endpointID within window, or 0 if none.
func (*MemoryStateStore) GetAvgTTFT ¶
func (s *MemoryStateStore) GetAvgTTFT(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
GetAvgTTFT returns average TTFT for endpointID within window, or 0 if none.
func (*MemoryStateStore) RateLimitIncr ¶
func (s *MemoryStateStore) RateLimitIncr(ctx context.Context, key string, tokens int64, window time.Duration) (int64, error)
RateLimitIncr adds tokens to the key counter and returns usage in the window. window sets the window size; if the key exists with a different window, it takes effect on the next reset.
func (*MemoryStateStore) RateLimitRefund ¶
RateLimitRefund refunds tokens to the key counter.
func (*MemoryStateStore) RateLimitTake ¶
func (s *MemoryStateStore) RateLimitTake(ctx context.Context, key string, tokens int64, rate int64, capacity int64, window time.Duration, now time.Time) (bool, int64, error)
RateLimitTake tries to consume tokens from a token bucket (smooth burst limit).
func (*MemoryStateStore) RecordLatency ¶
func (s *MemoryStateStore) RecordLatency(ctx context.Context, endpointID string, latency time.Duration) error
RecordLatency records a latency sample.
func (*MemoryStateStore) RecordTTFT ¶
func (s *MemoryStateStore) RecordTTFT(ctx context.Context, endpointID string, ttft time.Duration) error
RecordTTFT records a TTFT sample on a series separate from total latency.
func (*MemoryStateStore) StickyGet ¶
StickyGet returns the endpointID for sessionKey, or ErrKeyNotFound if expired/missing.
type RedisStateStore ¶
type RedisStateStore struct {
// contains filtered or unexported fields
}
RedisStateStore is a Redis-backed StateStore for multi-instance production. Uses redis.Cmdable (works with redis.Client and redis.ClusterClient).
func NewRedisStateStore ¶
func NewRedisStateStore(client redis.Cmdable, cfg *RedisStateStoreConfig) *RedisStateStore
NewRedisStateStore creates a RedisStateStore. client may be *redis.Client or *redis.ClusterClient.
func (*RedisStateStore) Close ¶
func (s *RedisStateStore) Close() error
Close closes the Redis connection if the client is closable.
func (*RedisStateStore) GetAvgLatency ¶
func (s *RedisStateStore) GetAvgLatency(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
GetAvgLatency returns average latency for endpointID within window, or 0 if none.
func (*RedisStateStore) GetAvgTTFT ¶
func (s *RedisStateStore) GetAvgTTFT(ctx context.Context, endpointID string, window time.Duration) (time.Duration, error)
GetAvgTTFT returns average TTFT for endpointID within window, or 0 if none.
func (*RedisStateStore) RateLimitIncr ¶
func (s *RedisStateStore) RateLimitIncr(ctx context.Context, key string, tokens int64, window time.Duration) (int64, error)
RateLimitIncr adds tokens to the key counter and returns usage in the window.
func (*RedisStateStore) RateLimitRefund ¶
RateLimitRefund refunds tokens to the key counter.
func (*RedisStateStore) RateLimitTake ¶
func (s *RedisStateStore) RateLimitTake(ctx context.Context, key string, tokens int64, rate int64, capacity int64, window time.Duration, now time.Time) (bool, int64, error)
RateLimitTake tries to consume tokens from a token bucket (smooth burst limit).
func (*RedisStateStore) RecordLatency ¶
func (s *RedisStateStore) RecordLatency(ctx context.Context, endpointID string, latency time.Duration) error
RecordLatency records a latency sample in a Redis sorted set. Score is timestamp (ms); member is "<latency_ns>:<rand>". Atomically adds the sample, sets TTL, and trims old samples.
func (*RedisStateStore) RecordTTFT ¶
func (s *RedisStateStore) RecordTTFT(ctx context.Context, endpointID string, ttft time.Duration) error
RecordTTFT records a time-to-first-token sample on a key separate from total latency.
func (*RedisStateStore) StickyGet ¶
StickyGet returns the endpointID for sessionKey, or ErrKeyNotFound.
type RedisStateStoreConfig ¶
type RedisStateStoreConfig struct {
// KeyPrefix is the Redis key namespace (default "aigw").
KeyPrefix string
// LatencyMaxSamples is the max latency samples retained (default 1000).
LatencyMaxSamples int
}
RedisStateStoreConfig configures RedisStateStore.
Source Files