server

package
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 23, 2026 License: MIT Imports: 34 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type AttemptMetric 

type AttemptMetric struct {
	Backend       string
	Outcome       string
	ErrorCategory string
}

type BackendHealthSnapshot 

type BackendHealthSnapshot struct {
	Name              string    `json:"name"`
	State             string    `json:"state"`
	ConsecutiveFails  int       `json:"consecutive_fails"`
	LastFailTime      time.Time `json:"last_fail_time,omitempty"`
	HalfOpenInFlight  int       `json:"half_open_in_flight"`
	Priority          int       `json:"priority"`
	Weight            int       `json:"weight"`
	Protocol          string    `json:"protocol"`
	URL               string    `json:"url"`
	PathPrefix        string    `json:"path_prefix,omitempty"`
	ActiveHealthCheck bool      `json:"active_health_check"`
}

type BackendSelector 

type BackendSelector struct {
	// contains filtered or unexported fields
}

BackendSelector picks healthy backends from candidate lists using priority-based selection with weighted round-robin within the same priority.

func NewBackendSelector 

func NewBackendSelector(health *HealthManager) *BackendSelector

func (*BackendSelector) Select 

func (bs *BackendSelector) Select(candidates []modelEntry) *candidateResult

Select picks the best single healthy backend from the candidate list. Candidates must already be sorted by priority (ascending). Algorithm: 1. Group by priority 2. From highest priority group, filter healthy backends 3. Weighted round-robin among healthy backends in that group 4. If group is empty, fall through to next priority group 5. Returns nil if all backends are unhealthy.

func (*BackendSelector) SelectFallback 

func (bs *BackendSelector) SelectFallback(candidates []modelEntry) []candidateResult

SelectFallback returns all candidates ordered by priority, ignoring health status. Used when no healthy backend is available and there is no alternative to route to.

func (*BackendSelector) SelectOrdered 

func (bs *BackendSelector) SelectOrdered(candidates []modelEntry) []candidateResult

SelectOrdered returns all healthy candidates ordered by priority then weight, suitable for retry iteration.

type CanonicalEvent 

type CanonicalEvent struct {
	EventType string          // SSE event name, e.g. "response.created", "response.output_text.delta"
	Data      json.RawMessage // raw JSON data payload
}

CanonicalEvent is a structured representation of a Responses SSE event.

type ConsoleRouteOptions 

type ConsoleRouteOptions struct {
	BasePath string
}

type DefaultOpts 

type DefaultOpts struct {
	MaxTokens      int      // inject max_tokens / max_output_tokens if client didn't set
	Temperature    *float64 // inject temperature if client didn't set
	NormalizeXHigh bool     // normalize xhigh → high for OpenAI family
}

DefaultOpts carries options for lightweight same-protocol default injection.

type EncodeOpts 

type EncodeOpts struct {
	ReasoningEffort string // default reasoning effort from config
	SystemPrompt    string // system prompt to inject
}

EncodeOpts carries options for cross-protocol request conversion.

type HealthManager 

type HealthManager struct {
	// contains filtered or unexported fields
}

func NewHealthManager 

func NewHealthManager(cfg HealthManagerConfig, client *http.Client, debug *logging.DebugLog) *HealthManager

func (*HealthManager) AllBackendsDown 

func (hm *HealthManager) AllBackendsDown() bool

AllBackendsDown returns true if all known backends are in tripped state.

func (*HealthManager) CircuitState 

func (hm *HealthManager) CircuitState(name string) string

CircuitState returns current circuit breaker state for a backend.

func (*HealthManager) IsHealthy 

func (hm *HealthManager) IsHealthy(name string) bool

IsHealthy returns whether the backend can currently receive traffic. - healthy: healthy - tripped: unhealthy until RecoveryTimeout passes - probing: healthy only while probe slots are available

func (*HealthManager) ReleaseProbe 

func (hm *HealthManager) ReleaseProbe(name string)

ReleaseProbe releases one probing slot when the request result should not affect circuit state (for example client-side 4xx or canceled requests).

func (*HealthManager) ReportFailure 

func (hm *HealthManager) ReportFailure(name string) bool

ReportFailure records a failure. Returns true if the backend just transitioned to tripped.

func (*HealthManager) ReportSuccess 

func (hm *HealthManager) ReportSuccess(name string)

ReportSuccess marks a backend as healthy and resets any tripped circuit.

func (*HealthManager) Snapshot 

func (hm *HealthManager) Snapshot(backends []config.Backend) []BackendHealthSnapshot

func (*HealthManager) StartActiveChecks 

func (hm *HealthManager) StartActiveChecks(backends []config.Backend)

StartActiveChecks launches goroutines for backends with health_check configured.

func (*HealthManager) Stop 

func (hm *HealthManager) Stop()

Stop shuts down all active check goroutines.

func (*HealthManager) TryAcquireProbe 

func (hm *HealthManager) TryAcquireProbe(name string) bool

TryAcquireProbe reserves one probe slot for probing backends. Healthy backends always allow requests.

type HealthManagerConfig 

type HealthManagerConfig struct {
	FailThreshold       int
	RecoveryTimeout     time.Duration
	HalfOpenMaxRequests int
}

type ModelIndex 

type ModelIndex struct {
	// contains filtered or unexported fields
}

ModelIndex provides O(1) model alias → backend resolution. Each alias may map to multiple backends (candidates), sorted by priority.

func BuildModelIndex 

func BuildModelIndex(backends []config.Backend) *ModelIndex

BuildModelIndex constructs the lookup index from config backends. Model names and their aliases are both registered as lookup keys. When the same alias exists in multiple backends, all are stored as candidates and sorted by backend priority (ascending = higher priority first).

func (*ModelIndex) AllAliases 

func (idx *ModelIndex) AllAliases() []string

AllAliases returns all registered model aliases (for model list endpoints).

func (*ModelIndex) DefaultBackend 

func (idx *ModelIndex) DefaultBackend() *config.Backend

DefaultBackend returns the default backend, or nil if none is configured.

func (*ModelIndex) MatchByPrefix 

func (idx *ModelIndex) MatchByPrefix(path string) (*config.Backend, bool)

MatchByPrefix finds a backend by longest path prefix match.

func (*ModelIndex) Resolve 

func (idx *ModelIndex) Resolve(alias string) (backend *config.Backend, backendModel string, found bool)

Resolve finds the backend and real model name for a given alias or model name. When multiple backends serve the same alias, returns the highest priority one.

func (*ModelIndex) ResolveCandidates 

func (idx *ModelIndex) ResolveCandidates(alias string) ([]modelEntry, bool)

ResolveCandidates returns all candidate backends for a given alias, sorted by priority.

func (*ModelIndex) ResolveWithinBackend 

func (idx *ModelIndex) ResolveWithinBackend(alias string, backend *config.Backend) ([]modelEntry, bool)

ResolveWithinBackend finds model entries for alias scoped to a specific backend.

type Observer 

type Observer interface {
	ObserveAttempt(metric AttemptMetric)
	ObserveRequest(metric RequestMetric)
}

func NewNoopObserver 

func NewNoopObserver() Observer

func NewPrometheusObserver 

func NewPrometheusObserver() (Observer, http.Handler, error)

type ProtocolCodec 

type ProtocolCodec interface {
	// Name returns the protocol identifier (e.g. "openai", "anthropic", "openai-responses").
	Name() string

	// DetectInbound returns true if the given request matches this protocol.
	DetectInbound(path string, headers http.Header) bool

	// ToCanonical converts a request body from this protocol to canonical (Responses) format.
	// For the Responses codec this is a no-op.
	ToCanonical(body []byte, opts EncodeOpts) ([]byte, error)

	// FromCanonical converts a canonical request body to this protocol's outbound format.
	// Returns (body, pathOverride, error).
	FromCanonical(body []byte) ([]byte, string, error)

	// InjectDefaults performs lightweight default injection on a raw protocol body
	// (used in same-protocol passthrough to avoid full decode/encode).
	InjectDefaults(body []byte, opts DefaultOpts) []byte

	// InjectSystemPrompt injects a system prompt into a raw protocol body
	// (used in same-protocol passthrough).
	InjectSystemPrompt(body []byte, prompt string) []byte

	// ResponseToCanonical converts a response body from this protocol to canonical format.
	ResponseToCanonical(body []byte, statusCode int) ([]byte, error)

	// ResponseFromCanonical converts a canonical response body to this protocol's format.
	ResponseFromCanonical(body []byte, statusCode int) ([]byte, error)

	// NewStreamDecoder creates a decoder: this protocol's SSE → canonical (Responses) events.
	NewStreamDecoder() StreamDecoder

	// NewStreamEncoder creates an encoder: canonical events → this protocol's SSE written to w.
	NewStreamEncoder(w http.ResponseWriter) StreamEncoder

	// ExtractUsage extracts usage info from a non-streaming response body in this protocol.
	ExtractUsage(body []byte) logging.UsageInfo
}

ProtocolCodec encapsulates all encode/decode behavior for a single protocol. Canonical format = OpenAI Responses API JSON / SSE event format.

type RequestMetric 

type RequestMetric struct {
	ClientProtocol string
	Backend        string
	Result         string
	StatusCode     int
	ErrorCategory  string
	Duration       time.Duration
	RetryCount     int
}

type Server 

type Server struct {
	Cfg        config.Config
	ConfigPath string
	ModelIndex *ModelIndex
	TokenLog   *logging.TokenLogger
	Debug      *logging.DebugLog
	Client     *http.Client
	Health     *HealthManager
	Selector   *BackendSelector
	Observer   Observer
	Codecs     map[string]ProtocolCodec // protocol name → codec
	// contains filtered or unexported fields
}

func New 

func New(cfg config.Config, tokenLog *logging.TokenLogger, debug *logging.DebugLog, client *http.Client, health *HealthManager, observer Observer) *Server

func (*Server) AllBackendsDown 

func (s *Server) AllBackendsDown() bool

func (*Server) Close 

func (s *Server) Close() error

func (*Server) CurrentConfig 

func (s *Server) CurrentConfig() config.Config

func (*Server) Handle 

func (s *Server) Handle(w http.ResponseWriter, r *http.Request)

func (*Server) RegisterConsoleRoutes 

func (s *Server) RegisterConsoleRoutes(mux *http.ServeMux)

func (*Server) RegisterConsoleRoutesWithOptions 

func (s *Server) RegisterConsoleRoutesWithOptions(mux *http.ServeMux, opts ConsoleRouteOptions)

func (*Server) ReloadConfig 

func (s *Server) ReloadConfig(newCfg config.Config) error

func (*Server) SaveAndReloadConfig 

func (s *Server) SaveAndReloadConfig(path string, cfg config.Config) error

func (*Server) SetConfigPath 

func (s *Server) SetConfigPath(path string)

type StreamDecoder 

type StreamDecoder interface {
	// Decode processes one SSE text line and returns zero or more canonical events.
	Decode(line string) ([]CanonicalEvent, error)
	// Flush returns any remaining events and accumulated usage info.
	Flush() ([]CanonicalEvent, logging.UsageInfo)
}

StreamDecoder converts a backend protocol's SSE lines into canonical events.

type StreamEncoder 

type StreamEncoder interface {
	// Encode writes one canonical event to the client.
	Encode(event CanonicalEvent) error
	// Close writes any final/closing events.
	Close() error
}

StreamEncoder writes canonical events as client-protocol SSE.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.