llmproxy

Documentation

Overview

Package llmproxy implements a multi-provider LLM proxy server

Index

• type ChatChunk
• type ChatRequest
• type ChatResponse
• type Choice
• type Config
  • func DefaultConfig() *Config
• type Cost
• type CostBudgetConfig
• type FunctionCall
• type FunctionSpec
• type LogEntry
• type Message
• type MessageDelta
• type Model
• type Provider
• type ProviderConfig
• type ProviderType
  • func (p ProviderType) IsValid() bool
  • func (p ProviderType) String() string
• type RateLimitConfig
• type Role
  • func (r Role) String() string
• type StreamError
  • func (e *StreamError) Error() string
• type StreamEvent
• type Tool
• type ToolCall
• type Usage

Types

type ChatChunk

type ChatChunk struct {
	ID       string       `json:"id"`
	Object   string       `json:"object"`
	Created  int64        `json:"created"`
	Model    string       `json:"model"`
	Choices  []Choice     `json:"choices"`
	Provider ProviderType `json:"_provider,omitempty"`
}

ChatChunk represents a chunk in a streaming response

type ChatRequest

type ChatRequest struct {
	Model           string       `json:"model"`
	Messages        []Message    `json:"messages"`
	Tools           []Tool       `json:"tools,omitempty"`
	ToolChoice      any          `json:"tool_choice,omitempty"`
	Temperature     float64      `json:"temperature,omitempty"`
	MaxTokens       int          `json:"max_tokens,omitempty"`
	TopP            float64      `json:"top_p,omitempty"`
	Stream          bool         `json:"stream,omitempty"`
	Stop            []string     `json:"stop,omitempty"`
	Provider        ProviderType `json:"-"` // Set by proxy based on routing
	ProviderAPIKey  string       `json:"-"` // API key for the provider
	ProviderBaseURL string       `json:"-"` // Optional custom base URL
}

ChatRequest is a request to generate a chat completion

type ChatResponse

type ChatResponse struct {
	ID               string       `json:"id"`
	Object           string       `json:"object"`
	Created          int64        `json:"created"`
	Model            string       `json:"model"`
	Choices          []Choice     `json:"choices"`
	Usage            Usage        `json:"usage"`
	Provider         ProviderType `json:"_provider,omitempty"`
	ProviderResponse any          `json:"_provider_response,omitempty"`
}

ChatResponse is the response from a chat completion

type Choice

type Choice struct {
	Index        int           `json:"index"`
	Message      Message       `json:"message"`
	FinishReason string        `json:"finish_reason"`
	Delta        *MessageDelta `json:"delta,omitempty"`
}

Choice represents a choice in the response

type Config

type Config struct {
	ListenAddr   string                          `json:"listen_addr" yaml:"listen_addr"`
	LogLevel     string                          `json:"log_level" yaml:"log_level"`
	Providers    map[ProviderType]ProviderConfig `json:"providers" yaml:"providers"`
	DefaultModel string                          `json:"default_model,omitempty" yaml:"default_model,omitempty"`
	ModelMapping map[string]ProviderType         `json:"model_mapping,omitempty" yaml:"model_mapping,omitempty"`

	// Rate limiting
	RateLimit RateLimitConfig `json:"rate_limit,omitempty" yaml:"rate_limit,omitempty"`

	// Cost tracking
	CostBudget CostBudgetConfig `json:"cost_budget,omitempty" yaml:"cost_budget,omitempty"`

	// Logging
	LogRequests  bool `json:"log_requests" yaml:"log_requests"`
	LogResponses bool `json:"log_responses" yaml:"log_responses"`
	RedactPII    bool `json:"redact_pii" yaml:"redact_pii"`
}

Config holds the proxy server configuration

func DefaultConfig

func DefaultConfig() *Config

DefaultConfig returns a default configuration

type Cost

type Cost struct {
	InputCost    float64      `json:"input_cost"`
	OutputCost   float64      `json:"output_cost"`
	TotalCost    float64      `json:"total_cost"`
	Currency     string       `json:"currency"`
	InputTokens  int          `json:"input_tokens"`
	OutputTokens int          `json:"output_tokens"`
	TotalTokens  int          `json:"total_tokens"`
	Model        string       `json:"model"`
	Provider     ProviderType `json:"provider"`
	Timestamp    time.Time    `json:"timestamp"`
}

Cost represents cost information for a request

type CostBudgetConfig

type CostBudgetConfig struct {
	HourlyLimit float64 `json:"hourly_limit" yaml:"hourly_limit"`
	DailyLimit  float64 `json:"daily_limit" yaml:"daily_limit"`
	Enabled     bool    `json:"enabled" yaml:"enabled"`
}

CostBudgetConfig holds cost budget configuration

type FunctionCall

type FunctionCall struct {
	Name      string `json:"name"`
	Arguments string `json:"arguments"`
}

FunctionCall represents a function call

type FunctionSpec

type FunctionSpec struct {
	Name        string                 `json:"name"`
	Description string                 `json:"description"`
	Parameters  map[string]interface{} `json:"parameters"`
}

FunctionSpec represents a function specification

type LogEntry

type LogEntry struct {
	ID         string        `json:"id"`
	Timestamp  time.Time     `json:"timestamp"`
	Provider   ProviderType  `json:"provider"`
	Model      string        `json:"model"`
	RequestID  string        `json:"request_id"`
	Request    ChatRequest   `json:"request,omitempty"`
	Response   ChatResponse  `json:"response,omitempty"`
	Error      string        `json:"error,omitempty"`
	Duration   time.Duration `json:"duration"`
	TokenUsage Usage         `json:"token_usage,omitempty"`
	Cost       *Cost         `json:"cost,omitempty"`
	IPAddress  string        `json:"ip_address,omitempty"`
	UserAgent  string        `json:"user_agent,omitempty"`
}

LogEntry represents a logged request/response

type Message

type Message struct {
	Role    Role   `json:"role"`
	Content string `json:"content"`

	// Tool-related fields
	ToolCallID string `json:"tool_call_id,omitempty"`
	Name       string `json:"name,omitempty"`

	// Tool calls (for assistant messages)
	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}

Message represents a chat message

type MessageDelta

type MessageDelta struct {
	Role      Role       `json:"role,omitempty"`
	Content   string     `json:"content,omitempty"`
	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}

MessageDelta represents a delta in a streaming response

type Model

type Model struct {
	ID          string       `json:"id"`
	Name        string       `json:"name"`
	Provider    ProviderType `json:"provider"`
	ContextSize int          `json:"context_size"`

	// Pricing (per 1M tokens)
	InputPrice  float64 `json:"input_price,omitempty"`
	OutputPrice float64 `json:"output_price,omitempty"`
}

Model represents an available model

type Provider

type Provider interface {
	// Name returns the provider name
	Name() ProviderType

	// StreamChat generates a streaming chat completion
	StreamChat(ctx context.Context, req *ChatRequest) (<-chan ChatChunk, error)

	// Chat generates a non-streaming chat completion
	Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)

	// GetModels returns a list of available models
	GetModels() []Model

	// CalculateCost calculates the cost for a given usage
	CalculateCost(model string, usage *Usage) (*Cost, error)

	// Validate validates the provider configuration
	Validate() error
}

Provider is the interface that all LLM providers must implement

type ProviderConfig

type ProviderConfig struct {
	Type       ProviderType      `json:"type" yaml:"type"`
	APIKey     string            `json:"api_key,omitempty" yaml:"api_key"`
	BaseURL    string            `json:"base_url,omitempty" yaml:"base_url"`
	Enabled    bool              `json:"enabled" yaml:"enabled"`
	ModelAlias map[string]string `json:"model_alias,omitempty" yaml:"model_alias,omitempty"`
}

ProviderConfig holds configuration for a single provider

type ProviderType

type ProviderType string

ProviderType identifies the LLM provider

const (
	ProviderAnthropic ProviderType = "anthropic"
	ProviderOpenAI    ProviderType = "openai"
	ProviderGLM       ProviderType = "glm"
	ProviderGroq      ProviderType = "groq"
	ProviderGrok      ProviderType = "grok"
)

func (ProviderType) IsValid

func (p ProviderType) IsValid() bool

IsValid checks if the provider type is valid

func (ProviderType) String

func (p ProviderType) String() string

String returns the string representation of the provider type

type RateLimitConfig

type RateLimitConfig struct {
	RequestsPerMinute int  `json:"requests_per_minute" yaml:"requests_per_minute"`
	TokensPerMinute   int  `json:"tokens_per_minute" yaml:"tokens_per_minute"`
	Enabled           bool `json:"enabled" yaml:"enabled"`
}

RateLimitConfig holds rate limiting configuration

type Role

type Role string

Role represents the role of a message sender

const (
	RoleSystem    Role = "system"
	RoleUser      Role = "user"
	RoleAssistant Role = "assistant"
	RoleTool      Role = "tool"
)

func (Role) String

func (r Role) String() string

String returns the string representation of the role

type StreamError

type StreamError struct {
	Provider ProviderType
	Message  string
	Code     string
}

StreamError represents an error during streaming

func (*StreamError) Error

func (e *StreamError) Error() string

Error returns the error message

type StreamEvent

type StreamEvent struct {
	Data  any    `json:"data"`
	Event string `json:"event,omitempty"` // Optional event type
	Error error  `json:"error,omitempty"`
}

StreamEvent is the event sent via SSE

type Tool

type Tool struct {
	Type     string       `json:"type"`
	Function FunctionSpec `json:"function"`
}

Tool represents a tool definition

type ToolCall

type ToolCall struct {
	ID       string       `json:"id"`
	Type     string       `json:"type"`
	Function FunctionCall `json:"function"`
}

ToolCall represents a tool/function call

type Usage

type Usage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

Usage represents token usage information