core

README

llmgw-core

Headless, embeddable LLM routing engine and resilient multi-provider proxy for Go.

Highlights

• Headless & Zero Database: Pure Go http.Handler. Mounts anywhere (net/http, Chi, Gin, Fiber, Echo) without requiring SQLite, PostgreSQL, or any storage engine.
• Bi-Directional Wire Translation: Full Anthropic Messages API (/v1/messages) support translated dynamically to OpenAI backends (and SSE streaming) via llm-translate.
• Resilient Multi-Provider Routing: Route aliases, ordered failover chains, rate-limit 429 detection, and circuit breaking.
• Pluggable Extension Hooks:
  • Authenticator: Custom token, session, or API key validation.
  • PolicyGate: RBAC, project-level, or model-level permission gates.
  • CredentialResolver: Dynamic or multi-tenant credential overrides (BYOC).
  • UsageHook: Telemetry, cost tracking, and audit logging.

Installation

go get github.com/xibodev/llmgw-core

Quick Start (Embedded in 15 Lines)

package main

import (
    "net/http"
    "github.com/xibodev/llmgw-core"
    "github.com/xibodev/llmgw-core/providers"
)

func main() {
    engine := core.NewEngine(core.Config{
        Routes: map[string]core.RouteConfig{
            "smart": {
                Targets: []core.Target{
                    {Provider: "groq", Model: "llama-3.3-70b-versatile"},
                    {Provider: "openai", Model: "gpt-4o"},
                },
            },
        },
    })

    // Register backends
    engine.RegisterProvider("openai", providers.NewOpenAIProvider("openai", "https://api.openai.com/v1", os.Getenv("OPENAI_API_KEY"), nil))
    engine.RegisterProvider("groq", providers.NewOpenAIProvider("groq", "https://api.groq.com/openai/v1", os.Getenv("GROQ_API_KEY"), nil))

    // Mount on standard HTTP server
    http.Handle("/v1/", engine)
    http.ListenAndServe(":8080", nil)
}

License

MIT

Documentation

Index

• type AllowAllPolicy
  • func (AllowAllPolicy) Allows(ctx context.Context, principal *Principal, target Target) (bool, string)
• type Authenticator
• type AuthenticatorFunc
  • func (f AuthenticatorFunc) Authenticate(r *http.Request) (*Principal, error)
• type Config
• type Credential
• type CredentialResolver
• type Engine
  • func NewEngine(cfg Config) *Engine
  • func (e *Engine) RegisterProvider(id string, provider any)
  • func (e *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request)
• type ModelInfo
• type PolicyGate
• type Principal
• type ProviderConfig
• type RouteConfig
• type StaticCredentialResolver
  • func (s StaticCredentialResolver) Resolve(ctx context.Context, principal *Principal, providerID string) (*Credential, error)
• type Target
• type UsageHook
• type UsageHookFunc
  • func (f UsageHookFunc) RecordUsage(ctx context.Context, record UsageRecord)
• type UsageRecord

Types

type AllowAllPolicy

type AllowAllPolicy struct{}

AllowAllPolicy is a PolicyGate that permits all requests.

func (AllowAllPolicy) Allows

func (AllowAllPolicy) Allows(ctx context.Context, principal *Principal, target Target) (bool, string)

type Authenticator

type Authenticator interface {
	Authenticate(r *http.Request) (*Principal, error)
}

Authenticator authenticates an incoming HTTP request. If not configured, requests proceed as anonymous.

type AuthenticatorFunc

type AuthenticatorFunc func(r *http.Request) (*Principal, error)

AuthenticatorFunc is a functional adapter for Authenticator.

func (AuthenticatorFunc) Authenticate

func (f AuthenticatorFunc) Authenticate(r *http.Request) (*Principal, error)

type Config

type Config struct {
	Providers    map[string]ProviderConfig `json:"providers"`
	Routes       map[string]RouteConfig    `json:"routes"`
	DefaultRoute string                    `json:"default_route,omitempty"`

	// Pluggable extension hooks
	Authenticator      Authenticator
	PolicyGate         PolicyGate
	CredentialResolver CredentialResolver
	UsageHook          UsageHook
}

Config configures the core LLM Gateway engine.

type Credential

type Credential struct {
	APIKey             string            `json:"api_key,omitempty"`
	Token              string            `json:"token,omitempty"`
	ConnectionID       string            `json:"connection_id,omitempty"`
	CredentialRevision int64             `json:"credential_revision,omitempty"`
	Headers            map[string]string `json:"headers,omitempty"`
}

Credential contains authentication secrets for an upstream provider.

type CredentialResolver

type CredentialResolver interface {
	Resolve(ctx context.Context, principal *Principal, providerID string) (*Credential, error)
}

CredentialResolver resolves upstream credentials for a principal and provider.

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the core LLM Gateway routing and proxy engine. It implements http.Handler and can be mounted directly on any router.

func NewEngine

func NewEngine(cfg Config) *Engine

func (*Engine) RegisterProvider

func (e *Engine) RegisterProvider(id string, provider any)

RegisterProvider registers an active provider backend.

func (*Engine) ServeHTTP

func (e *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request)

ServeHTTP implements http.Handler.

type ModelInfo

type ModelInfo struct {
	ID          string   `json:"id"`
	Object      string   `json:"object"`
	Created     int64    `json:"created"`
	OwnedBy     string   `json:"owned_by"`
	Description string   `json:"description,omitempty"`
	Tags        []string `json:"tags,omitempty"`
}

ModelInfo describes an available upstream model or alias.

type PolicyGate

type PolicyGate interface {
	Allows(ctx context.Context, principal *Principal, target Target) (bool, string)
}

PolicyGate controls whether a principal is authorized to access a given provider/model target.

type Principal

type Principal struct {
	ID        string         `json:"id"`
	Type      string         `json:"type"` // "user", "api_key", "anonymous", "service"
	Name      string         `json:"name,omitempty"`
	ProjectID string         `json:"project_id,omitempty"`
	Metadata  map[string]any `json:"metadata,omitempty"`
}

Principal represents the authenticated caller making a request.

type ProviderConfig

type ProviderConfig struct {
	Type          string            `json:"type"`                     // "openai_compatible", "anthropic", "googleai", "ollama", "github_copilot", "codex", "edgetts"
	BaseURL       string            `json:"base_url,omitempty"`       // Upstream base endpoint
	APIKey        string            `json:"api_key,omitempty"`        // Static API key or token
	Timeout       time.Duration     `json:"timeout,omitempty"`        // Request timeout
	Headers       map[string]string `json:"headers,omitempty"`        // Extra HTTP headers
	AllowedModels []string          `json:"allowed_models,omitempty"` // Whitelist of models
	Disabled      bool              `json:"disabled,omitempty"`       // Disable provider
}

ProviderConfig configures a single upstream provider backend.

type RouteConfig

type RouteConfig struct {
	Targets     []Target `json:"targets"`               // Ordered targets to try
	Fallback    bool     `json:"fallback,omitempty"`    // Fall back to subsequent targets on error
	Description string   `json:"description,omitempty"` // User-facing description
}

RouteConfig configures an alias (failover chain) across multiple provider targets.

type StaticCredentialResolver

type StaticCredentialResolver struct {
	Credentials map[string]Credential
}

StaticCredentialResolver returns pre-configured credentials from a map.

func (StaticCredentialResolver) Resolve

func (s StaticCredentialResolver) Resolve(ctx context.Context, principal *Principal, providerID string) (*Credential, error)

type Target

type Target struct {
	Provider string `json:"provider"`
	Model    string `json:"model"`
}

Target represents a resolved provider and model.

type UsageHook

type UsageHook interface {
	RecordUsage(ctx context.Context, record UsageRecord)
}

UsageHook receives telemetry records after each completed request.

type UsageHookFunc

type UsageHookFunc func(ctx context.Context, record UsageRecord)

UsageHookFunc is a functional adapter for UsageHook.

func (UsageHookFunc) RecordUsage

func (f UsageHookFunc) RecordUsage(ctx context.Context, record UsageRecord)

type UsageRecord

type UsageRecord struct {
	RequestID    string        `json:"request_id"`
	PrincipalID  string        `json:"principal_id,omitempty"`
	ProjectID    string        `json:"project_id,omitempty"`
	Provider     string        `json:"provider"`
	Model        string        `json:"model"`
	InputTokens  int           `json:"input_tokens"`
	OutputTokens int           `json:"output_tokens"`
	Duration     time.Duration `json:"duration"`
	Stream       bool          `json:"stream"`
	StatusCode   int           `json:"status_code"`
	Error        string        `json:"error,omitempty"`
}

UsageRecord holds metrics and token counts from a completed LLM call.