llmgateway

README

llmgateway

llmgateway is an embeddable Go engine for authenticated, policy-controlled, multi-provider LLM inference. It owns protocol normalization, routing, retries, stream commit semantics, guardrails, caching decisions, and accounting state transitions. Hosts provide persistence, identity, secrets, distributed counters, and telemetry through explicit interfaces.

The module intentionally has no dependency on Daptin, Gin, api2go, SQL, database drivers, or distributed-cache clients. Daptin consumes tagged releases and supplies those host integrations from its own repository.

The compatibility surface is declared in compatibility/manifest.json. An endpoint or provider is supported only when its manifest entry is backed by its conformance suite.

Development

go test ./...
go test -race ./...

The example in examples/basic proves that the catalog and lifecycle can be embedded by a service that has no Daptin dependency.

Documentation

Index

• Variables
• type AccountingStore
• type Authenticator
• type Authorizer
• type CatalogSource
• type Clock
• type CounterStore
• type Dependencies
• type Engine
  • func New(dependencies Dependencies, options Options) (*Engine, error)
  • func (e *Engine) Drain(context.Context) error
  • func (e *Engine) Invoke(ctx context.Context, principal contract.Principal, request contract.Request) (contract.Response, error)
  • func (e *Engine) Reload(ctx context.Context) error
  • func (e *Engine) Snapshot() (*catalog.Snapshot, error)
• type Options
• type ResponseCache
• type SecretResolver
• type Selector
• type TelemetryEvent
• type TelemetrySink
• type Timer

Variables

var (
	ErrDraining = errors.New("llmgateway is draining")
	ErrNotReady = errors.New("llmgateway has no valid catalog snapshot")
)

Types

type AccountingStore

type AccountingStore interface {
	Admit(context.Context, contract.Admission) (contract.ReservationToken, error)
	Finalize(context.Context, contract.Completion) error
	Cancel(context.Context, contract.Cancellation) error
	ReapExpired(context.Context, time.Time, int) (contract.ReapResult, error)
}

type Authenticator

type Authenticator interface {
	Authenticate(context.Context, string) (contract.Principal, error)
}

type Authorizer

type Authorizer interface {
	Authorize(context.Context, contract.Principal, catalog.Model) error
}

type CatalogSource

type CatalogSource interface {
	Load(context.Context, uint64) (catalog.Document, error)
}

type Clock

type Clock interface {
	Now() time.Time
	NewTimer(time.Duration) Timer
}

type CounterStore

type CounterStore interface {
	Add(context.Context, string, int64, time.Duration) (int64, error)
	Acquire(context.Context, string, int64, time.Duration) (string, error)
	Release(context.Context, string) error
}

type Dependencies

type Dependencies struct {
	Catalog    CatalogSource
	Secrets    SecretResolver
	Adapters   *adapter.Registry
	Authorizer Authorizer
	Accounting AccountingStore
	Selector   Selector
	Clock      Clock
}

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

func New

func New(dependencies Dependencies, options Options) (*Engine, error)

func (*Engine) Drain

func (e *Engine) Drain(context.Context) error

func (*Engine) Invoke

func (e *Engine) Invoke(ctx context.Context, principal contract.Principal, request contract.Request) (contract.Response, error)

func (*Engine) Reload

func (e *Engine) Reload(ctx context.Context) error

func (*Engine) Snapshot

func (e *Engine) Snapshot() (*catalog.Snapshot, error)

type Options

type Options struct {
	MaxAttempts         int
	FinalizationTimeout time.Duration
}

type ResponseCache

type ResponseCache interface {
	Get(context.Context, string) ([]byte, bool, error)
	Set(context.Context, string, []byte, time.Duration) error
	Delete(context.Context, string) error
}

type SecretResolver

type SecretResolver interface {
	ResolveSecret(context.Context, string) ([]byte, error)
}

type Selector

type Selector interface {
	Intn(int) int
}

type TelemetryEvent

type TelemetryEvent struct {
	Name       string
	RequestID  contract.ID
	Revision   uint64
	Attributes map[string]string
	Measures   map[string]int64
}

type TelemetrySink

type TelemetrySink interface {
	Record(context.Context, TelemetryEvent)
}

type Timer

type Timer interface {
	C() <-chan time.Time
	Stop() bool
}