server

Documentation

Index

• Constants
• func NewResearchTools(opts ResearchOptions) []gopherllm.AgenticTool
• func Serve(initialRunner *gopherllm.Runner, opts ServeOptions) error
• type APIMessage
• type DeploymentMode
  • func ParseDeploymentMode(value string) (DeploymentMode, error)
• type EmbeddingsRequest
  • func (e EmbeddingsRequest) Inputs() []string
• type GenerateRequest
  • func (g GenerateRequest) ToMessagesAndOptions(def gopherllm.GenerationOptions) ([]gopherllm.ChatMessage, gopherllm.GenerationOptions)
• type Handler
  • func HandlerForModel(m *gopherllm.Model, opts HandlerOptions) *Handler
  • func NewHandler(initialRunner *gopherllm.Runner, opts HandlerOptions) *Handler
  • func (h *Handler) Close() error
  • func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request)
• type HandlerOptions
• type OllamaChatRequest
  • func (o OllamaChatRequest) ChatMessages() []gopherllm.ChatMessage
  • func (o OllamaChatRequest) GenerationOptions(def gopherllm.GenerationOptions) gopherllm.GenerationOptions
• type OllamaEmbedRequest
  • func (o OllamaEmbedRequest) Inputs() []string
• type OllamaEmbeddingRequest
  • func (o OllamaEmbeddingRequest) Inputs() []string
• type OllamaGenerateRequest
  • func (o OllamaGenerateRequest) GenerationOptions(def gopherllm.GenerationOptions) gopherllm.GenerationOptions
• type OllamaMessage
• type OllamaOptions
• type OpenAIChatRequest
  • func (o OpenAIChatRequest) ChatMessages() []gopherllm.ChatMessage
  • func (o OpenAIChatRequest) ContextWindowMode() (gopherllm.ContextWindowMode, error)
  • func (o OpenAIChatRequest) Options(def gopherllm.GenerationOptions) gopherllm.GenerationOptions
  • func (o OpenAIChatRequest) SkillsEnabled() bool
• type OpenAICompletionRequest
  • func (o OpenAICompletionRequest) Options(def gopherllm.GenerationOptions) gopherllm.GenerationOptions
  • func (o OpenAICompletionRequest) PromptString() string
• type OpenAIStreamOpts
• type ResearchOptions
• type ServeOptions

Constants

const (
	// AdminTokenHeader is the preferred header for API clients and the Web UI.
	// Authorization: Bearer <token> is accepted as well for standard tooling.
	AdminTokenHeader = "X-GopherLLM-Admin-Token"
)

const OSMUsageNotice = "" /* 320-byte string literal not displayed */

OSMUsageNotice must be shown to anyone enabling the public Nominatim endpoint. It is intentionally prominent because the public service permits only user-triggered, low-volume lookups and must not receive personal data.

Functions

func NewResearchTools

func NewResearchTools(opts ResearchOptions) []gopherllm.AgenticTool

NewResearchTools returns opt-in factual research tools for direct Go use with RunAgenticChatWithTools. A caller that does not enable any source gets nil, so importing server never causes network activity on its own.

func Serve

func Serve(initialRunner *gopherllm.Runner, opts ServeOptions) error

Serve builds the API handler and runs a blocking http.Server on opts.Addr. Library consumers who want to control the server lifecycle, add middleware, TLS, or mount the API under a path prefix should use NewHandler instead:

handler := gopherllm.NewHandler(model.Runner(), gopherllm.HandlerOptions{...})
mux.Handle("/llm/", http.StripPrefix("/llm", handler))

Types

type APIMessage

type APIMessage struct {
	Role       string               `json:"role"`
	Content    any                  `json:"content"`
	ToolCalls  []gopherllm.ToolCall `json:"tool_calls,omitempty"`
	ToolCallID string               `json:"tool_call_id,omitempty"`
	Name       string               `json:"name,omitempty"`
}

type DeploymentMode

type DeploymentMode string

DeploymentMode describes where GopherLLM runs and who may change shared server state. It deliberately does not model end-user authentication: a managed deployment can put its normal users behind an identity-aware reverse proxy while this package keeps its own small, explicit admin boundary.

const (
	// DeploymentLocal is the single-user profile. Serve only accepts a loopback
	// address for it. Embedders that create their own listener are responsible
	// for binding it to loopback as well.
	DeploymentLocal DeploymentMode = "local"
	// DeploymentManaged is for a server shared by users. Generation remains
	// publicly available to the surrounding deployment, but shared settings and
	// host-side actions require the configured administrator token.
	DeploymentManaged DeploymentMode = "managed"
	// DeploymentBrowser serves the chat application and its WASM runtime, but
	// deliberately never runs a model in the server process. Each browser tab
	// loads its own GGUF and uses WebGPU when available.
	DeploymentBrowser DeploymentMode = "browser"
)

func ParseDeploymentMode

func ParseDeploymentMode(value string) (DeploymentMode, error)

ParseDeploymentMode parses a stable deployment-mode value. The empty value intentionally preserves the original, local-only behavior for existing HandlerOptions users.

type EmbeddingsRequest

type EmbeddingsRequest struct {
	Model string `json:"model"`
	Input any    `json:"input"`
}

func (EmbeddingsRequest) Inputs

func (e EmbeddingsRequest) Inputs() []string

type GenerateRequest

type GenerateRequest struct {
	Prompt        string                     `json:"prompt"`
	Messages      []APIMessage               `json:"messages"`
	MaxTokens     *int                       `json:"max_tokens"`
	Temp          *float32                   `json:"temp"`
	Temperature   *float32                   `json:"temperature"`
	TopP          *float32                   `json:"top_p"`
	TopK          *int                       `json:"top_k"`
	MinP          *float32                   `json:"min_p"`
	RepeatPenalty *float32                   `json:"repeat_penalty"`
	Seed          *uint64                    `json:"seed"`
	SystemPrompt  *string                    `json:"system_prompt"`
	Stop          any                        `json:"stop"`
	Tools         []gopherllm.ToolDefinition `json:"tools"`
	ToolChoice    any                        `json:"tool_choice"`
	Wikimedia     bool                       `json:"gopherllm_wikimedia"`
	OpenStreetMap bool                       `json:"gopherllm_openstreetmap"`
}

func (GenerateRequest) ToMessagesAndOptions

func (g GenerateRequest) ToMessagesAndOptions(def gopherllm.GenerationOptions) ([]gopherllm.ChatMessage, gopherllm.GenerationOptions)

type Handler

type Handler struct {
	// contains filtered or unexported fields
}

Handler is the mountable HTTP API returned by NewHandler. Close releases the currently active chat and embedding runners, including memory-mapped GGUF files installed through a hot-swap. Hosts should stop their HTTP server before calling Close so no new requests can enter.

func HandlerForModel

func HandlerForModel(m *gopherllm.Model, opts HandlerOptions) *Handler

HandlerForModel serves a Model opened through the high-level API. It is the replacement for the former gopherllm.Model.HTTPHandler method, which had to go when HTTP serving moved out of the inference package: the handler shares the Model's underlying Runner, so requests serialize with direct Model calls.

func NewHandler

func NewHandler(initialRunner *gopherllm.Runner, opts HandlerOptions) *Handler

NewHandler returns the complete GopherLLM HTTP API (OpenAI-compatible, Ollama-compatible, and native endpoints — see the README's endpoint table) as a mountable http.Handler. It owns no listener, manages runners installed by model hot-swaps, and writes nothing except to opts.LogWriter, so it composes with any router, middleware stack, or server the host application already has. After stopping that server, call Handler.Close to release the active runners and their memory-mapped GGUF files.

func (*Handler) Close

func (h *Handler) Close() error

func (*Handler) ServeHTTP

func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request)

type HandlerOptions

type HandlerOptions struct {
	// DeploymentMode selects the ownership boundary for inference and shared
	// server settings. The zero value is DeploymentLocal for backward
	// compatibility. See DeploymentMode for the semantics of local, managed,
	// and browser deployments.
	DeploymentMode DeploymentMode
	// AdminToken protects server-wide management routes in managed mode. It is
	// never returned by an endpoint or inserted into the Web UI. API clients may
	// send it through X-GopherLLM-Admin-Token or Authorization: Bearer.
	AdminToken string
	// Defaults are the generation settings requests inherit unless they
	// override individual fields.
	Defaults gopherllm.GenerationOptions
	// MaxConcurrentRequests bounds in-flight generation requests (default 8).
	// Requests beyond the bound queue rather than failing.
	MaxConcurrentRequests int
	// ChatUI serves the embedded browser chat at /chat (plus its assets).
	ChatUI bool
	// ChatHistoryPath enables the opt-in server-side browser workspace. The
	// history is compressed and atomically replaced at this path; an empty path
	// keeps browser-local IndexedDB/localStorage as the default.
	ChatHistoryPath string
	// ModelDir enables GET /models discovery and POST /models/load hot-swap.
	// A load request is resolved against this directory's discovered,
	// supported GGUF files; arbitrary filesystem paths are never loaded.
	ModelDir string
	// ModelPath is the initially loaded model's path (reported by /models).
	ModelPath string
	// WasmDir, if it contains both gopherllm.wasm and wasm_exec.js (see
	// `make wasm-build`), enables the chat UI's "run locally in this
	// browser tab" mode: those two files are served at /wasm/gopherllm.wasm
	// and /wasm/wasm_exec.js, and chatTemplateData.HasLocalRuntime is set so
	// the page can offer the toggle. Either file missing (or WasmDir unset)
	// means that mode simply isn't offered — the rest of the server is
	// unaffected either way.
	WasmDir string
	// ModelLoadOptions are retained for catalog hot-swaps, so a server started
	// in out-of-core mode does not accidentally load the next model eagerly.
	ModelLoadOptions gopherllm.LoadOptions
	// ModelLoaded is called after a local model has been successfully loaded or
	// hot-swapped. It is useful for hosts that persist the active selection.
	// Callback failures are the host's responsibility and never undo a load.
	ModelLoaded func(path string)
	// SkillsDir, if set, is scanned once at handler construction for SKILL.md
	// files (see skills.go). Every chat/generate endpoint offers a load_skill
	// tool and resolves it server-side via gopherllm.RunAgenticChat.
	SkillsDir string
	// AppliedAutoTune, if set, is a tuning already applied to initialRunner
	// before the handler was built (e.g. by the CLI's --auto flag). GET
	// /autotune reports it as active from the very first request, rather than
	// only after someone hits POST /autotune/run through the web UI.
	AppliedAutoTune *gopherllm.AutoTuneResult
	// BaselineRuntimeTuning is the process-wide configuration to restore after
	// a hot-swap to an uncalibrated model. When absent, NewHandler captures the
	// settings visible at construction time.
	BaselineRuntimeTuning *gopherllm.RuntimeTuning
	// LogWriter receives handler diagnostics (skill load notes). Defaults to
	// io.Discard.
	LogWriter io.Writer
	// AgentOS enables the agentic OS-command feature (a model proposes a local
	// shell command, this Runner's Policy decides whether it needs a human
	// click before it runs) when non-nil. Nil, the default, registers no
	// /agentos endpoints at all — the feature does not exist on this server
	// unless an operator deliberately configured a policy for it. See the
	// agentos package for the safety model.
	AgentOS *agentos.Runner
	// OSMSearchURL optionally replaces the public Nominatim endpoint with an
	// operator-managed compatible endpoint. The source remains disabled until
	// a request explicitly sets gopherllm_openstreetmap to true.
	OSMSearchURL string
}

HandlerOptions configures the mountable HTTP API handler.

type OllamaChatRequest

type OllamaChatRequest struct {
	Model         string                     `json:"model"`
	Messages      []OllamaMessage            `json:"messages"`
	Stream        *bool                      `json:"stream"`
	Options       OllamaOptions              `json:"options"`
	Tools         []gopherllm.ToolDefinition `json:"tools"`
	Wikimedia     bool                       `json:"gopherllm_wikimedia"`
	OpenStreetMap bool                       `json:"gopherllm_openstreetmap"`
}

func (OllamaChatRequest) ChatMessages

func (o OllamaChatRequest) ChatMessages() []gopherllm.ChatMessage

func (OllamaChatRequest) GenerationOptions

func (o OllamaChatRequest) GenerationOptions(def gopherllm.GenerationOptions) gopherllm.GenerationOptions

type OllamaEmbedRequest

type OllamaEmbedRequest struct {
	Model string `json:"model"`
	Input any    `json:"input"`
}

OllamaEmbedRequest is the request body for /api/embed, the batched successor to the deprecated single-prompt /api/embeddings.

func (OllamaEmbedRequest) Inputs

func (o OllamaEmbedRequest) Inputs() []string

type OllamaEmbeddingRequest

type OllamaEmbeddingRequest struct {
	Model  string `json:"model"`
	Prompt string `json:"prompt"`
	Input  any    `json:"input"`
}

func (OllamaEmbeddingRequest) Inputs

func (o OllamaEmbeddingRequest) Inputs() []string

type OllamaGenerateRequest

type OllamaGenerateRequest struct {
	Model         string        `json:"model"`
	Prompt        string        `json:"prompt"`
	System        string        `json:"system"`
	Stream        *bool         `json:"stream"`
	Options       OllamaOptions `json:"options"`
	Stop          any           `json:"stop"`
	Wikimedia     bool          `json:"gopherllm_wikimedia"`
	OpenStreetMap bool          `json:"gopherllm_openstreetmap"`
}

func (OllamaGenerateRequest) GenerationOptions

func (o OllamaGenerateRequest) GenerationOptions(def gopherllm.GenerationOptions) gopherllm.GenerationOptions

type OllamaMessage

type OllamaMessage struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type OllamaOptions

type OllamaOptions struct {
	// NumCtx is accepted for wire compatibility (real Ollama clients set it
	// routinely) but not actionable here: a gopherllm.Runner's KV cache is sized once
	// from the loaded GGUF's context_length at model-load time, and this
	// server has no per-request context-window resize.
	NumCtx        *int     `json:"num_ctx"`
	NumPredict    *int     `json:"num_predict"`
	Temperature   *float32 `json:"temperature"`
	TopP          *float32 `json:"top_p"`
	TopK          *int     `json:"top_k"`
	MinP          *float32 `json:"min_p"`
	RepeatPenalty *float32 `json:"repeat_penalty"`
	Seed          *uint64  `json:"seed"`
	Stop          any      `json:"stop"`
}

type OpenAIChatRequest

type OpenAIChatRequest struct {
	Model               string                     `json:"model"`
	Messages            []APIMessage               `json:"messages"`
	Stream              bool                       `json:"stream"`
	StreamOptions       *OpenAIStreamOpts          `json:"stream_options"`
	MaxTokens           *int                       `json:"max_tokens"`
	MaxCompletionTokens *int                       `json:"max_completion_tokens"`
	Temperature         *float32                   `json:"temperature"`
	TopP                *float32                   `json:"top_p"`
	TopK                *int                       `json:"top_k"`
	MinP                *float32                   `json:"min_p"`
	RepeatPenalty       *float32                   `json:"repeat_penalty"`
	Seed                *uint64                    `json:"seed"`
	SystemPrompt        *string                    `json:"system_prompt"`
	Stop                any                        `json:"stop"`
	Tools               []gopherllm.ToolDefinition `json:"tools"`
	ToolChoice          any                        `json:"tool_choice"`
	// GopherLLMContextMode is an opt-in extension for local clients. Omitting
	// it preserves normal OpenAI-compatible full-history semantics.
	GopherLLMContextMode string `json:"gopherllm_context_mode"`
	Wikimedia            bool   `json:"gopherllm_wikimedia"`
	OpenStreetMap        bool   `json:"gopherllm_openstreetmap"`
	// Skills is a pointer so an absent field keeps the historical default
	// (skills offered whenever --skills-dir is configured) while a client that
	// wants them off can say so.
	Skills *bool `json:"gopherllm_skills"`
}

func (OpenAIChatRequest) ChatMessages

func (o OpenAIChatRequest) ChatMessages() []gopherllm.ChatMessage

func (OpenAIChatRequest) ContextWindowMode

func (o OpenAIChatRequest) ContextWindowMode() (gopherllm.ContextWindowMode, error)

gopherllm.ContextWindowMode parses GopherLLM's local context-window extension. It is deliberately separate from Options so existing callers that use Options directly retain the zero-value (full history) behavior.

func (OpenAIChatRequest) Options

func (o OpenAIChatRequest) Options(def gopherllm.GenerationOptions) gopherllm.GenerationOptions

func (OpenAIChatRequest) SkillsEnabled

func (o OpenAIChatRequest) SkillsEnabled() bool

SkillsEnabled reports whether this request wants the load_skill tool offered.

type OpenAICompletionRequest

type OpenAICompletionRequest struct {
	Model               string   `json:"model"`
	Prompt              any      `json:"prompt"`
	MaxTokens           *int     `json:"max_tokens"`
	MaxCompletionTokens *int     `json:"max_completion_tokens"`
	Temperature         *float32 `json:"temperature"`
	TopP                *float32 `json:"top_p"`
	TopK                *int     `json:"top_k"`
	MinP                *float32 `json:"min_p"`
	RepeatPenalty       *float32 `json:"repeat_penalty"`
	Seed                *uint64  `json:"seed"`
	SystemPrompt        *string  `json:"system_prompt"`
	Stop                any      `json:"stop"`
}

func (OpenAICompletionRequest) Options

func (o OpenAICompletionRequest) Options(def gopherllm.GenerationOptions) gopherllm.GenerationOptions

func (OpenAICompletionRequest) PromptString

func (o OpenAICompletionRequest) PromptString() string

type OpenAIStreamOpts

type OpenAIStreamOpts struct {
	IncludeUsage bool `json:"include_usage"`
}

OpenAIStreamOpts is the OpenAI "stream_options" object; IncludeUsage gates whether the final SSE chunk carries a "usage" field (off by default, per spec — unlike a non-streaming response, which always includes usage).

type ResearchOptions

type ResearchOptions struct {
	Wikimedia     bool
	OpenStreetMap bool
	HTTPClient    *http.Client
	OSMSearchURL  string
}

ResearchOptions selects bounded, server-owned factual source tools for Go applications. Every source defaults to disabled; callers choose explicitly which sources their product and privacy policy permit.

type ServeOptions

type ServeOptions struct {
	// Context controls the lifetime of the listener. Cancelling it gracefully
	// stops accepting requests and releases the active runner.
	Context                  context.Context
	Addr                     string
	DeploymentMode           DeploymentMode
	AdminToken               string
	Defaults                 gopherllm.GenerationOptions
	MaxConcurrentConnections int
	ChatUI                   bool
	ChatHistoryPath          string
	ChatHistoryLock          *sync.Mutex
	ModelDir                 string
	ModelPath                string
	// WasmDir is forwarded to HandlerOptions.WasmDir.
	WasmDir          string
	ModelLoadOptions gopherllm.LoadOptions
	ModelLoaded      func(path string)
	SkillsDir        string
	// AppliedAutoTune carries forward a tuning already applied before Serve
	// was called (e.g. by --auto), so GET /autotune reports it from the start.
	AppliedAutoTune *gopherllm.AutoTuneResult
	// BaselineRuntimeTuning forwards the pre-auto runtime settings captured by
	// a host such as the CLI, so a later model hot-swap can restore them.
	BaselineRuntimeTuning *gopherllm.RuntimeTuning
	// LogWriter receives startup and handler diagnostics; Serve defaults it
	// to os.Stderr (CLI behavior), unlike NewHandler's io.Discard.
	LogWriter io.Writer
	// AgentOS enables the agentic OS-command feature; see HandlerOptions.AgentOS.
	AgentOS *agentos.Runner
	// OSMSearchURL is forwarded to HandlerOptions.OSMSearchURL.
	OSMSearchURL string
}

ServeOptions is HandlerOptions plus the listen address, for the Serve convenience wrapper (used by the CLI). ChatHistoryLock remains for source compatibility with older hosts; the handler serializes its own file access.