inference

Documentation

Index

• Constants
• Variables
• func AllowedCacheTypes() []string
• func HTTPErrorMessage(err error) string
• func HTTPStatusCode(err error) int
• func IsROCMHost() bool
• func ModelMaxPositionEmbeddings(modelDir string) int
• func NewHTTPStatusError(status int, body string) error
• func NormalizeCacheType(value string) (string, error)
• func NormalizeNGPULayers(requested int) (int, error)
• func ROCMSingleEngineMode() bool
• func ResolveEmbeddingPooling(modelName string) string
• func ResolveNGPULayers(requested int) int
• func ResolveNumCtx(modelDir string, requested int) int
• func ResolveNumParallel(requested int) int
• func SupportsNativeToolStreaming(eng Engine) bool
• type ChatCompletionProxier
• type ContentPart
• type ConvertProgressFunc
• type EmbeddingsProxier
• type Engine
  • func LoadEmbeddingEngineWithProgress(modelDir string, lm *model.LocalModel, progress ConvertProgressFunc, ...) (Engine, error)
  • func LoadEngine(modelDir string, lm *model.LocalModel) (Engine, error)
  • func LoadEngineWithProgress(modelDir string, lm *model.LocalModel, progress ConvertProgressFunc, ...) (Engine, error)
  • func NewOpenAICompatibleEngine(baseURL, modelName, token string) Engine
  • func NewOpenAIEngine(baseURL, modelName, token string) Engine
  • func NewRemoteEngine(baseURL, modelName string, numCtx, numParallel, nGPULayers int, ...) Engine
• type ImageURL
• type Message
• type NativeToolStreamer
• type Options
  • func DefaultOptions() Options
• type Session
  • func NewSession(engine Engine, opts Options) *Session
  • func (s *Session) Engine() Engine
  • func (s *Session) Messages() []Message
  • func (s *Session) Options() Options
  • func (s *Session) Send(ctx context.Context, userMsg string, onToken TokenCallback) (string, error)
  • func (s *Session) SetSystemPrompt(prompt string)
• type TokenCallback

Constants

const ROCMSingleEngineEnv = "CSGHUB_LITE_ROCM_SINGLE_ENGINE"

Variables

var ErrUnsupportedFormat = errors.New("unsupported model format for inference")

Functions

func AllowedCacheTypes

func AllowedCacheTypes() []string

AllowedCacheTypes returns the llama-server KV cache dtypes accepted by csghub-lite.

func HTTPErrorMessage

func HTTPErrorMessage(err error) string

func HTTPStatusCode

func HTTPStatusCode(err error) int

func IsROCMHost

func IsROCMHost() bool

func ModelMaxPositionEmbeddings

func ModelMaxPositionEmbeddings(modelDir string) int

func NewHTTPStatusError

func NewHTTPStatusError(status int, body string) error

func NormalizeCacheType

func NormalizeCacheType(value string) (string, error)

NormalizeCacheType returns a lower-case llama-server cache dtype or "" when unset.

func NormalizeNGPULayers

func NormalizeNGPULayers(requested int) (int, error)

NormalizeNGPULayers accepts -1 (unset) or any non-negative llama-server --n-gpu-layers value.

func ROCMSingleEngineMode

func ROCMSingleEngineMode() bool

func ResolveEmbeddingPooling

func ResolveEmbeddingPooling(modelName string) string

ResolveEmbeddingPooling returns the llama-server pooling strategy for common embedding model families. The env override is intentionally kept as an escape hatch because GGUF metadata and model cards occasionally disagree.

func ResolveNGPULayers

func ResolveNGPULayers(requested int) int

ResolveNGPULayers returns the effective llama-server GPU layer offload count. Explicit requests win; otherwise GPU-capable hosts default to offloading all layers and CPU-only hosts leave the flag unset.

func ResolveNumCtx

func ResolveNumCtx(modelDir string, requested int) int

ResolveNumCtx returns the effective llama-server context window (--ctx-size / -c). Explicit requests win, then CSGHUB_LITE_LLAMA_NUM_CTX, then a model-aware fallback.

func ResolveNumParallel

func ResolveNumParallel(requested int) int

ResolveNumParallel returns the effective number of parallel slots for llama-server. Explicit requests win, then CSGHUB_LITE_LLAMA_NUM_PARALLEL, then defaultLlamaParallel.

func SupportsNativeToolStreaming

func SupportsNativeToolStreaming(eng Engine) bool

SupportsNativeToolStreaming reports whether eng can stream tool-call responses directly from its backend.

Types

type ChatCompletionProxier

type ChatCompletionProxier interface {
	ChatCompletion(ctx context.Context, reqBody map[string]interface{}) (*http.Response, error)
}

ChatCompletionProxier exposes direct access to the underlying OpenAI-compatible /v1/chat/completions API for advanced use cases such as native Ollama tool-calling compatibility.

type ContentPart

type ContentPart struct {
	Type     string    `json:"type"`
	Text     string    `json:"text,omitempty"`
	ImageURL *ImageURL `json:"image_url,omitempty"`
}

ContentPart represents one part of a multimodal message.

type ConvertProgressFunc

type ConvertProgressFunc func(step string, current, total int)

ConvertProgressFunc receives conversion progress updates. If nil, conversion progress is not reported.

type EmbeddingsProxier

type EmbeddingsProxier interface {
	Embeddings(ctx context.Context, reqBody map[string]interface{}) (*http.Response, error)
}

EmbeddingsProxier exposes direct access to the underlying OpenAI-compatible /v1/embeddings API.

type Engine

type Engine interface {
	// Generate produces text from a prompt, calling onToken for each generated token.
	Generate(ctx context.Context, prompt string, opts Options, onToken TokenCallback) (string, error)

	// Chat produces a response from a conversation history.
	Chat(ctx context.Context, messages []Message, opts Options, onToken TokenCallback) (string, error)

	// Close releases the model resources.
	Close() error

	// ModelName returns the loaded model identifier.
	ModelName() string
}

Engine is the interface for model inference backends.

func LoadEmbeddingEngineWithProgress

func LoadEmbeddingEngineWithProgress(modelDir string, lm *model.LocalModel, progress ConvertProgressFunc, verbose bool, numCtx, numParallel, nGPULayers int, cacheTypeK, cacheTypeV, dtype string) (Engine, error)

LoadEmbeddingEngineWithProgress is like LoadEngineWithProgress but starts llama-server in embedding mode for OpenAI-compatible /v1/embeddings.

func LoadEngine

func LoadEngine(modelDir string, lm *model.LocalModel) (Engine, error)

LoadEngine loads a model and returns an Engine for inference. If the model is SafeTensors, it auto-converts to GGUF first. By default, llama-server output is not mirrored to stderr, but it is still captured for diagnostics and appended to the llama-server log file.

func LoadEngineWithProgress

func LoadEngineWithProgress(modelDir string, lm *model.LocalModel, progress ConvertProgressFunc, verbose bool, numCtx, numParallel, nGPULayers int, cacheTypeK, cacheTypeV, dtype string) (Engine, error)

LoadEngineWithProgress is like LoadEngine but accepts a progress callback for SafeTensors → GGUF conversion. When verbose is true, llama-server output is printed to stderr.

func NewOpenAICompatibleEngine

func NewOpenAICompatibleEngine(baseURL, modelName, token string) Engine

func NewOpenAIEngine

func NewOpenAIEngine(baseURL, modelName, token string) Engine

func NewRemoteEngine

func NewRemoteEngine(baseURL, modelName string, numCtx, numParallel, nGPULayers int, cacheTypeK, cacheTypeV, dtype string) Engine

NewRemoteEngine creates an Engine that delegates to a running csghub-lite server.

type ImageURL

type ImageURL struct {
	URL string `json:"url"`
}

type Message

type Message struct {
	Role             string      `json:"role"`
	Content          interface{} `json:"content"`
	ReasoningContent string      `json:"reasoning_content,omitempty"`
}

Message represents a chat message. Content can be a string for text-only, or an array of content parts for multimodal (e.g., image + text) messages.

type NativeToolStreamer

type NativeToolStreamer interface {
	SupportsNativeToolStreaming() bool
}

NativeToolStreamer marks engines whose backend natively emits OpenAI-compatible streaming tool-call deltas, so tool requests can be proxied with stream enabled instead of being aggregated and normalized locally after the full completion.

type Options

type Options struct {
	Temperature float64
	TopP        float64
	TopK        int
	MaxTokens   int
	Seed        int
	NumCtx      int
	Stop        []string
	// DisableThinking forces routing-style requests to skip provider thinking
	// modes (Qwen enable_thinking=false, GLM/Kimi/DeepSeek thinking.type=disabled).
	DisableThinking bool
}

Options controls generation parameters.

func DefaultOptions

func DefaultOptions() Options

DefaultOptions returns sensible defaults. MaxTokens follows Ollama and llama.cpp semantics: -1 means no explicit generation cap.

type Session

type Session struct {
	// contains filtered or unexported fields
}

Session holds a conversation with context.

func NewSession

func NewSession(engine Engine, opts Options) *Session

NewSession creates a new chat session with the given engine.

func (*Session) Engine

func (s *Session) Engine() Engine

Engine returns the session's inference engine.

func (*Session) Messages

func (s *Session) Messages() []Message

Messages returns the conversation history.

func (*Session) Options

func (s *Session) Options() Options

Options returns the session's generation options.

func (*Session) Send

func (s *Session) Send(ctx context.Context, userMsg string, onToken TokenCallback) (string, error)

Send sends a user message and returns the assistant response via streaming.

func (*Session) SetSystemPrompt

func (s *Session) SetSystemPrompt(prompt string)

SetSystemPrompt sets or replaces the system prompt.

type TokenCallback

type TokenCallback func(token string)

TokenCallback is called for each generated token during streaming.