llm

Documentation

Overview

SPDX-License-Identifier: MIT Purpose: NVIDIA NIM-specific helpers. NIM exposes an OpenAI-compatible chat/completions endpoint at https://integrate.api.nvidia.com/v1 and serves many vendor models. Friendly aliases like "haiku" or "qwen" resolve to a working NIM model ID.

SPDX-License-Identifier: MIT Purpose: generic OpenAI-compatible LLM client. Single-shot chat completion request with bearer auth, JSON marshaling, and typed response decoding.

SPDX-License-Identifier: MIT Purpose: LLM provider definitions for NIM, OpenAI, Anthropic, Ollama, Groq, and any OpenAI-compatible endpoint. Each Provider has a base URL, a default model, and an env-var name for the API key. Agents reference providers by name in their agent.toml.

Index

• Constants
• Variables
• func ListProviderNames() []string
• func ResolveModel(name string) string
• type ChatRequest
• type ChatResponse
  • func (r *ChatResponse) ExtractText() string
• type Client
  • func NewClient(baseURL, apiKey string) *Client
  • func ProviderFromConfig(name, baseURLOverride, apiKeyOverride, modelOverride string, ...) (*Client, error)
  • func (c *Client) Chat(ctx context.Context, req ChatRequest) (*ChatResponse, error)
• type Message
• type Provider
  • func LookupProvider(name string) (Provider, error)

Constants

const NIMDefaultBaseURL = "https://integrate.api.nvidia.com/v1"

const NIMDefaultModel = "meta/llama-3.3-70b-instruct"

const NIMGptOssModel = "openai/gpt-oss-120b"

const NIMHaikuModel = "nvidia/llama-3.1-nemotron-nano-8b-v1"

const NIMKimiModel = "moonshotai/kimi-k2.6"

const NIMNemotronModel = "nvidia/nemotron-3-nano-30b-a3b"

const NIMQwenModel = "qwen/qwen3-coder-480b-a35b-instruct"

Variables

var NIMModelAliases = map[string]string{
	"haiku":         NIMHaikuModel,
	"kimi":          NIMKimiModel,
	"qwen":          NIMQwenModel,
	"nemotron":      NIMNemotronModel,
	"gpt-oss":       NIMGptOssModel,
	"llama-70b":     NIMDefaultModel,
	"llama-3.3-70b": NIMDefaultModel,
	"llama-8b":      "meta/llama-3.1-8b-instruct",
	"default":       NIMDefaultModel,
}

var Providers = map[string]Provider{
	"nim": {
		Name:         "nim",
		BaseURL:      "https://integrate.api.nvidia.com/v1",
		APIKeyEnv:    "SIN_NIM_API_KEY",
		DefaultModel: NIMDefaultModel,
		Description:  "NVIDIA NIM — cloud-hosted open models (Llama, Qwen, Kimi, etc.)",
	},
	"openai": {
		Name:         "openai",
		BaseURL:      "https://api.openai.com/v1",
		APIKeyEnv:    "OPENAI_API_KEY",
		DefaultModel: "gpt-4o",
		Description:  "OpenAI — GPT-4o, o1, etc.",
	},
	"anthropic": {
		Name:         "anthropic",
		BaseURL:      "https://api.anthropic.com/v1",
		APIKeyEnv:    "ANTHROPIC_API_KEY",
		DefaultModel: "claude-sonnet-4-5",
		Description:  "Anthropic — Claude (via OpenAI-compatible proxy or direct)",
	},
	"ollama": {
		Name:         "ollama",
		BaseURL:      "http://127.0.0.1:11434/v1",
		APIKeyEnv:    "",
		DefaultModel: "llama3.1",
		Description:  "Ollama — local models (no API key required)",
	},
	"groq": {
		Name:         "groq",
		BaseURL:      "https://api.groq.com/openai/v1",
		APIKeyEnv:    "GROQ_API_KEY",
		DefaultModel: "llama-3.3-70b-versatile",
		Description:  "Groq — fast inference for open models",
	},
	"custom": {
		Name:         "custom",
		BaseURL:      "",
		APIKeyEnv:    "",
		DefaultModel: "",
		Description:  "Custom OpenAI-compatible endpoint (set SIN_LLM_BASE_URL and SIN_LLM_API_KEY)",
	},
}

Functions

func ListProviderNames

func ListProviderNames() []string

ListProviderNames returns all provider names.

func ResolveModel

func ResolveModel(name string) string

Types

type ChatRequest

type ChatRequest struct {
	Model       string    `json:"model"`
	Messages    []Message `json:"messages"`
	MaxTokens   int       `json:"max_tokens,omitempty"`
	Temperature float64   `json:"temperature,omitempty"`
	Stream      bool      `json:"stream,omitempty"`
}

type ChatResponse

type ChatResponse struct {
	ID      string `json:"id"`
	Object  string `json:"object"`
	Created int64  `json:"created"`
	Model   string `json:"model"`
	Choices []struct {
		Index        int     `json:"index"`
		Message      Message `json:"message"`
		FinishReason string  `json:"finish_reason"`
	} `json:"choices"`
	Usage struct {
		PromptTokens     int `json:"prompt_tokens"`
		CompletionTokens int `json:"completion_tokens"`
		TotalTokens      int `json:"total_tokens"`
	} `json:"usage"`
}

func (*ChatResponse) ExtractText

func (r *ChatResponse) ExtractText() string

type Client

type Client struct {
	BaseURL string
	APIKey  string
	HTTP    *http.Client
}

func NewClient

func NewClient(baseURL, apiKey string) *Client

func ProviderFromConfig

func ProviderFromConfig(name, baseURLOverride, apiKeyOverride, modelOverride string, timeout time.Duration) (*Client, error)

ProviderFromConfig resolves a provider by name with optional overrides:

• baseURL override
• apiKey override
• model override

Returns a Client ready to chat.

func (*Client) Chat

func (c *Client) Chat(ctx context.Context, req ChatRequest) (*ChatResponse, error)

type Message

type Message struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type Provider

type Provider struct {
	Name         string
	BaseURL      string
	APIKeyEnv    string
	DefaultModel string
	Description  string
}

func LookupProvider

func LookupProvider(name string) (Provider, error)

LookupProvider returns the provider for a name, or an error if unknown.