litellm

package module
v1.5.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 17, 2026 License: Apache-2.0 Imports: 17 Imported by: 1

README

LiteLLM (Go) — Multi‑Provider LLM Client

中文 | English

LiteLLM is a small, typed Go client that lets you call multiple LLM providers through one API.

Get Started

Install
go get github.com/voocel/litellm
1) Prepare an API key
export OPENAI_API_KEY="your-key"
2) Quick examples (minimal runnable)
Text (chat)
package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"github.com/voocel/litellm"
)

func main() {
	client, err := litellm.NewWithProvider("openai", litellm.ProviderConfig{
		APIKey: os.Getenv("OPENAI_API_KEY"),
	})
	if err != nil {
		log.Fatal(err)
	}

	resp, err := client.Chat(context.Background(), &litellm.Request{
		Model:    "gpt-4o-mini",
		Messages: []litellm.Message{litellm.NewUserMessage("Explain AI in one sentence.")},
	})
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(resp.Content)
}
Tool calling
package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"github.com/voocel/litellm"
)

func main() {
	client, err := litellm.NewWithProvider("openai", litellm.ProviderConfig{
		APIKey: os.Getenv("OPENAI_API_KEY"),
	})
	if err != nil {
		log.Fatal(err)
	}

	tools := []litellm.Tool{
		litellm.NewTool("get_weather", "Get weather for a city.", map[string]any{
			"type": "object",
			"properties": map[string]any{
				"city": map[string]any{"type": "string"},
			},
			"required": []string{"city"},
		}),
	}

	resp, err := client.Chat(context.Background(), &litellm.Request{
		Model:      "gpt-4o-mini",
		Messages:   []litellm.Message{litellm.NewUserMessage("Weather in Tokyo?")},
		Tools:      tools,
		ToolChoice: "auto",
	})
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(resp.Content)
}
Streaming (collect)
package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"github.com/voocel/litellm"
)

func main() {
	client, err := litellm.NewWithProvider("openai", litellm.ProviderConfig{
		APIKey: os.Getenv("OPENAI_API_KEY"),
	})
	if err != nil {
		log.Fatal(err)
	}

	stream, err := client.Stream(context.Background(), &litellm.Request{
		Model:    "gpt-4o-mini",
		Messages: []litellm.Message{litellm.NewUserMessage("Tell me a joke.")},
	})
	if err != nil {
		log.Fatal(err)
	}
	defer stream.Close()

resp, err := litellm.CollectStream(stream)
if err != nil {
	log.Fatal(err)
}
fmt.Println(resp.Content)
}

If you need real-time streaming and a final aggregated response:

resp, err := litellm.CollectStreamWithHandler(stream, func(chunk *litellm.StreamChunk) {
	if chunk.Type == litellm.ChunkTypeContent && chunk.Content != "" {
		fmt.Print(chunk.Content)
	}
	if chunk.Reasoning != nil && chunk.Reasoning.Done {
		fmt.Print("\n[reasoning done]")
	}
})
if err != nil {
	log.Fatal(err)
}
fmt.Println("\n---")
fmt.Println(resp.Content)

Notes

  • The providers package is an internal implementation detail. End users should only import github.com/voocel/litellm.
  • LiteLLM does not auto-discover providers or auto-route models. You must configure providers explicitly.

Core API

  • New(provider, opts...) builds a client with an explicit provider.
  • NewWithProvider(name, config, opts...) builds a client from a provider name and config.
  • Request is provider‑agnostic: set Model and Messages, then optional controls like MaxTokens, Temperature, TopP, Stop, etc.
  • Chat(ctx, req) returns a unified Response.
  • Stream(ctx, req) returns a StreamReader (not goroutine‑safe). Always defer stream.Close().
  • CollectStream(stream) collects a stream into a unified Response.
  • CollectStreamWithHandler(stream, onChunk) collects and also handles each chunk.
  • CollectStreamWithCallbacks(stream, callbacks) adds content/reasoning/tool callbacks.
  • Request.Thinking controls thinking output (default enabled; set to disabled to turn off).
  • ListModels(ctx) lists available models for the current provider (only some providers; fields are best‑effort).
Streaming (minimal)
stream, err := client.Stream(ctx, &litellm.Request{
	Model: "gpt-4o-mini",
	Messages: []litellm.Message{
		{Role: "user", Content: "Tell me a joke."},
	},
})
if err != nil {
	log.Fatal(err)
}
defer stream.Close()

resp, err := litellm.CollectStream(stream)
if err != nil {
	log.Fatal(err)
}
fmt.Print(resp.Content)

Advanced Features (optional)

Each feature below works across providers. Longer runnable examples live in examples/.

Model listing (some providers)

Notes

  • Supported today: OpenAI / Anthropic / Gemini / OpenRouter / DeepSeek / Bedrock
  • Returned fields vary by provider; ModelInfo is best‑effort
  • Gemini model IDs are normalized (the models/ prefix is removed)
  • Bedrock control plane can be overridden via ProviderConfig.Extra["control_plane_base_url"]
models, err := client.ListModels(ctx)
if err != nil {
	log.Fatal(err)
}
for _, m := range models {
	fmt.Println(m.ID, m.Name)
}
Structured outputs
schema := map[string]any{
	"type": "object",
	"properties": map[string]any{
		"name": map[string]any{"type": "string"},
		"age":  map[string]any{"type": "integer"},
	},
	"required": []string{"name", "age"},
}

resp, err := client.Chat(ctx, &litellm.Request{
	Model: "gpt-4o-mini",
	Messages: []litellm.Message{{Role: "user", Content: "Generate a person."}},
	ResponseFormat: litellm.NewResponseFormatJSONSchema("person", "", schema, true),
})
_ = resp
Function calling
tools := []litellm.Tool{
	{
		Type: "function",
		Function: litellm.FunctionDef{
			Name: "get_weather",
			Parameters: map[string]any{
				"type": "object",
				"properties": map[string]any{
					"city": map[string]any{"type": "string"},
				},
				"required": []string{"city"},
			},
		},
	},
}

resp, err := client.Chat(ctx, &litellm.Request{
	Model: "gpt-4o-mini",
	Messages: []litellm.Message{{Role: "user", Content: "Weather in Tokyo?"}},
	Tools: tools,
	ToolChoice: "auto",
})
_ = resp
Thinking output (default enabled)
resp, err := client.Chat(ctx, &litellm.Request{
	Model:    "claude-haiku-4-5-20251001",
	Messages: []litellm.Message{litellm.NewUserMessage("Explain the tradeoffs.")},
	Thinking: litellm.NewThinkingEnabled(1024),
})
_ = resp

To disable:

req := &litellm.Request{
	Model:    "claude-haiku-4-5-20251001",
	Messages: []litellm.Message{litellm.NewUserMessage("Explain the tradeoffs.")},
	Thinking: litellm.NewThinkingDisabled(),
}
_ = req
OpenAI Responses API
resp, err := client.Responses(ctx, &litellm.OpenAIResponsesRequest{
	Model: "o3-mini",
	Messages: []litellm.Message{{Role: "user", Content: "Solve 15*8 step by step."}},
	ReasoningEffort:  "medium",
	ReasoningSummary: "auto",
	Thinking:         litellm.NewThinkingEnabled(0),
	MaxOutputTokens:  litellm.IntPtr(800),
})
_ = resp
Retries & timeouts
res := litellm.DefaultResilienceConfig()
res.MaxRetries = 3
res.InitialDelay = 1 * time.Second
res.RequestTimeout = 60 * time.Second

client, _ := litellm.NewWithProvider("openai", litellm.ProviderConfig{
	APIKey:     os.Getenv("OPENAI_API_KEY"),
	Resilience: res,
})
_ = client
Provider‑specific knobs

Request.Extra is validated per provider. Unsupported providers will return an error.

Supported keys:

  • Gemini: tool_name (string) for tool response naming
  • GLM: enable_thinking (bool) or thinking (object with type string)
Cost calculation

Calculate request costs based on token usage. Pricing data is fetched from BerriAI/litellm and loaded automatically on first use.

resp, err := client.Chat(ctx, req)
if err != nil {
	log.Fatal(err)
}

// Calculate cost (pricing data loads automatically)
if cost, err := litellm.CalculateCostForResponse(resp); err == nil {
	fmt.Printf("Cost: $%.6f (input: $%.6f, output: $%.6f)\n",
		cost.TotalCost, cost.InputCost, cost.OutputCost)
}

// Or use the standalone function
cost, err := litellm.CalculateCost(resp.Model, resp.Usage)

// Set custom pricing for unlisted models
litellm.SetModelPricing("my-model", litellm.ModelPricing{
	InputCostPerToken:  0.000001,
	OutputCostPerToken: 0.000002,
})

Custom Providers

Implement litellm.Provider and register it:

type MyProvider struct {
	name   string
	config litellm.ProviderConfig
}

func (p *MyProvider) Name() string                     { return p.name }
func (p *MyProvider) Validate() error                 { return nil }

func (p *MyProvider) Chat(ctx context.Context, req *litellm.Request) (*litellm.Response, error) {
	return &litellm.Response{Content: "hello", Model: req.Model, Provider: p.name}, nil
}
func (p *MyProvider) Stream(ctx context.Context, req *litellm.Request) (litellm.StreamReader, error) {
	return nil, fmt.Errorf("streaming not implemented")
}

func init() {
	litellm.RegisterProvider("myprovider", func(cfg litellm.ProviderConfig) litellm.Provider {
		return &MyProvider{name: "myprovider", config: cfg}
	})
}

Supported Providers

Builtin providers: OpenAI, Anthropic, Google Gemini, DeepSeek, Qwen (DashScope), GLM, AWS Bedrock, OpenRouter.

LiteLLM does not rewrite model IDs. Always use official model IDs.

Configuration

Configure providers explicitly:

client, err := litellm.NewWithProvider("openai", litellm.ProviderConfig{
	APIKey:  os.Getenv("OPENAI_API_KEY"),
	BaseURL: os.Getenv("OPENAI_BASE_URL"), // optional
})
_ = client

License

Apache License

Documentation

Overview

Package litellm provides a predictable, explicit client for multiple LLM providers.

Design Principles

  • Explicit configuration: no environment auto-discovery, no auto-routing
  • Single-provider client: each Client binds exactly one Provider
  • Predictable behavior: fail fast instead of guessing

Quick Start

Create a client explicitly: 

client, err := litellm.NewWithProvider("openai", litellm.ProviderConfig{
    APIKey: os.Getenv("OPENAI_API_KEY"),
})
if err != nil {
    log.Fatal(err)
}

resp, err := client.Chat(context.Background(), &litellm.Request{
    Model: "gpt-4o-mini",
    Messages: []litellm.Message{
        {Role: "user", Content: "Explain AI in one sentence."},
    },
})

Streaming 

stream, err := client.Stream(ctx, &litellm.Request{
    Model: "gpt-4o-mini",
    Messages: []litellm.Message{
        {Role: "user", Content: "Tell me a joke."},
    },
})
if err != nil {
    log.Fatal(err)
}
defer stream.Close()

for {
    chunk, err := stream.Next()
    if err != nil || chunk.Done {
        break
    }
    fmt.Print(chunk.Content)
}

Or collect the full response: 

resp, err := litellm.CollectStream(stream)
if err != nil {
    log.Fatal(err)
}
fmt.Print(resp.Content)

Or stream while aggregating: 

resp, err := litellm.CollectStreamWithHandler(stream, func(chunk *litellm.StreamChunk) {
    if chunk.Type == litellm.ChunkTypeContent && chunk.Content != "" {
        fmt.Print(chunk.Content)
    }
})
if err != nil {
    log.Fatal(err)
}
fmt.Print(resp.Content)

OpenAI Responses API

Use a dedicated request type for Responses API: 

resp, err := client.Responses(ctx, &litellm.OpenAIResponsesRequest{
    Model: "o3-mini",
    Messages: []litellm.Message{
        {Role: "user", Content: "Solve 15*8 step by step."},
    },
    ReasoningEffort:  "medium",
    ReasoningSummary: "auto",
    MaxOutputTokens:  litellm.IntPtr(800),
})
_ = resp

Custom Providers

Implement the Provider interface and register it: 

type MyProvider struct{}
func (p *MyProvider) Name() string { return "myprovider" }
func (p *MyProvider) Validate() error { return nil }
func (p *MyProvider) Chat(ctx context.Context, req *litellm.Request) (*litellm.Response, error) { ... }
func (p *MyProvider) Stream(ctx context.Context, req *litellm.Request) (litellm.StreamReader, error) { ... }

litellm.RegisterProvider("myprovider", func(cfg litellm.ProviderConfig) litellm.Provider {
    return &MyProvider{}
})

Thread Safety

Client is safe for concurrent use. StreamReader is not goroutine-safe and must be consumed by a single goroutine.

Index

Constants

View Source 
const (
	CacheTypeEphemeral  = "ephemeral"
	CacheTypePersistent = "persistent"
)

CacheControl type constants.

View Source 
const (
	ChunkTypeContent       = "content"
	ChunkTypeToolCallDelta = "tool_call_delta"
	ChunkTypeReasoning     = "reasoning"
)

Stream chunk type constants.

View Source 
const (
	ResponseFormatText       = "text"
	ResponseFormatJSONObject = "json_object"
	ResponseFormatJSONSchema = "json_schema"
)

ResponseFormat type constants.

View Source 
const (
	ThinkingEnabled  = "enabled"
	ThinkingDisabled = "disabled"
)

Thinking type constants.

View Source 
const (
	FinishReasonStop     = providers.FinishReasonStop
	FinishReasonLength   = providers.FinishReasonLength
	FinishReasonToolCall = providers.FinishReasonToolCall
	FinishReasonError    = providers.FinishReasonError
	FinishReasonSafety   = providers.FinishReasonSafety
)

FinishReason constants — canonical values returned by all providers.

View Source 
const PricingURL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"

Variables

This section is empty.

Functions

func BoolPtr added in v1.2.1 

func BoolPtr(v bool) *bool

BoolPtr returns a pointer to a bool value Example: enabled := litellm.BoolPtr(true)

func Float64Ptr 

func Float64Ptr(v float64) *float64

Float64Ptr returns a pointer to a float64 value Example: req.Temperature = litellm.Float64Ptr(0.7)

func GetRetryAfter added in v1.5.0 

func GetRetryAfter(err error) int

func IntPtr 

func IntPtr(v int) *int

IntPtr returns a pointer to an int value Example: req.MaxTokens = litellm.IntPtr(2048)

func IsAuthError added in v1.5.0 

func IsAuthError(err error) bool

func IsModelError added in v1.5.0 

func IsModelError(err error) bool

func IsNetworkError added in v1.5.0 

func IsNetworkError(err error) bool

func IsPricingLoaded added in v1.5.5 

func IsPricingLoaded() bool

IsPricingLoaded returns whether registry data has been loaded.

func IsProviderRegistered added in v1.5.0 

func IsProviderRegistered(name string) bool

IsProviderRegistered checks if a provider is registered (built-in or custom)

func IsRateLimitError added in v1.5.0 

func IsRateLimitError(err error) bool

func IsRetryableError added in v1.5.0 

func IsRetryableError(err error) bool

func IsValidationError added in v1.5.0 

func IsValidationError(err error) bool

func ListRegisteredProviders added in v1.5.0 

func ListRegisteredProviders() []string

ListRegisteredProviders returns all registered provider names

func LoadPricing added in v1.5.5 

func LoadPricing(ctx context.Context) error

LoadPricing fetches model registry data from BerriAI/litellm GitHub.

func LoadPricingFromReader added in v1.5.5 

func LoadPricingFromReader(r io.Reader) error

LoadPricingFromReader loads model registry data from any io.Reader.

func NormalizeFinishReason added in v1.5.7 

func NormalizeFinishReason(raw string) string

NormalizeFinishReason maps provider-specific stop reasons to canonical constants.

func RegisterProvider 

func RegisterProvider(name string, factory ProviderFactory) error

RegisterProvider registers a custom provider factory Returns an error if the name is empty or factory is nil

func SetModelPricing added in v1.5.5 

func SetModelPricing(model string, pricing ModelPricing)

SetModelPricing sets custom pricing for a model.

func StringPtr added in v1.5.2 

func StringPtr(v string) *string

StringPtr returns a pointer to a string value Example: req.User = litellm.StringPtr("user-123")

func WrapError added in v1.5.0 

func WrapError(err error, provider string) error

Types

type CacheControl added in v1.5.0 

type CacheControl = providers.CacheControl

Core types are sourced from providers; litellm re-exports them.

func NewEphemeralCache added in v1.5.0 

func NewEphemeralCache() *CacheControl

NewEphemeralCache creates an ephemeral cache control (TTL is provider-defined, typically ~5 minutes).

func NewPersistentCache added in v1.5.0 

func NewPersistentCache(ttlSeconds int) *CacheControl

NewPersistentCache creates a persistent cache control with a custom TTL (seconds).

type Client 

type Client struct {
	// contains filtered or unexported fields
}

Client is a minimal, predictable client bound to a single Provider.

func New 

func New(provider Provider, opts ...ClientOption) (*Client, error)

New creates a client with an explicit Provider (no implicit discovery).

func NewWithProvider added in v1.5.4 

func NewWithProvider(name string, config ProviderConfig, opts ...ClientOption) (*Client, error)

NewWithProvider creates a client from provider name and config.

func (*Client) Chat added in v1.5.0 

func (c *Client) Chat(ctx context.Context, req *Request) (*Response, error)

Chat executes a non-streaming request.

func (*Client) ListModels added in v1.5.5 

func (c *Client) ListModels(ctx context.Context) ([]ModelInfo, error)

ListModels returns the list of available models for the bound provider (if supported).

func (*Client) ProviderName added in v1.5.7 

func (c *Client) ProviderName() string

ProviderName returns the name of the bound provider.

func (*Client) Responses added in v1.5.4 

func (c *Client) Responses(ctx context.Context, req *OpenAIResponsesRequest) (*Response, error)

Responses executes an OpenAI Responses API request on an OpenAI provider.

func (*Client) ResponsesStream added in v1.5.4 

func (c *Client) ResponsesStream(ctx context.Context, req *OpenAIResponsesRequest) (StreamReader, error)

ResponsesStream executes a streaming OpenAI Responses API request.

func (*Client) Stream 

func (c *Client) Stream(ctx context.Context, req *Request) (StreamReader, error)

Stream executes a streaming request.

type ClientOption 

type ClientOption func(*Client) error

ClientOption configures the Client.

func WithDebug added in v1.5.5 

func WithDebug(enabled bool) ClientOption

WithDebug enables debug logging to stderr.

func WithDebugOutput added in v1.5.5 

func WithDebugOutput(w io.Writer) ClientOption

WithDebugOutput enables debug logging to a custom writer. If w is nil, debug logging is disabled.

func WithDefaults 

func WithDefaults(maxTokens int, temperature float64, topP float64) ClientOption

WithDefaults sets request-level defaults (applies only when fields are unset).

type CostResult added in v1.5.5 

type CostResult struct {
	InputCost  float64 `json:"input_cost"`
	OutputCost float64 `json:"output_cost"`
	TotalCost  float64 `json:"total_cost"`
	Currency   string  `json:"currency"`
}

CostResult contains the calculated cost for a request.

func CalculateCost added in v1.5.5 

func CalculateCost(model string, usage Usage) (*CostResult, error)

CalculateCost calculates the cost based on token usage. Registry data is loaded automatically on first call.

func CalculateCostForResponse added in v1.5.5 

func CalculateCostForResponse(resp *Response) (*CostResult, error)

CalculateCostForResponse calculates the cost for a response.

type DefaultConfig 

type DefaultConfig struct {
	MaxTokens   int     `json:"max_tokens"`
	Temperature float64 `json:"temperature"`
	TopP        float64 `json:"top_p"`
}

DefaultConfig holds request-level defaults.

type ErrorType added in v1.5.0 

type ErrorType = providers.ErrorType

Error types and constructors are sourced from providers; this file is a thin re-export. 

const (
	ErrorTypeAuth       ErrorType = providers.ErrorTypeAuth
	ErrorTypeRateLimit  ErrorType = providers.ErrorTypeRateLimit
	ErrorTypeNetwork    ErrorType = providers.ErrorTypeNetwork
	ErrorTypeValidation ErrorType = providers.ErrorTypeValidation
	ErrorTypeProvider   ErrorType = providers.ErrorTypeProvider
	ErrorTypeTimeout    ErrorType = providers.ErrorTypeTimeout
	ErrorTypeQuota      ErrorType = providers.ErrorTypeQuota
	ErrorTypeModel      ErrorType = providers.ErrorTypeModel
	ErrorTypeInternal   ErrorType = providers.ErrorTypeInternal
)

type FunctionCall 

type FunctionCall = providers.FunctionCall

Core types are sourced from providers; litellm re-exports them.

type FunctionDef added in v1.5.0 

type FunctionDef = providers.FunctionDef

Core types are sourced from providers; litellm re-exports them.

type JSONSchema added in v1.2.1 

type JSONSchema = providers.JSONSchema

Core types are sourced from providers; litellm re-exports them.

type LiteLLMError added in v1.5.0 

type LiteLLMError = providers.LiteLLMError

func NewAuthError added in v1.5.0 

func NewAuthError(provider, message string) *LiteLLMError

func NewError added in v1.5.0 

func NewError(errorType ErrorType, message string) *LiteLLMError

func NewErrorWithCause added in v1.5.0 

func NewErrorWithCause(errorType ErrorType, message string, cause error) *LiteLLMError

func NewHTTPError added in v1.5.0 

func NewHTTPError(provider string, statusCode int, message string) *LiteLLMError

func NewModelError added in v1.5.0 

func NewModelError(provider, model, message string) *LiteLLMError

func NewNetworkError added in v1.5.0 

func NewNetworkError(provider, message string, cause error) *LiteLLMError

func NewProviderError added in v1.5.0 

func NewProviderError(provider string, errorType ErrorType, message string) *LiteLLMError

func NewRateLimitError added in v1.5.0 

func NewRateLimitError(provider, message string, retryAfter int) *LiteLLMError

func NewTimeoutError added in v1.5.0 

func NewTimeoutError(provider, message string) *LiteLLMError

func NewValidationError added in v1.5.0 

func NewValidationError(provider, message string) *LiteLLMError

type Message 

type Message = providers.Message

Core types are sourced from providers; litellm re-exports them.

func AssistantMessage added in v1.5.2 

func AssistantMessage(content string) Message

AssistantMessage creates an assistant message Example: litellm.AssistantMessage("Hello! How can I help you?")

func MultiContentMessage added in v1.5.4 

func MultiContentMessage(role string, contents ...MessageContent) Message

MultiContentMessage creates a message with multiple content items (text, images, etc).

Example: 

msg := litellm.MultiContentMessage("user",
    litellm.TextContent("What's in this image?"),
    litellm.ImageContent("https://example.com/image.png"),
)

func SystemMessage added in v1.5.2 

func SystemMessage(content string) Message

SystemMessage creates a system message Example: litellm.SystemMessage("You are a helpful assistant.")

func ToolMessage added in v1.5.2 

func ToolMessage(toolCallID, content string) Message

ToolMessage creates a tool response message Example: litellm.ToolMessage("call_abc123", `{"result": "success"}`)

func UserMessage added in v1.5.2 

func UserMessage(content string) Message

UserMessage creates a user message Example: litellm.UserMessage("Hello, AI!")

type MessageContent added in v1.5.3 

type MessageContent = providers.MessageContent

Core types are sourced from providers; litellm re-exports them.

func ImageContent added in v1.5.4 

func ImageContent(url string) MessageContent

ImageContent creates an image content item from a URL.

func ImageContentWithDetail added in v1.5.4 

func ImageContentWithDetail(url, detail string) MessageContent

ImageContentWithDetail creates an image content item with detail level. Detail can be "auto", "low", or "high".

func TextContent added in v1.5.4 

func TextContent(text string) MessageContent

TextContent creates a text content item for multi-content messages.

type MessageImageURL added in v1.5.3 

type MessageImageURL = providers.MessageImageURL

Core types are sourced from providers; litellm re-exports them.

type ModelCapabilities added in v1.5.7 

type ModelCapabilities struct {
	Provider          string `json:"litellm_provider"`
	MaxInputTokens    int    `json:"max_input_tokens"`
	MaxOutputTokens   int    `json:"max_output_tokens"`
	SupportsTools     bool   `json:"supports_function_calling"`
	SupportsVision    bool   `json:"supports_vision"`
	SupportsReasoning bool   `json:"supports_reasoning"`
}

ModelCapabilities contains capability and limit metadata for a model.

func GetModelCapabilities added in v1.5.7 

func GetModelCapabilities(model string) (*ModelCapabilities, bool)

GetModelCapabilities returns capability metadata for a model.

type ModelInfo 

type ModelInfo = providers.ModelInfo

Core types are sourced from providers; litellm re-exports them.

type ModelLister added in v1.5.5 

type ModelLister = providers.ModelLister

Core types are sourced from providers; litellm re-exports them.

type ModelPricing added in v1.5.5 

type ModelPricing struct {
	InputCostPerToken  float64 `json:"input_cost_per_token"`
	OutputCostPerToken float64 `json:"output_cost_per_token"`
}

ModelPricing contains pricing information for a model.

func GetModelPricing added in v1.5.5 

func GetModelPricing(model string) (*ModelPricing, bool)

GetModelPricing returns the pricing for a model.

type OpenAIResponsesRequest added in v1.5.4 

type OpenAIResponsesRequest = providers.OpenAIResponsesRequest

Core types are sourced from providers; litellm re-exports them.

type Provider 

type Provider = providers.Provider

Core types are sourced from providers; litellm re-exports them.

type ProviderConfig 

type ProviderConfig = providers.ProviderConfig

Core types are sourced from providers; litellm re-exports them.

type ProviderFactory 

type ProviderFactory func(config ProviderConfig) Provider

ProviderFactory is used to register custom providers.

type ReasoningChunk 

type ReasoningChunk = providers.ReasoningChunk

Core types are sourced from providers; litellm re-exports them.

type ReasoningData 

type ReasoningData = providers.ReasoningData

Core types are sourced from providers; litellm re-exports them.

type Request 

type Request = providers.Request

Core types are sourced from providers; litellm re-exports them.

func NewRequest added in v1.5.4 

func NewRequest(model, prompt string, opts ...RequestOption) *Request

NewRequest creates a new Request with the given model and user prompt. Additional options can be passed to configure the request.

Example: 

req := litellm.NewRequest("gpt-4", "Hello",
    litellm.WithSystemPrompt("You are helpful"),
    litellm.WithMaxTokens(1024),
    litellm.WithTemperature(0.7),
)

func NewRequestWithMessages added in v1.5.4 

func NewRequestWithMessages(model string, messages []Message, opts ...RequestOption) *Request

NewRequestWithMessages creates a new Request with the given model and messages. Use this when you need full control over the message history.

Example: 

req := litellm.NewRequestWithMessages("gpt-4",
    []litellm.Message{
        litellm.SystemMessage("You are helpful"),
        litellm.UserMessage("Hello"),
    },
    litellm.WithMaxTokens(1024),
)

type RequestOption added in v1.5.4 

type RequestOption func(*Request)

RequestOption configures a Request using the functional options pattern.

func WithExtra added in v1.5.4 

func WithExtra(key string, value any) RequestOption

WithExtra sets a provider-specific parameter.

func WithExtras added in v1.5.4 

func WithExtras(extras map[string]any) RequestOption

WithExtras sets multiple provider-specific parameters.

func WithJSONMode added in v1.5.4 

func WithJSONMode() RequestOption

WithJSONMode enables JSON object output mode. The model will return valid JSON without enforcing a specific schema.

func WithJSONSchema added in v1.5.4 

func WithJSONSchema(name, description string, schema any, strict bool) RequestOption

WithJSONSchema enables structured JSON output with schema validation.

func WithMaxTokens added in v1.5.4 

func WithMaxTokens(n int) RequestOption

WithMaxTokens sets the maximum number of tokens to generate.

func WithResponseFormat added in v1.5.4 

func WithResponseFormat(format *ResponseFormat) RequestOption

WithResponseFormat sets a custom response format.

func WithStop added in v1.5.4 

func WithStop(sequences ...string) RequestOption

WithStop sets the stop sequences.

func WithSystemPrompt added in v1.5.4 

func WithSystemPrompt(content string) RequestOption

WithSystemPrompt prepends a system message to the request. If a system message already exists, it will be replaced.

func WithTemperature added in v1.5.4 

func WithTemperature(t float64) RequestOption

WithTemperature sets the sampling temperature (0.0 to 2.0). Higher values make output more random, lower values more deterministic.

func WithThinking added in v1.5.4 

func WithThinking(budget int) RequestOption

WithThinking enables thinking/reasoning mode with an optional token budget. Set budget to 0 for provider default.

func WithToolChoice added in v1.5.4 

func WithToolChoice(choice any) RequestOption

WithToolChoice sets the tool choice behavior. Accepts: "auto", "none", "required", or a specific tool name.

func WithTools added in v1.5.4 

func WithTools(tools ...Tool) RequestOption

WithTools adds tool definitions to the request.

func WithTopP added in v1.5.4 

func WithTopP(p float64) RequestOption

WithTopP sets the nucleus sampling parameter (0.0 to 1.0).

func WithoutThinking added in v1.5.4 

func WithoutThinking() RequestOption

WithoutThinking explicitly disables thinking/reasoning mode.

type ResilienceConfig added in v1.2.2 

type ResilienceConfig = providers.ResilienceConfig

ResilienceConfig and defaults are sourced from providers; re-exported here to keep the public API small.

func DefaultResilienceConfig added in v1.2.2 

func DefaultResilienceConfig() ResilienceConfig

type ResilientHTTPClient added in v1.2.2 

type ResilientHTTPClient struct {
	// contains filtered or unexported fields
}

ResilientHTTPClient wraps http.Client with retry logic

func NewResilientHTTPClient added in v1.2.2 

func NewResilientHTTPClient(config ResilienceConfig) *ResilientHTTPClient

NewResilientHTTPClient creates a new resilient HTTP client

func (*ResilientHTTPClient) Do added in v1.2.2 

func (c *ResilientHTTPClient) Do(req *http.Request) (*http.Response, error)

Do executes HTTP request with retry logic

type Response 

type Response = providers.Response

Core types are sourced from providers; litellm re-exports them.

func CollectStream added in v1.5.4 

func CollectStream(stream StreamReader) (*Response, error)

CollectStream consumes a StreamReader and returns a unified Response. Callers are responsible for closing the stream.

func CollectStreamWithCallbacks added in v1.5.4 

func CollectStreamWithCallbacks(stream StreamReader, callbacks StreamCallbacks) (*Response, error)

CollectStreamWithCallbacks consumes a StreamReader, calls callbacks for each chunk, and returns a unified Response. Callers are responsible for closing the stream.

func CollectStreamWithHandler added in v1.5.4 

func CollectStreamWithHandler(stream StreamReader, onChunk func(*StreamChunk)) (*Response, error)

CollectStreamWithHandler consumes a StreamReader, calls onChunk for each chunk, and returns a unified Response. Callers are responsible for closing the stream.

type ResponseFormat added in v1.2.1 

type ResponseFormat = providers.ResponseFormat

Core types are sourced from providers; litellm re-exports them.

func NewResponseFormatJSONObject added in v1.2.1 

func NewResponseFormatJSONObject() *ResponseFormat

NewResponseFormatJSONObject creates a JSON object response format This ensures the model returns valid JSON without enforcing a specific schema

func NewResponseFormatJSONSchema added in v1.2.1 

func NewResponseFormatJSONSchema(name, description string, schema any, strict bool) *ResponseFormat

NewResponseFormatJSONSchema creates a JSON schema response format with strict validation enabled/disabled

Parameters:

  • name: Schema name (required)
  • description: Schema description (optional, can be empty)
  • schema: JSON Schema definition as a map[string]interface{}
  • strict: Enable strict schema validation (OpenAI only)

Example: 

schema := map[string]interface{}{
    "type": "object",
    "properties": map[string]interface{}{
        "name": map[string]interface{}{"type": "string"},
        "age": map[string]interface{}{"type": "integer"},
    },
    "required": []string{"name", "age"},
}
format := litellm.NewResponseFormatJSONSchema("person", "A person object", schema, true)

func NewResponseFormatText added in v1.2.1 

func NewResponseFormatText() *ResponseFormat

NewResponseFormatText creates a text response format

type StreamCallbacks added in v1.5.4 

type StreamCallbacks struct {
	OnChunk     func(*StreamChunk)
	OnContent   func(string)
	OnReasoning func(*ReasoningChunk)
	OnToolCall  func(*ToolCallDelta)
}

StreamCallbacks provides optional per-chunk handlers during stream collection.

type StreamChunk 

type StreamChunk = providers.StreamChunk

Core types are sourced from providers; litellm re-exports them.

type StreamReader 

type StreamReader = providers.StreamReader

Core types are sourced from providers; litellm re-exports them.

type ThinkingConfig added in v1.5.4 

type ThinkingConfig = providers.ThinkingConfig

Core types are sourced from providers; litellm re-exports them.

func NewThinkingDisabled added in v1.5.4 

func NewThinkingDisabled() *ThinkingConfig

NewThinkingDisabled disables thinking output.

func NewThinkingEnabled added in v1.5.4 

func NewThinkingEnabled(budgetTokens int) *ThinkingConfig

NewThinkingEnabled enables thinking with an optional budget.

func NewThinkingWithLevel added in v1.5.7 

func NewThinkingWithLevel(level string) *ThinkingConfig

NewThinkingWithLevel enables thinking with a reasoning level. Level is provider-specific: "low", "medium", "high" etc.

type Tool 

type Tool = providers.Tool

Core types are sourced from providers; litellm re-exports them.

func NewTool added in v1.5.4 

func NewTool(name, description string, parameters any) Tool

NewTool creates a function tool definition.

type ToolCall 

type ToolCall = providers.ToolCall

Core types are sourced from providers; litellm re-exports them.

type ToolCallAccumulator added in v1.5.7 

type ToolCallAccumulator struct {
	// contains filtered or unexported fields
}

ToolCallAccumulator reconstructs complete ToolCall objects from streaming deltas. Safe for single-goroutine use only.

func NewToolCallAccumulator added in v1.5.7 

func NewToolCallAccumulator() *ToolCallAccumulator

NewToolCallAccumulator creates an empty accumulator.

func (*ToolCallAccumulator) Apply added in v1.5.7 

func (a *ToolCallAccumulator) Apply(delta *ToolCallDelta)

Apply processes a single ToolCallDelta, creating or updating the corresponding ToolCall.

func (*ToolCallAccumulator) Build added in v1.5.7 

func (a *ToolCallAccumulator) Build() []ToolCall

Build returns the completed ToolCall list in first-seen order.

func (*ToolCallAccumulator) Started added in v1.5.7 

func (a *ToolCallAccumulator) Started(index int) bool

Started reports whether a delta with the given index has been received.

type ToolCallDelta 

type ToolCallDelta = providers.ToolCallDelta

Core types are sourced from providers; litellm re-exports them.

type Usage 

type Usage = providers.Usage

Core types are sourced from providers; litellm re-exports them.

Source Files

View all Source files

Directories

Path Synopsis
examples
anthropic command
bedrock command
deepseek command
gemini command
glm command
openai command
openrouter command
qwen command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.