llm

Documentation

Overview

Package llm provides a unified interface for interacting with LLMs.

Index

• Constants
• func ContentsAttr(contents []Content) slog.Attr
• func CostUSDFromResponse(headers http.Header) float64
• func DumpToFile(typ string, url string, content []byte) error
• func EmptySchema() json.RawMessage
• func MustSchema(schema string) json.RawMessage
• func UseSimplifiedPatch(svc Service) bool
• type Content
  • func StringContent(s string) Content
  • func TextContent(text string) []Content
• type ContentType
  • func (i ContentType) String() string
• type Message
  • func UserStringMessage(text string) Message
• type MessageRole
  • func (i MessageRole) String() string
• type Request
• type Response
  • func (m *Response) ToMessage() Message
• type Service
• type SimplifiedPatcher
• type StopReason
  • func (i StopReason) String() string
• type SystemContent
• type Tool
• type ToolChoice
• type ToolChoiceType
  • func (i ToolChoiceType) String() string
• type ToolOut
  • func ErrorToolOut(err error) ToolOut
  • func ErrorfToolOut(format string, args ...any) ToolOut
• type ToolUse
• type Usage
  • func (u *Usage) Add(other Usage)
  • func (u *Usage) Attr() slog.Attr
  • func (u *Usage) IsZero() bool
  • func (u *Usage) String() string

Constants

const (
	MessageRoleUser MessageRole = iota
	MessageRoleAssistant

	ContentTypeText ContentType = iota
	ContentTypeThinking
	ContentTypeRedactedThinking
	ContentTypeToolUse
	ContentTypeToolResult

	ToolChoiceTypeAuto ToolChoiceType = iota // default
	ToolChoiceTypeAny                        // any tool, but must use one
	ToolChoiceTypeNone                       // no tools allowed
	ToolChoiceTypeTool                       // must use the tool specified in the Name field

	StopReasonStopSequence StopReason = iota
	StopReasonMaxTokens
	StopReasonEndTurn
	StopReasonToolUse
	StopReasonRefusal
)

Functions

func ContentsAttr

func ContentsAttr(contents []Content) slog.Attr

ContentsAttr returns contents as a slog.Attr. It is meant for logging.

func CostUSDFromResponse

func CostUSDFromResponse(headers http.Header) float64

func DumpToFile

func DumpToFile(typ string, url string, content []byte) error

DumpToFile writes LLM communication content to a timestamped file in ~/.cache/sketch/. For requests, it includes the URL followed by the content. For responses, it only includes the content. The typ parameter is used as a prefix in the filename ("request", "response").

func EmptySchema

func EmptySchema() json.RawMessage

func MustSchema

func MustSchema(schema string) json.RawMessage

MustSchema validates that schema is a valid JSON schema and returns it as a json.RawMessage. It panics if the schema is invalid. The schema must have at least type="object" and a properties key.

func UseSimplifiedPatch

func UseSimplifiedPatch(svc Service) bool

Types

type Content

type Content struct {
	ID   string
	Type ContentType
	Text string

	// Media type for image content
	MediaType string

	// for thinking
	Thinking  string
	Data      string
	Signature string

	// for tool_use
	ToolName  string
	ToolInput json.RawMessage

	// for tool_result
	ToolUseID  string
	ToolError  bool
	ToolResult []Content

	// timing information for tool_result; added externally; not sent to the LLM
	ToolUseStartTime *time.Time
	ToolUseEndTime   *time.Time

	// Display is content to be displayed to the user, copied from ToolOut
	Display any

	Cache bool
}

func StringContent

func StringContent(s string) Content

func TextContent

func TextContent(text string) []Content

TextContent creates a simple text content for tool results. This is a helper function to create the most common type of tool result content.

type ContentType

type ContentType int

func (ContentType) String

func (i ContentType) String() string

type Message

type Message struct {
	Role    MessageRole
	Content []Content
	ToolUse *ToolUse // use to control whether/which tool to use
}

Message represents a message in the conversation.

func UserStringMessage

func UserStringMessage(text string) Message

UserStringMessage creates a user message with a single text content item.

type MessageRole

type MessageRole int

func (MessageRole) String

func (i MessageRole) String() string

type Request

type Request struct {
	Messages   []Message
	ToolChoice *ToolChoice
	Tools      []*Tool
	System     []SystemContent
}

type Response

type Response struct {
	ID           string
	Type         string
	Role         MessageRole
	Model        string
	Content      []Content
	StopReason   StopReason
	StopSequence *string
	Usage        Usage
	StartTime    *time.Time
	EndTime      *time.Time
}

func (*Response) ToMessage

func (m *Response) ToMessage() Message

type Service

type Service interface {
	// Do sends a request to an LLM.
	Do(context.Context, *Request) (*Response, error)
	// TokenContextWindow returns the maximum token context window size for this service
	TokenContextWindow() int
}

type SimplifiedPatcher

type SimplifiedPatcher interface {
	// UseSimplifiedPatch reports whether the service should use the simplified patch input schema.
	UseSimplifiedPatch() bool
}

type StopReason

type StopReason int

func (StopReason) String

func (i StopReason) String() string

type SystemContent

type SystemContent struct {
	Text  string
	Type  string
	Cache bool
}

type Tool

type Tool struct {
	Name string
	// Type is used by the text editor tool; see
	// https://docs.anthropic.com/en/docs/build-with-claude/tool-use/text-editor-tool
	Type        string
	Description string
	InputSchema json.RawMessage
	// EndsTurn indicates that this tool should cause the model to end its turn when used
	EndsTurn bool

	// The Run function is automatically called when the tool is used.
	// Run functions may be called concurrently with each other and themselves.
	// The input to Run function is the input to the tool, as provided by Claude, in compliance with the input schema.
	// The outputs from Run will be sent back to Claude.
	// If you do not want to respond to the tool call request from Claude, return ErrDoNotRespond.
	// ctx contains extra (rarely used) tool call information; retrieve it with ToolCallInfoFromContext.
	Run func(ctx context.Context, input json.RawMessage) ToolOut `json:"-"`
}

Tool represents a tool available to an LLM.

type ToolChoice

type ToolChoice struct {
	Type ToolChoiceType
	Name string
}

type ToolChoiceType

type ToolChoiceType int

func (ToolChoiceType) String

func (i ToolChoiceType) String() string

type ToolOut

type ToolOut struct {
	// LLMContent is the output of the tool to be sent back to the LLM.
	// May be nil on error.
	LLMContent []Content
	// Display is content to be displayed to the user.
	// The type of content is set by the tool and coordinated with the UIs.
	// It should be JSON-serializable.
	Display any
	// Error is the error (if any) that occurred during the tool run.
	// The text contents of the error will be sent back to the LLM.
	// If non-nil, LLMContent will be ignored.
	Error error
}

ToolOut represents the output of a tool run.

func ErrorToolOut

func ErrorToolOut(err error) ToolOut

func ErrorfToolOut

func ErrorfToolOut(format string, args ...any) ToolOut

type ToolUse

type ToolUse struct {
	ID   string
	Name string
}

ToolUse represents a tool use in the message content.

type Usage

type Usage struct {
	InputTokens              uint64  `json:"input_tokens"`
	CacheCreationInputTokens uint64  `json:"cache_creation_input_tokens"`
	CacheReadInputTokens     uint64  `json:"cache_read_input_tokens"`
	OutputTokens             uint64  `json:"output_tokens"`
	CostUSD                  float64 `json:"cost_usd"`
}

Usage represents the billing and rate-limit usage. Most LLM structs do not have JSON tags, to avoid accidental direct use in specific providers. However, the front-end uses this struct, and it relies on its JSON serialization. Do NOT use this struct directly when implementing an llm.Service.

func (*Usage) Add

func (u *Usage) Add(other Usage)

func (*Usage) Attr

func (u *Usage) Attr() slog.Attr

func (*Usage) IsZero

func (u *Usage) IsZero() bool

func (*Usage) String

func (u *Usage) String() string