gopherllm

package module

Go to main page

Versions in this module

v0

v0.3.1

Jul 19, 2026

v0.3.0

Jul 19, 2026

Changes in this version

+ func F32ToF16(f float32) uint16

+ func ForwardNemotronHBodyInto(cfg Config, weights NemotronHWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func ForwardNemotronHInto(cfg Config, weights NemotronHWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func LoadNemotronHModel(data []byte, gguf *GGUFFile, borrow, prepareQuantized, useMetal bool, ...) (Config, NemotronHWeights, error)

type Config

+ ExpertWeightsNorm bool

+ ExpertWeightsNormClip float32

+ ExpertWeightsScale float32

+ LayerFFNDim []int

+ LayerHeads []int

+ LayerKVHeads []int

+ SSMConv int

+ SSMGroups int

+ SSMHeads int

+ SSMInner int

+ SSMState int

type DecodeBuffer

+ ExpertHidden []float32

+ ExpertRow []float32

+ MambaB []float32

+ MambaC []float32

+ MambaConv []float32

+ MambaDT []float32

+ MambaIn []float32

+ MambaKernel []float32

+ MambaX []float32

+ MambaY []float32

+ MambaZ []float32

+ type ExpertWeight struct

+ Experts int

+ Input int

+ Output int

+ Weight Weight

type GGUFFile

+ func (g *GGUFFile) GetU32Array(key string) ([]uint32, bool)

type KVCache

+ F16 bool

+ K16 [][]uint16

+ Nemotron *NemotronHCache

+ V16 [][]uint16

+ func NewKVCacheF16(layers, kDim, vDim, maxLen int) *KVCache

type MetaValue

+ func (v MetaValue) AsU32Array() ([]uint32, bool)

+ type NemotronAttentionWeights struct

+ K Weight

+ O Weight

+ Q Weight

+ V Weight

+ type NemotronHCache struct

+ Channels int

+ Conv []float32

+ ConvLen int

+ HeadDim int

+ Heads int

+ Layers int

+ State []float32

+ StateSize int

+ type NemotronHLayerWeights struct

+ Attention NemotronAttentionWeights

+ Kind nemotronLayerKind

+ Mamba NemotronMambaWeights

+ MoE NemotronMoEWeights

+ Norm []float32

+ type NemotronHWeights struct

+ Layers []NemotronHLayerWeights

+ Output Weight

+ OutputNorm []float32

+ TokenEmbd Weight

+ type NemotronMambaWeights struct

+ A []float32

+ Conv Weight

+ ConvBias []float32

+ D []float32

+ DtBias []float32

+ In Weight

+ Norm []float32

+ Out Weight

+ type NemotronMoEWeights struct

+ Down ExpertWeight

+ LatentIn *Weight

+ LatentOut *Weight

+ Router Weight

+ RouterBias []float32

+ SharedDown *Weight

+ SharedUp *Weight

+ Up ExpertWeight

+ type OllamaEmbedRequest struct

+ Input any

+ Model string

+ func (o OllamaEmbedRequest) Inputs() []string

type OllamaOptions

+ NumCtx *int

type OpenAIChatRequest

+ StreamOptions *OpenAIStreamOpts

+ type OpenAIStreamOpts struct

+ IncludeUsage bool

v0.2.0

Jul 11, 2026

Changes in this version

+ func ArgmaxOutputToken(config Config, weights ModelWeights, buf *DecodeBuffer) (uint32, bool)

+ func MatvecQ4K2Q6KIntoWithXSums(aData []byte, aRows, aCols int, bData []byte, bRows, bCols int, cData []byte, ...) bool

+ func ProjectLogitsInto(config Config, weights ModelWeights, buf *DecodeBuffer, logits *[]float32)

type GenerationStats

+ TTFT time.Duration

type Weight

+ func (w Weight) ArgmaxMatvec(x []float32) (uint32, bool)

v0.1.0

Jul 4, 2026

Changes in this version

+ const LoadSkillToolName

+ const Version

+ var ErrGenerationCanceled = errors.New("generation canceled")

+ func ArchitectureSupported(arch string) bool

+ func AxpyF32(out []float32, alpha float32, x []float32)

+ func CosineSimilarity(a, b []float32) (float32, error)

+ func DefaultModelDir() string

+ func DequantRowMXFP4(row []byte, cols int) []float32

+ func DequantRowQ2K(row []byte, cols int) []float32

+ func DequantRowQ2KInto(row []byte, cols int, out []float32)

+ func DequantRowQ3K(row []byte, cols int) []float32

+ func DequantRowQ3KInto(row []byte, cols int, out []float32)

+ func DequantRowQ4K(row []byte, cols int) []float32

+ func DequantRowQ4KInto(row []byte, cols int, out []float32)

+ func DequantRowQ4_0(row []byte, cols int) []float32

+ func DequantRowQ4_1(row []byte, cols int) []float32

+ func DequantRowQ4_1Into(row []byte, cols int, out []float32)

+ func DequantRowQ5K(row []byte, cols int) []float32

+ func DequantRowQ5_0(row []byte, cols int) []float32

+ func DequantRowQ5_0Into(row []byte, cols int, out []float32)

+ func DequantRowQ5_1(row []byte, cols int) []float32

+ func DequantRowQ5_1Into(row []byte, cols int, out []float32)

+ func DequantRowQ6K(row []byte, cols int) []float32

+ func DequantRowQ6KInto(row []byte, cols int, out []float32)

+ func DequantRowQ8_0(row []byte, cols int) []float32

+ func DequantRowQ8_1(row []byte, cols int) []float32

+ func DequantRowQ8_1Into(row []byte, cols int, out []float32)

+ func DotF32(a, b []float32) float32

+ func DotMXFP4F32(row []byte, x []float32, cols int) float32

+ func DotQ2KF32(row []byte, x []float32, cols int) float32

+ func DotQ3KF32(row []byte, x []float32, cols int) float32

+ func DotQ4KF32(row []byte, x []float32, cols int) float32

+ func DotQ4_0F32(row []byte, x []float32, cols int) float32

+ func DotQ4_1F32(row []byte, x []float32, cols int) float32

+ func DotQ5KF32(row []byte, x []float32, cols int) float32

+ func DotQ5_0F32(row []byte, x []float32, cols int) float32

+ func DotQ5_1F32(row []byte, x []float32, cols int) float32

+ func DotQ6KF32(row []byte, x []float32, cols int) float32

+ func DotQ8_0F32(row []byte, x []float32, cols int) float32

+ func DotQ8_1F32(row []byte, x []float32, cols int) float32

+ func F16ToF32(h uint16) float32

+ func Forward(config Config, weights ModelWeights, cache *KVCache, buf *DecodeBuffer, ...) []float32

+ func ForwardBatchInto(config Config, weights ModelWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func ForwardBodyInto(config Config, weights ModelWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func ForwardGemma4Into(config Config, weights Gemma4Weights, cache *KVCache, buf *DecodeBuffer, ...)

+ func ForwardGptOssInto(config Config, weights GptOssWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func ForwardHidden(config Config, weights ModelWeights, cache *KVCache, buf *DecodeBuffer, ...) []float32

+ func ForwardHiddenGemma4(config Config, weights Gemma4Weights, cache *KVCache, buf *DecodeBuffer, ...) []float32

+ func ForwardHiddenGptOss(config Config, weights GptOssWeights, cache *KVCache, buf *DecodeBuffer, ...) []float32

+ func ForwardInto(config Config, weights ModelWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func ForwardPrefill(config Config, weights ModelWeights, cache *KVCache, buf *DecodeBuffer, ...)

+ func LoadGemma4Model(data []byte, gguf *GGUFFile, borrowQuantized, prepareQuantized, useMetal bool, ...) (Config, Gemma4Weights, error)

+ func LoadGptOssModel(data []byte, gguf *GGUFFile, borrowQuantized, prepareQuantized, useMetal bool, ...) (Config, GptOssWeights, error)

+ func LoadModel(data []byte, gguf *GGUFFile, borrowQuantized, prepareQuantized, useMetal bool, ...) (Config, ModelWeights, error)

+ func MatvecF32(data, x []float32, rows, cols int) []float32

+ func MatvecF32Into(data, x []float32, rows, cols int, out *[]float32)

+ func MatvecMXFP4Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecPreparedQ4K2IntoWithXSums(aData []byte, aPrep *PreparedQuantizedWeight, aRows, aCols int, bData []byte, ...) bool

+ func MatvecPreparedQ4K3IntoWithXSums(aData []byte, aPrep *PreparedQuantizedWeight, aRows, aCols int, bData []byte, ...) bool

+ func MatvecPreparedQ4KInto(data []byte, p *PreparedQuantizedWeight, x []float32, rows, cols int, ...) bool

+ func MatvecPreparedQ6KInto(data []byte, p *PreparedQuantizedWeight, x []float32, rows, cols int, ...) bool

+ func MatvecQ2KInto(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ3KInto(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ4K2Into(aData []byte, aRows, aCols int, bData []byte, bRows, bCols int, x []float32, ...) bool

+ func MatvecQ4K2IntoWithXSums(aData []byte, aRows, aCols int, bData []byte, bRows, bCols int, x []float32, ...) bool

+ func MatvecQ4KInto(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ4_0(data []byte, x []float32, rows, cols int) []float32

+ func MatvecQ4_0Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ4_1Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ5KInto(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ5_0Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ5_1Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ6K2Into(aData []byte, aRows, aCols int, bData []byte, bRows, bCols int, x []float32, ...) bool

+ func MatvecQ6K3Into(aData []byte, aRows, aCols int, bData []byte, bRows, bCols int, cData []byte, ...) bool

+ func MatvecQ6KInto(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ8_0(data []byte, x []float32, rows, cols int) []float32

+ func MatvecQ8_0Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MatvecQ8_1Into(data []byte, x []float32, rows, cols int, out *[]float32)

+ func MetalAvailable() bool

+ func MetalError() string

+ func NewHandler(initialRunner *Runner, opts HandlerOptions) http.Handler

+ func PrintModelList(entries []ModelEntry)

+ func Q4KMatvec3Into(wq, wk, wv Q4KMatrix, x []float32, q, k, v *[]float32) bool

+ func Q4KMatvec3IntoWithXSums(wq, wk, wv Q4KMatrix, x []float32, xSums *[]float32, q, k, v *[]float32) bool

+ func ResolveModelPath(selection *string, modelDir string) (string, error)

+ func RunKernelBench(r *Runner, modelPath string, runs, requestedLayer int, jsonOut bool) error

+ func RunnerFromPath(path string) (*Runner, LoadInfo, error)

+ func RunnerFromPathWithOptions(path string, options LoadOptions) (*Runner, LoadInfo, error)

+ func Sample(logits []float32, config SamplerConfig, rng *Rng, recent []uint32) uint32

+ func SampleWithScratch(logits []float32, config SamplerConfig, rng *Rng, recent []uint32, ...) uint32

+ func ScaleAddF32(out []float32, alpha float32, x []float32)

+ func ScaleF32(out []float32, alpha float32)

+ func Serve(initialRunner *Runner, opts ServeOptions) error

+ func SetNumThreads(n int)

+ type APIMessage struct

+ Content any

+ Name string

+ Role string

+ ToolCallID string

+ ToolCalls []ToolCall

+ type Analysis struct

+ Architecture string

+ BOSID uint32

+ BitsPerWeight float64

+ ChatTemplate bool

+ ContextLength int

+ DTypes []DTypeStat

+ Dim int

+ EOSID uint32

+ FileBytes int64

+ HeadDim int

+ Heads int

+ HiddenDim int

+ KVCacheBytesAt4K int64

+ KVCacheBytesAtFullContext int64

+ KVHeads int

+ LargestTensors []TensorStat

+ Layers int

+ Name string

+ Params int64

+ RopeScalingType string

+ RopeTheta float32

+ SWALayers int

+ SlidingWindow int

+ Supported bool

+ TemplateKind string

+ TokenizerModel string

+ TokenizerPre string

+ Version uint32

+ VocabSize int

+ func AnalyzeGGUF(g *GGUFFile, tok *Tokenizer) *Analysis

+ func (a *Analysis) WriteText(w io.Writer)

+ type ChatMessage struct

+ Content string

+ Name string

+ Role ChatRole

+ ToolCallID string

+ ToolCalls []ToolCall

+ func AssistantMessage(content string) ChatMessage

+ func ToolResultMessage(callID, name, content string) ChatMessage

+ func UserMessage(content string) ChatMessage

+ type ChatRole int

+ const ChatRoleAssistant

+ const ChatRoleSystem

+ const ChatRoleTool

+ const ChatRoleUser

+ type Config struct

+ Arch string

+ AttentionScale float32

+ AttnLogitSoftcap float32

+ Dim int

+ EmbeddingScale float32

+ ExpertCount int

+ ExpertUsedCount int

+ FinalLogitSoftcap float32

+ HeadDim int

+ HiddenDim int

+ KVDim int

+ KVMul int

+ LogitScale float32

+ MaxSeqLen int

+ NHeads int

+ NKVHeads int

+ NLayers int

+ RMSNormEps float32

+ ResidualScale float32

+ RopeAttentionFactor float32

+ RopeDimensionCount int

+ RopeFactorsLong []float32

+ RopeFactorsShort []float32

+ RopeOriginalContextLength int

+ RopeScalingFactor float32

+ RopeScalingType string

+ RopeTheta float32

+ RopeYarnBetaFast float32

+ RopeYarnBetaSlow float32

+ RopeYarnLogMultiplier float32

+ SWAPattern []bool

+ SlidingWindow int

+ UseGELU bool

+ ValueDim int

+ VocabSize int

+ func ConfigFromGGUF(gguf *GGUFFile) Config

+ type DTypeStat struct

+ Bytes int64

+ Params int64

+ Tensors int

+ Type GGMLType

+ type DecodeBuffer struct

+ AttnOut []float32

+ ExpertProbs []float32

+ Gate []float32

+ GateUp []float32

+ Hidden []float32

+ K []float32

+ MOE []float32

+ Proj []float32

+ Q []float32

+ Q4KXSums []float32

+ QKV []float32

+ RopeCos []float32

+ RopeGptOssConcentration float32

+ RopeGptOssInvFreq []float32

+ RopeInvFreq []float32

+ RopeMscale float32

+ RopeSin []float32

+ RouterLogits []float32

+ SamplerCandidates []TokenProb

+ TopExperts []ExpertScore

+ Up []float32

+ V []float32

+ X []float32

+ XN []float32

+ XN2 []float32

+ func NewDecodeBuffer(config Config, maxHeadDim, maxNKVHeads, maxValueDim int) *DecodeBuffer

+ type EmbeddingResult struct

+ Embedding []float32

+ TokenCount int

+ type EmbeddingsRequest struct

+ Input any

+ Model string

+ func (e EmbeddingsRequest) Inputs() []string

+ type ExpertScore struct

+ Index int

+ Score float32

+ type GGMLType uint32

+ const GGMLTypeBF16

+ const GGMLTypeF16

+ const GGMLTypeF32

+ const GGMLTypeMXFP4

+ const GGMLTypeQ2_K

+ const GGMLTypeQ3_K

+ const GGMLTypeQ4_0

+ const GGMLTypeQ4_1

+ const GGMLTypeQ4_K

+ const GGMLTypeQ5_0

+ const GGMLTypeQ5_1

+ const GGMLTypeQ5_K

+ const GGMLTypeQ6_K

+ const GGMLTypeQ8_0

+ const GGMLTypeQ8_1

+ const GGMLTypeQ8_K

+ const GGMLTypeUnknown

+ func (t GGMLType) BlockBytes() (int, bool)

+ func (t GGMLType) BlockSize() int

+ func (t GGMLType) DataSize(n int) (int, bool)

+ func (t GGMLType) String() string

+ type GGUFFile struct

+ DataOffset int

+ Metadata map[string]MetaValue

+ Tensors []TensorInfo

+ Version uint32

+ func ParseGGUF(data []byte) (*GGUFFile, error)

+ func ParseGGUFQuiet(data []byte) (*GGUFFile, error)

+ func (g *GGUFFile) GetF32(key string, def float32) float32

+ func (g *GGUFFile) GetString(key string) (string, bool)

+ func (g *GGUFFile) GetU32(key string, def uint32) uint32

+ type Gemma4LayerWeights struct

+ AttnK Weight

+ AttnNorm []float32

+ AttnOutput Weight

+ AttnQ Weight

+ AttnV Weight

+ FFNDown Weight

+ FFNGate Weight

+ FFNNorm []float32

+ FFNUp Weight

+ HasAttnV bool

+ HeadDim int

+ NKVHeads int

+ ValueDim int

+ type Gemma4Weights struct

+ Layers []Gemma4LayerWeights

+ Output Weight

+ OutputNorm []float32

+ Standard ModelWeights

+ TokenEmbd Weight

+ type GenOption func(*GenerationOptions)

+ func WithGenerationOptions(base GenerationOptions) GenOption

+ func WithMaxTokens(n int) GenOption

+ func WithMinP(p float32) GenOption

+ func WithRepeatPenalty(p float32) GenOption

+ func WithSeed(seed uint64) GenOption

+ func WithStop(sequences ...string) GenOption

+ func WithSystemPrompt(s string) GenOption

+ func WithTemperature(t float32) GenOption

+ func WithToolChoice(choice string) GenOption

+ func WithTools(tools ...ToolDefinition) GenOption

+ func WithTopK(k int) GenOption

+ func WithTopP(p float32) GenOption

+ type GenerateRequest struct

+ MaxTokens *int

+ Messages []APIMessage

+ MinP *float32

+ Prompt string

+ RepeatPenalty *float32

+ Seed *uint64

+ Stop any

+ SystemPrompt *string

+ Temp *float32

+ Temperature *float32

+ ToolChoice any

+ Tools []ToolDefinition

+ TopK *int

+ TopP *float32

+ func (g GenerateRequest) ToMessagesAndOptions(def GenerationOptions) ([]ChatMessage, GenerationOptions)

+ type GenerationOptions struct

+ MaxTokens int

+ Sampler SamplerConfig

+ Seed uint64

+ StopSequences []string

+ SystemPrompt string

+ ToolChoice string

+ Tools []ToolDefinition

+ func DefaultGenerationOptions() GenerationOptions

+ func (o GenerationOptions) Validate() error

+ type GenerationResult struct

+ FinishReason string

+ ReasoningText string

+ Stats GenerationStats

+ Text string

+ ToolCalls []ToolCall

+ func RunAgenticChat(r *Runner, messages []ChatMessage, options GenerationOptions, skills []Skill, ...) (GenerationResult, error)

+ type GenerationStats struct

+ DecodeTime time.Duration

+ GeneratedTokens int

+ PrefillTime time.Duration

+ PromptTokens int

+ TotalTime time.Duration

+ type GptOssWeights struct

+ Standard ModelWeights

+ type HandlerOptions struct

+ ChatUI bool

+ Defaults GenerationOptions

+ LogWriter io.Writer

+ MaxConcurrentRequests int

+ ModelDir string

+ ModelPath string

+ SkillsDir string

+ type KVCache struct

+ K [][]float32

+ MaxLen int

+ PerPosKDim int

+ PerPosVDim int

+ V [][]float32

+ func NewKVCache(layers, kDim, vDim, maxLen int) *KVCache

+ type KernelBenchRow struct

+ AvgMS float64

+ Cols int

+ DType string

+ Name string

+ Rows int

+ Runs int

+ TotalMS float64

+ type LayerWeights struct

+ AttnKNorm []float32

+ AttnNorm []float32

+ AttnQNorm []float32

+ BK []float32

+ BQ []float32

+ BV []float32

+ FFNNorm []float32

+ HasGateUp bool

+ HasQKV bool

+ PostAttnNorm []float32

+ PostFFNNorm []float32

+ W1 Weight

+ W2 Weight

+ W3 Weight

+ WGateUp Weight

+ WK Weight

+ WO Weight

+ WQ Weight

+ WQKV Weight

+ WV Weight

+ type LoadInfo struct

+ FileSizeBytes int

+ LoadTime time.Duration

+ type LoadOptions struct

+ LogWriter io.Writer

+ PrepareQuantized bool

+ UseMetal bool

+ type MetaValue struct

+ Kind string

+ Value any

+ func (v MetaValue) AsBool() (bool, bool)

+ func (v MetaValue) AsBoolArray() ([]bool, bool)

+ func (v MetaValue) AsF32() (float32, bool)

+ func (v MetaValue) AsF32Array() ([]float32, bool)

+ func (v MetaValue) AsString() (string, bool)

+ func (v MetaValue) AsStringArray() ([]string, bool)

+ func (v MetaValue) AsU32() (uint32, bool)

+ type MetalWeight struct

+ type MmapFile struct

+ func OpenMmap(path string) (*MmapFile, error)

+ func (m *MmapFile) Bytes() []byte

+ func (m *MmapFile) Close() error

+ func (m *MmapFile) Len() int

+ type Model struct

+ func Open(ctx context.Context, path string, opts ...Option) (*Model, error)

+ func OpenBytes(ctx context.Context, data []byte, opts ...Option) (*Model, error)

+ func (m *Model) Chat(ctx context.Context, messages []ChatMessage, opts ...GenOption) (GenerationResult, error)

+ func (m *Model) Close() error

+ func (m *Model) Config() Config

+ func (m *Model) Detokenize(ids []uint32) string

+ func (m *Model) Embed(ctx context.Context, text string) (EmbeddingResult, error)

+ func (m *Model) GGUF() *GGUFFile

+ func (m *Model) Generate(ctx context.Context, prompt string, opts ...GenOption) (GenerationResult, error)

+ func (m *Model) HTTPHandler(opts HandlerOptions) http.Handler

+ func (m *Model) Info() LoadInfo

+ func (m *Model) Name() string

+ func (m *Model) NearestTokens(token string, k int) ([]TokenMatch, error)

+ func (m *Model) Runner() *Runner

+ func (m *Model) Stream(ctx context.Context, messages []ChatMessage, onDelta func(delta string) error, ...) (GenerationResult, error)

+ func (m *Model) Tokenize(text string) []uint32

+ func (m *Model) Tokenizer() *Tokenizer

+ type ModelEntry struct

+ Architecture string

+ FileName string

+ ID string

+ IsProjector bool

+ IsSupported bool

+ ModelName string

+ Path string

+ Repository string

+ SizeBytes int64

+ func DiscoverModels(root string, logw io.Writer) ([]ModelEntry, error)

+ func PromptModelSelection(dir string, entries []ModelEntry, in io.Reader, out io.Writer) (ModelEntry, error)

+ func SelectModel(entries []ModelEntry, selector string) (ModelEntry, error)

+ func (m ModelEntry) Status() string

+ type ModelWeights struct

+ Layers []LayerWeights

+ Output Weight

+ OutputNorm []float32

+ TokenEmbd Weight

+ type OllamaChatRequest struct

+ Messages []OllamaMessage

+ Model string

+ Options OllamaOptions

+ Stream bool

+ Tools []ToolDefinition

+ func (o OllamaChatRequest) ChatMessages() []ChatMessage

+ func (o OllamaChatRequest) GenerationOptions(def GenerationOptions) GenerationOptions

+ type OllamaEmbeddingRequest struct

+ Input any

+ Model string

+ Prompt string

+ func (o OllamaEmbeddingRequest) Inputs() []string

+ type OllamaGenerateRequest struct

+ Model string

+ Options OllamaOptions

+ Prompt string

+ Stop any

+ Stream bool

+ System string

+ func (o OllamaGenerateRequest) GenerationOptions(def GenerationOptions) GenerationOptions

+ type OllamaMessage struct

+ Content string

+ Role string

+ type OllamaOptions struct

+ MinP *float32

+ NumPredict *int

+ RepeatPenalty *float32

+ Seed *uint64

+ Stop any

+ Temperature *float32

+ TopK *int

+ TopP *float32

+ type OpenAIChatRequest struct

+ MaxCompletionTokens *int

+ MaxTokens *int

+ Messages []APIMessage

+ MinP *float32

+ Model string

+ RepeatPenalty *float32

+ Seed *uint64

+ Stop any

+ Stream bool

+ SystemPrompt *string

+ Temperature *float32

+ ToolChoice any

+ Tools []ToolDefinition

+ TopK *int

+ TopP *float32

+ func (o OpenAIChatRequest) ChatMessages() []ChatMessage

+ func (o OpenAIChatRequest) Options(def GenerationOptions) GenerationOptions

+ type OpenAICompletionRequest struct

+ MaxCompletionTokens *int

+ MaxTokens *int

+ MinP *float32

+ Model string

+ Prompt any

+ RepeatPenalty *float32

+ Seed *uint64

+ Stop any

+ SystemPrompt *string

+ Temperature *float32

+ TopK *int

+ TopP *float32

+ func (o OpenAICompletionRequest) Options(def GenerationOptions) GenerationOptions

+ func (o OpenAICompletionRequest) PromptString() string

+ type Option func(*loadSettings)

+ func WithLogWriter(w io.Writer) Option

+ func WithMetal(enabled bool) Option

+ func WithPrepareQuantized(enabled bool) Option

+ func WithThreads(n int) Option

+ type Pair struct

+ Left string

+ Right string

+ type PreparedQuantizedWeight struct

+ Blocks int

+ Cols int

+ Mins []float32

+ Rows int

+ Scales []float32

+ Type GGMLType

+ func PrepareQuantizedWeight(data []byte, typ GGMLType, rows, cols int) *PreparedQuantizedWeight

+ type Q4KMatrix struct

+ Cols int

+ Data []byte

+ Rows int

+ type Rng struct

+ func NewRng(seed uint64) *Rng

+ func (r *Rng) NextF32() float32

+ type Runner struct

+ func RunnerFromGGUFBytes(data []byte) (*Runner, error)

+ func RunnerFromGGUFBytesWithOptions(data []byte, options LoadOptions) (*Runner, error)

+ func (r *Runner) Architecture() string

+ func (r *Runner) Close() error

+ func (r *Runner) Config() Config

+ func (r *Runner) Embed(text string) (EmbeddingResult, error)

+ func (r *Runner) GGUF() *GGUFFile

+ func (r *Runner) Generate(prompt string, options GenerationOptions) (GenerationResult, error)

+ func (r *Runner) GenerateChat(messages []ChatMessage, options GenerationOptions) (GenerationResult, error)

+ func (r *Runner) GenerateChatStream(messages []ChatMessage, options GenerationOptions, onToken func(string)) (GenerationResult, error)

+ func (r *Runner) GenerateChatStreamUntil(messages []ChatMessage, options GenerationOptions, onToken func(string) bool) (GenerationResult, error)

+ func (r *Runner) GenerateStream(prompt string, options GenerationOptions, onToken func(string)) (GenerationResult, error)

+ func (r *Runner) ModelName() (string, bool)

+ func (r *Runner) NearestTokens(id uint32, k int) ([]TokenMatch, error)

+ func (r *Runner) Tokenizer() *Tokenizer

+ type SamplerConfig struct

+ MinP float32

+ RepeatPenalty float32

+ Temperature float32

+ TopK int

+ TopP float32

+ func DefaultSamplerConfig() SamplerConfig

+ type ServeOptions struct

+ Addr string

+ ChatHistoryLock *sync.Mutex

+ ChatHistoryPath string

+ ChatUI bool

+ Defaults GenerationOptions

+ LogWriter io.Writer

+ MaxConcurrentConnections int

+ ModelDir string

+ ModelPath string

+ SkillsDir string

+ type Skill struct

+ Body string

+ Description string

+ Name string

+ Path string

+ func LoadSkills(dir string) ([]Skill, error)

+ type TensorInfo struct

+ DType GGMLType

+ Dims []uint64

+ Name string

+ Offset uint64

+ func (t TensorInfo) Numel() int

+ type TensorStat struct

+ Bytes int64

+ Name string

+ Params int64

+ Type GGMLType

+ type TokenMatch struct

+ ID uint32

+ Score float32

+ Text string

+ func SearchTokens(tok *Tokenizer, query string, limit int) []TokenMatch

+ type TokenProb struct

+ Prob float32

+ Token int

+ type Tokenizer struct

+ AddBOS bool

+ BOSID uint32

+ ByteDecoder map[rune]byte

+ ByteEncoder map[byte]rune

+ EOSID uint32

+ MergeRanks map[Pair]int

+ Mode TokenizerMode

+ Pre string

+ Scores []float32

+ TokenToID map[string]uint32

+ Vocab []string

+ func TokenizerFromMetadata(metadata map[string]MetaValue) (*Tokenizer, error)

+ func (t *Tokenizer) DecodeToken(id uint32) string

+ func (t *Tokenizer) Encode(text string) []uint32

+ func (t *Tokenizer) EncodeWithoutBOS(text string) []uint32

+ func (t *Tokenizer) SpecialID(token string) (uint32, bool)

+ func (t *Tokenizer) VocabSize() int

+ type TokenizerMode int

+ const TokenizerGPT2BPE

+ const TokenizerSentencePiece

+ type ToolCall struct

+ Function ToolCallFunction

+ ID string

+ Type string

+ type ToolCallFunction struct

+ Arguments string

+ Name string

+ type ToolDefinition struct

+ Function ToolFunctionDef

+ Type string

+ type ToolFunctionDef struct

+ Description string

+ Name string

+ Parameters json.RawMessage

+ type Weight struct

+ Cols int

+ F32 []float32

+ Metal *MetalWeight

+ Prepared *PreparedQuantizedWeight

+ Raw []byte

+ Rows int

+ Type GGMLType

+ func (w Weight) Matvec(x []float32) []float32

+ func (w Weight) MatvecInto(x []float32, out *[]float32)

+ func (w Weight) Row(row, cols int) []float32

+ func (w Weight) RowF32(row, cols int) []float32

+ func (w Weight) RowInto(row, cols int, out *[]float32)

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL