llm

package
v0.3.11-rc2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 16, 2024 License: MIT Imports: 34 Imported by: 8

Documentation

Index

Constants

View Source 
const (
	// Magic constant for `ggml` files (unversioned).
	FILE_MAGIC_GGML = 0x67676d6c
	// Magic constant for `ggml` files (versioned, ggmf).
	FILE_MAGIC_GGMF = 0x67676d66
	// Magic constant for `ggml` files (versioned, ggjt).
	FILE_MAGIC_GGJT = 0x67676a74
	// Magic constant for `ggla` files (LoRA adapter).
	FILE_MAGIC_GGLA = 0x67676C61
	// Magic constant for `gguf` files (versioned, gguf)
	FILE_MAGIC_GGUF_LE = 0x46554747
	FILE_MAGIC_GGUF_BE = 0x47475546
)

Variables

View Source 
var ErrUnsupportedFormat = errors.New("unsupported model format")
View Source 
var LlamaServerSysProcAttr = &syscall.SysProcAttr{}

Functions

func DetectGGMLType added in v0.1.35 

func DetectGGMLType(b []byte) string

func ParseFileType added in v0.1.35 

func ParseFileType(s string) (fileType, error)

func PredictServerFit added in v0.1.33 

func PredictServerFit(allGpus gpu.GpuInfoList, ggml *GGML, adapters, projectors []string, opts api.Options) (bool, uint64)

This algorithm looks for a complete fit to determine if we need to unload other models

func Quantize added in v0.1.32 

func Quantize(infile, outfile string, ftype fileType) error

func SystemInfo added in v0.1.32 

func SystemInfo() string

SystemInfo is an unused example of calling llama.cpp functions using CGo

func WriteGGUF added in v0.3.3 

func WriteGGUF(ws io.WriteSeeker, kv KV, ts []Tensor) error

Types

type CompletionRequest added in v0.1.32 

type CompletionRequest struct {
	Prompt  string
	Format  string
	Images  []ImageData
	Options *api.Options
}

type CompletionResponse added in v0.1.32 

type CompletionResponse struct {
	Content            string
	DoneReason         string
	Done               bool
	PromptEvalCount    int
	PromptEvalDuration time.Duration
	EvalCount          int
	EvalDuration       time.Duration
}

type DetokenizeRequest 

type DetokenizeRequest struct {
	Tokens []int `json:"tokens"`
}

type DetokenizeResponse 

type DetokenizeResponse struct {
	Content string `json:"content"`
}

type EmbeddingRequest 

type EmbeddingRequest struct {
	Content string `json:"content"`
}

type EmbeddingResponse 

type EmbeddingResponse struct {
	Embedding []float32 `json:"embedding"`
}

type GGML 

type GGML struct {
	// contains filtered or unexported fields
}

func DecodeGGML 

func DecodeGGML(rs io.ReadSeeker, maxArraySize int) (*GGML, int64, error)

DecodeGGML decodes a GGML model from the given reader.

It collects array values for arrays with a size less than or equal to maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If the maxArraySize is negative, all arrays are collected.

func LoadModel added in v0.1.33 

func LoadModel(model string, maxArraySize int) (*GGML, error)

LoadModel will load a model from disk. The model must be in the GGML format.

It collects array values for arrays with a size less than or equal to maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If the maxArraySize is negative, all arrays are collected.

func (GGML) GraphSize added in v0.1.32 

func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload uint64)

type ImageData 

type ImageData struct {
	Data []byte `json:"data"`
	ID   int    `json:"id"`
}

type KV 

type KV map[string]any

func (KV) Architecture added in v0.1.32 

func (kv KV) Architecture() string

func (KV) BlockCount added in v0.1.32 

func (kv KV) BlockCount() uint64

func (KV) ChatTemplate added in v0.1.42 

func (kv KV) ChatTemplate() string

func (KV) ContextLength added in v0.1.32 

func (kv KV) ContextLength() uint64

func (KV) EmbeddingHeadCount added in v0.1.45 

func (kv KV) EmbeddingHeadCount() uint64

func (KV) EmbeddingHeadCountK added in v0.1.45 

func (kv KV) EmbeddingHeadCountK() uint64

func (KV) EmbeddingHeadCountV added in v0.1.45 

func (kv KV) EmbeddingHeadCountV() uint64

func (KV) EmbeddingLength added in v0.1.32 

func (kv KV) EmbeddingLength() uint64

func (KV) FileType added in v0.1.32 

func (kv KV) FileType() fileType

func (KV) GQA added in v0.1.32 

func (kv KV) GQA() uint64

func (KV) HeadCount added in v0.1.32 

func (kv KV) HeadCount() uint64

func (KV) HeadCountKV added in v0.1.32 

func (kv KV) HeadCountKV() uint64

func (KV) Kind added in v0.3.7 

func (kv KV) Kind() string

func (KV) ParameterCount added in v0.1.32 

func (kv KV) ParameterCount() uint64

type Layer added in v0.1.32 

type Layer map[string]*Tensor

type LlamaServer added in v0.1.32 

type LlamaServer interface {
	Ping(ctx context.Context) error
	WaitUntilRunning(ctx context.Context) error
	Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error
	Embedding(ctx context.Context, input string) ([]float32, error)
	Tokenize(ctx context.Context, content string) ([]int, error)
	Detokenize(ctx context.Context, tokens []int) (string, error)
	Close() error
	EstimatedVRAM() uint64 // Total VRAM across all GPUs
	EstimatedTotal() uint64
	EstimatedVRAMByGPU(gpuID string) uint64
}

func NewLlamaServer added in v0.1.32 

func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error)

NewLlamaServer will run a server for the given GPUs The gpu list must be a single family.

type MemoryEstimate added in v0.1.45 

type MemoryEstimate struct {
	// How many layers we predict we can load
	Layers int

	// The size of the graph which occupies the main GPU
	Graph uint64

	// How much VRAM will be allocated given the number of layers we predict
	VRAMSize uint64

	// The total size of the model if loaded into VRAM.  If all layers are loaded, VRAMSize == TotalSize
	TotalSize uint64

	// For multi-GPU scenarios, this provides the tensor split parameter
	TensorSplit string

	// For multi-GPU scenarios, this is the size in bytes per GPU
	GPUSizes []uint64
	// contains filtered or unexported fields
}

func EstimateGPULayers added in v0.1.33 

func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts api.Options) MemoryEstimate

Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size The GPUs provided must all be the same Library

type ServerStatus added in v0.1.32 

type ServerStatus int
const (
	ServerStatusReady ServerStatus = iota
	ServerStatusNoSlotsAvailable
	ServerStatusLoadingModel
	ServerStatusNotResponding
	ServerStatusError
)

func (ServerStatus) ToString added in v0.1.33 

func (s ServerStatus) ToString() string

type ServerStatusResp added in v0.1.32 

type ServerStatusResp struct {
	Status          string  `json:"status"`
	SlotsIdle       int     `json:"slots_idle"`
	SlotsProcessing int     `json:"slots_processing"`
	Error           string  `json:"error"`
	Progress        float32 `json:"progress"`
}

type StatusWriter added in v0.1.32 

type StatusWriter struct {
	LastErrMsg string
	// contains filtered or unexported fields
}

StatusWriter is a writer that captures error messages from the llama runner process

func NewStatusWriter added in v0.1.32 

func NewStatusWriter(out *os.File) *StatusWriter

func (*StatusWriter) Write added in v0.1.32 

func (w *StatusWriter) Write(b []byte) (int, error)

type Tensor 

type Tensor struct {
	Name   string `json:"name"`
	Kind   uint32 `json:"kind"`
	Offset uint64 `json:"-"`

	// Shape is the number of elements in each dimension
	Shape []uint64 `json:"shape"`

	io.WriterTo `json:"-"`
}

func (Tensor) Size 

func (t Tensor) Size() uint64

type Tensors added in v0.1.32 

type Tensors struct {
	Items  []*Tensor
	Offset uint64
}

func (Tensors) Layers added in v0.1.32 

func (ts Tensors) Layers() map[string]Layer

type TokenizeRequest 

type TokenizeRequest struct {
	Content string `json:"content"`
}

type TokenizeResponse 

type TokenizeResponse struct {
	Tokens []int `json:"tokens"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.