llm

Documentation

Index

• Variables
• func LoadModel(model string, maxArraySize int) (*ggml.GGML, error)
• func PredictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projectors []string, ...) (bool, uint64)
• type CompletionRequest
• type CompletionResponse
• type DetokenizeRequest
• type DetokenizeResponse
• type DoneReason
  • func (d DoneReason) String() string
• type EmbeddingRequest
• type EmbeddingResponse
• type ImageData
• type LlamaServer
  • func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, ...) (LlamaServer, error)
• type MemoryEstimate
  • func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []string, opts api.Options, ...) MemoryEstimate
  • func (m MemoryEstimate) LogValue() slog.Value
• type ServerStatus
  • func (s ServerStatus) String() string
• type ServerStatusResponse
• type StatusWriter
  • func NewStatusWriter(out *os.File) *StatusWriter
  • func (w *StatusWriter) Write(b []byte) (int, error)
• type TokenizeRequest
• type TokenizeResponse

Variables

var LlamaServerSysProcAttr = &syscall.SysProcAttr{}

Functions

func LoadModel

func LoadModel(model string, maxArraySize int) (*ggml.GGML, error)

LoadModel will load a model from disk. The model must be in the GGML format.

It collects array values for arrays with a size less than or equal to maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If the maxArraySize is negative, all arrays are collected.

func PredictServerFit

func PredictServerFit(allGpus discover.GpuInfoList, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64)

This algorithm looks for a complete fit to determine if we need to unload other models

Types

type CompletionRequest

type CompletionRequest struct {
	Prompt  string
	Format  json.RawMessage
	Images  []ImageData
	Options *api.Options

	Grammar string // set before sending the request to the subprocess
}

type CompletionResponse

type CompletionResponse struct {
	Content            string        `json:"content"`
	DoneReason         DoneReason    `json:"done_reason"`
	Done               bool          `json:"done"`
	PromptEvalCount    int           `json:"prompt_eval_count"`
	PromptEvalDuration time.Duration `json:"prompt_eval_duration"`
	EvalCount          int           `json:"eval_count"`
	EvalDuration       time.Duration `json:"eval_duration"`
}

type DetokenizeRequest

type DetokenizeRequest struct {
	Tokens []int `json:"tokens"`
}

type DetokenizeResponse

type DetokenizeResponse struct {
	Content string `json:"content"`
}

type DoneReason

type DoneReason int

DoneReason represents the reason why a completion response is done

const (
	// DoneReasonStop indicates the completion stopped naturally
	DoneReasonStop DoneReason = iota
	// DoneReasonLength indicates the completion stopped due to length limits
	DoneReasonLength
	// DoneReasonConnectionClosed indicates the completion stopped due to the connection being closed
	DoneReasonConnectionClosed
)

func (DoneReason) String

func (d DoneReason) String() string

type EmbeddingRequest

type EmbeddingRequest struct {
	Content string `json:"content"`
}

type EmbeddingResponse

type EmbeddingResponse struct {
	Embedding []float32 `json:"embedding"`
}

type ImageData

type ImageData struct {
	Data []byte `json:"data"`
	ID   int    `json:"id"`
}

type LlamaServer

type LlamaServer interface {
	Ping(ctx context.Context) error
	WaitUntilRunning(ctx context.Context) error
	Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error
	Embedding(ctx context.Context, input string) ([]float32, error)
	Tokenize(ctx context.Context, content string) ([]int, error)
	Detokenize(ctx context.Context, tokens []int) (string, error)
	Close() error
	EstimatedVRAM() uint64 // Total VRAM across all GPUs
	EstimatedTotal() uint64
	EstimatedVRAMByGPU(gpuID string) uint64
	Pid() int
}

func NewLlamaServer

func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error)

NewLlamaServer will run a server for the given GPUs The gpu list must be a single family.

type MemoryEstimate

type MemoryEstimate struct {
	// How many layers we predict we can load
	Layers int

	// The size of the graph which occupies the main GPU
	Graph uint64

	// How much VRAM will be allocated given the number of layers we predict
	VRAMSize uint64

	// The total size of the model if loaded into VRAM.  If all layers are loaded, VRAMSize == TotalSize
	TotalSize uint64

	// For multi-GPU scenarios, this provides the tensor split parameter
	TensorSplit string

	// For multi-GPU scenarios, this is the size in bytes per GPU
	GPUSizes []uint64
	// contains filtered or unexported fields
}

func EstimateGPULayers

func EstimateGPULayers(gpus []discover.GpuInfo, f *ggml.GGML, projectors []string, opts api.Options, numParallel int) MemoryEstimate

Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size The GPUs provided must all be the same Library

func (MemoryEstimate) LogValue

func (m MemoryEstimate) LogValue() slog.Value

type ServerStatus

type ServerStatus int

const (
	ServerStatusReady ServerStatus = iota
	ServerStatusNoSlotsAvailable
	ServerStatusLoadingModel
	ServerStatusNotResponding
	ServerStatusError
)

func (ServerStatus) String

func (s ServerStatus) String() string

type ServerStatusResponse

type ServerStatusResponse struct {
	Status   ServerStatus `json:"status"`
	Progress float32      `json:"progress"`
}

type StatusWriter

type StatusWriter struct {
	LastErrMsg string
	// contains filtered or unexported fields
}

StatusWriter is a writer that captures error messages from the llama runner process

func NewStatusWriter

func NewStatusWriter(out *os.File) *StatusWriter

func (*StatusWriter) Write

func (w *StatusWriter) Write(b []byte) (int, error)

type TokenizeRequest

type TokenizeRequest struct {
	Content string `json:"content"`
}

type TokenizeResponse

type TokenizeResponse struct {
	Tokens []int `json:"tokens"`
}