llm

Documentation

Index

• Constants
• Variables
• func Init(workdir string) error
• type DetokenizeRequest
• type DetokenizeResponse
• type EmbeddingRequest
• type EmbeddingResponse
• type GGML
  • func DecodeGGML(r io.ReadSeeker) (*GGML, error)
• type ImageData
• type LLM
  • func New(workDir, model string, adapters, projectors []string, opts api.Options) (LLM, error)
• type PredictOpts
• type PredictResult
• type TokenizeRequest
• type TokenizeResponse

Constants

const (
	// Magic constant for `ggml` files (unversioned).
	FILE_MAGIC_GGML = 0x67676d6c
	// Magic constant for `ggml` files (versioned, ggmf).
	FILE_MAGIC_GGMF = 0x67676d66
	// Magic constant for `ggml` files (versioned, ggjt).
	FILE_MAGIC_GGJT = 0x67676a74
	// Magic constant for `ggla` files (LoRA adapter).
	FILE_MAGIC_GGLA = 0x67676C61
	// Magic constant for `gguf` files (versioned, gguf)
	FILE_MAGIC_GGUF_LE = 0x46554747
	FILE_MAGIC_GGUF_BE = 0x47475546
)

Variables

var ErrUnsupportedFormat = errors.New("unsupported model format")

Functions

func Init

func Init(workdir string) error

Give any native cgo implementations an opportunity to initialize

Types

type DetokenizeRequest

type DetokenizeRequest struct {
	Tokens []int `json:"tokens"`
}

type DetokenizeResponse

type DetokenizeResponse struct {
	Content string `json:"content"`
}

type EmbeddingRequest

type EmbeddingRequest struct {
	Content string `json:"content"`
}

type EmbeddingResponse

type EmbeddingResponse struct {
	Embedding []float64 `json:"embedding"`
}

type GGML

type GGML struct {
	Size int64
	// contains filtered or unexported fields
}

func DecodeGGML

func DecodeGGML(r io.ReadSeeker) (*GGML, error)

type ImageData

type ImageData struct {
	Data []byte `json:"data"`
	ID   int    `json:"id"`
}

type LLM

type LLM interface {
	Predict(context.Context, PredictOpts, func(PredictResult)) error
	Embedding(context.Context, string) ([]float64, error)
	Encode(context.Context, string) ([]int, error)
	Decode(context.Context, []int) (string, error)
	Close()
}

func New

func New(workDir, model string, adapters, projectors []string, opts api.Options) (LLM, error)

type PredictOpts

type PredictOpts struct {
	Prompt  string
	Format  string
	Images  []ImageData
	Options api.Options
}

type PredictResult

type PredictResult struct {
	Content            string
	Done               bool
	PromptEvalCount    int
	PromptEvalDuration time.Duration
	EvalCount          int
	EvalDuration       time.Duration
}

type TokenizeRequest

type TokenizeRequest struct {
	Content string `json:"content"`
}

type TokenizeResponse

type TokenizeResponse struct {
	Tokens []int `json:"tokens"`
}