llama

Documentation

Index

• Constants
• Variables
• func Colorize(format string, opts ...interface{}) (n int, err error)
• func Eval(lctx *Context, tokens []uint32, tokensCount uint32, pastCount uint32, ...) error
• func Resize(slice []float32, size int) []float32
• func ResizeInplace(slice *[]float32, size int)
• func SampleTopPTopK(lctx *Context, lastNTokens []uint32, lastNTokensSize uint32, topK uint32, ...) uint32
• type Context
  • func LoadModel(fileName string, silent bool) (*Context, error)
  • func NewContext() *Context
• type ContextParams
• type HParams
• type KVCache
• type Layer
• type Model
  • func NewModel() *Model
• type ModelType

Constants

const (
	LLAMA_FILE_VERSION           = 1
	LLAMA_FILE_MAGIC             = 0x67676a74 // 'ggjt' in hex
	LLAMA_FILE_MAGIC_OLD         = 0x67676d66 // 'ggmf' in hex
	LLAMA_FILE_MAGIC_UNVERSIONED = 0x67676d6c // 'ggml' pre-versioned files
)

Variables

var (
	// determine number of model parts based on the dimension
	LLAMA_N_PARTS = map[uint32]uint32{
		4096: 1,
		5120: 2,
		6656: 4,
		8192: 8,
	}
)

Functions

func Colorize

func Colorize(format string, opts ...interface{}) (n int, err error)

func Eval

func Eval(

	lctx *Context,
	tokens []uint32,
	tokensCount uint32,
	pastCount uint32,
	threadsCount int) error

func Resize

func Resize(slice []float32, size int) []float32

Safe Resize() for using instead of C++ std::vector:resize() https://go.dev/play/p/VlQ7N75E5AD

func ResizeInplace

func ResizeInplace(slice *[]float32, size int)

NB! This do not clear the underlying array when resizing https://go.dev/play/p/DbK4dFqwrZn

func SampleTopPTopK

func SampleTopPTopK(
	lctx *Context,
	lastNTokens []uint32,
	lastNTokensSize uint32,
	topK uint32,
	topP float32,
	temp float32,
	repeatPenalty float32,
) uint32

std::mt19937 = A Mersenne Twister pseudo-random generator of 32-bit numbers with a state size of 19937 bits.

Types

type Context

type Context struct {
	Model *Model
	Vocab *ml.Vocab

	// decode output (2-dimensional array: [n_tokens][n_vocab])
	Logits    []float32
	LogitsAll bool

	// input embedding (1-dimensional array: [n_embd])
	Embedding []float32
}

func LoadModel

func LoadModel(
	fileName string,

	silent bool,
) (*Context, error)

func NewContext

func NewContext() *Context

type ContextParams

type ContextParams struct {
	CtxSize    uint32 // text context
	PartsCount int    // -1 for default
	Seed       int    // RNG seed, 0 for random
	LogitsAll  bool   // the llama_eval() call computes all logits, not just the last one
	VocabOnly  bool   // only load the vocabulary, no weights
	UseLock    bool   // force system to keep model in RAM
	Embedding  bool   // embedding mode only
}

struct llama_context_params {

type HParams

type HParams struct {
	// contains filtered or unexported fields
}

default hparams (LLaMA 7B)

type KVCache

type KVCache struct {
	K *ml.Tensor
	V *ml.Tensor

	N uint32 // number of tokens currently in the cache
}

type Layer

type Layer struct {
	// contains filtered or unexported fields
}

type Model

type Model struct {
	Type ModelType
	// contains filtered or unexported fields
}

func NewModel

func NewModel() *Model

type ModelType

type ModelType uint8

const (
	MODEL_UNKNOWN ModelType = iota
	MODEL_7B
	MODEL_13B
	MODEL_30B
	MODEL_65B
)

available llama models