llamabench

Documentation

Index

• func BuildArgs(c FitConfig) []string
• type Acquirer
  • func NewAcquirer(cacheDir string) *Acquirer
  • func (a *Acquirer) Resolve(explicitPath string) (string, error)
• type BoolLike
  • func (b *BoolLike) UnmarshalJSON(data []byte) error
• type Entry
  • func ParseJSON(data []byte) ([]Entry, error)
  • func RunFit(r Runner, binaryPath string, cfg FitConfig) (*Entry, error)
  • func RunFitAll(r Runner, binaryPath string, cfg FitConfig) ([]Entry, error)
  • func (e Entry) FlashAttnBool() bool
• type ExecRunner
  • func (r ExecRunner) Run(ctx context.Context, binaryPath string, args []string) ([]byte, []byte, error)
• type FitConfig
  • func DefaultFitConfig(modelPath string) FitConfig
• type Runner

Functions

func BuildArgs

func BuildArgs(c FitConfig) []string

Types

type Acquirer

type Acquirer struct {
	CacheDir         string
	Version          string
	Repo             string
	GitHubAPIBaseURL string
	HTTPClient       *http.Client
}

func NewAcquirer

func NewAcquirer(cacheDir string) *Acquirer

func (*Acquirer) Resolve

func (a *Acquirer) Resolve(explicitPath string) (string, error)

type BoolLike

type BoolLike bool

func (*BoolLike) UnmarshalJSON

func (b *BoolLike) UnmarshalJSON(data []byte) error

type Entry

type Entry struct {
	BuildCommit  string    `json:"build_commit"`
	BuildNumber  int       `json:"build_number"`
	ModelSize    int64     `json:"model_size"`
	ModelNParams int64     `json:"model_n_params"`
	NBatch       int       `json:"n_batch"`
	NUBatch      int       `json:"n_ubatch"`
	NThreads     int       `json:"n_threads"`
	NGPULayers   int       `json:"n_gpu_layers"`
	NCPUMoE      int       `json:"n_cpu_moe"`
	TypeK        string    `json:"type_k"`
	TypeV        string    `json:"type_v"`
	FlashAttn    BoolLike  `json:"flash_attn"`
	FitTarget    int       `json:"fit_target"`
	FitMinCtx    int       `json:"fit_min_ctx"`
	NPrompt      int       `json:"n_prompt"`
	NGen         int       `json:"n_gen"`
	AvgTS        float64   `json:"avg_ts"`
	StddevTS     float64   `json:"stddev_ts"`
	SamplesNS    []float64 `json:"samples_ns"`
	SamplesTS    []float64 `json:"samples_ts"`
}

func ParseJSON

func ParseJSON(data []byte) ([]Entry, error)

func RunFit

func RunFit(r Runner, binaryPath string, cfg FitConfig) (*Entry, error)

func RunFitAll

func RunFitAll(r Runner, binaryPath string, cfg FitConfig) ([]Entry, error)

func (Entry) FlashAttnBool

func (e Entry) FlashAttnBool() bool

type ExecRunner

type ExecRunner struct{}

func (ExecRunner) Run

func (r ExecRunner) Run(ctx context.Context, binaryPath string, args []string) ([]byte, []byte, error)

type FitConfig

type FitConfig struct {
	ModelPath  string
	Prompt     int
	Predict    int
	FitCtx     int
	FitTarget  *int
	Timeout    time.Duration
	ExtraFlags []string
}

func DefaultFitConfig

func DefaultFitConfig(modelPath string) FitConfig

type Runner

type Runner interface {
	Run(ctx context.Context, binaryPath string, args []string) ([]byte, []byte, error)
}