bench

Documentation

Index

• type DurationStats
• type Expectation
• type Fixture
  • func Fixtures() ([]Fixture, error)
  • func LoadFixtures(readFile ReadFileFunc, specs []Spec) ([]Fixture, error)
  • func MustFixtures() []Fixture
  • func MustLoadFixtures(load func() ([]Fixture, error)) []Fixture
  • func (f Fixture) RawCombined() string
• type Harness
  • func NewHarness(maxLines int) *Harness
  • func NewHarnessWithProfiles(list []engine.Profile) *Harness
  • func (h *Harness) Benchmark(fixture Fixture, samples int) (Measurement, error)
  • func (h *Harness) Measure(fixture Fixture) (Measurement, error)
  • func (h *Harness) Profile(name string) (engine.Profile, bool)
  • func (h *Harness) Render(fixture Fixture) (string, error)
• type Measurement
• type Quality
• type ReadFileFunc
• type Spec
  • func Specs() []Spec

Types

type DurationStats

type DurationStats struct {
	Samples int           `json:"samples"`
	Min     time.Duration `json:"min"`
	P50     time.Duration `json:"p50"`
	P95     time.Duration `json:"p95"`
	Max     time.Duration `json:"max"`
	Total   time.Duration `json:"total"`
}

type Expectation

type Expectation struct {
	ContainsOK      bool     `json:"contains_ok"`
	TokenSavingsOK  bool     `json:"token_savings_ok"`
	QualityOK       bool     `json:"quality_ok"`
	MissingContains []string `json:"missing_contains,omitempty"`
	OK              bool     `json:"ok"`
}

type Fixture

type Fixture struct {
	Name             string
	Class            string
	Description      string
	ProfileName      string
	Invocation       engine.Invocation
	Execution        engine.Execution
	ExpectedContains []string
	MinTokenSavings  float64
	MinQualityScore  int
}

func Fixtures

func Fixtures() ([]Fixture, error)

func LoadFixtures

func LoadFixtures(readFile ReadFileFunc, specs []Spec) ([]Fixture, error)

func MustFixtures

func MustFixtures() []Fixture

func MustLoadFixtures

func MustLoadFixtures(load func() ([]Fixture, error)) []Fixture

func (Fixture) RawCombined

func (f Fixture) RawCombined() string

type Harness

type Harness struct {
	// contains filtered or unexported fields
}

func NewHarness

func NewHarness(maxLines int) *Harness

func NewHarnessWithProfiles

func NewHarnessWithProfiles(list []engine.Profile) *Harness

func (*Harness) Benchmark

func (h *Harness) Benchmark(fixture Fixture, samples int) (Measurement, error)

func (*Harness) Measure

func (h *Harness) Measure(fixture Fixture) (Measurement, error)

func (*Harness) Profile

func (h *Harness) Profile(name string) (engine.Profile, bool)

func (*Harness) Render

func (h *Harness) Render(fixture Fixture) (string, error)

type Measurement

type Measurement struct {
	FixtureName        string        `json:"fixture_name"`
	Class              string        `json:"class"`
	ProfileName        string        `json:"profile_name"`
	CommandFingerprint string        `json:"command_fingerprint"`
	Duration           time.Duration `json:"duration"`
	Durations          DurationStats `json:"durations"`
	RawCombined        string        `json:"-"`
	Rendered           string        `json:"-"`
	RawBytes           int           `json:"raw_bytes"`
	ParsedBytes        int           `json:"parsed_bytes"`
	FilteredBytes      int           `json:"filtered_bytes"`
	EmittedBytes       int           `json:"emitted_bytes"`
	SavedBytes         int           `json:"saved_bytes"`
	ByteRatio          float64       `json:"byte_ratio"`
	ByteSavingsPct     float64       `json:"byte_savings_pct"`
	RawTokens          int           `json:"raw_tokens"`
	FilteredTokens     int           `json:"filtered_tokens"`
	SavedTokens        int           `json:"saved_tokens"`
	TokenRatio         float64       `json:"token_ratio"`
	TokenSavingsPct    float64       `json:"token_savings_pct"`
	FallbackCount      int           `json:"fallback_count"`
	FallbackRate       float64       `json:"fallback_rate_pct"`
	TeeCount           int           `json:"tee_count"`
	TeeRate            float64       `json:"tee_rate_pct"`
	FailureCount       int           `json:"failure_count"`
	FailureRate        float64       `json:"failure_rate_pct"`
	Quality            Quality       `json:"quality"`
	Expectation        Expectation   `json:"expectation"`
}

type Quality

type Quality struct {
	Score              int      `json:"score"`
	ActionableLines    int      `json:"actionable_lines"`
	FailureIdentifiers int      `json:"failure_identifiers"`
	PreservedFailures  int      `json:"preserved_failures"`
	FallbackRate       float64  `json:"fallback_rate"`
	ProfileConfidence  string   `json:"profile_confidence"`
	Issues             []string `json:"issues"`
}

type ReadFileFunc

type ReadFileFunc func(name string) ([]byte, error)

type Spec

type Spec struct {
	Name             string
	Class            string
	Description      string
	ProfileName      string
	Command          []string
	Display          []string
	Cwd              string
	ExitCode         int
	StdoutFile       string
	StderrFile       string
	ExpectedContains []string
	MinTokenSavings  float64
	MinQualityScore  int
}

func Specs

func Specs() []Spec