compare

Documentation

Overview

Package compare diffs two mcpbench RunResults and applies regression thresholds for CI gating.

Index

• type Config
  • func Default() Config
• type Diff
• type Report
  • func Compare(baseline, candidate *RunFile, cfg Config) Report
  • func (r Report) WriteJSON(w io.Writer) error
  • func (r Report) WriteText(w io.Writer)
• type RunFile
  • func LoadRun(path string) (*RunFile, error)
• type Threshold
  • func ParseThreshold(s string) (Threshold, error)
• type ThresholdKind

Types

type Config

type Config struct {
	P95Threshold   *Threshold
	P99Threshold   *Threshold
	ErrorThreshold *Threshold // must be KindPercentagePoint
	// PerTool, when true, applies thresholds to per-tool entries as well as
	// overall. Default true.
	PerTool bool
}

Config controls the comparison behaviour.

func Default

func Default() Config

Default returns sensible defaults for a CI gate.

type Diff

type Diff struct {
	Tool            string   `json:"tool"`
	BaselineCount   int      `json:"baseline_count"`
	CandidateCount  int      `json:"candidate_count"`
	BaselineP95     float64  `json:"baseline_p95_ms"`
	CandidateP95    float64  `json:"candidate_p95_ms"`
	DeltaP95Rel     float64  `json:"delta_p95_rel"`
	DeltaP95MS      float64  `json:"delta_p95_ms"`
	BaselineP99     float64  `json:"baseline_p99_ms"`
	CandidateP99    float64  `json:"candidate_p99_ms"`
	DeltaP99Rel     float64  `json:"delta_p99_rel"`
	DeltaP99MS      float64  `json:"delta_p99_ms"`
	BaselineErrPct  float64  `json:"baseline_err_pct"`
	CandidateErrPct float64  `json:"candidate_err_pct"`
	DeltaErrPP      float64  `json:"delta_err_pp"`
	Regressed       bool     `json:"regressed"`
	Reasons         []string `json:"reasons,omitempty"`
}

Diff is the resulting per-tool diff.

type Report

type Report struct {
	Overall     Diff     `json:"overall"`
	PerTool     []Diff   `json:"per_tool"`
	UniqueToA   []string `json:"only_in_baseline,omitempty"`
	UniqueToB   []string `json:"only_in_candidate,omitempty"`
	Regressed   bool     `json:"regressed"`
	SummaryNote string   `json:"summary"`
}

Report is the full comparison output.

func Compare

func Compare(baseline, candidate *RunFile, cfg Config) Report

Compare builds a Report by diffing baseline against candidate using cfg.

func (Report) WriteJSON

func (r Report) WriteJSON(w io.Writer) error

WriteJSON writes the report as indented JSON.

func (Report) WriteText

func (r Report) WriteText(w io.Writer)

WriteText writes a human-readable comparison report.

type RunFile

type RunFile struct {
	Version    string                   `json:"version"`
	Scenario   string                   `json:"scenario"`
	StartedAt  time.Time                `json:"started_at"`
	EndedAt    time.Time                `json:"ended_at"`
	Throughput float64                  `json:"throughput_per_sec"`
	Summary    metrics.AggregateSummary `json:"summary"`
}

RunFile mirrors the JSON structure emitted by the `run` subcommand; see internal/report.

func LoadRun

func LoadRun(path string) (*RunFile, error)

LoadRun reads a serialized mcpbench RunResult from disk.

type Threshold

type Threshold struct {
	Kind  ThresholdKind
	Value float64 // interpretation depends on Kind
}

Threshold is a parsed comparison threshold.

func ParseThreshold

func ParseThreshold(s string) (Threshold, error)

ParseThreshold reads a threshold string such as "+20%", "-5%", "+10ms", "+2pp" and returns a parsed Threshold.

type ThresholdKind

type ThresholdKind int

ThresholdKind enumerates threshold varieties accepted by the CLI.

const (
	// ThresholdKindRelative represents a relative change (e.g. "+20%").
	ThresholdKindRelative ThresholdKind = iota
	// ThresholdKindAbsoluteMS represents a duration delta in milliseconds
	// (e.g. "+10ms").
	ThresholdKindAbsoluteMS
	// ThresholdKindPercentagePoint represents a percentage-point delta (e.g.
	// "+2pp") — used for error rate deltas.
	ThresholdKindPercentagePoint
)