Versions in this module Expand all Collapse all v0 v0.5.0 Aug 15, 2026 v0.4.0 Jun 28, 2026 Changes in this version + type Bench struct + Concurrency int + Metrics []Metric + OnResult func(task Task, s Sample, err error) + Target Target + Tasks []Task + Timeout time.Duration + func (b Bench) Run(ctx context.Context) (Report, []Sample) + type Comparison struct + Concurrency int + Gate []string + Metrics []Metric + OnResult func(target string, task Task, s Sample, err error) + Targets []NamedTarget + Tasks []Task + Timeout time.Duration + func (c Comparison) Run(ctx context.Context) ComparisonReport + type ComparisonReport struct + Gate []string + Targets []TargetReport + func (cr ComparisonReport) Grid() map[string]map[string]bool + func (cr ComparisonReport) RenderGrid(w io.Writer) + type ExecTarget struct + Command []string + Dir string + Env map[string]string + func (e ExecTarget) Run(ctx context.Context, task Task) (RunOutput, error) + type ExecTargetSpec struct + Command []string + Dir string + Env map[string]string + Name string + type NamedTarget struct + Name string + Target Target + func LoadTargets(r io.Reader) ([]NamedTarget, error) + type RunOutput struct + Context []string + Output string + Plan string + ToolCalls []ToolCall + Trajectory []string + type Runner struct + Concurrency int + OnResult func(task Task, s Sample, err error) + Target Target + Timeout time.Duration + func (r Runner) Run(ctx context.Context, tasks []Task) []Sample + type Target interface + Run func(ctx context.Context, task Task) (RunOutput, error) + type TargetFunc func(ctx context.Context, task Task) (RunOutput, error) + func (f TargetFunc) Run(ctx context.Context, task Task) (RunOutput, error) + type TargetReport struct + Name string + Report Report + Samples []Sample + type Task struct + Context []string + Expected string + ExpectedTools []string + Files map[string]string + Input string + Meta map[string]string + Name string + Rubric string + func LoadTasks(r io.Reader) ([]Task, error) + func (t Task) Sample(out RunOutput) Sample v0.3.0 Jun 27, 2026 Changes in this version + var ErrNoLabels = errors.New("no samples carry a label for this metric") + type AlignmentReport []AlignmentResult + func (a AlignmentReport) Failing(minKappa, minF1 float64) []string + func (a AlignmentReport) WriteConsole(w io.Writer, maxDisagreements int) + func (a AlignmentReport) WriteJSON(w io.Writer) error + type AlignmentResult struct + Accuracy float64 + BestThreshold float64 + Disagreements []Disagreement + Errored int + F1 float64 + FN int + FP int + Kappa float64 + MAE float64 + Metric string + N int + Pearson float64 + Precision float64 + RMSE float64 + Recall float64 + Spearman float64 + TN int + TP int + func AlignMetric(ctx context.Context, samples []Sample, metric Metric, labelKey string) (AlignmentResult, error) + type Disagreement struct + Human float64 + JudgePassed bool + JudgeScore float64 + Reason string + Sample string type Sample + Labels map[string]float64 v0.2.0 Jun 26, 2026 Changes in this version + var AllAttackKinds = []AttackKind + func AttackKinds() []string + func EstimateTokens(s string) int + type AttackKind string + const HarmfulRequest + const Jailbreak + const PIIExtraction + const PromptInjection + type ChangeStatus string + const Added + const Declined + const Fixed + const Improved + const Regressed + const Removed + const Unchanged + type DAGNode interface + func Branch(question string, choices map[string]DAGNode, fallback DAGNode) DAGNode + func Leaf(score float64, reason string) DAGNode + func YesNo(question string, yes, no DAGNode) DAGNode + type Diff []ScoreDelta + func DiffReports(old, new Report) Diff + func (d Diff) Regressions() int + func (d Diff) WriteConsole(w io.Writer) type JudgeFunc + func Cache(j JudgeFunc, dir string) JudgeFunc + type Meter struct + func NewMeter(costInPerM, costOutPerM float64) *Meter + func (m *Meter) Usage() Usage + func (m *Meter) Wrap(j JudgeFunc) JudgeFunc type Metric + func ArgumentCorrectness(judge JudgeFunc, passThreshold float64) Metric + func AttackResistance(judge JudgeFunc, passThreshold float64) Metric + func Bias(judge JudgeFunc, passThreshold float64) Metric + func ContextualRecall(judge JudgeFunc, passThreshold float64) Metric + func ContextualRelevancy(judge JudgeFunc, passThreshold float64) Metric + func ConversationCompleteness(judge JudgeFunc, passThreshold float64) Metric + func ConversationRelevancy(judge JudgeFunc, passThreshold float64) Metric + func DAG(judge JudgeFunc, name string, passThreshold float64, root DAGNode) Metric + func Hallucination(judge JudgeFunc, passThreshold float64) Metric + func KnowledgeRetention(judge JudgeFunc, passThreshold float64) Metric + func PIILeakage(judge JudgeFunc) Metric + func PlanAdherence(judge JudgeFunc, passThreshold float64) Metric + func PlanQuality(judge JudgeFunc, passThreshold float64) Metric + func RefusalPresent() Metric + func RoleAdherence(judge JudgeFunc, passThreshold float64) Metric + func StepEfficiency(judge JudgeFunc, passThreshold float64) Metric + func Summarization(judge JudgeFunc, passThreshold float64) Metric + func TaskCompletion(judge JudgeFunc, passThreshold float64) Metric + func ToolCorrectness() Metric + func Toxicity(judge JudgeFunc, passThreshold float64) Metric + type RedTeam struct + Enhance int + Judge JudgeFunc + Kinds []AttackKind + func (rt RedTeam) Generate(ctx context.Context) ([]Sample, error) type Report + Usage *Usage + func LoadReport(r io.Reader) (Report, error) type Sample + ExpectedTools []string + Persona string + Plan string + ToolCalls []ToolCall + Trajectory []string + Turns []Turn + func Filter(samples []Sample, keep func(Sample) bool) []Sample + func FilterMeta(samples []Sample, key, value string) []Sample + func LoadCSV(r io.Reader) ([]Sample, error) + func LoadJSON(r io.Reader) ([]Sample, error) + func LoadJSONL(r io.Reader) ([]Sample, error) + type ScoreDelta struct + Metric string + New float64 + NewPassed bool + Old float64 + OldPassed bool + Sample string + Status ChangeStatus + type Synthesizer struct + Evolutions int + Judge JudgeFunc + PerContext int + Rubric string + func (sy Synthesizer) FromContexts(ctx context.Context, groups [][]string) ([]Sample, error) + func (sy Synthesizer) FromDocuments(ctx context.Context, docs []string, chunkSize int) ([]Sample, error) + type ToolCall struct + Args map[string]any + Name string + Output string + type Turn struct + Content string + Role string + type Usage struct + Calls int + CompletionTokens int + Cost float64 + JudgeSeconds float64 + PromptTokens int + TotalTokens int v0.1.1 Jun 25, 2026 v0.1.0 Jun 25, 2026 Changes in this version + func RegisteredMetrics() []string + func RunT(t *testing.T, s Suite) + type JudgeFunc func(ctx context.Context, prompt string) (string, error) + func RateLimit(j JudgeFunc, rps float64, burst int) JudgeFunc + type Metric interface + Name func() string + Score func(ctx context.Context, s Sample) (Result, error) + func AnswerRelevancy(judge JudgeFunc, passThreshold float64) Metric + func BuildMetrics(names []string, spec MetricSpec) ([]Metric, error) + func CitationPresent() Metric + func Contains(substr string) Metric + func ContextualPrecision(judge JudgeFunc, passThreshold float64) Metric + func ExactMatch() Metric + func Faithfulness(judge JudgeFunc) Metric + func ForbidsRegex(pattern string) Metric + func JSONHasFields(fields ...string) Metric + func MatchesRegex(pattern string) Metric + func NonEmpty() Metric + func RubricJudge(judge JudgeFunc, passThreshold float64) Metric + func ValidJSON() Metric + type MetricFunc struct + Fn func(ctx context.Context, s Sample) (Result, error) + MetricName string + func (m MetricFunc) Name() string + func (m MetricFunc) Score(ctx context.Context, s Sample) (Result, error) + type MetricSpec struct + Judge JudgeFunc + Threshold float64 + type MetricSummary struct + MeanScore float64 + Metric string + PassRate float64 + Passed int + Total int + type Report struct + Samples []SampleReport + func (r Report) Failed() bool + func (r Report) Summary() []MetricSummary + func (r Report) WriteConsole(w io.Writer) + func (r Report) WriteJSON(w io.Writer) error + type Result struct + Err string + Metric string + Passed bool + Reason string + Score float64 + type Sample struct + Context []string + Expected string + Input string + Meta map[string]string + Name string + Output string + Rubric string + type SampleReport struct + Meta map[string]string + Passed bool + Results []Result + Sample string + type Suite struct + Concurrency int + Metrics []Metric + Samples []Sample + func (s Suite) Run(ctx context.Context) Report
Go to main page