agenteval

package module

Go to main page

Versions in this module

v0

v0.0.10

Oct 2, 2026

Changes in this version

+ const UnjudgedA

+ const UnjudgedB

+ const UnjudgedBoth

+ func SampleName(task string, sample int) string

type Comparison

+ Unjudged int

type Pair

+ Unjudged string

type Runner

+ func (r *Runner) JudgeGroups(ctx context.Context, suite *Suite) (*Report, error)

type Summary

+ Unjudged int

v0.0.9

Oct 1, 2026

Changes in this version

+ const SampleMeta

type ConfigDiff

+ FoldFailed *Change

+ type GroupJudge interface

+ JudgeGroup func(ctx context.Context, group []Member, task Task) ([]Score, error)

+ Name func() string

+ type GroupJudgeFunc struct

+ Fn func(ctx context.Context, group []Member, task Task) ([]Score, error)

+ JudgeName string

+ func (j GroupJudgeFunc) JudgeGroup(ctx context.Context, group []Member, task Task) ([]Score, error)

+ func (j GroupJudgeFunc) Name() string

+ type Member struct

+ Scores []Score

+ Trajectory export.Trajectory

type OutcomeDetails

+ Sample int

type Pair

+ Sample int

type Report

+ func (r *Report) Sample(task string, sample int) (*Result, bool)

type Result

+ Sample int

type Runner

+ GroupJudges []GroupJudge

+ JudgeFailedRuns bool

+ Samples int

type TaskRecord

+ GroupJudges []string

+ Sample int

+ Samples int

v0.0.8

Oct 1, 2026

v0.0.7

Sep 30, 2026

v0.0.6

Sep 29, 2026

Changes in this version

type Runner

+ Build func(ctx context.Context, t Task, rec *session.Recorder) (cfg agentturn.Config, close func() error, err error)

+ SessionOptions func(Task) []session.Option

v0.0.5

Sep 29, 2026

Changes in this version

type ConfigDiff

+ Folded *Change

type Result

+ ResumeBound bool

v0.0.4

Sep 28, 2026

v0.0.3

Sep 24, 2026

v0.0.2

Sep 24, 2026

Changes in this version

+ const DefaultMaxResumes

+ var ErrUnreplayable = errors.New("agenteval: the run cannot be replayed strictly")

type ConfigDiff

+ BeyondSettings bool

type Result

+ Resumes int

type Runner

+ Answer func(context.Context, *agentturn.RunEnd) ([]agentturn.Answer, error)

+ ConfigWith func(Task, *session.Recorder) agentturn.Config

+ MaxResumes int

v0.0.1

Sep 21, 2026

Changes in this version

+ const SuiteFile

+ const TaskNS

+ var ErrNoPrompts = errors.New("agenteval: session has no user prompts on its path")

+ func NewOutcome(score Score, target, task string) (*agentsession.OutcomeEntry, error)

+ func ReadOutcome(e *agentsession.OutcomeEntry) (Score, OutcomeDetails, bool)

+ func Summarize(results []Result) map[string]Summary

+ type Change struct

+ A any

+ B any

+ type Comparison struct

+ A *Report

+ B *Report

+ ByJudge map[string]float64

+ Config ConfigDiff

+ Manifest Manifest

+ Pairs []Pair

+ Suite string

+ Uniform bool

+ func Compare(ctx context.Context, suite *Suite, a, b *Runner) (*Comparison, error)

+ func (c *Comparison) WriteJSON(w io.Writer) error

+ type ConfigDiff struct

+ Extra map[string]Change

+ Instructions *Change

+ Model *Change

+ Reasoning *Change

+ Text *Change

+ ToolsAdded []string

+ ToolsChanged []string

+ ToolsRemoved []string

+ func DiffSettings(a, b agentsession.Settings) ConfigDiff

+ func (d ConfigDiff) Empty() bool

+ type Judge interface

+ Judge func(ctx context.Context, t export.Trajectory, task Task) (Score, error)

+ Name func() string

+ type JudgeFunc struct

+ Fn func(ctx context.Context, t export.Trajectory, task Task) (Score, error)

+ JudgeName string

+ func (j JudgeFunc) Judge(ctx context.Context, t export.Trajectory, task Task) (Score, error)

+ func (j JudgeFunc) Name() string

+ type Manifest struct

+ Files map[string]string

+ Location string

+ type OutcomeDetails struct

+ Judge json.RawMessage

+ Reason string

+ Session string

+ Task string

+ type Pair struct

+ A *Result

+ B *Result

+ Config ConfigDiff

+ Delta map[string]float64

+ Task string

+ type Report struct

+ ByJudge map[string]Summary

+ Manifest Manifest

+ Results []Result

+ Suite string

+ func ReadReport(rd io.Reader) (*Report, error)

+ func (r *Report) Judges() []string

+ func (r *Report) Result(task string) (*Result, bool)

+ func (r *Report) WriteJSON(w io.Writer) error

+ type Result struct

+ CostUSD *float64

+ Ends []*agentturn.RunEnd

+ Err error

+ Reason agentturn.Reason

+ Runs int

+ Scores []Score

+ SessionID string

+ Target string

+ Task Task

+ Usage openresponses.Usage

+ func (r *Result) UnmarshalJSON(data []byte) error

+ func (r Result) MarshalJSON() ([]byte, error)

+ type Runner struct

+ Config func(Task) agentturn.Config

+ Cost func(model string, usage openresponses.Usage) (float64, bool)

+ Header func(Task) agentsession.Header

+ Judges []Judge

+ Parallel int

+ Store agentsession.Store

+ func (r *Runner) Run(ctx context.Context, suite *Suite) (*Report, error)

+ type Score struct

+ Details json.RawMessage

+ Judge string

+ Pass bool

+ Reason string

+ Session string

+ Value float64

+ type Suite struct

+ Manifest Manifest

+ Name string

+ Tasks []Task

+ func LoadSuite(fsys fs.FS, location string) (*Suite, error)

+ type Summary struct

+ Count int

+ Mean float64

+ PassRate float64

+ Passed int

+ type Task struct

+ Expect json.RawMessage

+ ID string

+ Instruction string

+ Meta map[string]string

+ Prompts openresponses.Items

+ Setup map[string]string

+ func FromSession(s *agentsession.Session) (Task, error)

+ func FromSessionAt(s *agentsession.Session, leaf string) (Task, error)

+ func (t Task) Inputs() openresponses.Items

+ type TaskRecord struct

+ Location string

+ Meta map[string]string

+ Setup map[string]string

+ Suite string

+ Task string

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL