report

package
v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 16, 2026 License: MIT Imports: 26 Imported by: 0

Documentation

Overview

Package report turns content-addressed agentic benchmark artifacts into a deterministic, self-contained HTML report. Formal and pilot measurements are loaded from harness artifacts; diagnostic canaries are deliberately kept in a separate, visibly watermarked input channel.

Index

Constants

View Source 
const (
	InputSchemaVersion                 = "agentic-bench/report-input-v1"
	OptimizationSchemaVersion          = "agentic-bench/optimization-ledger-v1"
	ReportStatisticsMethod             = "paired-task-cluster-bootstrap-v1"
	ReportStatisticsResamples          = 10_000
	ReportStatisticsSeed               = int64(20_260_726)
	ReportConfidenceLevel              = 0.95
	BenchmarkContractDeepSWEV11Pilot5  = "deepswe-v1.1-pilot5-development"
	BenchmarkContractDeepSWEV11Full113 = "deepswe-v1.1-full113"
	// ComparableCostBasisDevelopmentNonBilling identifies the development-only
	// same-gateway token estimate. It is deliberately distinct from the formal
	// all-transport basis so a pilot report cannot be mistaken for an invoice.
	ComparableCostBasisDevelopmentNonBilling = "development_non_billing_same_gateway_frozen_token_estimate"
)

Variables

This section is empty.

Functions

func Generate 

func Generate(inputPath string, writer io.Writer) error

func GenerateFile 

func GenerateFile(inputPath, outputPath string) error

func LoadInput 

func LoadInput(path string) (Input, OptimizationLedger, error)

func Render 

func Render(writer io.Writer, data Data) error

func ValidateInput 

func ValidateInput(input Input) error

func ValidateOptimizationLedger 

func ValidateOptimizationLedger(ledger OptimizationLedger) error

Types

type Ablation 

type Ablation struct {
	Status   AblationStatus      `json:"status"`
	Endpoint *ExperimentEndpoint `json:"endpoint,omitempty"`
	Note     string              `json:"note"`
}

type AblationStatus 

type AblationStatus string
const (
	AblationMeasured AblationStatus = "measured"
	AblationNotRun   AblationStatus = "not_run"
	AblationNA       AblationStatus = "not_applicable"
)

type AgentData 

type AgentData struct {
	ExperimentID string
	AgentID      string
	Variant      string
	Runs         int
	Tasks        int
	Passed       *int
	PassRate     *float64
	PassCI       *ConfidenceInterval
	Metrics      MetricData
	Tools        []ToolData
}

type ArtifactSource 

type ArtifactSource struct {
	ID               string          `json:"id"`
	Label            string          `json:"label"`
	Class            ExperimentClass `json:"class"`
	Root             string          `json:"root"`
	LedgerFileSHA256 string          `json:"ledger_file_sha256"`
	Description      string          `json:"description"`
}

ArtifactSource points at an artifact root produced by harness.Runner. The file names inside the root are intentionally not configurable: manifest.json, plan.json, scorecard.json, plus the state/ledger paths frozen by the manifest.

type AttributionScope 

type AttributionScope string
const (
	AttributionDiagnosticAssociation AttributionScope = "diagnostic_association"
	AttributionCausalFeatureAblation AttributionScope = "causal_feature_ablation"
	AttributionDesignRationale       AttributionScope = "design_rationale"
)

type ComparisonMetric 

type ComparisonMetric string
const (
	MetricPassRate               ComparisonMetric = "pass_rate"
	MetricWallTime               ComparisonMetric = "wall_time_seconds"
	MetricTrialDuration          ComparisonMetric = "trial_duration_seconds"
	MetricLLMCallsStarted        ComparisonMetric = "llm_calls_started"
	MetricProviderRounds         ComparisonMetric = "provider_rounds"
	MetricProviderErrors         ComparisonMetric = "provider_errors"
	MetricToolBearingRounds      ComparisonMetric = "tool_bearing_rounds"
	MetricToolInvocations        ComparisonMetric = "tool_invocations"
	MetricPhysicalToolOperations ComparisonMetric = "physical_tool_operations"
	MetricNativeEvents           ComparisonMetric = "native_events"
	MetricToolErrors             ComparisonMetric = "tool_errors"
	MetricInputTokens            ComparisonMetric = "input_tokens"
	MetricCachedInputTokens      ComparisonMetric = "cached_input_tokens"
	MetricCacheWriteInputTokens  ComparisonMetric = "cache_write_input_tokens"
	MetricUncachedInputTokens    ComparisonMetric = "uncached_input_tokens"
	MetricOutputTokens           ComparisonMetric = "output_tokens"
	MetricReasoningTokens        ComparisonMetric = "reasoning_output_tokens"
	MetricTokenCacheHit          ComparisonMetric = "token_weighted_cache_hit"
	MetricRequestCacheHit        ComparisonMetric = "request_cache_hit"
	MetricComparableCost         ComparisonMetric = "comparable_cost"
	MetricProviderCost           ComparisonMetric = "provider_reported_cost"
)

type ConfidenceInterval 

type ConfidenceInterval struct {
	Estimate        float64
	Lower           float64
	Upper           float64
	ConfidenceLevel float64
	Method          string
	Tasks           int
	Pairs           int
	Resamples       int
	Seed            int64
}

type Data 

type Data struct {
	Meta              ReportMeta
	Statistics        StatisticsSpec
	Experiments       []ExperimentData
	PublicReferences  []PublicReference
	Optimizations     []OptimizationData
	FailureSummary    []FailureData
	Reproduction      []RenderedCommand
	Limitations       []string
	HasFormal         bool
	HasPilot          bool
	HasDiagnosticOnly bool
	// DevelopmentContract is true only for the preregistered five-task
	// optimization set. It is a hard presentation and verdict boundary: the
	// selected tasks are contaminated by iterative development and can never be
	// presented as a formal public score.
	DevelopmentContract bool
	Verdict             VerdictData
}

func Compile 

func Compile(inputPath string) (Data, error)

Compile always applies the registered benchmark contract. The validator is injectable only through an unexported function so tests can exercise a tiny synthetic artifact bundle without creating a production contract bypass.

type DiagnosticExperiment 

type DiagnosticExperiment struct {
	ID          string          `json:"id"`
	Label       string          `json:"label"`
	Class       ExperimentClass `json:"class"`
	Description string          `json:"description"`
	SourceNote  string          `json:"source_note"`
	Runs        []DiagnosticRun `json:"runs"`
}

DiagnosticExperiment is the only accepted source for pre-formal canary numbers. Every optional scalar is a pointer so absent telemetry remains unknown instead of becoming a misleading zero after JSON decoding.

type DiagnosticRun 

type DiagnosticRun struct {
	TaskID          string             `json:"task_id"`
	AgentID         string             `json:"agent_id"`
	Variant         string             `json:"variant"`
	Provider        string             `json:"provider"`
	Model           string             `json:"model"`
	ReasoningEffort string             `json:"reasoning_effort"`
	Repetition      int                `json:"repetition"`
	Passed          *bool              `json:"passed,omitempty"`
	Metrics         OptionalMetrics    `json:"metrics"`
	Tools           []OptionalToolStat `json:"tools"`
}

type EvidenceGrade 

type EvidenceGrade string
const (
	EvidenceMeasuredAblation EvidenceGrade = "measured_ablation"
	EvidenceDiagnosticBundle EvidenceGrade = "diagnostic_bundle"
	EvidenceNotRun           EvidenceGrade = "not_run"
)

type ExperimentClass 

type ExperimentClass string
const (
	ClassDiagnosticCanary ExperimentClass = "diagnostic_canary"
	ClassPilot            ExperimentClass = "pilot"
	ClassFormal           ExperimentClass = "formal"
)

type ExperimentData 

type ExperimentData struct {
	ID              string
	Label           string
	Class           ExperimentClass
	Description     string
	SourceNote      string
	Manifest        *ManifestData
	Runs            []RunData
	Agents          []AgentData
	Comparisons     []PairedComparison
	Gates           []GateData
	Hashes          []HashData
	ToolStats       []ToolData
	ProviderRounds  []RoundData
	OrderStrata     []OrderStratumData
	PublicScorecard *harness.DeepSWEPublicScorecard
}

type ExperimentEndpoint 

type ExperimentEndpoint struct {
	ExperimentID string `json:"experiment_id"`
	AgentID      string `json:"agent_id"`
}

type FailureAnnotation 

type FailureAnnotation struct {
	ExperimentID string          `json:"experiment_id"`
	TaskID       string          `json:"task_id"`
	AgentID      string          `json:"agent_id"`
	Repetition   int             `json:"repetition"`
	Category     FailureCategory `json:"category"`
	Summary      string          `json:"summary"`
	Evidence     []string        `json:"evidence"`
}

type FailureCategory 

type FailureCategory string
const (
	FailureImplementation FailureCategory = "implementation"
	FailureIncomplete     FailureCategory = "incomplete"
	FailureRegression     FailureCategory = "regression"
	FailureValidation     FailureCategory = "validation"
	FailureTimeout        FailureCategory = "timeout"
	FailureInfrastructure FailureCategory = "infrastructure"
	FailureProtocol       FailureCategory = "protocol"
	FailureUnknown        FailureCategory = "unknown"
)

type FailureData 

type FailureData struct {
	ExperimentID string
	TaskID       string
	AgentID      string
	Repetition   int
	Category     FailureCategory
	Summary      string
	Evidence     []string
}

type FileReference 

type FileReference struct {
	Path   string `json:"path"`
	SHA256 string `json:"sha256"`
}

type GateData 

type GateData struct {
	Name   string
	Status GateStatus
	Detail string
}

type GateStatus 

type GateStatus string
const (
	GatePass    GateStatus = "pass"
	GateFail    GateStatus = "fail"
	GateUnknown GateStatus = "unknown"
)

type HashData 

type HashData struct {
	Name   string
	Value  string
	Source string
}

type ImplementationRef 

type ImplementationRef struct {
	Path    string `json:"path"`
	SHA256  string `json:"sha256,omitempty"`
	Summary string `json:"summary"`
}

type Input 

type Input struct {
	SchemaVersion         string                 `json:"schema_version"`
	Report                ReportMeta             `json:"report"`
	Statistics            StatisticsSpec         `json:"statistics"`
	ArtifactSources       []ArtifactSource       `json:"artifact_sources"`
	DiagnosticExperiments []DiagnosticExperiment `json:"diagnostic_experiments"`
	OptimizationLedger    FileReference          `json:"optimization_ledger"`
	PublicReferences      []PublicReference      `json:"public_references"`
	FailureAnnotations    []FailureAnnotation    `json:"failure_annotations"`
	Reproduction          []ReproductionCommand  `json:"reproduction"`
	Limitations           []string               `json:"limitations"`
}

Input is the small report assembly manifest. It contains no copied formal scores: formal and pilot results are reloaded and cross-checked from their manifest, plan, state, scorecard, evidence, and artifact ledger.

type ManifestAgentData 

type ManifestAgentData struct {
	ID                         string
	Provider                   string
	Model                      string
	ReasoningEffort            string
	ServiceTier                string
	ServiceTierRequestEncoding string
	BinarySHA256               string
	SourceBaseCommit           string
	SourceTreeOID              string
	SourcePatchSHA             string
	SourceArchiveSHA           string
	BuildReceiptSHA            string
	BuildArgv                  []string
	BuildToolchain             string
	Argv                       []string
	// ArchivedSourceFiles maps canonical source.tar paths to file SHA-256.
	// It is intentionally not rendered wholesale; optimization references use
	// it to prove that a cited implementation file belongs to the executed
	// source archive.
	ArchivedSourceFiles map[string]string
}

type ManifestData 

type ManifestData struct {
	ExperimentID                 string
	DatasetName                  string
	DatasetRepository            string
	DatasetCommit                string
	DatasetTreeSHA               string
	EvaluatorName                string
	EvaluatorCommit              string
	EvaluatorVersion             string
	EvaluatorProtocol            string
	ScoringProfile               string
	SelectionMode                string
	ExpectedTasks                int
	Repetitions                  int
	PairingSeed                  uint64
	MaxParallelPairs             int
	TaskNetwork                  string
	VerifierNetwork              string
	NetworkAttestation           string
	EgressProxyImage             string
	EgressProxyImageID           string
	HostEnvAllowlist             []string
	AgentEgressHosts             []string
	CPUs                         int
	MemoryMB                     int
	StorageMB                    int
	HostStorageGuard             harness.HostStorageGuardSpec
	GuestStorageGuard            harness.GuestStorageGuardSpec
	StoragePreflight             harness.StorageAdmissionReceipt
	AgentTimeout                 int
	VerifierTimeout              int
	PricingCurrency              string
	PricingUnit                  int64
	PricingAt                    time.Time
	PricingObservedAt            time.Time
	PricingSource                string
	ProviderOrigin               string
	ProviderSemanticsSHA         string
	ProviderObservationAuthority string
	ProviderTLSRequired          bool
	ProviderWebSocketAllowed     bool
	PricingRates                 []harness.PricingRate
	Agents                       []ManifestAgentData
}

type MeasurementLayer 

type MeasurementLayer string
const (
	MeasurementControllerEndToEnd MeasurementLayer = "controller_end_to_end"
	MeasurementTrial              MeasurementLayer = "trial"
	MeasurementAgent              MeasurementLayer = "agent"
	MeasurementProvider           MeasurementLayer = "provider"
	MeasurementTool               MeasurementLayer = "tool"
	MeasurementMixed              MeasurementLayer = "mixed"
)

type MetricComparison 

type MetricComparison struct {
	Metric         ComparisonMetric
	Baseline       *float64
	Contender      *float64
	Difference     *float64
	RelativeChange *float64
	CI             *ConfidenceInterval
	Pairs          int
	Tasks          int
	Note           string
}

type MetricData 

type MetricData struct {
	WallTimeSeconds                  *float64
	TrialDurationSeconds             *float64
	TransportAttempts                *int
	PrewarmAttempts                  *int
	PrewarmErrors                    *int
	LLMCallsStarted                  *int
	CompletedLLMResponses            *int
	RetryAmplification               *float64
	HTTPInferenceRequests            *int
	WebSocketInferenceRequests       *int
	WebSocketConnections             *int
	PrewarmUsageObservations         *int
	PrewarmInputTokens               *int64
	PrewarmCachedInputTokens         *int64
	PrewarmOutputTokens              *int64
	PrewarmUnknownCostAttempts       *int
	CostReceiptObserved              int
	CostReceiptTotal                 int
	UnknownCostAttempts              *int
	CostIdentityUnknownAttempts      *int
	KnownCatalogCostLowerBound       *float64
	AllExecutedInputTokens           *int64
	AllExecutedCachedTokens          *int64
	AllExecutedUncachedTokens        *int64
	AllExecutedNonCachedBaseTokens   *int64
	AllExecutedOutputTokens          *int64
	AllExecutedCacheWriteInputTokens *int64
	AllExecutedCacheHit              *float64
	AllExecutedUsageObserved         int
	AllExecutedUsageTotal            int
	AllExecutedCacheWriteObserved    int
	AllExecutedCacheWriteTotal       int
	AllExecutedUnreportedCacheWrite  int
	ProviderRequests                 *int
	ProviderRounds                   *int
	ProviderErrors                   *int
	ToolBearingRounds                *int
	ToolInvocations                  *int
	ToolTraceMatched                 *int
	ToolTraceUnmatched               *int
	PhysicalToolOperations           *int
	PhysicalToolObserved             int
	PhysicalToolTotal                int
	NativeEvents                     *int
	ToolErrors                       *int
	ToolCriticalPathMS               *int64
	ToolCriticalObserved             int
	ToolCriticalTotal                int
	ToolTotalLatencyMS               *int64
	ToolTotalObserved                int
	ToolTotalTotal                   int
	ToolQueueMS                      *int64
	ToolQueueObserved                int
	ToolQueueTotal                   int
	InputTokens                      *int64
	CachedInputTokens                *int64
	CacheWriteInputTokens            *int64
	CacheMissTokens                  *int64
	OutputTokens                     *int64
	ReasoningOutputTokens            *int64
	ReasoningTokenObserved           int
	ReasoningTokenTotal              int
	CacheWriteTokenObserved          int
	CacheWriteTokenTotal             int
	UnreportedCacheWriteRounds       int
	KnownCacheWriteSurcharge         *float64
	TokenWeightedCacheHit            *float64
	RequestCacheHit                  *float64
	RequestCacheHits                 *int
	RequestCacheObserved             *int
	CachePolicyObservedRequests      *int
	CacheKeyPresentRequests          *int
	CacheUniqueKeyCount              *int
	CacheKeyTransitions              *int
	CacheLineageStable               *bool
	CatalogCost                      *float64
	CatalogCostPartial               *float64
	ComparableCost                   *float64
	ComparableCostBasis              string
	ProviderReportedCost             *float64
	ProviderCostPartial              *float64
	ProviderCostObserved             int
	ProviderCostTotal                int
	TokenUsageObserved               int
	TokenUsageTotal                  int
	ToolErrorObserved                int
	ToolErrorTotal                   int
	ToolTimingObserved               int
	ToolTimingTotal                  int
}

type OptimizationData 

type OptimizationData struct {
	Entry           OptimizationEntry
	Before          *AgentData
	After           *AgentData
	Ablation        *AgentData
	Metrics         []MetricComparison
	AblationMetrics []MetricComparison
	ClassValid      bool
}

type OptimizationEntry 

type OptimizationEntry struct {
	ID               string              `json:"id"`
	Title            string              `json:"title"`
	Summary          string              `json:"summary"`
	DesignDefect     string              `json:"design_defect"`
	Mechanism        string              `json:"mechanism"`
	Value            string              `json:"value"`
	AttributionScope AttributionScope    `json:"attribution_scope"`
	MeasurementLayer MeasurementLayer    `json:"measurement_layer"`
	EvidenceGrade    EvidenceGrade       `json:"evidence_grade"`
	ExpectedEffect   string              `json:"expected_effect"`
	ObservedEffect   string              `json:"observed_effect"`
	Confounders      []string            `json:"confounders"`
	Risks            []string            `json:"risks"`
	Implementation   []ImplementationRef `json:"implementation"`
	Before           ExperimentEndpoint  `json:"before"`
	After            ExperimentEndpoint  `json:"after"`
	Ablation         Ablation            `json:"ablation"`
	Metrics          []ComparisonMetric  `json:"metrics"`
	EvidenceClass    ExperimentClass     `json:"evidence_class"`
}

type OptimizationLedger 

type OptimizationLedger struct {
	SchemaVersion string              `json:"schema_version"`
	Entries       []OptimizationEntry `json:"entries"`
}

type OptionalMetrics 

type OptionalMetrics struct {
	WallTimeSeconds        *float64              `json:"wall_time_seconds,omitempty"`
	TrialDurationSeconds   *float64              `json:"trial_duration_seconds,omitempty"`
	LLMCallsStarted        *int                  `json:"llm_calls_started,omitempty"`
	ProviderRounds         *int                  `json:"provider_rounds,omitempty"`
	ProviderErrors         *int                  `json:"provider_errors,omitempty"`
	ToolBearingRounds      *int                  `json:"tool_bearing_rounds,omitempty"`
	ToolInvocations        *int                  `json:"tool_invocations,omitempty"`
	PhysicalToolOperations *int                  `json:"physical_tool_operations,omitempty"`
	NativeEvents           *int                  `json:"native_events,omitempty"`
	ToolErrors             *int                  `json:"tool_errors,omitempty"`
	ToolCriticalPathMS     *int64                `json:"tool_critical_path_ms,omitempty"`
	ToolTotalLatencyMS     *int64                `json:"tool_total_latency_ms,omitempty"`
	ToolQueueMS            *int64                `json:"tool_queue_ms,omitempty"`
	InputTokens            *int64                `json:"input_tokens,omitempty"`
	CachedInputTokens      *int64                `json:"cached_input_tokens,omitempty"`
	CacheWriteInputTokens  *int64                `json:"cache_write_input_tokens,omitempty"`
	OutputTokens           *int64                `json:"output_tokens,omitempty"`
	ReasoningOutputTokens  *int64                `json:"reasoning_output_tokens,omitempty"`
	RequestCache           *RequestCacheCoverage `json:"request_cache,omitempty"`
	ComparableCost         *float64              `json:"comparable_cost,omitempty"`
	ComparableCostBasis    string                `json:"comparable_cost_basis,omitempty"`
	ProviderReportedCost   *float64              `json:"provider_reported_cost,omitempty"`
}

type OptionalToolStat 

type OptionalToolStat struct {
	Name       string `json:"name"`
	Calls      *int   `json:"calls,omitempty"`
	Errors     *int   `json:"errors,omitempty"`
	DurationMS *int64 `json:"duration_ms,omitempty"`
}

type OrderStratumData 

type OrderStratumData struct {
	ExperimentID           string
	AgentID                string
	Position               string
	Raw                    int
	Scored                 int
	Excluded               int
	Passed                 int
	PassRate               *float64
	MeanTrialSeconds       *float64
	TrialObserved          int
	MeanComparableCost     *float64
	ComparableCostObserved int
}

OrderStratumData exposes the crossover-position sensitivity separately from the pooled score. Time and cost are means over observed raw attempts; their coverage counts remain explicit so missing telemetry is never coerced to zero.

type PairedComparison 

type PairedComparison struct {
	ExperimentID  string
	Baseline      string
	Contender     string
	Tasks         int
	Pairs         int
	Metrics       []MetricComparison
	QualityWins   int
	QualityLosses int
	QualityTies   int
}

type PublicReference 

type PublicReference struct {
	ID                    string                                  `json:"id"`
	Benchmark             string                                  `json:"benchmark"`
	Version               string                                  `json:"version"`
	Agent                 string                                  `json:"agent"`
	Model                 string                                  `json:"model"`
	ReasoningEffort       string                                  `json:"reasoning_effort"`
	ComputedArtifact      string                                  `json:"computed_artifact,omitempty"`
	Score                 *float64                                `json:"score,omitempty"`
	Passed                *int                                    `json:"passed,omitempty"`
	Total                 *int                                    `json:"total,omitempty"`
	CostPerTask           *float64                                `json:"cost_per_task,omitempty"`
	MinutesPerTask        *float64                                `json:"minutes_per_task,omitempty"`
	TurnsPerTask          *float64                                `json:"turns_per_task,omitempty"`
	TokensPerTask         *float64                                `json:"tokens_per_task,omitempty"`
	TokenWeightedCacheHit *float64                                `json:"token_weighted_cache_hit,omitempty"`
	Components            []ReferenceComponent                    `json:"components"`
	SourceURL             string                                  `json:"source_url"`
	AccessedAt            time.Time                               `json:"accessed_at"`
	Notes                 string                                  `json:"notes"`
	Computed              *harness.DeepSWEPublicReferenceArtifact `json:"-"`
}

type ReferenceComponent 

type ReferenceComponent struct {
	Name  string   `json:"name"`
	Score *float64 `json:"score,omitempty"`
}

type RenderedCommand 

type RenderedCommand struct {
	Label   string
	Command string
}

type ReportMeta 

type ReportMeta struct {
	Title               string    `json:"title"`
	Subtitle            string    `json:"subtitle"`
	Benchmark           string    `json:"benchmark"`
	BenchmarkVersion    string    `json:"benchmark_version"`
	BenchmarkContractID string    `json:"benchmark_contract_id"`
	Language            string    `json:"language"`
	BaselineAgentID     string    `json:"baseline_agent_id"`
	ContenderAgentID    string    `json:"contender_agent_id"`
	AsOf                time.Time `json:"as_of"`
}

type ReproductionCommand 

type ReproductionCommand struct {
	Label string   `json:"label"`
	Argv  []string `json:"argv"`
}

ReproductionCommand stores argv, not a shell program. Rendering applies POSIX quoting, preventing presentation-time interpolation or hidden commands.

type RequestCacheCoverage 

type RequestCacheCoverage struct {
	Hits     int `json:"hits"`
	Observed int `json:"observed"`
}

type RoundData 

type RoundData struct {
	ExperimentID                         string
	AgentID                              string
	TaskID                               string
	Repetition                           int
	Round                                int
	Outcome                              string
	ErrorCode                            string
	Transport                            string
	ProviderAttemptKind                  string
	TransportDisposition                 string
	RequestIDHash                        string
	ResponseIDHash                       string
	RequestedReasoningMode               string
	ReasoningModeCanonical               string
	RequestedTextVerbosity               string
	MaxOutputTokensSpecified             bool
	MaxOutputTokens                      *int64
	RequestedServiceTier                 string
	RequestedServiceTierPresent          bool
	RequestedServiceTierCanonical        string
	RequestedServiceTierRepresentation   string
	ClientCanonicalizationProofSHA256    string
	ClientAgentID                        string
	OriginalRequestBodySHA256            string
	ForwardedRequestBodySHA256           string
	OriginalRequestCanonicalSHA256       string
	ForwardedRequestCanonicalSHA256      string
	OriginalWithoutTierSHA256            string
	ForwardedWithoutTierSHA256           string
	OriginalServiceTierPresent           bool
	OriginalServiceTier                  string
	ForwardedServiceTierPresent          bool
	ForwardedServiceTier                 string
	ForwardedRequestBytes                int64
	ServiceTierTransformation            string
	ServiceTierTransformationExactDiff   bool
	ServiceTierTransformationProofSHA256 string
	ResponseServiceTier                  string
	ResponseServiceTierCanonical         string
	ServiceTierComparable                bool
	ResponseCreatedModel                 string
	ResponseModel                        string
	StartedAt                            time.Time
	FinishedAt                           time.Time
	HeadersMS                            *float64
	FirstByteMS                          *float64
	StreamMS                             *float64
	ProviderMS                           float64
	PostRoundGapMS                       *float64
	ToolCalls                            int
	PhysicalTools                        *int
	ToolCriticalMS                       *int64
	ToolTotalMS                          *int64
	ToolQueueMS                          *int64
	InputTokens                          *int64
	CachedInputTokens                    *int64
	CacheWriteTokens                     *int64
	OutputTokens                         *int64
	CacheHit                             *bool
	PromptCacheKeyPresent                bool
	PromptCacheKeyHash                   string
	CachePolicyObserved                  bool
	PromptCacheOptionsPresent            bool
	PromptCacheOptionsMode               string
	PromptCacheTTLSeconds                *int64
	PromptCacheRetentionPresent          bool
	PromptCacheRetention                 string
	CacheBreakpointCount                 int
	CacheBreakpointPositionHashes        []string
}

type RunData 

type RunData struct {
	AttemptID             string
	ExperimentID          string
	Class                 ExperimentClass
	PairID                string
	TaskID                string
	AgentID               string
	Variant               string
	Provider              string
	Model                 string
	ReasoningEffort       string
	Repetition            int
	Attempt               int
	ExecutionPosition     string
	Disposition           string
	FailureCategory       string
	Passed                *bool
	ExitClass             string
	ExitCode              *int
	AttemptStartedAt      time.Time
	ControllerStartedAt   time.Time
	ControllerFinishedAt  time.Time
	ControllerRecovered   bool
	ProviderAttemptState  string
	ProviderAttemptCount  uint64
	StartedAt             time.Time
	FinishedAt            time.Time
	TrialStartedAt        time.Time
	TrialFinishedAt       time.Time
	TrialDurationSeconds  *float64
	Metrics               MetricData
	Tools                 []ToolData
	Rounds                []RoundData
	ToolCatalogObserved   int
	NestedToolDefinitions int
	ServiceTierEvidence   harness.ServiceTierCanonicalizationEvidence
	StorageAdmission      harness.StorageAdmissionReceipt
	HostStorageEvidence   harness.StorageResourceEvidence
	GuestStorageEvidence  []harness.GuestStorageResourceEvidence
	CacheInitialState     string
	CacheEvidenceClass    string
	Failure               *FailureData
}

type StatisticsSpec 

type StatisticsSpec struct {
	ConfidenceLevel float64 `json:"confidence_level"`
	Method          string  `json:"method"`
	Resamples       int     `json:"resamples"`
	Seed            int64   `json:"seed"`
}

type ToolData 

type ToolData struct {
	ExperimentID string
	AgentID      string
	Name         string
	Calls        *int
	Errors       *int
	DurationMS   *int64
	ErrorKnown   int
	ErrorTotal   int
	TimingKnown  int
	TimingTotal  int
}

type VerdictCriterion 

type VerdictCriterion struct {
	Metric ComparisonMetric
	Passed *bool
	Detail string
}

type VerdictData 

type VerdictData struct {
	Status   string
	Criteria []VerdictCriterion
}

Source Files

View all Source files

Directories

Path Synopsis
cmd
devreport command
reportgen command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.