agenticbench

package
v0.53.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 8, 2026 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Overview

Package agenticbench folds the #868 agentic benchmark artifacts into one parent gate. It is intentionally read-only: the live benchmark lanes remain owned by their native harnesses.

Index

Constants

View Source 
const (
	ALEExperimentIdentitySchema = "fak.ale-experiment-identity.v1"
	ALEResumeReceiptSchema      = "fak.ale-resume-preflight.v1"
	ALEIdentitySidecar          = "fak-experiment-identity.json"
	ALEOutputIdentityFile       = "fak-output-identity.json"
)
View Source 
const (
	ReasonALEIdentityInvalid         = "ALE_IDENTITY_INVALID"
	ReasonALEOutputIdentityMissing   = "ALE_OUTPUT_IDENTITY_MISSING"
	ReasonALEOutputIdentityMismatch  = "ALE_OUTPUT_IDENTITY_MISMATCH"
	ReasonALEResumeIdentityMissing   = "ALE_RESUME_IDENTITY_MISSING"
	ReasonALEResumeIdentityMismatch  = "ALE_RESUME_IDENTITY_MISMATCH"
	ReasonALEResumeSourceAmbiguous   = "ALE_RESUME_SOURCE_AMBIGUOUS"
	ReasonALEResumeReadFailed        = "ALE_RESUME_READ_FAILED"
	ReasonALECrossoverResumeUnscoped = "ALE_CROSSOVER_RESUME_UNSCOPED"
)
View Source 
const (
	ReasonTimeoutRetryForbidden = "TIMEOUT_RETRY_FORBIDDEN"
	ReasonRetryRegimeMismatch   = "RETRY_REGIME_MISMATCH"
	ReasonHiddenAttempt         = "HIDDEN_ATTEMPT"
	ReasonDuplicateOrdinal      = "DUPLICATE_ORDINAL"
	ReasonNonContiguousOrdinals = "NON_CONTIGUOUS_ORDINALS"
	ReasonDuplicateAttemptID    = "DUPLICATE_ATTEMPT_ID"
	ReasonDuplicateArtifactRoot = "DUPLICATE_ARTIFACT_ROOT"
	ReasonInvalidTerminalStatus = "INVALID_TERMINAL_STATUS"
	ReasonMaxAttemptsExceeded   = "MAX_ATTEMPTS_EXCEEDED"
	ReasonRetryForbidden        = "RETRY_FORBIDDEN"
	ReasonInvalidSelectionRule  = "INVALID_SELECTION_RULE"
	ReasonInvalidRetryRegime    = "INVALID_RETRY_REGIME"
	ReasonUnitIDMismatch        = "UNIT_ID_MISMATCH"
	ReasonEmptyAttempts         = "EMPTY_ATTEMPTS"
	ReasonInvalidScore          = "INVALID_SCORE"
	ReasonInvalidRetryReason    = "INVALID_RETRY_REASON"
	ReasonAggregateMismatch     = "AGGREGATE_MISMATCH"
)

Reason constants for attempt lineage validation and refusal.

View Source 
const (
	TerminalStatusSuccess = "success"
	TerminalStatusFailed  = "failed"
	TerminalStatusTimeout = "timeout"
	TerminalStatusError   = "error"
)
View Source 
const (
	FrozenCohortSchema       = "fak.agentic-benchmark-planned-cohort.v1"
	DenominatorReceiptSchema = "fak.agentic-benchmark-denominator-receipt.v1"
	DenominatorReportSchema  = "fak.agentic-benchmark-denominator-report.v1"
)
View Source 
const (
	EvaluatorArtifactALEEvalResult       = "ale_eval_result"
	EvaluatorArtifactStructuredBreakdown = "structured_breakdown"
	EvaluatorArtifactRawGraderPayload    = "raw_grader_payload"

	EvaluatorScorerDeterministic = "deterministic"
	EvaluatorScorerLLMJudge      = "llm_judge"

	EvaluatorEvidenceGatePass = "PASS"

	EvaluatorEvidenceReasonSchema            = "EVALUATOR_EVIDENCE_SCHEMA"
	EvaluatorEvidenceReasonLevelMismatch     = "EVALUATOR_EVIDENCE_LEVEL_MISMATCH"
	EvaluatorEvidenceReasonArtifactMissing   = "EVALUATOR_EVIDENCE_ARTIFACT_MISSING"
	EvaluatorEvidenceReasonHashMismatch      = "EVALUATOR_EVIDENCE_HASH_MISMATCH"
	EvaluatorEvidenceReasonRawIsScalar       = "EVALUATOR_EVIDENCE_RAW_IS_SCALAR"
	EvaluatorEvidenceReasonModelMissing      = "EVALUATOR_EVIDENCE_MODEL_MISSING"
	EvaluatorEvidenceReasonScorerProvenance  = "EVALUATOR_EVIDENCE_SCORER_PROVENANCE"
	EvaluatorEvidenceReasonMalformedArtifact = "EVALUATOR_EVIDENCE_MALFORMED_ARTIFACT"
	EvaluatorEvidenceReasonOverclaim         = "EVALUATOR_EVIDENCE_OVERCLAIM"
)
View Source 
const (
	EvaluatorSuccess = "success"
	EvaluatorFailure = "failure"
	EvaluatorTimeout = "timeout"
)
View Source 
const (
	FailureAnalysisCardSchema = "fak.agenticbench.failure-analysis-card.v1"
	FailureAnalysisAdvisory   = "ADVISORY"
)
View Source 
const (
	ResultPacketSchema     = "fak.agentic-benchmark-result-packet.v1"
	DefaultResultPacketDir = "experiments/agent-live/agentic-benchmark-result-packets"
)
View Source 
const AccountingReportSchema = "fak.agentic-benchmark-accounting-report.v1"
View Source 
const AttemptLineageSchema = "fak.agentic-benchmark-attempt-lineage.v1"
View Source 
const EvaluatorEvidenceSchema = "fak.agenticbench-evaluator-evidence.v1"
View Source 
const (
	ExternalHarnessQueueSchema = "fak.agentic-benchmark-external-harness-queue.v1"
)
View Source 
const Schema = "fak.agentic-benchmark-epic-rollup.v1"

Variables

This section is empty.

Functions

func ALEManifestIdentity added in v0.45.0 

func ALEManifestIdentity(identity ALEExperimentIdentity) (string, error)

ALEManifestIdentity projects ALE's wider launch contract into a valid ArmBench manifest. The source content hash binds the complete ALE projection, while ArmBench supplies the canonical immutable manifest identity.

func CompareArmLineage added in v0.46.0 

func CompareArmLineage(armA, armB AttemptLineagePacket) error

CompareArmLineage verifies that two arms are comparable by ensuring equal retry policies and maximum attempt limits, refusing comparison with RETRY_REGIME_MISMATCH if they diverge.

func ComputeAggregate added in v0.46.0 

func ComputeAggregate(packet *AttemptLineagePacket)

ComputeAggregate computes total cost, tokens, and duration across all attempts, ensuring failed attempts are never hidden from spend totals, and sets the final score based on the declared selection rule.

func DiscoverAttemptDirectories added in v0.46.0 

func DiscoverAttemptDirectories(unitDir string) ([]string, error)

DiscoverAttemptDirectories returns sorted directory paths under a unit directory.

func FormatAccountingMarkdown added in v0.45.0 

func FormatAccountingMarkdown(report AccountingReport) string

func FormatDenominatorMarkdown added in v0.45.0 

func FormatDenominatorMarkdown(report DenominatorReport) string

func IsALEAliasReason added in v0.45.0 

func IsALEAliasReason(err error, reason string) bool

func IsLineageReason added in v0.46.0 

func IsLineageReason(err error, reason string) bool

IsLineageReason tests if an error carries a given refusal reason.

func ReconcileDiscoveredAttempts added in v0.46.0 

func ReconcileDiscoveredAttempts(packet AttemptLineagePacket, discoveredRoots []string) error

ReconcileDiscoveredAttempts checks that every discovered attempt directory matches a declared attempt in the packet without hidden or missing directories.

func RenderEvaluatorEvidence added in v0.45.0 

func RenderEvaluatorEvidence(report EvaluatorEvidenceReport) string

func RenderExternalHarnessQueueMarkdown 

func RenderExternalHarnessQueueMarkdown(q *ExternalHarnessQueue) string

func RenderFailureAnalysisJSON added in v0.45.0 

func RenderFailureAnalysisJSON(runRoot string, card *FailureAnalysisCard) ([]byte, error)

RenderFailureAnalysisJSON validates the card before emitting stable JSON.

func RenderFailureAnalysisMarkdown added in v0.45.0 

func RenderFailureAnalysisMarkdown(runRoot string, card *FailureAnalysisCard) (string, error)

RenderFailureAnalysisMarkdown validates the card before emitting the five mandatory advisory sections.

func RenderMarkdown 

func RenderMarkdown(r *Report) string

func ResolveFailureAnalysisEvidence added in v0.45.0 

func ResolveFailureAnalysisEvidence(runRoot string, evidence FailureAnalysisEvidence) (json.RawMessage, error)

ResolveFailureAnalysisEvidence returns the cited JSON value or JSONL event after path, size, and digest checks succeed.

func ValidateAttemptLineage added in v0.46.0 

func ValidateAttemptLineage(packet AttemptLineagePacket) error

ValidateAttemptLineage enforces attempt lineage invariants: contiguous ordinals, unique IDs and roots, valid terminal statuses, retry policy bounds, and exact match with discovered attempt directories.

func ValidateFailureAnalysisCard added in v0.45.0 

func ValidateFailureAnalysisCard(runRoot string, card *FailureAnalysisCard) error

ValidateFailureAnalysisCard resolves every cited field or event inside the declared run root before the advisory card can be promoted.

Types

type ALEAliasError added in v0.45.0 

type ALEAliasError struct {
	Reason string
	Detail string
}

func (*ALEAliasError) Error added in v0.45.0 

func (e *ALEAliasError) Error() string

type ALEExperimentIdentity added in v0.45.0 

type ALEExperimentIdentity struct {
	TaskPath           string `json:"task_path"`
	SourceRepo         string `json:"source_repo"`
	SourceSHA          string `json:"source_sha"`
	Harness            string `json:"harness"`
	AgentID            string `json:"agent_id"`
	Model              string `json:"model"`
	Effort             string `json:"effort"`
	Endpoint           string `json:"endpoint"`
	Arm                string `json:"arm"`
	Snapshot           string `json:"snapshot"`
	BudgetSeconds      int    `json:"budget_seconds"`
	MaxTokens          int    `json:"max_tokens"`
	RetryPolicy        string `json:"retry_policy"`
	PromptSuffixSHA256 string `json:"prompt_suffix_sha256"`
	Repetition         int    `json:"repetition"`
	Variant            int    `json:"variant"`
}

ALEExperimentIdentity pins every experiment term that can make two native ALE run units look equal while making their measurements incomparable.

type ALEIdentityRecord added in v0.45.0 

type ALEIdentityRecord struct {
	Schema   string                `json:"schema"`
	Identity ALEExperimentIdentity `json:"identity"`
}

ALEIdentityRecord is written beside an ALE run before launch. Keeping the full projection, not only its digest, makes a refusal independently auditable.

type ALELaunchSpec added in v0.45.0 

type ALELaunchSpec struct {
	Identity      ALEExperimentIdentity `json:"identity"`
	Output        ALEOutputRoot         `json:"output_root"`
	Crossover     bool                  `json:"crossover"`
	DisableResume bool                  `json:"disable_resume"`
}

type ALEOutputRoot added in v0.45.0 

type ALEOutputRoot struct {
	Path             string `json:"path"`
	ManifestIdentity string `json:"manifest_identity,omitempty"`
}

ALEOutputRoot binds an arm/repetition-specific root to the same immutable identity as the launch. A blank ManifestIdentity denotes a shared ALE root.

type ALEResumeDisposition added in v0.45.0 

type ALEResumeDisposition string
const (
	ALEResumeFresh   ALEResumeDisposition = "fresh"
	ALEResumed       ALEResumeDisposition = "resumed"
	ALEResumeRefused ALEResumeDisposition = "refused"
)

type ALEResumeReceipt added in v0.45.0 

type ALEResumeReceipt struct {
	Schema           string               `json:"schema"`
	Decision         ALEResumeDisposition `json:"decision"`
	ManifestIdentity string               `json:"manifest_identity,omitempty"`
	OutputRoot       string               `json:"output_root"`
	SourceRunID      string               `json:"source_run_id,omitempty"`
	SourceRunPath    string               `json:"source_run_path,omitempty"`
	Detail           string               `json:"detail"`
}

ALEResumeReceipt is embedded unchanged in the comparison manifest. Refusals are receipts too, so a skipped launch cannot disappear from the experiment.

func CheckALEResumeAlias added in v0.45.0 

func CheckALEResumeAlias(spec ALELaunchSpec) (ALEResumeReceipt, error)

CheckALEResumeAlias runs before ALE's native resume filter. It mirrors the native on-disk lookup, then requires every terminal collision to carry the exact full identity before allowing ALE to skip the unit.

type AccountingArm added in v0.45.0 

type AccountingArm struct {
	Arm     string                     `json:"arm"`
	Receipt armbench.AccountingReceipt `json:"receipt"`
}

AccountingArm is the narrow ArmBench-to-AgenticBench adapter. It carries the shared receipt unchanged so the benchmark layer cannot reinterpret missing usage or silently choose a different authority.

func AccountingArmFromSummary added in v0.45.0 

func AccountingArmFromSummary(summary armbench.ArmSummary) AccountingArm

type AccountingClaimState added in v0.45.0 

type AccountingClaimState string
const (
	AccountingClaimAllowed AccountingClaimState = "ALLOWED"
	AccountingClaimRefused AccountingClaimState = "REFUSED"
)

type AccountingComparison added in v0.45.0 

type AccountingComparison struct {
	LeftArm  string                    `json:"left_arm"`
	RightArm string                    `json:"right_arm"`
	Metric   armbench.AccountingMetric `json:"metric"`
	State    AccountingClaimState      `json:"state"`
	Left     armbench.AccountingField  `json:"left"`
	Right    armbench.AccountingField  `json:"right"`
	Delta    *float64                  `json:"delta"`
	Detail   string                    `json:"detail"`
}

type AccountingComparisonRequest added in v0.45.0 

type AccountingComparisonRequest struct {
	LeftArm  string                    `json:"left_arm"`
	RightArm string                    `json:"right_arm"`
	Metric   armbench.AccountingMetric `json:"metric"`
}

type AccountingReport added in v0.45.0 

type AccountingReport struct {
	Schema      string                 `json:"schema"`
	Arms        []AccountingArm        `json:"arms"`
	Comparisons []AccountingComparison `json:"comparisons"`
}

func BuildAccountingReport added in v0.45.0 

func BuildAccountingReport(arms []AccountingArm, requests []AccountingComparisonRequest) (AccountingReport, error)

BuildAccountingReport applies the shared comparison gate to every requested token, cache, or cost claim. A refusal remains a report row rather than an error because it is evidence about the benchmark, not a malformed request.

type AggregateAccounting added in v0.46.0 

type AggregateAccounting struct {
	TotalCostUSD      float64  `json:"total_cost_usd"`
	TotalTokens       int64    `json:"total_tokens"`
	TotalDurationMS   int64    `json:"total_duration_ms"`
	TotalAttempts     int      `json:"total_attempts"`
	FinalScore        *float64 `json:"final_score"`
	SelectedAttemptID string   `json:"selected_attempt_id,omitempty"`
}

AggregateAccounting represents total resource spend and selected outcome across all attempts.

type AnalysisRule added in v0.45.0 

type AnalysisRule struct {
	Scope               ComparisonScope `json:"scope"`
	MaxMissingRateDelta float64         `json:"max_missing_rate_delta"`
}

type ArmComparison added in v0.45.0 

type ArmComparison struct {
	LeftArm    string          `json:"left_arm"`
	RightArm   string          `json:"right_arm"`
	State      ClaimState      `json:"state"`
	Scope      ComparisonScope `json:"scope"`
	LeftScore  ScoreSummary    `json:"left_score"`
	RightScore ScoreSummary    `json:"right_score"`
	Detail     string          `json:"detail"`
}

func CompareArms added in v0.45.0 

func CompareArms(left, right DenominatorReceipt, rule AnalysisRule) ArmComparison

type ArmLatencyStatus added in v0.45.0 

type ArmLatencyStatus struct {
	Role            string                      `json:"role,omitempty"`
	Name            string                      `json:"name"`
	Total           LatencyMeasurement          `json:"total"`
	QueueWait       LatencyMeasurement          `json:"queue_wait"`
	AgentExecution  LatencyMeasurement          `json:"agent_execution"`
	Evaluation      LatencyMeasurement          `json:"evaluation"`
	GatewayRequests []GatewayLatencyObservation `json:"gateway_requests,omitempty"`
}

type ArmRun added in v0.45.0 

type ArmRun struct {
	Arm     string       `json:"arm"`
	Results []UnitResult `json:"results"`
}

type AttemptLineagePacket added in v0.46.0 

type AttemptLineagePacket struct {
	Schema          string              `json:"schema,omitempty"`
	UnitID          string              `json:"unit_id"`
	RetryRegime     NativeRetryRegime   `json:"retry_policy"`
	MaxAttempts     int                 `json:"max_attempts"`
	SelectionRule   SelectionRule       `json:"selection_rule"`
	Attempts        []AttemptReceipt    `json:"attempts"`
	Aggregate       AggregateAccounting `json:"aggregate"`
	DiscoveredRoots []string            `json:"discovered_roots,omitempty"`
}

AttemptLineagePacket captures full attempt lineage, retry policy, and aggregate spend for an external benchmark unit.

func ReadAttemptLineagePacket added in v0.46.0 

func ReadAttemptLineagePacket(path string) (AttemptLineagePacket, error)

ReadAttemptLineagePacket reads and unmarshals an AttemptLineagePacket from disk.

type AttemptReceipt added in v0.46.0 

type AttemptReceipt struct {
	AttemptID      string   `json:"attempt_id"`
	Ordinal        int      `json:"ordinal"`
	RunID          string   `json:"run_id"`
	UnitID         string   `json:"unit_id"`
	IsClean        bool     `json:"is_clean"`
	TerminalStatus string   `json:"terminal_status"`
	Score          *float64 `json:"score"`
	CostUSD        float64  `json:"cost_usd"`
	Tokens         int64    `json:"tokens"`
	DurationMS     int64    `json:"duration_ms"`
	ArtifactRoot   string   `json:"artifact_root"`
	ArtifactSHA256 string   `json:"artifact_sha256"`
	RetryReason    string   `json:"retry_reason,omitempty"`
}

AttemptReceipt records the complete execution, cost, duration, and artifact identity for one attempt within a benchmark unit run.

type ChildStatus 

type ChildStatus struct {
	Issue              int      `json:"issue"`
	Packet             string   `json:"packet"`
	Title              string   `json:"title"`
	Artifact           string   `json:"artifact"`
	Gate               string   `json:"gate"`
	Status             string   `json:"status"`
	EvidenceClass      string   `json:"evidence_class,omitempty"`
	ResultClaimAllowed bool     `json:"result_claim_allowed"`
	Detail             string   `json:"detail"`
	Missing            []string `json:"missing,omitempty"`
}

type ClaimState added in v0.45.0 

type ClaimState string
const (
	ClaimAllowed               ClaimState = "ALLOWED"
	ClaimLimitedToScoredSubset ClaimState = "LIMITED_TO_SCORED_SUBSET"
	ClaimRefused               ClaimState = "REFUSED"
)

type CohortUnit added in v0.45.0 

type CohortUnit struct {
	ID      string `json:"id"`
	Stratum string `json:"stratum,omitempty"`
}

type ComparisonRequest added in v0.45.0 

type ComparisonRequest struct {
	LeftArm  string       `json:"left_arm"`
	RightArm string       `json:"right_arm"`
	Rule     AnalysisRule `json:"rule"`
}

type ComparisonScope added in v0.45.0 

type ComparisonScope string
const (
	ScopeFullCohort   ComparisonScope = "full_cohort"
	ScopeScoredSubset ComparisonScope = "scored_subset"
)

type DenominatorReceipt added in v0.45.0 

type DenominatorReceipt struct {
	Schema          string                `json:"schema"`
	CohortDigest    string                `json:"cohort_digest"`
	Arm             string                `json:"arm"`
	Counts          FunnelCounts          `json:"counts"`
	MissingByReason map[MissingReason]int `json:"missing_by_reason"`
	Score           ScoreSummary          `json:"score"`
	Results         []UnitResult          `json:"results"`
}

func ReconcileArm added in v0.45.0 

func ReconcileArm(cohort FrozenCohort, run ArmRun) (DenominatorReceipt, error)

func (DenominatorReceipt) Validate added in v0.45.0 

func (r DenominatorReceipt) Validate() error

type DenominatorReport added in v0.45.0 

type DenominatorReport struct {
	Schema      string               `json:"schema"`
	Cohort      FrozenCohort         `json:"cohort"`
	Arms        []DenominatorReceipt `json:"arms"`
	Comparisons []ArmComparison      `json:"comparisons"`
}

func BuildDenominatorReport added in v0.45.0 

func BuildDenominatorReport(cohort FrozenCohort, runs []ArmRun, requests []ComparisonRequest) (DenominatorReport, error)

type EvaluatorEvidenceArtifact added in v0.45.0 

type EvaluatorEvidenceArtifact struct {
	Kind      string                 `json:"kind"`
	Level     EvaluatorEvidenceLevel `json:"level"`
	Authority string                 `json:"authority"`
	Path      string                 `json:"path"`
	SHA256    string                 `json:"sha256"`
}

type EvaluatorEvidenceError added in v0.45.0 

type EvaluatorEvidenceError struct {
	Reason string
	Detail string
}

func (*EvaluatorEvidenceError) Error added in v0.45.0 

func (e *EvaluatorEvidenceError) Error() string

type EvaluatorEvidenceLevel added in v0.45.0 

type EvaluatorEvidenceLevel string
const (
	EvaluatorEvidenceOfficialScalar      EvaluatorEvidenceLevel = "official_scalar"
	EvaluatorEvidenceStructuredBreakdown EvaluatorEvidenceLevel = "structured_breakdown"
	EvaluatorEvidenceRawGraderPayload    EvaluatorEvidenceLevel = "raw_grader_payload"
)

type EvaluatorEvidenceLevelStatus added in v0.45.0 

type EvaluatorEvidenceLevelStatus struct {
	Level             EvaluatorEvidenceLevel `json:"level"`
	Available         bool                   `json:"available"`
	Authority         string                 `json:"authority,omitempty"`
	ArtifactPath      string                 `json:"artifact_path,omitempty"`
	ArtifactSHA256    string                 `json:"artifact_sha256,omitempty"`
	UnavailableReason string                 `json:"unavailable_reason,omitempty"`
}

type EvaluatorEvidenceManifest added in v0.45.0 

type EvaluatorEvidenceManifest struct {
	Schema        string                      `json:"schema"`
	DeclaredLevel EvaluatorEvidenceLevel      `json:"declared_level"`
	Artifacts     []EvaluatorEvidenceArtifact `json:"artifacts"`
	Scorer        EvaluatorScorerProvenance   `json:"scorer,omitempty"`
}

type EvaluatorEvidenceReport added in v0.45.0 

type EvaluatorEvidenceReport struct {
	Schema        string                         `json:"schema"`
	ManifestPath  string                         `json:"manifest_path"`
	Gate          string                         `json:"gate"`
	DeclaredLevel EvaluatorEvidenceLevel         `json:"declared_level"`
	ResolvedLevel EvaluatorEvidenceLevel         `json:"resolved_level"`
	EvalStatus    string                         `json:"eval_status"`
	Score         *float64                       `json:"score"`
	Scorer        EvaluatorScorerProvenance      `json:"scorer,omitempty"`
	Levels        []EvaluatorEvidenceLevelStatus `json:"levels"`
}

func LoadEvaluatorEvidence added in v0.45.0 

func LoadEvaluatorEvidence(root, manifestPath string) (EvaluatorEvidenceReport, error)

LoadEvaluatorEvidence resolves every declared artifact from disk before it grants the manifest's evidence rung. Stock ALE eval_result.json is always typed as an official scalar projection, regardless of what the caller claims.

type EvaluatorHashedArtifact added in v0.45.0 

type EvaluatorHashedArtifact struct {
	Path   string `json:"path"`
	SHA256 string `json:"sha256"`
}

type EvaluatorReceipt added in v0.45.0 

type EvaluatorReceipt struct {
	Status     string
	Score      *float64
	MinScore   float64
	MaxScore   float64
	DurationMS int64
	ErrorRef   string
}

func (EvaluatorReceipt) Validate added in v0.45.0 

func (r EvaluatorReceipt) Validate() error

type EvaluatorScorerProvenance added in v0.45.0 

type EvaluatorScorerProvenance struct {
	Kind      string                   `json:"kind,omitempty"`
	Version   string                   `json:"version,omitempty"`
	Code      EvaluatorHashedArtifact  `json:"code,omitempty"`
	Reference *EvaluatorHashedArtifact `json:"reference,omitempty"`
	Model     string                   `json:"model,omitempty"`
	Prompt    *EvaluatorHashedArtifact `json:"prompt,omitempty"`
	Rubric    *EvaluatorHashedArtifact `json:"rubric,omitempty"`
}

type ExternalHarnessQueue 

type ExternalHarnessQueue struct {
	Schema             string                     `json:"schema"`
	GeneratedAt        string                     `json:"generated_at"`
	Epic               int                        `json:"epic"`
	Status             string                     `json:"status"`
	ResultClaimAllowed bool                       `json:"result_claim_allowed"`
	Summary            ExternalHarnessSummary     `json:"summary"`
	Items              []ExternalHarnessQueueItem `json:"items"`
	ClaimBoundary      string                     `json:"claim_boundary"`
}

func BuildExternalHarnessQueue 

func BuildExternalHarnessQueue(root string, now time.Time) (*ExternalHarnessQueue, error)

func WriteExternalHarnessQueue 

func WriteExternalHarnessQueue(root, jsonPath, markdownPath string, now time.Time) (*ExternalHarnessQueue, error)

type ExternalHarnessQueueItem 

type ExternalHarnessQueueItem struct {
	Issue               int              `json:"issue"`
	Packet              string           `json:"packet"`
	Title               string           `json:"title"`
	SourceArtifact      string           `json:"source_artifact"`
	Gate                string           `json:"gate"`
	Status              string           `json:"status"`
	ExternalState       string           `json:"external_state"`
	ClaimBoundary       string           `json:"claim_boundary,omitempty"`
	Commands            []HarnessCommand `json:"commands,omitempty"`
	Arms                []HarnessArm     `json:"arms,omitempty"`
	RequiredBeforeClaim []string         `json:"required_before_claim"`
	CompareMetrics      []string         `json:"compare_metrics,omitempty"`
	ResultClaimAllowed  bool             `json:"result_claim_allowed"`
}

type ExternalHarnessSummary 

type ExternalHarnessSummary struct {
	ItemsTotal              int   `json:"items_total"`
	ReadyItems              int   `json:"ready_items"`
	BlockedItems            int   `json:"blocked_items"`
	RequiredReturnArtifacts int   `json:"required_return_artifacts"`
	PendingIssues           []int `json:"pending_issues"`
}

type FailureAnalysisArtifact added in v0.45.0 

type FailureAnalysisArtifact struct {
	ArtifactPath string `json:"artifact_path"`
	SHA256       string `json:"sha256"`
}

type FailureAnalysisCard added in v0.45.0 

type FailureAnalysisCard struct {
	Schema         string                        `json:"schema"`
	Label          string                        `json:"label"`
	RunID          string                        `json:"run_id"`
	Generator      FailureAnalysisGenerator      `json:"generator"`
	Artifacts      []FailureAnalysisArtifact     `json:"artifacts"`
	Verdict        FailureAnalysisVerdict        `json:"verdict"`
	Task           FailureAnalysisTask           `json:"task"`
	Strengths      []FailureAnalysisClaim        `json:"strengths"`
	Failures       FailureAnalysisFailures       `json:"failures"`
	Scoring        FailureAnalysisScoring        `json:"scoring"`
	Classification FailureAnalysisClassification `json:"classification"`
}

FailureAnalysisCard is an advisory sidecar. It deliberately has no hook into official scoring, retry, routing, or policy decisions.

func DecodeFailureAnalysisCard added in v0.45.0 

func DecodeFailureAnalysisCard(data []byte) (FailureAnalysisCard, error)

DecodeFailureAnalysisCard rejects schema drift instead of silently ignoring fields emitted by a newer or malformed generator.

type FailureAnalysisClaim added in v0.45.0 

type FailureAnalysisClaim struct {
	Text        string                    `json:"text"`
	Kind        FailureAnalysisClaimKind  `json:"kind"`
	Uncertainty string                    `json:"uncertainty,omitempty"`
	Evidence    []FailureAnalysisEvidence `json:"evidence"`
}

type FailureAnalysisClaimKind added in v0.45.0 

type FailureAnalysisClaimKind string
const (
	FailureAnalysisObservation FailureAnalysisClaimKind = "observation"
	FailureAnalysisInference   FailureAnalysisClaimKind = "inference"
)

type FailureAnalysisClassification added in v0.45.0 

type FailureAnalysisClassification struct {
	Class       string               `json:"class"`
	Subcategory string               `json:"subcategory"`
	Confidence  float64              `json:"confidence"`
	Uncertainty string               `json:"uncertainty"`
	Rationale   FailureAnalysisClaim `json:"rationale"`
}

type FailureAnalysisEvidence added in v0.45.0 

type FailureAnalysisEvidence struct {
	ArtifactPath string `json:"artifact_path"`
	JSONPointer  string `json:"json_pointer,omitempty"`
	EventOrdinal *int   `json:"event_ordinal,omitempty"`
	ArtifactHash string `json:"artifact_hash"`
}

type FailureAnalysisFailures added in v0.45.0 

type FailureAnalysisFailures struct {
	Observations []FailureAnalysisClaim `json:"observations"`
	Inferences   []FailureAnalysisClaim `json:"inferences"`
}

type FailureAnalysisGenerator added in v0.45.0 

type FailureAnalysisGenerator struct {
	Recipe          string `json:"recipe"`
	Model           string `json:"model"`
	PromptVersion   string `json:"prompt_version"`
	SoftwareVersion string `json:"software_version"`
	GeneratedAt     string `json:"generated_at"`
}

type FailureAnalysisScoring added in v0.45.0 

type FailureAnalysisScoring struct {
	FinalScore  FailureAnalysisClaim   `json:"final_score"`
	Breakdown   []FailureAnalysisClaim `json:"breakdown"`
	Confidence  float64                `json:"confidence"`
	Uncertainty string                 `json:"uncertainty"`
}

type FailureAnalysisTask added in v0.45.0 

type FailureAnalysisTask struct {
	Summary     FailureAnalysisClaim   `json:"summary"`
	Constraints []FailureAnalysisClaim `json:"constraints"`
}

type FailureAnalysisVerdict added in v0.45.0 

type FailureAnalysisVerdict struct {
	Outcome              string               `json:"outcome"`
	Summary              FailureAnalysisClaim `json:"summary"`
	MostImportantProblem FailureAnalysisClaim `json:"most_important_problem"`
}

type FrozenCohort added in v0.45.0 

type FrozenCohort struct {
	Schema string       `json:"schema"`
	Digest string       `json:"digest"`
	Units  []CohortUnit `json:"units"`
}

func FreezeCohort added in v0.45.0 

func FreezeCohort(units []CohortUnit) (FrozenCohort, error)

FreezeCohort sorts the declared population before hashing so callers cannot create different cohort identities merely by enumerating the same plan in a different order.

func (FrozenCohort) Validate added in v0.45.0 

func (c FrozenCohort) Validate() error

type FunnelCounts added in v0.45.0 

type FunnelCounts struct {
	Planned           int `json:"planned"`
	Admitted          int `json:"admitted"`
	Attempted         int `json:"attempted"`
	AgentTerminal     int `json:"agent_terminal"`
	EvaluatorTerminal int `json:"evaluator_terminal"`
	Scored            int `json:"scored"`
	Missing           int `json:"missing"`
}

type FunnelStage added in v0.45.0 

type FunnelStage string
const (
	StageDeclared          FunnelStage = "planned"
	StageAdmitted          FunnelStage = "admitted"
	StageAttempted         FunnelStage = "attempted"
	StageAgentTerminal     FunnelStage = "agent_terminal"
	StageEvaluatorTerminal FunnelStage = "evaluator_terminal"
	StageScored            FunnelStage = "scored"
)

type Gate 

type Gate struct {
	Name   string `json:"name"`
	OK     bool   `json:"ok"`
	Detail string `json:"detail"`
}

type GatewayLatencyObservation added in v0.45.0 

type GatewayLatencyObservation struct {
	Name string `json:"name,omitempty"`
	LatencyMeasurement
	Additive bool `json:"additive"`
}

type HarnessArm 

type HarnessArm struct {
	Name              string   `json:"name"`
	Harness           string   `json:"harness,omitempty"`
	Command           string   `json:"command"`
	OutputDir         string   `json:"output_dir,omitempty"`
	RequiredArtifacts []string `json:"required_artifacts,omitempty"`
}

type HarnessCommand 

type HarnessCommand struct {
	ID             string   `json:"id"`
	Description    string   `json:"description,omitempty"`
	Command        string   `json:"command"`
	Artifacts      []string `json:"artifacts,omitempty"`
	Required       bool     `json:"required"`
	ArtifactStatus string   `json:"artifact_status,omitempty"`
	ArtifactDetail string   `json:"artifact_detail,omitempty"`
}

type LatencyMeasurement added in v0.45.0 

type LatencyMeasurement struct {
	Duration      *float64 `json:"duration,omitempty"`
	Unit          string   `json:"unit,omitempty"`
	Source        string   `json:"source,omitempty"`
	Start         *float64 `json:"start,omitempty"`
	End           *float64 `json:"end,omitempty"`
	UnknownReason string   `json:"unknown_reason,omitempty"`
}

type LineageError added in v0.46.0 

type LineageError struct {
	Reason string
	Detail string
}

LineageError records structured reasons for attempt lineage refusals and validation failures.

func (*LineageError) Error added in v0.46.0 

func (e *LineageError) Error() string

type MissingReason added in v0.45.0 

type MissingReason string
const (
	MissingAdmissionRefused   MissingReason = "admission_refused"
	MissingLaunchFailed       MissingReason = "launch_failed"
	MissingAgentTimeout       MissingReason = "agent_timeout"
	MissingEvaluatorFailed    MissingReason = "evaluator_failed"
	MissingArtifactIncomplete MissingReason = "artifact_incomplete"
)

type NativeRetryRegime added in v0.46.0 

type NativeRetryRegime string
const (
	RetryOnFailureOnly NativeRetryRegime = "failure_only" // ALE native: failures retry, timeouts do NOT
	NoRetry            NativeRetryRegime = "no_retry"
	RetryAll           NativeRetryRegime = "retry_all"
)

type Report 

type Report struct {
	Schema             string               `json:"schema"`
	GeneratedAt        string               `json:"generated_at"`
	Epic               int                  `json:"epic"`
	Status             string               `json:"status"`
	ResultClaimAllowed bool                 `json:"result_claim_allowed"`
	Summary            Summary              `json:"summary"`
	Children           []ChildStatus        `json:"children"`
	ResultPackets      []ResultPacketStatus `json:"result_packets,omitempty"`
	Acceptance         []Gate               `json:"acceptance"`
	ClaimBoundary      string               `json:"claim_boundary"`
}

func Build 

func Build(root string, now time.Time) (*Report, error)

type ResultPacketStatus 

type ResultPacketStatus struct {
	Path               string             `json:"path"`
	Issue              int                `json:"issue"`
	Packet             string             `json:"packet,omitempty"`
	Status             string             `json:"status"`
	ResultClaimAllowed bool               `json:"result_claim_allowed"`
	Gate               string             `json:"gate"`
	Detail             string             `json:"detail"`
	Missing            []string           `json:"missing,omitempty"`
	Artifacts          []string           `json:"artifacts,omitempty"`
	Latency            []ArmLatencyStatus `json:"latency_phases,omitempty"`
}

type ScoreSummary added in v0.45.0 

type ScoreSummary struct {
	Metric           string   `json:"metric"`
	Numerator        string   `json:"numerator"`
	NumeratorValue   float64  `json:"numerator_value"`
	Denominator      string   `json:"denominator"`
	DenominatorCount int      `json:"denominator_count"`
	Value            *float64 `json:"value,omitempty"`
}

type SelectionRule added in v0.46.0 

type SelectionRule string
const (
	FinalAttempt SelectionRule = "final_attempt" // official score is the final attempt's score
	BestOfN      SelectionRule = "best_of_n"     // must be declared explicitly
)

type Summary 

type Summary struct {
	ChildrenTotal          int   `json:"children_total"`
	ChildrenParsed         int   `json:"children_parsed"`
	LocalEvidenceArtifacts int   `json:"local_evidence_artifacts"`
	ResultClaimArtifacts   int   `json:"result_claim_artifacts"`
	ResultPacketsTotal     int   `json:"result_packets_total"`
	ResultPacketsPassed    int   `json:"result_packets_passed"`
	ResultPacketsFailed    int   `json:"result_packets_failed,omitempty"`
	PendingChildren        []int `json:"pending_children,omitempty"`
	FailedChildren         []int `json:"failed_children,omitempty"`
}

type UnitResult added in v0.45.0 

type UnitResult struct {
	UnitID        string        `json:"unit_id"`
	FurthestStage FunnelStage   `json:"furthest_stage"`
	MissingReason MissingReason `json:"missing_reason,omitempty"`
	Score         *float64      `json:"score,omitempty"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.