eval

Documentation

Overview

Package eval defines the attribution evaluation corpus and runner. It tests the domain packages (events, scoring, reporting) directly without any database, blob store, or service orchestration.

Index

• Variables
• func FormatSummary(s Summary) string
• type CaseResult
  • func RunCase(tc EvalCase) CaseResult
• type EvalCase
• type ExpectedFile
• type ExpectedResult
• type FileResult
• type Summary
  • func RunCorpus(cases []EvalCase) Summary

Variables

var Corpus = []EvalCase{

	{
		Name:        "claude-exact-only",
		Description: "Claude writes two new files, all lines match exactly",
		RepoRoot:    "/repo",
		Diff: "diff --git a/main.go b/main.go\n" +
			"--- /dev/null\n" +
			"+++ b/main.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package main\n" +
			"+\n" +
			"+func main() {}\n" +
			"diff --git a/util.go b/util.go\n" +
			"--- /dev/null\n" +
			"+++ b/util.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func helper() {}\n",
		Events: []events.EventRow{
			claudeEvent("Write", "main.go", "package main\n\nfunc main() {}\n", "/repo"),
			claudeEvent("Write", "util.go", "package main\nfunc helper() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "main.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "util.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
			},
		},
	},

	{
		Name:        "formatter-churn-normalized",
		Description: "Claude writes code, formatter changes whitespace, normalized matches needed",
		RepoRoot:    "/repo",
		Diff: "diff --git a/handler.go b/handler.go\n" +
			"--- /dev/null\n" +
			"+++ b/handler.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package api\n" +
			"+func Handle() {\n" +
			"+}\n",
		Events: []events.EventRow{

			claudeEvent("Write", "handler.go", "package api\nfunc Handle(){\n}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "handler.go", AILines: 3, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceNormalized,
					},
					Notes: "package api and } match exactly; func Handle() { matches via normalization",
				},
			},
		},
	},

	{
		Name:        "cursor-file-touch-only",
		Description: "Cursor edits a file, no line-level evidence, file-touch fallback",
		RepoRoot:    "/repo",
		Diff: "diff --git a/config.go b/config.go\n" +
			"--- /dev/null\n" +
			"+++ b/config.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package main\n" +
			"+var port = 8080\n" +
			"+var host = \"localhost\"\n",
		Events: []events.EventRow{
			{
				Provider: "cursor", Role: "assistant",
				ToolUses: `{"content_types":["cursor_file_edit"],"tools":[{"name":"cursor_file_edit","file_path":"config.go","file_op":"edit"}]}`,
			},
		},
		Expected: ExpectedResult{

			AIPercentage:  0,
			Evidence:      "Low",
			FallbackCount: 1,
			Files: []ExpectedFile{
				{
					Path: "config.go", AILines: 0, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceProviderTouch,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceProviderTouch,
					},
					Notes: "Provider-touch-only files: lines go into ProviderOnlyLines, not " +
						"ModifiedLines. AILines (exact+formatted+modified) is 0, AIPercentage is 0. " +
						"PrimaryEvidence is 'provider_touch' because there is no line-level " +
						"evidence. The file still counts toward FilesAITouched in the change list.",
				},
			},
		},
	},

	{
		Name:        "mixed-human-and-ai",
		Description: "One AI file (exact) and one human-only file in the same commit",
		RepoRoot:    "/repo",
		Diff: "diff --git a/ai.go b/ai.go\n" +
			"--- /dev/null\n" +
			"+++ b/ai.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func generated() {}\n" +
			"diff --git a/human.go b/human.go\n" +
			"--- /dev/null\n" +
			"+++ b/human.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func handwritten() {}\n",
		Events: []events.EventRow{
			claudeEvent("Write", "ai.go", "package main\nfunc generated() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  50,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "ai.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "human.go", AILines: 0, HumanLines: 2,
					PrimaryEvidence:      reporting.EvidenceNone,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceNone},
				},
			},
		},
	},

	{
		Name:        "group-overlap-modified",
		Description: "One exact match in a group promotes neighbors to AI-Modified",
		RepoRoot:    "/repo",
		Diff: "diff --git a/service.go b/service.go\n" +
			"--- /dev/null\n" +
			"+++ b/service.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+func Start() {\n" +
			"+\tlog.Println(\"starting\")\n" +
			"+\tgo run()\n",
		Events: []events.EventRow{

			claudeEvent("Write", "service.go", "func Start() {\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "Medium",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "service.go", AILines: 3, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceModified,
					},
					Notes: "1 exact + 2 modified from group overlap. Primary evidence is 'exact' " +
						"(highest wins). Evidence is Medium because LineScore = 0.7 " +
						"(modified lines pull it below the 0.75 High threshold).",
				},
			},
		},
	},

	{
		Name:        "carry-forward-success",
		Description: "File attributed via carry-forward with actual AI lines from historical window",
		RepoRoot:    "/repo",
		Diff: "diff --git a/utils.go b/utils.go\n" +
			"--- /dev/null\n" +
			"+++ b/utils.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package utils\n" +
			"+func Helper() {}\n",
		Events: []events.EventRow{
			claudeEvent("Write", "utils.go", "package utils\nfunc Helper() {}\n", "/repo"),
		},
		CarryForwardFiles: map[string]bool{"utils.go": true},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 1,
			Files: []ExpectedFile{
				{
					Path: "utils.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceCarryForward,
					},
					Notes: "Exact line match wins for display, but carry-forward is a contributing " +
						"class because the lines came from a historical window. CommitEvidence walks " +
						"AllEvidence so the carry-forward signal counts as a fallback even when an " +
						"exact match wins for PrimaryEvidence. Strength stays High because " +
						"penalty (0.25/1) leaves score = 0.75.",
				},
			},
		},
	},

	{
		Name:        "carry-forward-zero-score",
		Description: "Carry-forward attempted but no AI lines found in either window",
		RepoRoot:    "/repo",
		Diff: "diff --git a/config.yaml b/config.yaml\n" +
			"--- /dev/null\n" +
			"+++ b/config.yaml\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+port: 8080\n" +
			"+host: localhost\n",
		Events: []events.EventRow{

			claudeEvent("Write", "other.go", "package other\n", "/repo"),
		},

		Expected: ExpectedResult{
			AIPercentage:  0,
			Evidence:      "",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "config.yaml", AILines: 0, HumanLines: 2,
					PrimaryEvidence:      reporting.EvidenceNone,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceNone},
					Notes: "File was carry-forward eligible but scored zero AI lines. " +
						"The orchestrator excluded it from CarryForwardFiles (actualCF).",
				},
			},
		},
	},

	{
		Name:        "bash-deletion-inference",
		Description: "File deleted via Claude bash command, tracked in FilesDeleted",
		RepoRoot:    "/repo",
		Diff: "diff --git a/old.go b/old.go\n" +
			"--- a/old.go\n" +
			"+++ /dev/null\n" +
			"@@ -1,2 +0,0 @@\n" +
			"-package main\n" +
			"-func deprecated() {}\n",
		Events: []events.EventRow{
			claudeEvent("Bash", "", "rm /repo/old.go", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  0,
			Evidence:      "Low",
			FallbackCount: 1,
			Files: []ExpectedFile{
				{
					Path: "old.go", AILines: 0, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceDeletion,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceDeletion},
					Notes: "File has zero added lines but is in the AI-touched set via bash deletion. " +
						"Evidence class is 'deletion' (inferential). The file appears in both " +
						"Files (with zero lines) and FilesDeleted in the commit result.",
				},
			},
		},
	},

	{
		Name:        "kiro-ide-line-level",
		Description: "Kiro IDE Write (create) and Edit (replace) emit canonical tool_uses; lines score line-level",
		RepoRoot:    "/repo",
		Diff: "diff --git a/main.go b/main.go\n" +
			"--- /dev/null\n" +
			"+++ b/main.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func main() {}\n" +
			"diff --git a/util.go b/util.go\n" +
			"--- a/util.go\n" +
			"+++ b/util.go\n" +
			"@@ -1,1 +1,2 @@\n" +
			" package main\n" +
			"+func helper() {}\n",
		Events: []events.EventRow{
			kiroIDEEvent("Write", "main.go", "package main\nfunc main() {}\n", "/repo"),
			kiroIDEEvent("Edit", "util.go", "package main\nfunc helper() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "main.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "util.go", AILines: 1, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
			},
		},
	},

	{
		Name:        "kiro-cli-line-level",
		Description: "Kiro CLI Write (create) and Edit (strReplace/insert) emit canonical tool_uses; lines score line-level",
		RepoRoot:    "/repo",
		Diff: "diff --git a/main.go b/main.go\n" +
			"--- /dev/null\n" +
			"+++ b/main.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func main() {}\n" +
			"diff --git a/util.go b/util.go\n" +
			"--- a/util.go\n" +
			"+++ b/util.go\n" +
			"@@ -1,1 +1,2 @@\n" +
			" package main\n" +
			"+func helper() {}\n",
		Events: []events.EventRow{
			kiroCLIEvent("Write", "main.go", "package main\nfunc main() {}\n", "/repo"),
			kiroCLIEvent("Edit", "util.go", "package main\nfunc helper() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "main.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "util.go", AILines: 1, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
			},
		},
	},

	{
		Name: "duplicate-common-lines-overcount",
		Description: "AI writes function A; human adds function B with a duplicate `return nil` line. " +
			"Current scorer counts B's `return nil` and `}` as exact, then promotes B's func " +
			"signature to modified via the overlap rule, attributing all of B to the AI.",
		RepoRoot: "/repo",
		Diff: "diff --git a/handler.go b/handler.go\n" +
			"--- /dev/null\n" +
			"+++ b/handler.go\n" +
			"@@ -0,0 +1,6 @@\n" +
			"+func A() error {\n" +
			"+\treturn nil\n" +
			"+}\n" +
			"+func B() error {\n" +
			"+\treturn nil\n" +
			"+}\n",
		Events: []events.EventRow{

			claudeEvent("Write", "handler.go", "func A() error {\n\treturn nil\n}\n", "/repo"),
		},
		Expected: ExpectedResult{

			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path:            "handler.go",
					AILines:         6,
					HumanLines:      0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceModified,
					},
					Notes: "Known overcount: AI set holds {func A() error {, return nil, }}. " +
						"B's `return nil` and `}` match by trimmed text (exact overcount). " +
						"B's `func B() error {` does not match but the hunk has overlap, so " +
						"it counts as 'modified' under the current same-hunk-overlap rule. " +
						"Ground truth would be 3 AI lines + 3 human lines (50% headline). " +
						"Fixing needs occurrence accounting or positional tracking in the " +
						"scorer; this case documents current behavior so a future change can " +
						"confirm the headline drops to ~50%.",
				},
			},
		},
	},

	{
		Name:        "codex-apply-patch-line-level",
		Description: "Codex apply_patch Add produces line-level evidence via the synthesized Write blob",
		RepoRoot:    "/repo",
		Diff: "diff --git a/main.go b/main.go\n" +
			"--- /dev/null\n" +
			"+++ b/main.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package main\n" +
			"+\n" +
			"+func main() {}\n",
		Events: []events.EventRow{
			codexApplyPatchEvent("main.go", "package main\n\nfunc main() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "main.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
			},
		},
	},

	{
		Name:        "codex-file-touch-delete",
		Description: "Codex apply_patch Delete records the file as provider-touch with no line-level evidence",
		RepoRoot:    "/repo",
		Diff: "diff --git a/legacy.go b/legacy.go\n" +
			"--- a/legacy.go\n" +
			"+++ /dev/null\n" +
			"@@ -1,2 +0,0 @@\n" +
			"-package legacy\n" +
			"-func Old() {}\n",
		Events: []events.EventRow{
			codexFileTouchEvent("legacy.go", "delete"),
		},
		Expected: ExpectedResult{
			AIPercentage:  0,
			Evidence:      "Low",
			FallbackCount: 1,
			Files: []ExpectedFile{
				{
					Path: "legacy.go", AILines: 0, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceProviderTouch,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceProviderTouch},
					Notes:                "Lines land in ProviderOnlyLines (provider-touch sidecar); headline AI% excludes them.",
				},
			},
		},
	},

	{
		Name:        "codex-bash-rm-touches-file",
		Description: "Codex Bash rm extracts the deleted file path into provider-touch",
		RepoRoot:    "/repo",
		Diff: "diff --git a/cleanup.go b/cleanup.go\n" +
			"--- a/cleanup.go\n" +
			"+++ /dev/null\n" +
			"@@ -1,1 +0,0 @@\n" +
			"-package cleanup\n",
		Events: []events.EventRow{
			codexBashEvent("rm cleanup.go"),
		},
		Expected: ExpectedResult{
			AIPercentage:  0,
			Evidence:      "Low",
			FallbackCount: 1,
			Files: []ExpectedFile{
				{
					Path: "cleanup.go", AILines: 0, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceProviderTouch,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceProviderTouch},
					Notes:                "Bash rm command extracted by ExtractDeletedPaths; provider-touch only.",
				},
			},
		},
	},

	{
		Name:        "codex-cross-provider-mixed",
		Description: "Codex writes one file, Claude writes another; per-provider attribution survives",
		RepoRoot:    "/repo",
		Diff: "diff --git a/codex_file.go b/codex_file.go\n" +
			"--- /dev/null\n" +
			"+++ b/codex_file.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package codexfile\n" +
			"+func FromCodex() {}\n" +
			"diff --git a/claude_file.go b/claude_file.go\n" +
			"--- /dev/null\n" +
			"+++ b/claude_file.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package claudefile\n" +
			"+func FromClaude() {}\n",
		Events: []events.EventRow{
			codexApplyPatchEvent("codex_file.go", "package codexfile\nfunc FromCodex() {}\n", "/repo"),
			claudeEvent("Write", "claude_file.go", "package claudefile\nfunc FromClaude() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "codex_file.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "claude_file.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
			},
		},
	},
}

Corpus is the evaluation case set. Start small (5 cases), scale after the schema proves correct.

Functions

func FormatSummary

func FormatSummary(s Summary) string

FormatSummary returns a human-readable summary string.

Types

type CaseResult

type CaseResult struct {
	Name   string
	Passed bool
	Errors []string

	// Observed values for summary reporting.
	AIPercentage  float64
	Evidence      string
	FallbackCount int
	FileResults   []FileResult
}

CaseResult is the outcome of running a single evaluation case.

func RunCase

func RunCase(tc EvalCase) CaseResult

RunCase runs a single evaluation case through the domain pipeline.

type EvalCase

type EvalCase struct {
	Name        string
	Description string

	// Inputs
	Diff     string            // unified diff text
	Events   []events.EventRow // event rows with inline payloads
	RepoRoot string            // repo root for path normalization

	// Evidence context (optional)
	CarryForwardFiles    map[string]bool                  // files known to come from historical lookback
	TouchOriginOverrides map[string]reporting.TouchOrigin // explicit touch origin overrides for testing

	// Expected outcomes
	Expected ExpectedResult
}

EvalCase is a single attribution evaluation fixture.

type ExpectedFile

type ExpectedFile struct {
	Path                 string
	AILines              int
	HumanLines           int
	PrimaryEvidence      reporting.EvidenceClass
	ContributingEvidence []reporting.EvidenceClass // all evidence classes that should appear
	Notes                string                    // optional: known ambiguity or edge case
}

ExpectedFile defines the expected attribution for a single file.

type ExpectedResult

type ExpectedResult struct {
	AIPercentage  float64 // headline AI% (within tolerance)
	Evidence      string  // commit-level: "High", "Medium", "Low"
	FallbackCount int     // expected number of fallback files

	Files []ExpectedFile // per-file expectations
}

ExpectedResult defines the ground-truth attribution for a case.

type FileResult

type FileResult struct {
	Path            string
	AILines         int
	HumanLines      int
	PrimaryEvidence reporting.EvidenceClass
	AllEvidence     []reporting.EvidenceClass
	Provider        string // primary provider for this file (for summary breakdown)
}

FileResult is the per-file observed vs expected comparison.

type Summary

type Summary struct {
	Total   int
	Passed  int
	Failed  int
	Results []CaseResult

	// Per-evidence-class usage counts (how often each class was primary).
	EvidenceUsage map[reporting.EvidenceClass]int
	// Files with multiple contributing classes.
	MultiEvidenceFiles int
	// Files where any contributing evidence class is a fallback
	// class. Mirrors the CommitEvidence aggregation: line-level
	// primary plus weaker corroboration still counts. Each file
	// contributes at most once.
	FallbackFiles int
	// Files where primary (display) class is stronger than the weakest
	// contributing class. Shows how often the "highest wins" rule hides
	// weaker evidence from the user.
	PrimaryStrongerThanWeakest int
	// Provider breakdown: provider -> primary evidence class -> count.
	ProviderEvidence map[string]map[reporting.EvidenceClass]int
}

Summary aggregates results across all cases.

func RunCorpus

func RunCorpus(cases []EvalCase) Summary

RunCorpus executes all evaluation cases and returns a summary.