eval

Documentation

Overview

Package eval defines the attribution evaluation corpus and runner. It tests the domain packages (events, scoring, reporting) directly without any database, blob store, or service orchestration.

Index

• Variables
• func FormatSummary(s Summary) string
• type CaseResult
  • func RunCase(tc EvalCase) CaseResult
• type EvalCase
• type ExpectedFile
• type ExpectedResult
• type FileResult
• type Summary
  • func RunCorpus(cases []EvalCase) Summary

Variables

var Corpus = []EvalCase{

	{
		Name:        "claude-exact-only",
		Description: "Claude writes two new files, all lines match exactly",
		RepoRoot:    "/repo",
		Diff: "diff --git a/main.go b/main.go\n" +
			"--- /dev/null\n" +
			"+++ b/main.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package main\n" +
			"+\n" +
			"+func main() {}\n" +
			"diff --git a/util.go b/util.go\n" +
			"--- /dev/null\n" +
			"+++ b/util.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func helper() {}\n",
		Events: []events.EventRow{
			claudeEvent("Write", "main.go", "package main\n\nfunc main() {}\n", "/repo"),
			claudeEvent("Write", "util.go", "package main\nfunc helper() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "main.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "util.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
			},
		},
	},

	{
		Name:        "formatter-churn-normalized",
		Description: "Claude writes code, formatter changes whitespace, normalized matches needed",
		RepoRoot:    "/repo",
		Diff: "diff --git a/handler.go b/handler.go\n" +
			"--- /dev/null\n" +
			"+++ b/handler.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package api\n" +
			"+func Handle() {\n" +
			"+}\n",
		Events: []events.EventRow{

			claudeEvent("Write", "handler.go", "package api\nfunc Handle(){\n}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "handler.go", AILines: 3, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceNormalized,
					},
					Notes: "package api and } match exactly; func Handle() { matches via normalization",
				},
			},
		},
	},

	{
		Name:        "cursor-file-touch-only",
		Description: "Cursor edits a file, no line-level evidence, file-touch fallback",
		RepoRoot:    "/repo",
		Diff: "diff --git a/config.go b/config.go\n" +
			"--- /dev/null\n" +
			"+++ b/config.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+package main\n" +
			"+var port = 8080\n" +
			"+var host = \"localhost\"\n",
		Events: []events.EventRow{
			{
				Provider: "cursor", Role: "assistant",
				ToolUses: `{"content_types":["cursor_file_edit"],"tools":[{"name":"cursor_file_edit","file_path":"config.go","file_op":"edit"}]}`,
			},
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "Medium",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "config.go", AILines: 3, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceModified,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceModified},
					Notes: "All lines become ModifiedLines via provider-touch-only path in ScoreFiles. " +
						"Primary evidence is 'modified' (not fallback) because ScoreFiles actually scores the lines.",
				},
			},
		},
	},

	{
		Name:        "mixed-human-and-ai",
		Description: "One AI file (exact) and one human-only file in the same commit",
		RepoRoot:    "/repo",
		Diff: "diff --git a/ai.go b/ai.go\n" +
			"--- /dev/null\n" +
			"+++ b/ai.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func generated() {}\n" +
			"diff --git a/human.go b/human.go\n" +
			"--- /dev/null\n" +
			"+++ b/human.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package main\n" +
			"+func handwritten() {}\n",
		Events: []events.EventRow{
			claudeEvent("Write", "ai.go", "package main\nfunc generated() {}\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  50,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "ai.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceExact},
				},
				{
					Path: "human.go", AILines: 0, HumanLines: 2,
					PrimaryEvidence:      reporting.EvidenceNone,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceNone},
				},
			},
		},
	},

	{
		Name:        "group-overlap-modified",
		Description: "One exact match in a group promotes neighbors to AI-Modified",
		RepoRoot:    "/repo",
		Diff: "diff --git a/service.go b/service.go\n" +
			"--- /dev/null\n" +
			"+++ b/service.go\n" +
			"@@ -0,0 +1,3 @@\n" +
			"+func Start() {\n" +
			"+\tlog.Println(\"starting\")\n" +
			"+\tgo run()\n",
		Events: []events.EventRow{

			claudeEvent("Write", "service.go", "func Start() {\n", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "Medium",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "service.go", AILines: 3, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceModified,
					},
					Notes: "1 exact + 2 modified from group overlap. Primary evidence is 'exact' " +
						"(highest wins). Evidence is Medium because LineScore = 0.7 " +
						"(modified lines pull it below the 0.75 High threshold).",
				},
			},
		},
	},

	{
		Name:        "carry-forward-success",
		Description: "File attributed via carry-forward with actual AI lines from historical window",
		RepoRoot:    "/repo",
		Diff: "diff --git a/utils.go b/utils.go\n" +
			"--- /dev/null\n" +
			"+++ b/utils.go\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+package utils\n" +
			"+func Helper() {}\n",
		Events: []events.EventRow{
			claudeEvent("Write", "utils.go", "package utils\nfunc Helper() {}\n", "/repo"),
		},
		CarryForwardFiles: map[string]bool{"utils.go": true},
		Expected: ExpectedResult{
			AIPercentage:  100,
			Evidence:      "High",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "utils.go", AILines: 2, HumanLines: 0,
					PrimaryEvidence: reporting.EvidenceExact,
					ContributingEvidence: []reporting.EvidenceClass{
						reporting.EvidenceExact,
						reporting.EvidenceCarryForward,
					},
					Notes: "Exact line match wins for display, but carry-forward is a contributing " +
						"class because the lines came from a historical window.",
				},
			},
		},
	},

	{
		Name:        "carry-forward-zero-score",
		Description: "Carry-forward attempted but no AI lines found in either window",
		RepoRoot:    "/repo",
		Diff: "diff --git a/config.yaml b/config.yaml\n" +
			"--- /dev/null\n" +
			"+++ b/config.yaml\n" +
			"@@ -0,0 +1,2 @@\n" +
			"+port: 8080\n" +
			"+host: localhost\n",
		Events: []events.EventRow{

			claudeEvent("Write", "other.go", "package other\n", "/repo"),
		},

		Expected: ExpectedResult{
			AIPercentage:  0,
			Evidence:      "",
			FallbackCount: 0,
			Files: []ExpectedFile{
				{
					Path: "config.yaml", AILines: 0, HumanLines: 2,
					PrimaryEvidence:      reporting.EvidenceNone,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceNone},
					Notes: "File was carry-forward eligible but scored zero AI lines. " +
						"The orchestrator excluded it from CarryForwardFiles (actualCF).",
				},
			},
		},
	},

	{
		Name:        "bash-deletion-inference",
		Description: "File deleted via Claude bash command, tracked in FilesDeleted",
		RepoRoot:    "/repo",
		Diff: "diff --git a/old.go b/old.go\n" +
			"--- a/old.go\n" +
			"+++ /dev/null\n" +
			"@@ -1,2 +0,0 @@\n" +
			"-package main\n" +
			"-func deprecated() {}\n",
		Events: []events.EventRow{
			claudeEvent("Bash", "", "rm /repo/old.go", "/repo"),
		},
		Expected: ExpectedResult{
			AIPercentage:  0,
			Evidence:      "Low",
			FallbackCount: 1,
			Files: []ExpectedFile{
				{
					Path: "old.go", AILines: 0, HumanLines: 0,
					PrimaryEvidence:      reporting.EvidenceDeletion,
					ContributingEvidence: []reporting.EvidenceClass{reporting.EvidenceDeletion},
					Notes: "File has zero added lines but is in the AI-touched set via bash deletion. " +
						"Evidence class is 'deletion' (inferential). The file appears in both " +
						"Files (with zero lines) and FilesDeleted in the commit result.",
				},
			},
		},
	},
}

Corpus is the evaluation case set. Start small (5 cases), scale after the schema proves correct.

Functions

func FormatSummary

func FormatSummary(s Summary) string

FormatSummary returns a human-readable summary string.

Types

type CaseResult

type CaseResult struct {
	Name   string
	Passed bool
	Errors []string

	// Observed values for summary reporting.
	AIPercentage  float64
	Evidence      string
	FallbackCount int
	FileResults   []FileResult
}

CaseResult is the outcome of running a single evaluation case.

func RunCase

func RunCase(tc EvalCase) CaseResult

RunCase runs a single evaluation case through the domain pipeline.

type EvalCase

type EvalCase struct {
	Name        string
	Description string

	// Inputs
	Diff     string            // unified diff text
	Events   []events.EventRow // event rows with inline payloads
	RepoRoot string            // repo root for path normalization

	// Evidence context (optional)
	CarryForwardFiles    map[string]bool                  // files known to come from historical lookback
	TouchOriginOverrides map[string]reporting.TouchOrigin // explicit touch origin overrides for testing

	// Expected outcomes
	Expected ExpectedResult
}

EvalCase is a single attribution evaluation fixture.

type ExpectedFile

type ExpectedFile struct {
	Path                 string
	AILines              int
	HumanLines           int
	PrimaryEvidence      reporting.EvidenceClass
	ContributingEvidence []reporting.EvidenceClass // all evidence classes that should appear
	Notes                string                    // optional: known ambiguity or edge case
}

ExpectedFile defines the expected attribution for a single file.

type ExpectedResult

type ExpectedResult struct {
	AIPercentage  float64 // headline AI% (within tolerance)
	Evidence      string  // commit-level: "High", "Medium", "Low"
	FallbackCount int     // expected number of fallback files

	Files []ExpectedFile // per-file expectations
}

ExpectedResult defines the ground-truth attribution for a case.

type FileResult

type FileResult struct {
	Path            string
	AILines         int
	HumanLines      int
	PrimaryEvidence reporting.EvidenceClass
	AllEvidence     []reporting.EvidenceClass
	Provider        string // primary provider for this file (for summary breakdown)
}

FileResult is the per-file observed vs expected comparison.

type Summary

type Summary struct {
	Total   int
	Passed  int
	Failed  int
	Results []CaseResult

	// Per-evidence-class usage counts (how often each class was primary).
	EvidenceUsage map[reporting.EvidenceClass]int
	// Files with multiple contributing classes.
	MultiEvidenceFiles int
	// Files where primary evidence is a fallback class.
	FallbackFiles int
	// Files where primary (display) class is stronger than the weakest
	// contributing class. Shows how often the "highest wins" rule hides
	// weaker evidence from the user.
	PrimaryStrongerThanWeakest int
	// Provider breakdown: provider -> primary evidence class -> count.
	ProviderEvidence map[string]map[reporting.EvidenceClass]int
}

Summary aggregates results across all cases.

func RunCorpus

func RunCorpus(cases []EvalCase) Summary

RunCorpus executes all evaluation cases and returns a summary.