benchmark

package
v1.2.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 27, 2026 License: MIT Imports: 16 Imported by: 0

Documentation

Overview

Package benchmark provides benchmark execution and tracking services.

Package benchmark provides benchmark execution and tracking services.

Index

Constants

View Source 
const (
	// DefaultBridgeURL is the default Python bridge server URL
	DefaultBridgeURL = "http://localhost:9876"

	// BridgeTimeout is the maximum time to wait for bridge responses
	BridgeTimeout = 30 * time.Minute
)

Variables

View Source 
var (
	// ErrDirtyWorkTree is returned when git working tree has uncommitted changes
	ErrDirtyWorkTree = errors.New("git working tree is dirty - commit or stash changes first")

	// ErrRunNotFound is returned when a benchmark run is not found
	ErrRunNotFound = errors.New("benchmark run not found")

	// ErrRunAlreadyRunning is returned when trying to start a run while one is running
	ErrRunAlreadyRunning = errors.New("a benchmark run is already in progress")

	// ErrNoLoopRunning is returned when trying to stop a loop that isn't running
	ErrNoLoopRunning = errors.New("no autonomous loop is currently running")

	// ErrLoopAlreadyRunning is returned when trying to start a loop while one is running
	ErrLoopAlreadyRunning = errors.New("an autonomous loop is already running")

	// ErrPythonBridgeNotAvailable is returned when the Python bridge server is not responding
	ErrPythonBridgeNotAvailable = errors.New("Python benchmark bridge is not available - start it with 'make server' in benchmark/locomo/")

	// ErrBenchmarkFailed is returned when benchmark execution fails
	ErrBenchmarkFailed = errors.New("benchmark execution failed")

	// ErrInvalidBenchmarkType is returned for unsupported benchmark types
	ErrInvalidBenchmarkType = errors.New("invalid benchmark type - supported: locomo")

	// ErrMaxIterationsReached is returned when loop reaches max iterations
	ErrMaxIterationsReached = errors.New("maximum iterations reached")

	// ErrConverged is returned when loop converges (minimal improvement)
	ErrConverged = errors.New("loop converged - improvements are below threshold")

	// ErrNoImprovement is returned when multiple consecutive iterations show no improvement
	ErrNoImprovement = errors.New("no improvement after multiple iterations")
)

Functions

func CommitChanges 

func CommitChanges(repoPath, message string) (string, error)

CommitChanges commits current changes with a message

func ComparisonToJSON 

func ComparisonToJSON(comp *Comparison) (string, error)

ComparisonToJSON converts comparison to JSON

func CreateBenchmarkBranch 

func CreateBenchmarkBranch(repoPath, branchName string) error

CreateBenchmarkBranch creates a new branch for benchmark iteration

func DefaultCatalogDir 

func DefaultCatalogDir() string

DefaultCatalogDir returns the default catalog directory

func FormatComparison 

func FormatComparison(comp *Comparison) string

FormatComparison formats a comparison for display

func FormatResults 

func FormatResults(results *RunResults) string

FormatResults formats results for display

func GenerateImprovementSuggestions 

func GenerateImprovementSuggestions(results *RunResults) []string

GenerateImprovementSuggestions generates suggestions based on weak areas

func GetCommitMessage 

func GetCommitMessage(repoPath, commitHash string) (string, error)

GetCommitMessage gets the commit message for a hash

func RequireCleanWorkTree 

func RequireCleanWorkTree(repoPath string) error

RequireCleanWorkTree checks if the git working tree is clean

func ResultsToJSON 

func ResultsToJSON(results *RunResults) (string, error)

ResultsToJSON converts results to JSON

func RevertToCommit 

func RevertToCommit(repoPath, commitHash string) error

RevertToCommit reverts to a specific commit (hard reset)

func SwitchBranch 

func SwitchBranch(repoPath, branchName string) error

SwitchBranch switches to an existing branch

Types

type BridgeRunRequest 

type BridgeRunRequest struct {
	RunID         string   `json:"run_id"`
	BenchmarkType string   `json:"benchmark_type"`
	MaxQuestions  int      `json:"max_questions"`
	Categories    []string `json:"categories,omitempty"`
	RandomSample  bool     `json:"random_sample"`
	Seed          *int     `json:"seed,omitempty"`
	Verbose       bool     `json:"verbose"`
}

BridgeRunRequest is the request to the Python bridge

type BridgeRunResponse 

type BridgeRunResponse struct {
	Success bool   `json:"success"`
	Error   string `json:"error,omitempty"`
	Results struct {
		Overall struct {
			LLMJudgeAccuracy float64 `json:"llm_judge_accuracy"`
			F1Score          float64 `json:"f1_score"`
			BLEU1Score       float64 `json:"bleu1_score"`
			TotalQuestions   int     `json:"total_questions"`
			TotalCorrect     int     `json:"total_correct"`
		} `json:"overall"`
		ByCategory map[string]struct {
			LLMJudgeAccuracy float64 `json:"llm_judge_accuracy"`
			F1Score          float64 `json:"f1_score"`
			BLEU1Score       float64 `json:"bleu1_score"`
			Count            int     `json:"count"`
			Correct          int     `json:"correct"`
		} `json:"by_category"`
		Questions []struct {
			ID              string  `json:"id"`
			Category        string  `json:"category"`
			Question        string  `json:"question"`
			GoldAnswer      string  `json:"gold_answer"`
			GeneratedAnswer string  `json:"generated_answer"`
			LLMJudgeLabel   int     `json:"llm_judge_label"`
			F1Score         float64 `json:"f1_score"`
			BLEU1Score      float64 `json:"bleu1_score"`
		} `json:"questions"`
		// Enhanced metrics
		Latency *struct {
			MeanSeconds   float64 `json:"mean_latency_seconds"`
			MedianSeconds float64 `json:"median_latency_seconds"`
			P95Seconds    float64 `json:"p95_latency_seconds"`
			P99Seconds    float64 `json:"p99_latency_seconds"`
			MinSeconds    float64 `json:"min_latency_seconds"`
			MaxSeconds    float64 `json:"max_latency_seconds"`
			StdDevSeconds float64 `json:"stdev_latency_seconds"`
		} `json:"latency,omitempty"`
		Tokens *struct {
			TotalInput  int     `json:"total_input_tokens"`
			TotalOutput int     `json:"total_output_tokens"`
			Total       int     `json:"total_tokens"`
			MeanInput   float64 `json:"mean_input_tokens"`
			MeanOutput  float64 `json:"mean_output_tokens"`
		} `json:"tokens,omitempty"`
		Cost *struct {
			InputCostUSD       float64 `json:"input_cost_usd"`
			OutputCostUSD      float64 `json:"output_cost_usd"`
			TotalCostUSD       float64 `json:"total_cost_usd"`
			CostPerQuestionUSD float64 `json:"cost_per_question_usd"`
		} `json:"cost_estimation,omitempty"`
		DurationSecs float64 `json:"duration_seconds,omitempty"`
	} `json:"results"`
}

BridgeRunResponse is the response from the Python bridge

type Catalog 

type Catalog struct {
	// contains filtered or unexported fields
}

Catalog manages file-based benchmark results storage

func NewCatalog 

func NewCatalog(baseDir string) *Catalog

NewCatalog creates a new catalog at the specified directory

func (*Catalog) CleanOldRuns 

func (c *Catalog) CleanOldRuns(daysToKeep int) (int, error)

CleanOldRuns removes runs older than the specified number of days

func (*Catalog) EnsureDirectories 

func (c *Catalog) EnsureDirectories() error

EnsureDirectories creates the catalog directory structure

func (*Catalog) GetBest 

func (c *Catalog) GetBest() (*CatalogedRun, error)

GetBest returns the current best run

func (*Catalog) GetRunByCommit 

func (c *Catalog) GetRunByCommit(commitHash string) (*CatalogedRun, error)

GetRunByCommit finds a run by git commit hash

func (*Catalog) ListRuns 

func (c *Catalog) ListRuns(limit int) ([]*CatalogedRun, error)

ListRuns returns all cataloged runs

func (*Catalog) SaveComparison 

func (c *Catalog) SaveComparison(comp *Comparison) (string, error)

SaveComparison saves a comparison between two runs

func (*Catalog) SaveRun 

func (c *Catalog) SaveRun(results *RunResults) (string, error)

SaveRun saves a benchmark run to the catalog

func (*Catalog) UpdateBest 

func (c *Catalog) UpdateBest(results *RunResults) error

UpdateBest updates the best run marker

type CatalogedRun 

type CatalogedRun struct {
	RunID       string                    `json:"run_id"`
	Timestamp   time.Time                 `json:"timestamp"`
	Git         GitState                  `json:"git"`
	Config      RunConfig                 `json:"config"`
	Overall     OverallScores             `json:"overall"`
	ByCategory  map[string]CategoryScores `json:"by_category"`
	DurationSec float64                   `json:"duration_seconds"`
	IsBest      bool                      `json:"is_best,omitempty"`
	Comparison  *ComparisonInfo           `json:"comparison,omitempty"`
}

CatalogedRun represents a cataloged benchmark run

type CategoryScores 

type CategoryScores struct {
	Category         string  `json:"category"`
	LLMJudgeAccuracy float64 `json:"llm_judge_accuracy"`
	F1Score          float64 `json:"f1_score"`
	BLEU1Score       float64 `json:"bleu1_score"`
	TotalQuestions   int     `json:"total_questions"`
	CorrectCount     int     `json:"correct_count"`
}

CategoryScores holds scores for a single category

func AnalyzeWeakCategories 

func AnalyzeWeakCategories(results *RunResults) []CategoryScores

AnalyzeWeakCategories returns categories with lowest accuracy, sorted worst-first

type Comparison 

type Comparison struct {
	RunA             string               `json:"run_a"`
	RunB             string               `json:"run_b"`
	OverallDiff      ScoreDiff            `json:"overall_diff"`
	CategoryDiffs    map[string]ScoreDiff `json:"category_diffs"`
	Improvements     []string             `json:"improvements"`
	Regressions      []string             `json:"regressions"`
	ChangedQuestions []QuestionComparison `json:"changed_questions,omitempty"`
}

Comparison holds comparison between two runs

func CompareRuns 

func CompareRuns(runA, runB *RunResults) *Comparison

CompareRuns compares two benchmark runs

type ComparisonInfo 

type ComparisonInfo struct {
	BaselineRunID    string  `json:"baseline_run_id"`
	BaselineAccuracy float64 `json:"baseline_accuracy"`
	Improvement      float64 `json:"improvement"`
}

ComparisonInfo holds baseline comparison data

type CostEstimation 

type CostEstimation struct {
	InputCostUSD       float64 `json:"input_cost_usd"`
	OutputCostUSD      float64 `json:"output_cost_usd"`
	TotalCostUSD       float64 `json:"total_cost_usd"`
	CostPerQuestionUSD float64 `json:"cost_per_question_usd"`
}

CostEstimation holds cost estimation for API usage

type GitState 

type GitState struct {
	CommitHash string `json:"commit_hash"`
	Branch     string `json:"branch"`
	Dirty      bool   `json:"dirty"`
	ShortHash  string `json:"short_hash"`
}

GitState captures the state of the git repository

func CaptureGitState 

func CaptureGitState(repoPath string) (*GitState, error)

CaptureGitState captures the current git repository state

type LatencyStats 

type LatencyStats struct {
	MeanSeconds   float64 `json:"mean_seconds"`
	MedianSeconds float64 `json:"median_seconds"`
	P95Seconds    float64 `json:"p95_seconds"`
	P99Seconds    float64 `json:"p99_seconds"`
	MinSeconds    float64 `json:"min_seconds"`
	MaxSeconds    float64 `json:"max_seconds"`
	StdDevSeconds float64 `json:"stdev_seconds"`
}

LatencyStats holds latency statistics for benchmark execution

type LoopConfig 

type LoopConfig struct {
	MaxIterations           int     `json:"max_iterations"`
	MinImprovementThreshold float64 `json:"min_improvement_threshold"`
	ConvergenceThreshold    float64 `json:"convergence_threshold"`
	TimeoutMinutes          int     `json:"timeout_minutes"`
}

LoopConfig holds configuration for autonomous improvement loop

func DefaultLoopConfig 

func DefaultLoopConfig() *LoopConfig

DefaultLoopConfig returns default loop configuration

type LoopManager 

type LoopManager struct {
	// contains filtered or unexported fields
}

LoopManager manages autonomous improvement loops

func NewLoopManager 

func NewLoopManager(service *Service, db *database.Database) *LoopManager

NewLoopManager creates a new loop manager

func (*LoopManager) GetState 

func (m *LoopManager) GetState() *LoopState

GetState returns the current loop state

func (*LoopManager) IsRunning 

func (m *LoopManager) IsRunning() bool

IsRunning returns true if a loop is currently running

func (*LoopManager) StartLoop 

func (m *LoopManager) StartLoop(ctx context.Context, config *LoopConfig) (*LoopState, error)

StartLoop begins an autonomous improvement loop

func (*LoopManager) StopLoop 

func (m *LoopManager) StopLoop() error

StopLoop stops the active loop

type LoopState 

type LoopState struct {
	ID               string     `json:"id"`
	Status           LoopStatus `json:"status"`
	CurrentIteration int        `json:"current_iteration"`
	MaxIterations    int        `json:"max_iterations"`
	BaselineScore    float64    `json:"baseline_score"`
	CurrentScore     float64    `json:"current_score"`
	BestScore        float64    `json:"best_score"`
	BestRunID        string     `json:"best_run_id"`
	LastChange       string     `json:"last_change"`
	StopReason       string     `json:"stop_reason,omitempty"`
	StartedAt        time.Time  `json:"started_at"`
	ElapsedMinutes   float64    `json:"elapsed_minutes"`
}

LoopState holds current state of an improvement loop

type LoopStatus 

type LoopStatus string

LoopStatus represents the status of an autonomous loop 

const (
	LoopRunning   LoopStatus = "running"
	LoopCompleted LoopStatus = "completed"
	LoopStopped   LoopStatus = "stopped"
	LoopFailed    LoopStatus = "failed"
)

type OverallScores 

type OverallScores struct {
	LLMJudgeAccuracy float64 `json:"llm_judge_accuracy"`
	F1Score          float64 `json:"f1_score"`
	BLEU1Score       float64 `json:"bleu1_score"`
	TotalQuestions   int     `json:"total_questions"`
	TotalCorrect     int     `json:"total_correct"`
}

OverallScores holds aggregate scores

type Progress 

type Progress struct {
	RunID              string    `json:"run_id"`
	Status             RunStatus `json:"status"`
	TotalQuestions     int       `json:"total_questions"`
	CompletedCount     int       `json:"completed_count"`
	CurrentQuestion    string    `json:"current_question,omitempty"`
	PercentComplete    float64   `json:"percent_complete"`
	ElapsedSecs        float64   `json:"elapsed_seconds"`
	EstimatedRemaining float64   `json:"estimated_remaining_seconds,omitempty"`
}

Progress represents benchmark execution progress

type QuestionComparison 

type QuestionComparison struct {
	QuestionID string `json:"question_id"`
	Category   string `json:"category"`
	WasCorrect bool   `json:"was_correct"`
	NowCorrect bool   `json:"now_correct"`
	Improved   bool   `json:"improved"`
	Regressed  bool   `json:"regressed"`
}

QuestionComparison compares a single question across runs

type QuestionResult 

type QuestionResult struct {
	QuestionID       string  `json:"question_id"`
	Category         string  `json:"category"`
	QuestionText     string  `json:"question_text"`
	GoldAnswer       string  `json:"gold_answer"`
	GeneratedAnswer  string  `json:"generated_answer"`
	LLMJudgeLabel    int     `json:"llm_judge_label"` // 0 or 1
	F1Score          float64 `json:"f1_score"`
	BLEU1Score       float64 `json:"bleu1_score"`
	ContextLength    int     `json:"context_length"`
	MemoriesUsed     int     `json:"memories_used"`
	RetrievalTimeMs  int     `json:"retrieval_time_ms"`
	GenerationTimeMs int     `json:"generation_time_ms"`
}

QuestionResult holds result for a single question

func IdentifyFailedQuestions 

func IdentifyFailedQuestions(results *RunResults) []QuestionResult

IdentifyFailedQuestions returns questions that were answered incorrectly

type RunConfig 

type RunConfig struct {
	BenchmarkType string   `json:"benchmark_type"` // "locomo10" or "locomo_mc10"
	MaxQuestions  int      `json:"max_questions"`  // 0 = all questions
	QuestionTypes []string `json:"question_types"` // Filter by category
	TopK          int      `json:"top_k"`          // Number of memories to retrieve
	Verbose       bool     `json:"verbose"`
	UseSummaries  bool     `json:"use_summaries"`
	Async         bool     `json:"async"`              // Run asynchronously
	ChangeDesc    string   `json:"change_description"` // Description of code change being tested
	RandomSample  bool     `json:"random_sample"`      // Randomly sample questions instead of first N
	Seed          *int     `json:"seed,omitempty"`     // Random seed for reproducible sampling
}

RunConfig holds configuration for a benchmark run

func DefaultRunConfig 

func DefaultRunConfig() *RunConfig

DefaultRunConfig returns default configuration

func (*RunConfig) ToJSON 

func (c *RunConfig) ToJSON() string

ToJSON serializes config to JSON

type RunResults 

type RunResults struct {
	RunID        string                    `json:"run_id"`
	Status       RunStatus                 `json:"status"`
	StartedAt    time.Time                 `json:"started_at"`
	CompletedAt  *time.Time                `json:"completed_at,omitempty"`
	DurationSecs float64                   `json:"duration_seconds"`
	Git          GitState                  `json:"git"`
	Config       RunConfig                 `json:"config"`
	Overall      OverallScores             `json:"overall"`
	ByCategory   map[string]CategoryScores `json:"by_category"`
	Questions    []QuestionResult          `json:"questions,omitempty"`
	ErrorMessage string                    `json:"error_message,omitempty"`
	// Enhanced metrics
	Latency *LatencyStats   `json:"latency,omitempty"`
	Tokens  *TokenStats     `json:"tokens,omitempty"`
	Cost    *CostEstimation `json:"cost,omitempty"`
}

RunResults holds complete results from a benchmark run

type RunStatus 

type RunStatus string

RunStatus represents the status of a benchmark run 

const (
	StatusPending   RunStatus = "pending"
	StatusRunning   RunStatus = "running"
	StatusCompleted RunStatus = "completed"
	StatusFailed    RunStatus = "failed"
	StatusCancelled RunStatus = "cancelled"
)

type Runner 

type Runner struct {
	// contains filtered or unexported fields
}

Runner handles benchmark execution

func NewRunner 

func NewRunner(repoPath string) *Runner

NewRunner creates a new benchmark runner

func (*Runner) Cancel 

func (r *Runner) Cancel(ctx context.Context, runID string) error

Cancel cancels a running benchmark

func (*Runner) CheckBridge 

func (r *Runner) CheckBridge() error

CheckBridge checks if the Python bridge is available

func (*Runner) GetProgress 

func (r *Runner) GetProgress(ctx context.Context, runID string) (*Progress, error)

GetProgress gets the progress of a running benchmark

func (*Runner) Run 

func (r *Runner) Run(ctx context.Context, config *RunConfig) (*RunResults, error)

Run executes a benchmark run

func (*Runner) SetBridgeURL 

func (r *Runner) SetBridgeURL(url string)

SetBridgeURL sets a custom bridge URL

type ScoreDiff 

type ScoreDiff struct {
	Before        float64 `json:"before"`
	After         float64 `json:"after"`
	Diff          float64 `json:"diff"`
	PercentChange float64 `json:"percent_change"`
	Improved      bool    `json:"improved"`
}

ScoreDiff holds the difference between two scores

type Service 

type Service struct {
	// contains filtered or unexported fields
}

Service manages benchmark operations

func NewService 

func NewService(db *database.Database, repoPath string) *Service

NewService creates a new benchmark service

func (*Service) CancelRun 

func (s *Service) CancelRun(ctx context.Context) error

CancelRun cancels an active run

func (*Service) CheckBridge 

func (s *Service) CheckBridge() error

CheckBridge checks if the Python bridge is available

func (*Service) Compare 

func (s *Service) Compare(runIDA, runIDB string) (*Comparison, error)

Compare compares two benchmark runs

func (*Service) GetActiveRunID 

func (s *Service) GetActiveRunID() string

GetActiveRunID returns the ID of the currently running benchmark

func (*Service) GetBestRun 

func (s *Service) GetBestRun(benchmarkType string) (*database.BenchmarkRun, error)

GetBestRun returns the best benchmark run

func (*Service) GetLoopState 

func (s *Service) GetLoopState() *LoopState

GetLoopState returns the current loop state

func (*Service) GetProgress 

func (s *Service) GetProgress(ctx context.Context) (*Progress, error)

GetProgress returns progress of active run

func (*Service) GetResults 

func (s *Service) GetResults(runID string) (*RunResults, error)

GetResults reconstructs full results from database

func (*Service) GetRun 

func (s *Service) GetRun(runID string) (*database.BenchmarkRun, error)

GetRun retrieves a benchmark run by ID

func (*Service) IsLoopRunning 

func (s *Service) IsLoopRunning() bool

IsLoopRunning returns true if a loop is active

func (*Service) IsRunning 

func (s *Service) IsRunning() bool

IsRunning returns true if a benchmark is currently running

func (*Service) ListRuns 

func (s *Service) ListRuns(filters *database.BenchmarkRunFilters) ([]*database.BenchmarkRun, error)

ListRuns lists benchmark runs with filtering

func (*Service) Run 

func (s *Service) Run(ctx context.Context, config *RunConfig) (*RunResults, error)

Run executes a benchmark and stores results

func (*Service) SetBridgeURL 

func (s *Service) SetBridgeURL(url string)

SetBridgeURL configures the Python bridge URL

func (*Service) StartLoop 

func (s *Service) StartLoop(ctx context.Context, config *LoopConfig) (*LoopState, error)

StartLoop starts an autonomous improvement loop

func (*Service) StopLoop 

func (s *Service) StopLoop() error

StopLoop stops the active improvement loop

type TokenStats 

type TokenStats struct {
	TotalInput  int     `json:"total_input"`
	TotalOutput int     `json:"total_output"`
	Total       int     `json:"total"`
	MeanInput   float64 `json:"mean_input"`
	MeanOutput  float64 `json:"mean_output"`
}

TokenStats holds token usage statistics

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.