eval

package
v0.0.0-...-75ec8e3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 28, 2026 License: MIT Imports: 21 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type CLITestCase 

type CLITestCase struct {
	ID          string   `json:"id"`
	Args        []string `json:"args"`
	Expected    string   `json:"expected"`
	Description string   `json:"description,omitempty"`
	Timeout     int      `json:"timeout,omitempty"` // timeout in seconds, default 30
}

CLITestCase defines a CLI test case

type ContainerConfig 

type ContainerConfig struct {
	Image        string
	WorkspaceDir string
	Timeout      time.Duration
	Env          map[string]string
}

ContainerConfig holds configuration for container execution

type ContainerEngine 

type ContainerEngine string

ContainerEngine represents the detected container runtime 

const (
	EngineDocker ContainerEngine = "docker"
	EnginePodman ContainerEngine = "podman"
)

type ContainerExecutor 

type ContainerExecutor struct {
	// contains filtered or unexported fields
}

ContainerExecutor handles Docker/Podman operations

func NewContainerExecutor 

func NewContainerExecutor() (*ContainerExecutor, error)

NewContainerExecutor auto-detects available container engine

func (*ContainerExecutor) BuildImage 

func (ce *ContainerExecutor) BuildImage(ctx context.Context, imageName string, baseImage string) error

BuildImage builds the scriptschnell image with multi-stage Dockerfile in a temp directory

func (*ContainerExecutor) RunCLITest 

func (ce *ContainerExecutor) RunCLITest(ctx context.Context, config *ContainerConfig, execPath string, args []string, timeout time.Duration) (*TestResult, error)

RunCLITest executes a single CLI test case in container

func (*ContainerExecutor) RunEval 

func (ce *ContainerExecutor) RunEval(ctx context.Context, config *ContainerConfig, prompt, modelID, provider string, runID int64) (*ContainerResult, error)

RunEval executes eval in container with LLM integration using environment variables

type ContainerResult 

type ContainerResult struct {
	Output   string
	ExitCode int
}

ContainerResult holds result of container execution

type Database 

type Database struct {
	// contains filtered or unexported fields
}

Database handles SQLite operations for the eval system

func NewDatabase 

func NewDatabase(dbPath string) (*Database, error)

NewDatabase creates a new database connection

func (*Database) AddModel 

func (d *Database) AddModel(model *EvalModel) error

AddModel adds a model to the database

func (*Database) ClearModelSelection 

func (d *Database) ClearModelSelection() error

ClearModelSelection clears all model selections

func (*Database) Close 

func (d *Database) Close() error

Close closes the database connection

func (*Database) CreateEvalResult 

func (d *Database) CreateEvalResult(result *EvalResult) error

CreateEvalResult creates a new eval result

func (*Database) CreateEvalRun 

func (d *Database) CreateEvalRun(evalID, modelID string) (*EvalRun, error)

CreateEvalRun creates a new eval run

func (*Database) DeleteModel 

func (d *Database) DeleteModel(modelID string) error

DeleteModel removes a model from the database

func (*Database) GetConfig 

func (d *Database) GetConfig(key string) (string, error)

GetConfig gets a configuration value

func (*Database) GetEvalConfig 

func (d *Database) GetEvalConfig() (*EvalConfig, error)

GetEvalConfig gets the eval configuration

func (*Database) GetEvalResults 

func (d *Database) GetEvalResults(runID int64) ([]EvalResult, error)

GetEvalResults gets all results for a run

func (*Database) GetEvalRun 

func (d *Database) GetEvalRun(id int64) (*EvalRun, error)

GetEvalRun gets an eval run by ID

func (*Database) GetEvalRuns 

func (d *Database) GetEvalRuns(evalID string) ([]*EvalRun, error)

GetEvalRuns gets all eval runs, optionally filtered by eval_id

func (*Database) GetEvalStats 

func (d *Database) GetEvalStats(evalID string) ([]EvalStats, error)

GetEvalStats gets aggregated stats for eval runs

func (*Database) GetModels 

func (d *Database) GetModels() ([]*EvalModel, error)

GetModels gets all models from the database

func (*Database) GetSelectedModels 

func (d *Database) GetSelectedModels() ([]*EvalModel, error)

GetSelectedModels gets all selected models

func (*Database) SetConfig 

func (d *Database) SetConfig(key, value string) error

SetConfig sets a configuration value

func (*Database) SetEvalConfig 

func (d *Database) SetEvalConfig(config *EvalConfig) error

SetEvalConfig sets the eval configuration

func (*Database) UpdateEvalRunAgentResult 

func (d *Database) UpdateEvalRunAgentResult(id int64, output string, exitCode int) error

UpdateEvalRunAgentResult updates the agent output and exit code

func (*Database) UpdateEvalRunStatus 

func (d *Database) UpdateEvalRunStatus(id int64, status string) error

UpdateEvalRunStatus updates the status of an eval run

func (*Database) UpdateModelSelection 

func (d *Database) UpdateModelSelection(modelID string, selected bool) error

UpdateModelSelection updates the selection status of a model

type EvalConfig 

type EvalConfig struct {
	OpenRouterToken string `json:"openrouter_token" db:"openrouter_token"`
}

EvalConfig stores configuration

type EvalDefinition 

type EvalDefinition struct {
	ID            string            `json:"id"`
	Name          string            `json:"name"`
	Description   string            `json:"description"`
	Image         string            `json:"image"` // Docker/Podman base image
	UserPrompt    string            `json:"user_prompt"`
	ExpectedFiles map[string]string `json:"expected_files"` // path -> content
	RunFile       string            `json:"run_file"`       // which file to execute
	CLITests      []CLITestCase     `json:"cli_tests"`
}

EvalDefinition defines an evaluation test case

func LoadEvalDefinition 

func LoadEvalDefinition(data []byte) (*EvalDefinition, error)

LoadEvalDefinition loads an eval definition from JSON data

func (*EvalDefinition) Validate 

func (e *EvalDefinition) Validate() error

Validate validates the eval definition

type EvalModel 

type EvalModel struct {
	ID            string `json:"id" db:"id"`
	Name          string `json:"name" db:"name"`
	Provider      string `json:"provider" db:"provider"`
	Selected      bool   `json:"selected" db:"selected"`
	Description   string `json:"description,omitempty" db:"description"`
	ContextWindow int    `json:"context_window,omitempty" db:"context_window"`
}

EvalModel represents a model available for evaluation

type EvalResult 

type EvalResult struct {
	ID                    int64   `json:"id" db:"id"`
	RunID                 int64   `json:"run_id" db:"run_id"`
	TestCaseID            string  `json:"test_case_id" db:"test_case_id"`
	Passed                bool    `json:"passed" db:"passed"`
	ActualOutput          string  `json:"actual_output" db:"actual_output"`
	ExpectedOutput        string  `json:"expected_output" db:"expected_output"`
	Error                 string  `json:"error,omitempty" db:"error"`
	InputTokens           int     `json:"input_tokens" db:"input_tokens"`
	OutputTokens          int     `json:"output_tokens" db:"output_tokens"`
	EstimateCost          float64 `json:"estimate_cost" db:"estimate_cost"`      // in USD
	ResponseTime          int     `json:"response_time_ms" db:"response_time"`   // in milliseconds
	ExecutionTime         int     `json:"execution_time_ms" db:"execution_time"` // in milliseconds
	ContainerName         string  `json:"container_name,omitempty" db:"container_name"`
	RawOutput             string  `json:"raw_output,omitempty" db:"raw_output"`
	Errors                string  `json:"errors,omitempty" db:"errors"`
	DetailedExecutionInfo string  `json:"detailed_execution_info,omitempty" db:"detailed_execution_info"`
}

EvalResult represents the result of a single test case within an eval run

type EvalRun 

type EvalRun struct {
	ID            int64      `json:"id" db:"id"`
	EvalID        string     `json:"eval_id" db:"eval_id"`
	ModelID       string     `json:"model_id" db:"model_id"`
	Status        string     `json:"status" db:"status"` // pending, running, completed, failed
	AgentOutput   string     `json:"agent_output,omitempty" db:"agent_output"`
	AgentExitCode int        `json:"agent_exit_code,omitempty" db:"agent_exit_code"`
	StartedAt     time.Time  `json:"started_at" db:"started_at"`
	CompletedAt   *time.Time `json:"completed_at,omitempty" db:"completed_at"`
}

EvalRun represents a single evaluation run

func (*EvalRun) GetStats 

func (r *EvalRun) GetStats(results []EvalResult) EvalStats

GetStats aggregates statistics for an eval run

type EvalStats 

type EvalStats struct {
	RunID            int64      `json:"run_id"`
	EvalID           string     `json:"eval_id"`
	ModelID          string     `json:"model_id"`
	Status           string     `json:"status"`
	TotalCount       int        `json:"total_count"`
	PassedCount      int        `json:"passed_count"`
	FailedCount      int        `json:"failed_count"`
	PassRate         float64    `json:"pass_rate"`
	TotalTokens      int        `json:"total_tokens"`
	TotalCost        float64    `json:"total_cost"`
	AvgResponseTime  int        `json:"avg_response_time_ms"`
	AvgExecutionTime int        `json:"avg_execution_time_ms"`
	ResponseTime     int        `json:"response_time_ms"`
	ExecutionTime    int        `json:"execution_time_ms"`
	StartedAt        time.Time  `json:"started_at"`
	CompletedAt      *time.Time `json:"completed_at,omitempty"`
}

EvalStats provides aggregated statistics

type Loader 

type Loader struct {
	// contains filtered or unexported fields
}

Loader loads evaluation definitions from JSON files

func NewLoader 

func NewLoader(evalDir string) *Loader

NewLoader creates a new loader

func (*Loader) ListAvailableEvals 

func (l *Loader) ListAvailableEvals() ([]string, error)

ListAvailableEvals returns a list of available eval IDs

func (*Loader) LoadAll 

func (l *Loader) LoadAll() (map[string]*EvalDefinition, error)

LoadAll loads all eval definitions from the eval directory

func (*Loader) LoadEval 

func (l *Loader) LoadEval(evalID string) (*EvalDefinition, error)

LoadEval loads a specific eval definition by ID

func (*Loader) ValidateDirectory 

func (l *Loader) ValidateDirectory() error

ValidateDirectory checks if the eval directory exists and is readable

type Server 

type Server struct {
	// contains filtered or unexported fields
}

Server provides the HTTP interface for the eval web UI

func NewServer 

func NewServer(service *Service, port int) *Server

NewServer creates a new eval server

func (*Server) Start 

func (s *Server) Start() error

Start starts the HTTP server

func (*Server) Stop 

func (s *Server) Stop() error

Stop stops the HTTP server

type Service 

type Service struct {
	// contains filtered or unexported fields
}

Service manages the eval system

func NewService 

func NewService(dbPath, evalDir string) (*Service, error)

NewService creates a new eval service

func (*Service) ClearModelSelection 

func (s *Service) ClearModelSelection() error

ClearModelSelection clears all model selections

func (*Service) Close 

func (s *Service) Close() error

Close closes the service and cleans up resources

func (*Service) DeselectModel 

func (s *Service) DeselectModel(modelID string) error

DeselectModel removes a model from selection

func (*Service) GetConfig 

func (s *Service) GetConfig() (*EvalConfig, error)

GetConfig gets the current configuration

func (*Service) GetEval 

func (s *Service) GetEval(evalID string) (*EvalDefinition, error)

GetEval gets a specific evaluation definition

func (*Service) GetEvalResults 

func (s *Service) GetEvalResults(runID int64) ([]EvalResult, error)

GetEvalResults gets results for a specific run

func (*Service) GetEvalRuns 

func (s *Service) GetEvalRuns(evalID string) ([]*EvalRun, error)

GetEvalRuns gets evaluation runs

func (*Service) GetEvalStats 

func (s *Service) GetEvalStats(evalID string) ([]EvalStats, error)

GetEvalStats gets aggregated statistics

func (*Service) GetEvals 

func (s *Service) GetEvals() (map[string]*EvalDefinition, error)

GetEvals gets all available evaluation definitions

func (*Service) GetModels 

func (s *Service) GetModels() ([]*EvalModel, error)

GetModels gets all models from the database

func (*Service) GetSelectedModels 

func (s *Service) GetSelectedModels() ([]*EvalModel, error)

GetSelectedModels gets models selected for evaluation

func (*Service) RefreshModels 

func (s *Service) RefreshModels() ([]*EvalModel, error)

RefreshModels refreshes the model list from OpenRouter

func (*Service) RunEval 

func (s *Service) RunEval(evalID string) ([]*EvalRun, error)

RunEval starts an evaluation run

func (*Service) SelectModel 

func (s *Service) SelectModel(modelID string) error

SelectModel sets a model as selected for evaluation

func (*Service) SetConfig 

func (s *Service) SetConfig(config *EvalConfig) error

SetConfig updates the configuration

func (*Service) SetLogAssistant 

func (s *Service) SetLogAssistant(enabled bool)

SetLogAssistant sets whether to log assistant messages to console

type TestResult 

type TestResult struct {
	Output        string
	ExitCode      int
	ExecutionTime int
	Error         string
}

TestResult holds result of CLI test execution

type WorkspaceManager 

type WorkspaceManager struct {
	// contains filtered or unexported fields
}

WorkspaceManager handles temporary workspace creation and cleanup

func NewWorkspaceManager 

func NewWorkspaceManager(baseDir string) (*WorkspaceManager, error)

NewWorkspaceManager creates a workspace manager

func (*WorkspaceManager) CleanupWorkspace 

func (wm *WorkspaceManager) CleanupWorkspace(workspaceDir string) error

CleanupWorkspace removes a workspace after eval completes

func (*WorkspaceManager) CreateWorkspace 

func (wm *WorkspaceManager) CreateWorkspace(runID int64) (string, error)

CreateWorkspace creates a temporary workspace for an eval run

func (*WorkspaceManager) WaitForSignal 

func (wm *WorkspaceManager) WaitForSignal(workspaceDir string, timeout time.Duration) error

WaitForSignal waits for .build_done or .build_failed signal file

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.