Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CompositeEvaluator ¶
type CompositeEvaluator struct {
// contains filtered or unexported fields
}
CompositeEvaluator runs multiple child evaluators and returns a weighted average.
func NewCompositeEvaluator ¶
func NewCompositeEvaluator(name string, children []Evaluator, weights []float64) *CompositeEvaluator
NewCompositeEvaluator creates a composite evaluator from children and their weights. Weights do not need to sum to 1; they are normalized internally.
func (*CompositeEvaluator) Evaluate ¶
func (c *CompositeEvaluator) Evaluate(ctx context.Context, input models.TestCase, output models.GenerateResponse) (*models.Score, error)
Evaluate runs all children and returns the weighted average score. Individual child scores are stored in Score.RawValue.
func (*CompositeEvaluator) Name ¶
func (c *CompositeEvaluator) Name() string
func (*CompositeEvaluator) Type ¶
func (c *CompositeEvaluator) Type() models.EvalType
type Evaluator ¶
type Evaluator interface {
// Name returns the evaluator's unique name.
Name() string
// Type returns the evaluator type (metric, ai_judge, composite).
Type() models.EvalType
// Evaluate scores a single test case against the generated output.
Evaluate(ctx context.Context, input models.TestCase, output models.GenerateResponse) (*models.Score, error)
}
Evaluator is the interface every evaluator must implement.
func NewEvaluator ¶
func NewEvaluator(cfg models.EvaluatorConfig, resolver ProviderResolver) (Evaluator, error)
NewEvaluator constructs an Evaluator from config. The resolver is used to obtain provider functions for AI judge evaluators. It may be nil if no ai_judge evaluators are expected.
type JudgeEvaluator ¶
type JudgeEvaluator struct {
// contains filtered or unexported fields
}
JudgeEvaluator uses an AI model as a judge to score outputs.
func NewJudgeEvaluator ¶
func NewJudgeEvaluator(cfg models.EvaluatorConfig, provider ProviderFunc) (*JudgeEvaluator, error)
NewJudgeEvaluator creates a new AI-as-Judge evaluator.
func (*JudgeEvaluator) Evaluate ¶
func (j *JudgeEvaluator) Evaluate(ctx context.Context, input models.TestCase, output models.GenerateResponse) (*models.Score, error)
Evaluate runs the judge on a single test case.
func (*JudgeEvaluator) Name ¶
func (j *JudgeEvaluator) Name() string
func (*JudgeEvaluator) Type ¶
func (j *JudgeEvaluator) Type() models.EvalType
type MetricsEvaluator ¶
type MetricsEvaluator struct {
// contains filtered or unexported fields
}
MetricsEvaluator shells out to a Python metrics script.
func NewMetricsEvaluator ¶
func NewMetricsEvaluator(cfg models.EvaluatorConfig) (*MetricsEvaluator, error)
NewMetricsEvaluator creates a new Python metrics bridge evaluator.
func (*MetricsEvaluator) Evaluate ¶
func (m *MetricsEvaluator) Evaluate(ctx context.Context, input models.TestCase, output models.GenerateResponse) (*models.Score, error)
Evaluate shells out to the appropriate Python metric script.
func (*MetricsEvaluator) Name ¶
func (m *MetricsEvaluator) Name() string
func (*MetricsEvaluator) Type ¶
func (m *MetricsEvaluator) Type() models.EvalType
type ProviderFunc ¶
type ProviderFunc func(ctx context.Context, req models.GenerateRequest) (*models.GenerateResponse, error)
ProviderFunc is a function type that calls an AI provider. This decouples the evaluator package from any concrete provider implementation.
type ProviderResolver ¶
type ProviderResolver func(provider, model string) (ProviderFunc, error)
ProviderResolver returns a ProviderFunc for a given provider name and model.
Source Files