Versions in this module Expand all Collapse all v0 v0.5.0 Jul 2, 2026 Changes in this version + type AccountingSummary struct + CacheReadInputTokens int + CacheWriteInputTokens int + CostCount int + Currency string + EstimatedTotalCost float64 + InputTokens int + LongCacheWriteInputTokens int + OutputTokens int + ProviderReportedCost float64 + ProviderReportedCostCount int + ProviderReportedCurrency string + ThinkingTokens int + ToolUseInputTokens int + TotalTokens int + UsageCount int type ModelSummary + Accounting *AccountingSummary type Summary + Accounting *AccountingSummary v0.4.0 Jun 5, 2026 Changes in this version + const VarianceLayerBatchJudge + const VarianceLayerJudgeAlignment + const VarianceLayerSuite + var DefaultRubricRegistry = NewStandardRubricRegistry(DefaultRubrics) + var DefaultRubrics = []StandardRubric + func FormatInlineEvalPrompt(rubricPrompt string, input string, targetOutput string, groundTruth string) string + func GetRubric(idOrPrompt string) string + func GetRubricPrompt(idOrPrompt string) string + func WriteBatchJudgeSamplesJSONL(w io.Writer, run BatchJudgeResult) error + func WriteJudgeAlignmentSamplesJSONL(w io.Writer, run JudgeAlignmentRunResult) error + func WriteRunResultSamplesJSONL(w io.Writer, run RunResult) error + func WriteVarianceSamplesJSONL(w io.Writer, samples []VarianceSample) error + type BatchJudgeResult struct + ByModel map[string]BatchJudgeSummary + ByTag map[string]BatchJudgeSummary + EndedAt time.Time + Metadata map[string]any + Name string + Results []JudgeCaseResult + StartedAt time.Time + Summary BatchJudgeSummary + Version string + type BatchJudgeSpec struct + Cases []JudgeCase + Concurrency int + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Mode Mode + Name string + PassThreshold float64 + Repeats int + Rubric string + Target Target + TargetModel sigma.Model + TargetOptions []sigma.Option + TargetPrompt string + Version string + type BatchJudgeSummary struct + DurationMS int64 + Errors int + Failed int + MeanScore float64 + Passed int + ScoreCount int + Total int type Evaluator + func (e *Evaluator) EvaluateBatch(ctx context.Context, spec BatchJudgeSpec) (BatchJudgeResult, error) + type JudgeCase struct + GroundTruth string + ID string + Input string + Metadata map[string]any + Name string + Rubric string + Tags []string + TargetOutput string + TargetPrompt string + type JudgeCaseResult struct + CaseID string + CaseName string + DurationMS int64 + Error string + JSON string + Judge string + JudgeProvider sigma.ProviderID + Metadata map[string]any + Mode Mode + PassThreshold float64 + Passed bool + Rationale string + RawJudgeOutput string + Repeat int + Result *JudgeResult + Score float64 + Tags []string + Target string + TargetOutput string + TargetProvider sigma.ProviderID + type RubricRegistry interface + Get func(idOrPrompt string) RubricResolution + List func() []StandardRubric + type RubricResolution struct + Found bool + Input string + Prompt string + Rubric StandardRubric + func ResolveRubric(idOrPrompt string) RubricResolution + type StandardRubric struct + Aliases []string + Description string + ID string + Name string + Prompt string + Tags []string + type StandardRubricRegistry struct + func NewStandardRubricRegistry(rubrics []StandardRubric) *StandardRubricRegistry + func (r *StandardRubricRegistry) Get(idOrPrompt string) RubricResolution + func (r *StandardRubricRegistry) List() []StandardRubric + type VarianceCompareOptions struct + ConfidenceZ float64 + MinMeanScoreDelta float64 + MinPassRateDelta float64 + type VarianceComparison struct + BaselineName string + CurrentName string + Deltas []VarianceDelta + Options VarianceCompareOptions + Summary VarianceComparisonSummary + func CompareVarianceReports(baseline VarianceReport, current VarianceReport, ...) VarianceComparison + type VarianceComparisonSummary struct + Groups int + Improvements int + Missing int + New int + Regressions int + Stable int + type VarianceDelta struct + Baseline *VarianceGroupStats + Current *VarianceGroupStats + DeltaMeanDurationMS float64 + DeltaMeanScore float64 + DeltaPassRate float64 + DeltaStdDevScore float64 + Direction string + EffectSize float64 + Key VarianceGroupKey + PassRateZ float64 + ScoreZ float64 + Significant bool + type VarianceGroupKey struct + CaseID string + Layer string + Model string + Scorer string + type VarianceGroupStats struct + Count int + Errors int + Failed int + Key VarianceGroupKey + MaxScore float64 + MeanDurationMS float64 + MeanScore float64 + MeanScoreError float64 + MinScore float64 + OutputDiversity float64 + PassRate float64 + Passed int + StdDevScore float64 + StdDevScoreError float64 + StdErrScore float64 + UniqueOutputs int + type VarianceReport struct + GeneratedAt time.Time + Groups []VarianceGroupStats + Name string + Total VarianceStats + func BuildVarianceReport(name string, samples []VarianceSample) VarianceReport + func VarianceReportFromBatchJudgeResult(run BatchJudgeResult) VarianceReport + func VarianceReportFromJudgeAlignmentResult(run JudgeAlignmentRunResult) VarianceReport + func VarianceReportFromRunResult(run RunResult) VarianceReport + type VarianceSample struct + CaseID string + CaseName string + DurationMS int64 + Error string + ExpectedScore float64 + Layer string + Model string + Output string + OutputHash string + Passed bool + Provider string + Repeat int + RunName string + Score float64 + ScoreError float64 + Scorer string + Tags []string + func ReadVarianceSamplesJSONL(r io.Reader) ([]VarianceSample, error) + func VarianceSamplesFromBatchJudgeResult(run BatchJudgeResult) []VarianceSample + func VarianceSamplesFromJudgeAlignmentResult(run JudgeAlignmentRunResult) []VarianceSample + func VarianceSamplesFromRunResult(run RunResult) []VarianceSample + type VarianceStats struct + Count int + Errors int + Failed int + MeanScore float64 + OutputDiversity float64 + PassRate float64 + Passed int + StdDevScore float64 + StdErrScore float64 + UniqueOutputs int v0.3.0 Jun 5, 2026 Changes in this version type CaseResult + ErrorDetails *ErrorDetails + type ErrorDetails struct + API sigma.API + Class sigma.ErrorClass + Message string + Model sigma.ModelID + Provider sigma.ProviderID + ProviderCode string + RequestID string + RetryAfterMS int64 + Retryable bool + StatusCode int type TargetResult + ErrorDetails *ErrorDetails v0.2.0 May 31, 2026 v0.1.1 May 30, 2026 v0.1.0 May 30, 2026 Changes in this version + var ErrGEvalLogprobsRequired = errors.New("g-eval requires provider logprobs for score tokens 1-5") + var ErrInvalidJudgeResult = errors.New("invalid judge result") + var ErrInvalidOutput = errors.New("invalid model output") + func AssistantText(message sigma.AssistantMessage) (string, error) + func AssistantToolCalls(message sigma.AssistantMessage) []sigma.ToolCall + func EstimatePassAtK(total int, correct int, k int) float64 + func GEvalScore(logprobs []TokenLogprob) (float64, bool) + func GEvalScoreForOutput(logprobs []TokenLogprob, output string) (float64, bool) + func NormalizeAnswer(text string) string + func PassAtK(correct []bool, k int) float64 + func PickChoice(output string, choices []Choice) (string, bool) + func TraceToolCalls(trace Trace) []sigma.ToolCall + type AnswerMatchMode string + const MatchContains + const MatchExact + const MatchNormalized + type AnswerScorer struct + Mode AnswerMatchMode + func (s AnswerScorer) Name() string + func (s AnswerScorer) Score(_ context.Context, input ScoreInput) (Score, error) + type AutoScorer struct + func (AutoScorer) Name() string + func (AutoScorer) Score(ctx context.Context, input ScoreInput) (Score, error) + type CalibrationMetrics struct + BrierScore float64 + Count int + func ComputeCalibrationMetrics(expectedPassed []bool, actualScores []float64, minScore float64, ...) CalibrationMetrics + type Case struct + DataType EvalDataType + Expected Expected + ID string + Input string + Messages []sigma.Message + Metadata map[string]any + Name string + SystemPrompt string + Tags []string + Tools []sigma.Tool + Trace Trace + type CaseResult struct + CaseID string + CaseName string + Cost *sigma.Cost + DurationMS int64 + Error string + Message sigma.AssistantMessage + Model string + Output string + Provider sigma.ProviderID + ProviderMeta map[string]any + Repeat int + Request sigma.Request + ScorerVersion string + Scores []Score + Tags []string + Usage *sigma.Usage + type Choice struct + Label string + Text string + type ClassificationMetrics struct + Accuracy float64 + BalancedAccuracy float64 + CohenKappa float64 + Count int + F1 float64 + FalseNegative int + FalsePositive int + Precision float64 + Recall float64 + TrueNegative int + TruePositive int + func ComputeClassificationMetrics(expected []bool, actual []bool) ClassificationMetrics + type Completer interface + Complete func(context.Context, sigma.Model, sigma.Request, ...sigma.Option) (sigma.AssistantMessage, error) + type DefaultRenderer struct + func (DefaultRenderer) Render(_ context.Context, input RenderInput) (sigma.Request, error) + type EvalDataType string + const EvalDataFinalAnswer + const EvalDataFullTrace + const EvalDataReferenceAnswer + type EvaluateInput struct + GroundTruth string + Input string + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Mode Mode + PassThreshold float64 + Rubric string + Target Target + TargetModel sigma.Model + TargetOptions []sigma.Option + TargetPrompt string + type Evaluator struct + Client Completer + TargetCompleter TargetCompleter + func NewEvaluator(client Completer) *Evaluator + func NewTargetEvaluator(completer TargetCompleter) *Evaluator + func (e *Evaluator) Evaluate(ctx context.Context, input EvaluateInput) (JudgeResult, error) + func (e *Evaluator) EvaluateJudges(ctx context.Context, spec JudgeAlignmentSpec) (JudgeAlignmentRunResult, error) + func (e *Evaluator) Judge(ctx context.Context, input JudgeInput) (JudgeResult, error) + func (e *Evaluator) PairwiseJudge(ctx context.Context, input PairwiseJudgeInput) (PairwiseJudgeResult, error) + type ExistingOutput struct + CaseID string + Message sigma.AssistantMessage + Model sigma.Model + Output string + Repeat int + type Expected struct + Answers []string + Choices []Choice + CorrectChoices []string + JSON any + Metadata map[string]any + NegativeAnswers []string + Output string + Patterns []string + Rubric string + ToolCalls []ExpectedToolCall + type ExpectedToolCall struct + Arguments any + Name string + type FanoutResult struct + EndedAt time.Time + Metadata map[string]any + Request sigma.Request + Results []TargetResult + StartedAt time.Time + Summary FanoutSummary + func RunFanout(ctx context.Context, completer TargetCompleter, spec FanoutSpec) (FanoutResult, error) + type FanoutSpec struct + Concurrency int + Metadata map[string]any + Options []sigma.Option + Progress ProgressFunc + Repeats int + Request sigma.Request + Targets []Target + type FanoutSummary struct + DurationMS int64 + Failed int + Succeeded int + Total int + type JSONJudgeResult struct + JSON string + Passed bool + Rationale string + Score float64 + func ParseJSONJudgeResult(text string) (JSONJudgeResult, error) + type JSONMatchScorer struct + func (JSONMatchScorer) Name() string + func (JSONMatchScorer) Score(_ context.Context, input ScoreInput) (Score, error) + type JudgeAlignmentCase struct + ExpectedPassed bool + ExpectedScore float64 + GroundTruth string + ID string + Input string + Metadata map[string]any + Rubric string + Tags []string + TargetOutput string + type JudgeAlignmentCaseResult struct + ActualPassed bool + ActualScore float64 + CaseID string + DurationMS int64 + Error string + ExpectedPassed bool + ExpectedScore float64 + Model string + PassedMatch bool + Provider sigma.ProviderID + Result *JudgeResult + ScoreError float64 + WithinTolerance bool + type JudgeAlignmentRunResult struct + ByModel map[string]JudgeModelSummary + EndedAt time.Time + Name string + Results []JudgeAlignmentCaseResult + StartedAt time.Time + Summary JudgeAlignmentSummary + Version string + type JudgeAlignmentSpec struct + Cases []JudgeAlignmentCase + Concurrency int + JudgeModels []sigma.Model + JudgeTargets []Target + Mode Mode + Name string + Options []sigma.Option + PassThreshold float64 + Tolerance float64 + Version string + type JudgeAlignmentSummary struct + Calibration CalibrationMetrics + Classification ClassificationMetrics + Errors int + Regression RegressionMetrics + ScoreWithinTolerance int + ToleranceAccuracy float64 + Total int + type JudgeInput struct + GroundTruth string + Input string + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Mode Mode + PassThreshold float64 + Rubric string + TargetOutput string + type JudgeModelSummary struct + Calibration CalibrationMetrics + Classification ClassificationMetrics + DurationMS int64 + Errors int + Regression RegressionMetrics + ScoreWithinTolerance int + ToleranceAccuracy float64 + Total int + type JudgeResult struct + GroundTruth string + Input string + JSON string + JudgeMessage sigma.AssistantMessage + Logprobs []TokenLogprob + Mode Mode + PassThreshold float64 + Passed bool + Rationale string + RawJudgeOutput string + Score float64 + TargetMessage sigma.AssistantMessage + TargetOutput string + type LLMJudgeScorer struct + Client Completer + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Mode Mode + PassThreshold float64 + Rubric string + TargetCompleter TargetCompleter + func (s LLMJudgeScorer) Name() string + func (s LLMJudgeScorer) Score(ctx context.Context, input ScoreInput) (Score, error) + type Mode string + const ModeEvaluate + const ModeGEval + type ModelSummary struct + DurationMS int64 + Errors int + Failed int + MeanScore float64 + Passed int + ScoreCount int + Total int + type MultipleChoiceScorer struct + func (MultipleChoiceScorer) Name() string + func (MultipleChoiceScorer) Score(_ context.Context, input ScoreInput) (Score, error) + type PairwiseJudgeInput struct + AnswerA string + AnswerB string + Input string + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Reference string + Rubric string + type PairwiseJudgeResult struct + Consistent bool + FirstOrder PairwiseSingleResult + SwappedOrder PairwiseSingleResult + Winner PairwiseWinner + type PairwiseSingleResult struct + JudgeMessage sigma.AssistantMessage + Rationale string + RawJudgeOutput string + Winner PairwiseWinner + func ParsePairwiseJudgeOutput(text string) (PairwiseSingleResult, error) + type PairwiseWinner string + const PairwiseWinnerA + const PairwiseWinnerB + const PairwiseWinnerInconsistent + const PairwiseWinnerTie + type ProgressEvent struct + CaseID string + Kind ProgressKind + Repeat int + Result *CaseResult + Target Target + TargetResult *TargetResult + type ProgressFunc func(ProgressEvent) + type ProgressKind string + const ProgressResult + const ProgressStart + type RatingType string + const RatingFiveStar + const RatingPassFail + const RatingPassFailCritical + type RegexScorer struct + func (RegexScorer) Name() string + func (RegexScorer) Score(_ context.Context, input ScoreInput) (Score, error) + type RegressionMetrics struct + Count int + MeanAbsoluteError float64 + MeanSquaredError float64 + Pearson float64 + RootMeanSquaredError float64 + Spearman float64 + func ComputeRegressionMetrics(expected []float64, actual []float64) RegressionMetrics + type RenderInput struct + Case Case + Suite Suite + type Renderer interface + Render func(context.Context, RenderInput) (sigma.Request, error) + type RendererFunc func(context.Context, RenderInput) (sigma.Request, error) + func (f RendererFunc) Render(ctx context.Context, input RenderInput) (sigma.Request, error) + type Rubric struct + Description string + Dimensions []RubricDimension + Name string + Threshold float64 + func PresetRubric(preset RubricPreset, details string) (Rubric, error) + func (r Rubric) JSONSchema(allowFloat bool) map[string]any + func (r Rubric) ParseScores(text string) (RubricScores, error) + func (r Rubric) Prompt(input ScoreInput) string + func (r Rubric) ScoreRaw(raw map[string]any) (RubricScores, error) + func (r Rubric) WithCase(c Case) Rubric + type RubricDimension struct + Instruction string + Name string + Type RatingType + Weight float64 + func (d RubricDimension) JSONKey() string + type RubricGEvalScorer struct + Client Completer + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Rubric Rubric + TargetCompleter TargetCompleter + TopLogprobs int + func (s RubricGEvalScorer) Name() string + func (s RubricGEvalScorer) Score(ctx context.Context, input ScoreInput) (Score, error) + type RubricJudgeScorer struct + Client Completer + Judge Target + JudgeModel sigma.Model + JudgeOptions []sigma.Option + Rubric Rubric + TargetCompleter TargetCompleter + func (s RubricJudgeScorer) Name() string + func (s RubricJudgeScorer) Score(ctx context.Context, input ScoreInput) (Score, error) + type RubricPreset string + const PresetBias + const PresetDesiredBehaviour + const PresetFactualCorrectness + const PresetIssue + const PresetJailbreak + const PresetMaliciousness + const PresetRAG + const PresetRequirements + const PresetToolCall + const PresetToxicity + type RubricScores struct + Aggregate float64 + Normalized map[string]float64 + Raw map[string]any + Scores map[string]float64 + func RubricGEvalScoreForOutput(rubric Rubric, rawOutput string, logprobs []TokenLogprob) (RubricScores, error) + type RunResult struct + ByModel map[string]ModelSummary + ByTag map[string]Summary + EndedAt time.Time + Metadata map[string]any + Results []CaseResult + StartedAt time.Time + SuiteName string + SuiteVersion string + Summary Summary + func ScoreExisting(ctx context.Context, spec ScoreExistingSpec) (RunResult, error) + type RunSpec struct + Concurrency int + Models []sigma.Model + Options []sigma.Option + Renderer Renderer + Repeats int + Scorers []Scorer + Suite Suite + type Runner struct + Client Completer + func NewRunner(client Completer) *Runner + func (r *Runner) Run(ctx context.Context, spec RunSpec) (RunResult, error) + type Score struct + Details map[string]any + Name string + Passed bool + Rationale string + Score float64 + type ScoreExistingSpec struct + Outputs []ExistingOutput + Scorers []Scorer + Suite Suite + type ScoreInput struct + Case Case + Message sigma.AssistantMessage + Model sigma.Model + Output string + Repeat int + Request sigma.Request + Suite Suite + type Scorer interface + Name func() string + Score func(context.Context, ScoreInput) (Score, error) + type ScorerFunc struct + Func func(context.Context, ScoreInput) (Score, error) + ScorerName string + func (f ScorerFunc) Name() string + func (f ScorerFunc) Score(ctx context.Context, input ScoreInput) (Score, error) + type SigmaTargetCompleter struct + Client Completer + Registry *sigma.Registry + func NewSigmaTargetCompleter(client Completer) SigmaTargetCompleter + func (c SigmaTargetCompleter) CompleteTarget(ctx context.Context, input TargetRequest) (TargetResult, error) + type Suite struct + Cases []Case + DataType EvalDataType + Description string + Metadata map[string]any + Name string + SystemPrompt string + Version string + func LoadSuite(reader io.Reader) (Suite, error) + func LoadSuiteFile(path string) (Suite, error) + type Summary struct + Errors int + Failed int + Passed int + ScoreCount int + Total int + type Target struct + Label string + Metadata map[string]any + ModelConfig *sigma.Model + ModelID sigma.ModelID + Name string + Options map[string]any + Provider sigma.ProviderID + func ParseTarget(raw string) (Target, error) + func TargetFromModel(model sigma.Model) Target + type TargetCompleter interface + CompleteTarget func(context.Context, TargetRequest) (TargetResult, error) + type TargetRequest struct + Metadata map[string]any + Options []sigma.Option + Repeat int + Request sigma.Request + Target Target + type TargetResult struct + Cost *sigma.Cost + DurationMS int64 + Error string + Logprobs []TokenLogprob + Message sigma.AssistantMessage + Metadata map[string]any + Output string + ProviderMetadata map[string]any + Repeat int + Request sigma.Request + Target Target + Usage *sigma.Usage + type TargetRunSpec struct + Concurrency int + Options []sigma.Option + Progress ProgressFunc + Renderer Renderer + Repeats int + Scorers []Scorer + Suite Suite + Targets []Target + type TargetRunner struct + Completer TargetCompleter + func NewTargetRunner(completer TargetCompleter) *TargetRunner + func (r *TargetRunner) Run(ctx context.Context, spec TargetRunSpec) (RunResult, error) + type TokenF1Scorer struct + func (TokenF1Scorer) Name() string + func (TokenF1Scorer) Score(_ context.Context, input ScoreInput) (Score, error) + type TokenLogprob struct + Bytes []byte + Logprob float64 + Token string + TopLogprobs []TokenLogprob + func DecodeTokenLogprobs(value any) ([]TokenLogprob, bool) + func TokenLogprobsFromMetadata(metadata map[string]any) ([]TokenLogprob, bool) + type ToolCallScorer struct + func (ToolCallScorer) Name() string + func (ToolCallScorer) Score(_ context.Context, input ScoreInput) (Score, error) + type Trace struct + Events []TraceEvent + Messages []sigma.Message + Metadata map[string]any + type TraceEvent struct + Metadata map[string]any + Name string + Payload any + Type string
Go to main page