evaluator

Documentation

Index

• type Config
• type EvalLoopConfig
  • func NewLoopConfig(p provider.Provider, model string, logger telemetry.Logger, ...) EvalLoopConfig
  • func NewLoopConfigWithInput(p provider.Provider, model string, logger telemetry.Logger, input *aop.Message, ...) EvalLoopConfig
• type Evaluator
  • func New(cfg Config) *Evaluator
  • func (e *Evaluator) Evaluate(ctx context.Context, goal, criteria string, messages []*aop.Message, ...) (*Verdict, error)
• type Verdict
  • func RunWithEval(ctx context.Context, a *agent.Agent, cfg EvalLoopConfig, ...) (*agent.Result, *Verdict, error)

Types

type Config

type Config struct {
	Provider      provider.Provider
	Model         string
	MaxRetries    int
	ContextWindow int
	Logger        telemetry.Logger
}

type EvalLoopConfig

type EvalLoopConfig struct {
	Evaluator     *Evaluator
	MaxEvalRounds int
	Goal          string
	Criteria      string
	TurnID        string
	InitialInput  *aop.Message
}

func NewLoopConfig

func NewLoopConfig(p provider.Provider, model string, logger telemetry.Logger, goal, criteria string, maxRounds int) EvalLoopConfig

NewLoopConfig builds an EvalLoopConfig around a fresh Evaluator. A maxRounds of zero (or negative) defers to RunWithEval's default.

func NewLoopConfigWithInput

func NewLoopConfigWithInput(p provider.Provider, model string, logger telemetry.Logger, input *aop.Message, criteria string, maxRounds int) EvalLoopConfig

NewLoopConfigWithInput preserves transport controls and multimodal parts on the first evaluation round. Boundaries that already published the user input use this constructor so the original multimodal input is preserved in Goal mode.

type Evaluator

type Evaluator struct {
	// contains filtered or unexported fields
}

func New

func New(cfg Config) *Evaluator

func (*Evaluator) Evaluate

func (e *Evaluator) Evaluate(ctx context.Context, goal, criteria string, messages []*aop.Message, output string, turns, contextTokens int) (*Verdict, error)

type Verdict

type Verdict struct {
	Pass           bool   `json:"pass"`
	Reason         string `json:"reason"`
	Feedback       string `json:"feedback"`
	InheritContext bool   `json:"inherit_context"`
}

func RunWithEval

func RunWithEval(ctx context.Context, a *agent.Agent, cfg EvalLoopConfig, opts ...agent.RunOption) (*agent.Result, *Verdict, error)