trigger

Documentation

Index

• type Runner
  • func NewRunner(spec *TestSpec, engine execution.AgentEngine, cfg *config.BenchmarkConfig, ...) *Runner
  • func (r *Runner) Run(ctx context.Context) (*models.TriggerMetrics, error)
  • func (r *Runner) RunDetailed(ctx context.Context) ([]models.TriggerResult, *models.TriggerMetrics, error)
• type TestPrompt
• type TestSpec
  • func Discover(dir string) (*TestSpec, error)
  • func ParseSpec(data []byte) (*TestSpec, error)

Types

type Runner

type Runner struct {
	// contains filtered or unexported fields
}

Runner executes trigger tests and returns classification metrics.

func NewRunner

func NewRunner(spec *TestSpec, engine execution.AgentEngine, cfg *config.BenchmarkConfig, out io.Writer) *Runner

func (*Runner) Run

func (r *Runner) Run(ctx context.Context) (*models.TriggerMetrics, error)

func (*Runner) RunDetailed

func (r *Runner) RunDetailed(ctx context.Context) ([]models.TriggerResult, *models.TriggerMetrics, error)

type TestPrompt

type TestPrompt struct {
	Prompt     string `yaml:"prompt"`
	Reason     string `yaml:"reason"`
	Confidence string `yaml:"confidence"`
}

type TestSpec

type TestSpec struct {
	Skill                   string       `yaml:"skill"`
	ShouldTriggerPrompts    []TestPrompt `yaml:"should_trigger_prompts"`
	ShouldNotTriggerPrompts []TestPrompt `yaml:"should_not_trigger_prompts"`
}

func Discover

func Discover(dir string) (*TestSpec, error)

Discover searches dir for a trigger test file (trigger_tests.yaml) and returns the parsed spec. Returns nil, nil when no test file is found.

func ParseSpec

func ParseSpec(data []byte) (*TestSpec, error)