generate

README

generate command

This command is based on PromptPex, a test generation framework for prompts.

• Documentation
• Source
• Agentic implementation plan

In a nutshell, read https://microsoft.github.io/promptpex/reference/test-generation/

Documentation

Overview

Package generate provides a gh command to generate tests.

Index

• Constants
• Variables
• func AddCommandLineFlags(cmd *cobra.Command)
• func ApplyEffortConfiguration(options *PromptPexOptions, effort string)
• func ComputePromptHash(p *prompt.File) (string, error)
• func ExtractJSON(content string) string
• func IsUnassistedResponse(text string) bool
• func MergeStringMaps(maps ...map[string]string) map[string]string
• func NewGenerateCommand(cfg *command.Config) *cobra.Command
• func ParseFlags(cmd *cobra.Command, options *PromptPexOptions) error
• func ParseRules(text string) []string
• func RenderMessagesToString(messages []prompt.Message) string
• func SplitLines(text string) []string
• func StringSliceContains(slice []string, value string) bool
• func Unbracket(text string) string
• func Unfence(text string) string
• func Unxml(text string) string
• type EffortConfiguration
  • func GetEffortConfiguration(effort string) *EffortConfiguration
• type PromptPexContext
• type PromptPexModelAliases
• type PromptPexOptions
  • func GetDefaultOptions() *PromptPexOptions
• type PromptPexPrompts
• type PromptPexTest

Constants

const (
	EffortMin    = "min"
	EffortLow    = "low"
	EffortMedium = "medium"
	EffortHigh   = "high"
)

Effort levels

Variables

var BOX_END = "╰──"

var BOX_START = "╭──"

var COLOR_SECONDARY = ansi.ColorFunc(ansi.LightBlack)

var EVALUATOR_RULES_COMPLIANCE_ID = "output_rules_compliance"

Functions

func AddCommandLineFlags

func AddCommandLineFlags(cmd *cobra.Command)

func ApplyEffortConfiguration

func ApplyEffortConfiguration(options *PromptPexOptions, effort string)

ApplyEffortConfiguration applies effort configuration to options

func ComputePromptHash

func ComputePromptHash(p *prompt.File) (string, error)

ComputePromptHash computes a SHA256 hash of the prompt's messages, model, and model parameters

func ExtractJSON

func ExtractJSON(content string) string

ExtractJSON extracts JSON content from a string that might be wrapped in markdown

func IsUnassistedResponse

func IsUnassistedResponse(text string) bool

IsUnassistedResponse returns true if the text is an unassisted response, like "i'm sorry" or "i can't assist with that".

func MergeStringMaps

func MergeStringMaps(maps ...map[string]string) map[string]string

MergeStringMaps merges multiple string maps, with later maps taking precedence

func NewGenerateCommand

func NewGenerateCommand(cfg *command.Config) *cobra.Command

NewGenerateCommand returns a new command to generate tests using PromptPex.

func ParseFlags

func ParseFlags(cmd *cobra.Command, options *PromptPexOptions) error

ParseFlags parses command-line flags and applies them to the options

func ParseRules

func ParseRules(text string) []string

ParseRules removes numbering, bullets, and extraneous "Rules:" lines from a rules text block.

func RenderMessagesToString

func RenderMessagesToString(messages []prompt.Message) string

RenderMessagesToString converts a slice of Messages to a human-readable string representation

func SplitLines

func SplitLines(text string) []string

SplitLines splits text into lines.

func StringSliceContains

func StringSliceContains(slice []string, value string) bool

StringSliceContains checks if a string slice contains a value

func Unbracket

func Unbracket(text string) string

Unbracket removes leading and trailing square brackets.

func Unfence

func Unfence(text string) string

Unfence removes Markdown code fences and splits text into lines.

func Unxml

func Unxml(text string) string

Unxml removes leading and trailing XML tags, like `<foo>` and `</foo>`, from the given string.

Types

type EffortConfiguration

type EffortConfiguration struct {
	MaxRules     int
	TestsPerRule int
	RulesPerGen  int
}

EffortConfiguration defines the configuration for different effort levels

func GetEffortConfiguration

func GetEffortConfiguration(effort string) *EffortConfiguration

GetEffortConfiguration returns the configuration for a given effort level Based on the reference TypeScript implementation in constants.mts

type PromptPexContext

type PromptPexContext struct {
	RunID        string            `json:"runId" yaml:"runId"`
	Prompt       *prompt.File      `json:"prompt" yaml:"prompt"`
	PromptHash   string            `json:"promptHash" yaml:"promptHash"`
	Options      *PromptPexOptions `json:"options" yaml:"options"`
	Intent       *string           `json:"intent" yaml:"intent"`
	Rules        []string          `json:"rules" yaml:"rules"`
	InverseRules []string          `json:"inverseRules" yaml:"inverseRules"`
	InputSpec    *string           `json:"inputSpec" yaml:"inputSpec"`
	Tests        []PromptPexTest   `json:"tests" yaml:"tests"`
}

PromptPexContext represents the main context for PromptPex operations

type PromptPexModelAliases

type PromptPexModelAliases struct {
	Rules       string `yaml:"rules,omitempty" json:"rules,omitempty"`
	Tests       string `yaml:"tests,omitempty" json:"tests,omitempty"`
	Groundtruth string `yaml:"groundtruth,omitempty" json:"groundtruth,omitempty"`
	Eval        string `yaml:"eval,omitempty" json:"eval,omitempty"`
}

PromptPexModelAliases represents model aliases for different purposes

type PromptPexOptions

type PromptPexOptions struct {
	// Core options
	Instructions       *PromptPexPrompts      `yaml:"instructions,omitempty" json:"instructions,omitempty"`
	Models             *PromptPexModelAliases `yaml:"models,omitempty" json:"models,omitempty"`
	TestsPerRule       int                    `yaml:"testsPerRule,omitempty" json:"testsPerRule,omitempty"`
	RulesPerGen        int                    `yaml:"rulesPerGen,omitempty" json:"rulesPerGen,omitempty"`
	MaxRules           int                    `yaml:"maxRules,omitempty" json:"maxRules,omitempty"`
	IntentMaxTokens    int                    `yaml:"intentMaxTokens,omitempty" json:"intentMaxTokens,omitempty"`
	InputSpecMaxTokens int                    `yaml:"inputSpecMaxTokens,omitempty" json:"inputSpecMaxTokens,omitempty"`

	// CLI-specific options
	Effort string `yaml:"effort,omitempty" json:"effort,omitempty"`
	Prompt string `yaml:"prompt,omitempty" json:"prompt,omitempty"`

	// Loader options
	Verbose bool `yaml:"verbose,omitempty" json:"verbose,omitempty"`
}

PromptPexOptions contains all configuration options for PromptPex

func GetDefaultOptions

func GetDefaultOptions() *PromptPexOptions

GetDefaultOptions returns default options for PromptPex

type PromptPexPrompts

type PromptPexPrompts struct {
	InputSpec          string `yaml:"inputSpec,omitempty" json:"inputSpec,omitempty"`
	OutputRules        string `yaml:"outputRules,omitempty" json:"outputRules,omitempty"`
	InverseOutputRules string `yaml:"inverseOutputRules,omitempty" json:"inverseOutputRules,omitempty"`
	Intent             string `yaml:"intent,omitempty" json:"intent,omitempty"`
	Tests              string `yaml:"tests,omitempty" json:"tests,omitempty"`
}

PromptPexPrompts contains custom prompts for different stages

type PromptPexTest

type PromptPexTest struct {
	Input     string `json:"input" yaml:"input"`
	Expected  string `json:"expected,omitempty" yaml:"expected,omitempty"`
	Predicted string `json:"predicted,omitempty" yaml:"predicted,omitempty"`
	Reasoning string `json:"reasoning,omitempty" yaml:"reasoning,omitempty"`
	Scenario  string `json:"scenario,omitempty" yaml:"scenario,omitempty"`
}

PromptPexTest represents a single test case