studio

Documentation

Overview

Package studio implements App 4: Studio — prompt templates, experiments, benchmarks, snapshot tests.

Index

• type App
  • func New(conn *sql.DB) *App
  • func (a *App) Description() string
  • func (a *App) Migrate(conn *sql.DB) error
  • func (a *App) Name() string
  • func (a *App) RegisterRoutes(mux *http.ServeMux)
  • func (a *App) SetProxyPort(port int)
• type ExperimentResult
• type RunExperimentRequest
• type RunResult
• type Runner
  • func NewRunner(conn *sql.DB, proxyPort int) *Runner
  • func (r *Runner) Run(ctx context.Context, req RunExperimentRequest) (*ExperimentResult, error)
• type VariantResult

Types

type App

type App struct {
	// contains filtered or unexported fields
}

func New

func New(conn *sql.DB) *App

func (*App) Description

func (a *App) Description() string

func (*App) Migrate

func (a *App) Migrate(conn *sql.DB) error

func (*App) Name

func (a *App) Name() string

func (*App) RegisterRoutes

func (a *App) RegisterRoutes(mux *http.ServeMux)

func (*App) SetProxyPort

func (a *App) SetProxyPort(port int)

SetProxyPort configures the runner with the proxy's port for experiment execution.

type ExperimentResult

type ExperimentResult struct {
	ExperimentID int64           `json:"experiment_id"`
	Name         string          `json:"name"`
	Prompt       string          `json:"prompt"`
	Variants     []VariantResult `json:"variants"`
	Winner       string          `json:"winner"`
	WinReason    string          `json:"win_reason"`
	TotalCost    float64         `json:"total_cost_usd"`
	Duration     float64         `json:"duration_ms"`
}

ExperimentResult holds the full experiment results.

type RunExperimentRequest

type RunExperimentRequest struct {
	Name    string   `json:"name"`
	Prompt  string   `json:"prompt"`   // The user message to send
	System  string   `json:"system"`   // Optional system message
	Models  []string `json:"models"`   // Models to compare (e.g., ["gpt-4o", "claude-sonnet-4-5-20250929"])
	Runs    int      `json:"runs"`     // Number of runs per model (default 1)
	Eval    string   `json:"eval"`     // Evaluation method: "length", "contains", "manual", ""
	EvalArg string   `json:"eval_arg"` // Eval argument (e.g., substring for "contains")
	APIKey  string   `json:"api_key"`  // Optional: Stockyard API key to use for requests
}

RunExperimentRequest defines an experiment to run.

type RunResult

type RunResult struct {
	Content   string  `json:"content"`
	LatencyMs float64 `json:"latency_ms"`
	TokensIn  int     `json:"tokens_in"`
	TokensOut int     `json:"tokens_out"`
	CostUSD   float64 `json:"cost_usd"`
	Model     string  `json:"model"`
	Error     string  `json:"error,omitempty"`
	EvalScore float64 `json:"eval_score"`
}

RunResult holds the result of a single run.

type Runner

type Runner struct {
	// contains filtered or unexported fields
}

Runner executes experiments by sending prompts to multiple models via the proxy.

func NewRunner

func NewRunner(conn *sql.DB, proxyPort int) *Runner

NewRunner creates a new experiment runner.

func (*Runner) Run

func (r *Runner) Run(ctx context.Context, req RunExperimentRequest) (*ExperimentResult, error)

Run executes an experiment.

type VariantResult

type VariantResult struct {
	Model        string      `json:"model"`
	Provider     string      `json:"provider"`
	Runs         []RunResult `json:"runs"`
	AvgLatency   float64     `json:"avg_latency_ms"`
	AvgTokensIn  int         `json:"avg_tokens_in"`
	AvgTokensOut int         `json:"avg_tokens_out"`
	AvgCost      float64     `json:"avg_cost_usd"`
	EvalScore    float64     `json:"eval_score"`
	Errors       int         `json:"errors"`
}

VariantResult holds results for a single model variant.