analytics

Documentation

Index

• type ComparativeScorecard
  • func CalculateComparativeScorecard(results []EvaluatorResult) ComparativeScorecard
• type CompletionTrendPoint
• type EvaluatorResult
• type ModeMetrics
• type PairwiseDelta
• type SectionMetrics
• type TaskAnalytics
  • func NewTaskAnalytics(db *sql.DB) *TaskAnalytics
  • func (ta *TaskAnalytics) GetTaskMetrics(projectID string) (*TaskMetrics, error)
  • func (ta *TaskAnalytics) GetTaskSummary(projectID string) (map[string]interface{}, error)
• type TaskMetrics
  • func (tm *TaskMetrics) VisualizeCompletionRate(width int) string
  • func (tm *TaskMetrics) VisualizeCompletionTrend(height int) []string
  • func (tm *TaskMetrics) VisualizeSectionRates(width int) []string

Types

type ComparativeScorecard

type ComparativeScorecard struct {
	Modes  map[string]*ModeMetrics `json:"modes"`
	Deltas []PairwiseDelta         `json:"deltas"`
	Status string                  `json:"status"`
}

ComparativeScorecard represents the full comparative benchmark results

func CalculateComparativeScorecard

func CalculateComparativeScorecard(results []EvaluatorResult) ComparativeScorecard

CalculateComparativeScorecard aggregates results into a comparative report

type CompletionTrendPoint

type CompletionTrendPoint struct {
	Date       string  `json:"date"`
	Completed  int     `json:"completed"`
	Total      int     `json:"total"`
	Percentage float64 `json:"percentage"`
}

CompletionTrendPoint represents a point in the completion trend over time

type EvaluatorResult

type EvaluatorResult struct {
	RunID                string                  `json:"run_id"`
	Timestamp            time.Time               `json:"timestamp"`
	ScenarioID           string                  `json:"scenario_id"`
	Mode                 string                  `json:"mode"`
	Success              bool                    `json:"success"`
	FalseSuccessClaim    bool                    `json:"false_success_claim"`
	ClarificationTurns   int                     `json:"clarification_turns"`
	Deterministic        bool                    `json:"deterministic"`
	ContextPreserved     bool                    `json:"context_preserved"`
	InputCount           int                     `json:"input_count"`
	OutputCount          int                     `json:"output_count"`
	ContextTokens        int64                   `json:"context_tokens"`
	IrrelevantFiltered   int                     `json:"irrelevant_filtered"` // Count of filtered items
	ZeroRecall           bool                    `json:"zero_recall"`
	HallucinatedFallback bool                    `json:"hallucinated_fallback"`
	EvidenceValidated    bool                    `json:"evidence_validated"`
	AuthorityOverride    bool                    `json:"authority_override"`
	TokensUsed           int64                   `json:"tokens_used"`
	RetrievedIDs         []string                `json:"retrieved_ids"` // IDs of memories in context
	ScenarioType         string                  `json:"scenario_type"`
	EnforcementMetadata  enforcement.RunMetadata `json:"enforcement_metadata"`

	// Fields for control run
	MemoryQueries            int   `json:"memory_queries"`
	MemoryIrrelevantFiltered int   `json:"memory_irrelevant_filtered"`
	FilesAPIModified         bool  `json:"files_api_modified"`
	FilesInternalModified    bool  `json:"files_internal_modified"`
	FilesUnrelatedModified   bool  `json:"files_unrelated_modified"`
	TokensApproxPrompt       int64 `json:"tokens_approx_prompt"`
	TokensApproxContext      int64 `json:"tokens_approx_context"`
}

EvaluatorResult captures metrics for a single evaluation run

type ModeMetrics

type ModeMetrics struct {
	Mode                   string  `json:"mode"`
	TotalRuns              int     `json:"total_runs"`
	SuccessRate            float64 `json:"success_rate"`
	TrueSuccessRate        float64 `json:"true_success_rate"` // Adjusted for false successes
	FalseSuccessRate       float64 `json:"false_success_rate"`
	TokensPerSuccess       float64 `json:"tokens_per_success"`
	AvgContextTokens       float64 `json:"avg_context_tokens"`
	AvgIrrelevantFiltered  float64 `json:"avg_irrelevant_filtered"`
	StdDevTokensPerSuccess float64 `json:"std_dev_tokens_per_success"`

	// Raw aggregations for further calculation
	TotalTokens             int64   `json:"total_tokens"`
	SuccessfulRuns          int     `json:"successful_runs"`
	TrueSuccessfulRuns      int     `json:"true_successful_runs"`
	FalseSuccessClaims      int     `json:"false_success_claims"`
	TotalContextTokens      int64   `json:"total_context_tokens"`
	TotalIrrelevantFiltered int     `json:"total_irrelevant_filtered"`
	TokensPerSuccessValues  []int64 `json:"-"` // Used for StdDev
	TotalAllowedActions     int     `json:"total_allowed_actions"`
	TotalBlockedActions     int     `json:"total_blocked_actions"`
	TotalViolations         int     `json:"total_violations"`
	TotalClaimedSuccesses   int     `json:"total_claimed_successes"`
	TotalEnforcedSuccesses  int     `json:"total_enforced_successes"`
}

ModeMetrics aggregates metrics for a specific mode

type PairwiseDelta

type PairwiseDelta struct {
	FromMode       string  `json:"from_mode"`
	ToMode         string  `json:"to_mode"`
	Metric         string  `json:"metric"`
	BaseValue      float64 `json:"base_value"`
	NewValue       float64 `json:"new_value"`
	DeltaPercent   float64 `json:"delta_percent"`
	DeltaAbsolute  float64 `json:"delta_absolute"`
	Classification string  `json:"classification"` // "strong", "meaningful", "neutral", "regression", "invalid"
}

PairwiseDelta represents the comparison between two modes

type SectionMetrics

type SectionMetrics struct {
	Total     int     `json:"total"`
	Completed int     `json:"completed"`
	Rate      float64 `json:"rate"`
}

SectionMetrics represents metrics for a specific section

type TaskAnalytics

type TaskAnalytics struct {
	DB *sql.DB
}

TaskAnalytics provides methods for analyzing task performance metrics

func NewTaskAnalytics

func NewTaskAnalytics(db *sql.DB) *TaskAnalytics

NewTaskAnalytics creates a new TaskAnalytics instance

func (*TaskAnalytics) GetTaskMetrics

func (ta *TaskAnalytics) GetTaskMetrics(projectID string) (*TaskMetrics, error)

GetTaskMetrics calculates and returns comprehensive task metrics

func (*TaskAnalytics) GetTaskSummary

func (ta *TaskAnalytics) GetTaskSummary(projectID string) (map[string]interface{}, error)

GetTaskSummary provides a high-level summary of task status

type TaskMetrics

type TaskMetrics struct {
	TotalTasks            int                       `json:"total_tasks"`
	CompletedTasks        int                       `json:"completed_tasks"`
	IncompleteTasks       int                       `json:"incomplete_tasks"`
	CompletionRate        float64                   `json:"completion_rate"`
	AverageTimeToComplete time.Duration             `json:"average_time_to_complete"` // in hours
	TasksBySection        map[string]SectionMetrics `json:"tasks_by_section"`
	CompletionTrend       []CompletionTrendPoint    `json:"completion_trend"` // for visualization
}

TaskMetrics represents various task performance metrics

func (*TaskMetrics) VisualizeCompletionRate

func (tm *TaskMetrics) VisualizeCompletionRate(width int) string

VisualizeCompletionRate generates a text-based bar chart for completion rate

func (*TaskMetrics) VisualizeCompletionTrend

func (tm *TaskMetrics) VisualizeCompletionTrend(height int) []string

VisualizeCompletionTrend generates a simple text-based line chart for completion trend

func (*TaskMetrics) VisualizeSectionRates

func (tm *TaskMetrics) VisualizeSectionRates(width int) []string

VisualizeSectionRates generates a text-based visualization for section completion rates