types

Documentation

Overview

Package types defines shared domain types for the Vision Debugging Agent.

Index

• type Action
• type ActionType
• type Bounds
  • func (b Bounds) Center() (x, y int)
• type HealingEvent
  • func (e HealingEvent) ToMap() map[string]any
• type HealingStrategy
• type IterationRecord
• type MultiDeviceResult
• type ScreenType
• type Target
• type UICoordinate
• type UIElement
• type VisionResponse

Types

type Action

type Action struct {
	Action string  `json:"action"`
	Target *Target `json:"target,omitempty"`
	Value  string  `json:"value,omitempty"`
}

Action represents the physical action data.

type ActionType

type ActionType string

ActionType represents the type of UI action to perform.

const (
	ActionTap   ActionType = "tap"
	ActionSwipe ActionType = "swipe"
	ActionInput ActionType = "input"
	ActionKey   ActionType = "key"
	ActionBack  ActionType = "back"
	ActionHome  ActionType = "home"
)

type Bounds

type Bounds struct {
	X      int `json:"x"`
	Y      int `json:"y"`
	Width  int `json:"width"`
	Height int `json:"height"`
}

Bounds describes the bounding rectangle of a UI element.

func (Bounds) Center

func (b Bounds) Center() (x, y int)

Center returns the center point of the bounds.

type HealingEvent

type HealingEvent struct {
	StepIndex      int             `json:"step_index"`
	OriginalTarget string          `json:"original_target"`
	HealedTarget   string          `json:"healed_target"`
	StrategyUsed   HealingStrategy `json:"strategy_used"` // StrategyUIAutomator or StrategyVision
	Confidence     float64         `json:"confidence"`
	DurationMs     int64           `json:"duration_ms"`
	Duration       time.Duration   `json:"-"`       // Internal only; serialised via DurationMs
	Success        bool            `json:"success"` // Whether healing succeeded
	Reason         string          `json:"reason,omitempty"`
	ScreenshotPath string          `json:"screenshot_path,omitempty"`
	DismissedPopup bool            `json:"dismissed_popup,omitempty"`
}

HealingEvent records a self-healing action taken by the executor.

func (HealingEvent) ToMap

func (e HealingEvent) ToMap() map[string]any

ToMap converts the event to a map for legacy compatibility and logging.

type HealingStrategy

type HealingStrategy string

HealingStrategy represents the strategy used for element healing.

const (
	// StrategyUIAutomator uses UIAutomator dump to find similar elements.
	StrategyUIAutomator HealingStrategy = "uiautomator"
	// StrategyVision uses vision model to re-query element location.
	StrategyVision HealingStrategy = "vision"
)

type IterationRecord

type IterationRecord struct {
	Iteration    int             `json:"iteration"`
	Timestamp    time.Time       `json:"timestamp"`
	Screenshot   string          `json:"screenshot_b64,omitempty"` // Base64 encoded screenshot
	UIState      *VisionResponse `json:"ui_state"`
	ActionTaken  string          `json:"action_taken,omitempty"`
	ActionFailed bool            `json:"action_failed,omitempty"`
	Error        string          `json:"error,omitempty"`
	Duration     time.Duration   `json:"-"`           // Internal only; serialised via DurationMs
	DurationMs   int64           `json:"duration_ms"` // Milliseconds — single source of truth for JSON
	RawResponse  string          `json:"raw_response,omitempty"`
	ActionError  string          `json:"action_error,omitempty"`

	HealingEvents []HealingEvent `json:"healing_events,omitempty"`
}

IterationRecord represents a single step or iteration in a test run.

type MultiDeviceResult

type MultiDeviceResult struct {
	DeviceID string            `json:"device_id"`
	History  []IterationRecord `json:"history"`
	Error    string            `json:"error,omitempty"` // String, not error — marshals correctly to avoid empty {} objects
}

MultiDeviceResult groups execution history by device.

type ScreenType

type ScreenType string

ScreenType represents the type of screen currently displayed on the device.

const (
	ScreenLogin     ScreenType = "login"
	ScreenDashboard ScreenType = "dashboard"
	ScreenUnknown   ScreenType = "unknown"
	// Expo/React Native screen types per D-04
	ScreenExpoQRScanner  ScreenType = "expo_qr_scanner"  // QR code scanner in Expo Go
	ScreenRNErrorOverlay ScreenType = "rn_error_overlay" // React Native error display
	ScreenExpoDevMenu    ScreenType = "expo_dev_menu"    // Expo developer menu overlay
)

type Target

type Target struct {
	Label  string  `json:"label,omitempty"`
	Bounds *Bounds `json:"bounds,omitempty"`
}

Target describes the UI element to act upon.

type UICoordinate

type UICoordinate struct {
	X           int    `json:"x"`
	Y           int    `json:"y"`
	ElementType string `json:"element_type"`
	Label       string `json:"label,omitempty"`
}

UICoordinate represents a screen coordinate for a UI element.

type UIElement

type UIElement struct {
	ElementType string         `json:"element_type"`
	Label       string         `json:"label"`
	Coordinates *UICoordinate  `json:"coordinates,omitempty"`
	Attributes  map[string]any `json:"attributes,omitempty"`
}

UIElement represents a detected UI element on screen.

type VisionResponse

type VisionResponse struct {
	StepIndex   int        `json:"stepIndex,omitempty"`
	Found       bool       `json:"found"`
	X           int        `json:"x,omitempty"`
	Y           int        `json:"y,omitempty"`
	Width       int        `json:"width,omitempty"`
	Height      int        `json:"height,omitempty"`
	Reason      string     `json:"reason,omitempty"`
	ScreenType  ScreenType `json:"screenType,omitempty"`
	Action      string     `json:"action,omitempty"`
	Confidence  float64    `json:"confidence,omitempty"`
	Alternative []string   `json:"alternativeLabels,omitempty"`
	RawResponse string     `json:"rawResponse,omitempty"` // Original raw model output
}

VisionResponse represents the standard vision model analysis result.