helios

README

Helios

Helios is a GUI automation tool written in Go - it automates anything that you can see on your screen. Helios uses image recognition powered by OpenCV under-the-hood. You can use Helios to automate GUI testing, video games and more.

Example:

package main

import (
	. "github.com/j7mbo/helios"
	"time"
)

func main() {
    // Initialise helios without any configuration
    app := NewApplication(nil)
    // Load .gopher.png and expect to find it at 0.9 / 1.0 confidence
    gopherImage, _ := NewImage("./gopher.png", 0.9)
    // Find the gopher image on-screen
    gopher := app.GetScreen().Find(gopherImage)
    // Highlight it for two seconds
    gopher.Highlight(2 * time.Second)
    // Click at a random (X,Y) coordinate within the matched box
    gopher.Click()
}

The Gopher from Goland, highlighted for 2 seconds:

Installation

MacOS

brew install pkg-config
brew install opencv

Docs

TBD...

Upcoming work

• Ability to define Regions to search
• Abstraction for finder
• Works with any resolution
• Wait X seconds for an image to appear
• Offsets for regions
• Stable supported api
• Support and releases for Windows, Linux (MacOS already supported)
• Moving highlight box during highlighting
• Support for multiple screens
• Fix for issue #1 - no need to embed a highlighter binary
• Embedding OpenCV directly, 0 dependency binary (need help with this)
• API for interfacing with other languages
• OCR, text extraction
• GUI for region and image selection, scripting language (this'll be a fun one)
• Tests...

Documentation

Index

• Variables
• type App
  • func NewApplication(config *Config) *App
  • func (a *App) GetScreen() *Screen
• type Clicker
  • func (c *Clicker) Click(match *Match)
• type Config
  • func NewConfig(configs ...configuration) (*Config, error)
  • func (c *Config) GetPollInterval() *PollInterval
• type Finder
  • func NewFinder(screen *Screen, pollInterval *PollInterval) *Finder
  • func (f *Finder) Find(i *Image, r *Region) *Match
  • func (f *Finder) FindAll(i *Image, r *Region) []*Match
  • func (f *Finder) Wait(i *Image, r *Region, t time.Duration) *Match
• type HighlightRequest
  • func (r HighlightRequest) AsArgsArray() []string
• type Highlighter
  • func NewHighlighter(highlighterBinary []byte) *Highlighter
  • func (h *Highlighter) Highlight(r *HighlightRequest)
• type Image
  • func NewImage(path string, confidenceThreshold float64) (*Image, error)
  • func (i *Image) GetImage() i.Image
  • func (i *Image) GetPath() string
• type Match
  • func NewMatch(img *Image, confidence float32, screen *Screen, highlighter *Highlighter, ...) *Match
  • func (m *Match) Click() *Match
  • func (m *Match) GetConfidence() float32
  • func (m *Match) Highlight(t time.Duration)
• type Point
  • func NewPoint(x, y float64) *Point
  • func (p *Point) GetX() float64
  • func (p *Point) GetY() float64
• type PollInterval
• type Region
  • func NewRegion(topLeft *Point, width, height int, screen *Screen, finder *Finder) *Region
  • func (r *Region) Find(i *Image) *Match
  • func (r *Region) FindAll(i *Image) []*Match
  • func (r *Region) GetHeight() int
  • func (r *Region) GetTopLeft() *Point
  • func (r *Region) GetWidth() int
  • func (r *Region) Highlight(t time.Duration)
  • func (r *Region) Wait(i *Image, t time.Duration) *Match
• type Screen
  • func NewScreen(highlighter *Highlighter, interval *PollInterval) *Screen
  • func (s *Screen) Find(i *Image) *Match
  • func (s *Screen) FindAll(i *Image) []*Match
  • func (s *Screen) Highlight(t time.Duration)
  • func (s *Screen) NewRegion(topLeft *Point, width, height float64) *Region
  • func (s *Screen) Wait(i *Image, t time.Duration) *Match

Variables

var ImageNotFoundError = errors.New("file not found")

var ImageUnknownError = errors.New("unknown error")

var ImageUnsupportedFormatError = errors.New("unsupported format")

var InvalidConfigurationError = errors.New("invalid configuration provided")

Types

type App

type App struct {
	// contains filtered or unexported fields
}

func NewApplication

func NewApplication(config *Config) *App

func (*App) GetScreen

func (a *App) GetScreen() *Screen

type Clicker

type Clicker struct {
}

func (*Clicker) Click

func (c *Clicker) Click(match *Match)

Click clicks at a random X and Y coordinate within the given location. This doesn't use the standard Click() function of any library which is easy to detect as an automated click - rather it adds entropy throughout the mouse events.

type Config

type Config struct {
	// contains filtered or unexported fields
}

func NewConfig

func NewConfig(configs ...configuration) (*Config, error)

func (*Config) GetPollInterval

func (c *Config) GetPollInterval() *PollInterval

type Finder

type Finder struct {
	// contains filtered or unexported fields
}

func NewFinder

func NewFinder(screen *Screen, pollInterval *PollInterval) *Finder

func (*Finder) Find

func (f *Finder) Find(i *Image, r *Region) *Match

func (*Finder) FindAll

func (f *Finder) FindAll(i *Image, r *Region) []*Match

func (*Finder) Wait

func (f *Finder) Wait(i *Image, r *Region, t time.Duration) *Match

type HighlightRequest

type HighlightRequest struct {
	ScreenWidth  int     `arg:"--screen-width"`
	ScreenHeight int     `arg:"--screen-height"`
	X            float64 `arg:"--x"`
	Y            float64 `arg:"--y"`
	Width        float64 `arg:"--w"`
	Height       float64 `arg:"--h"`
	Duration     float64 `arg:"--d"`
}

func (HighlightRequest) AsArgsArray

func (r HighlightRequest) AsArgsArray() []string

type Highlighter

type Highlighter struct {
	// contains filtered or unexported fields
}

func NewHighlighter

func NewHighlighter(highlighterBinary []byte) *Highlighter

func (*Highlighter) Highlight

func (h *Highlighter) Highlight(r *HighlightRequest)

type Image

type Image struct {
	// contains filtered or unexported fields
}

func NewImage

func NewImage(path string, confidenceThreshold float64) (*Image, error)

func (*Image) GetImage

func (i *Image) GetImage() i.Image

func (*Image) GetPath

func (i *Image) GetPath() string

type Match

type Match struct {
	Region
	// contains filtered or unexported fields
}

func NewMatch

func NewMatch(img *Image, confidence float32, screen *Screen, highlighter *Highlighter, region *Region) *Match

func (*Match) Click

func (m *Match) Click() *Match

func (*Match) GetConfidence

func (m *Match) GetConfidence() float32

func (*Match) Highlight

func (m *Match) Highlight(t time.Duration)

type Point

type Point struct {
	// contains filtered or unexported fields
}

func NewPoint

func NewPoint(x, y float64) *Point

func (*Point) GetX

func (p *Point) GetX() float64

func (*Point) GetY

func (p *Point) GetY() float64

type PollInterval

type PollInterval struct {
	time.Duration
}

type Region

type Region struct {
	// contains filtered or unexported fields
}

func NewRegion

func NewRegion(topLeft *Point, width, height int, screen *Screen, finder *Finder) *Region

func (*Region) Find

func (r *Region) Find(i *Image) *Match

func (*Region) FindAll

func (r *Region) FindAll(i *Image) []*Match

func (*Region) GetHeight

func (r *Region) GetHeight() int

func (*Region) GetTopLeft

func (r *Region) GetTopLeft() *Point

func (*Region) GetWidth

func (r *Region) GetWidth() int

func (*Region) Highlight

func (r *Region) Highlight(t time.Duration)

func (*Region) Wait

func (r *Region) Wait(i *Image, t time.Duration) *Match

type Screen

type Screen struct {
	Region
	// contains filtered or unexported fields
}

func NewScreen

func NewScreen(highlighter *Highlighter, interval *PollInterval) *Screen

func (*Screen) Find

func (s *Screen) Find(i *Image) *Match

func (*Screen) FindAll

func (s *Screen) FindAll(i *Image) []*Match

func (*Screen) Highlight

func (s *Screen) Highlight(t time.Duration)

func (*Screen) NewRegion

func (s *Screen) NewRegion(topLeft *Point, width, height float64) *Region

func (*Screen) Wait

func (s *Screen) Wait(i *Image, t time.Duration) *Match