crawler

Documentation

Index

• type CrawlConfig
• type CrawlResult
  • func Crawl(config CrawlConfig) (*CrawlResult, error)
• type DiscoveredForm
• type DiscoveredURL
• type FormInput

Types

type CrawlConfig

type CrawlConfig struct {
	SeedURL      string
	MaxDepth     int
	Concurrency  int
	Timeout      int
	Proxy        string
	IgnoreRobots bool
	Headers      []string
	Cookies      string
}

CrawlConfig holds crawler configuration.

type CrawlResult

type CrawlResult struct {
	URLs      []DiscoveredURL
	Forms     []DiscoveredForm
	APIRoutes []string
	JSFiles   []string
}

CrawlResult holds all discovered resources.

func Crawl

func Crawl(config CrawlConfig) (*CrawlResult, error)

Crawl performs web crawling starting from the seed URL.

type DiscoveredForm

type DiscoveredForm struct {
	Action string
	Method string
	Inputs []FormInput
}

DiscoveredForm represents a found HTML form.

type DiscoveredURL

type DiscoveredURL struct {
	URL    string
	Method string
	Source string
	Params []string
}

DiscoveredURL represents a found URL.

type FormInput

type FormInput struct {
	Name  string
	Type  string
	Value string
}

FormInput represents a form input field.