availability

Documentation

Index

• func ColorizeOutput(enabled bool) func(*Printer)
• func CrawlerForSites(crawler Crawler) func(*Report)
• func DecodeOutput(enabled bool) func(*Printer)
• func HideError(disabled bool) func(*Printer)
• func HideRedirect(disabled bool) func(*Printer)
• func NewReadableEventBus(size int) chan event
• func NoCookie() func(*colly.Collector)
• func NoRedirect() func(*colly.Collector)
• func OnError(bus EventBus) func(*colly.Collector)
• func OnHTML(base *url.URL, bus EventBus) func(*colly.Collector)
• func OnRequest() func(*colly.Collector)
• func OnResponse(bus EventBus) func(*colly.Collector)
• func OutputForPrinting(output io.Writer) func(*Printer)
• type Crawler
  • func CrawlerColly(config CrawlerConfig) Crawler
• type CrawlerConfig
• type CrawlerFunc
  • func (fn CrawlerFunc) Visit(url string, bus EventBus) error
• type ErrorEvent
• type EventBus
• type Link
• type Page
• type Printer
  • func NewPrinter(options ...func(*Printer)) *Printer
  • func (p *Printer) For(report Reporter) *Printer
  • func (p *Printer) Print() error
• type ProblemEvent
• type Report
  • func NewReport(options ...func(*Report)) *Report
  • func (r *Report) Fill() *Report
  • func (r *Report) For(rawURLs []string) *Report
  • func (r *Report) Sites() <-chan Site
• type Reporter
• type ResponseEvent
• type Site
  • func NewSite(rawURL string) *Site
  • func (s *Site) Fetch(crawler Crawler) error
• type WalkEvent

Functions

func ColorizeOutput

func ColorizeOutput(enabled bool) func(*Printer)

ColorizeOutput sets the ink for the printer.

func CrawlerForSites

func CrawlerForSites(crawler Crawler) func(*Report)

CrawlerForSites sets the website crawler to a report builder.

func DecodeOutput

func DecodeOutput(enabled bool) func(*Printer)

DecodeOutput sets `net/url.PathUnescape` as a decoder.

func HideError

func HideError(disabled bool) func(*Printer)

HideError prevents URL's error output.

func HideRedirect

func HideRedirect(disabled bool) func(*Printer)

HideRedirect prevents URL's redirect output.

func NewReadableEventBus

func NewReadableEventBus(size int) chan event

NewReadableEventBus returns read/write-channel of events.

func NoCookie

func NoCookie() func(*colly.Collector)

NoCookie disables cookie for `github.com/gocolly/colly.Collector`.

func NoRedirect

func NoRedirect() func(*colly.Collector)

NoRedirect disables redirects for `github.com/gocolly/colly.Collector`.

func OnError

func OnError(bus EventBus) func(*colly.Collector)

OnError registers a callback by `github.com/gocolly/colly.Collector.OnError()`.

func OnHTML

func OnHTML(base *url.URL, bus EventBus) func(*colly.Collector)

OnHTML registers a callback by `github.com/gocolly/colly.Collector.OnHTML()`.

func OnRequest

func OnRequest() func(*colly.Collector)

OnRequest registers a callback by `github.com/gocolly/colly.Collector.OnRequest()`.

func OnResponse

func OnResponse(bus EventBus) func(*colly.Collector)

OnResponse registers a callback by `github.com/gocolly/colly.Collector.OnResponse()`.

func OutputForPrinting

func OutputForPrinting(output io.Writer) func(*Printer)

OutputForPrinting sets up printer output.

Types

type Crawler

type Crawler interface {
	// Visit starts to crawl a website starting with the passed URL.
	// All implementations must communicate with report builder
	// through the passed EventBus and close it when they're done.
	Visit(url string, bus EventBus) error
}

Crawler defines general behavior of website crawlers.

func CrawlerColly

func CrawlerColly(config CrawlerConfig) Crawler

CrawlerColly returns configured website crawler.

type CrawlerConfig

type CrawlerConfig struct {
	UserAgent string
	Verbose   bool
	Output    io.Writer
}

CrawlerConfig holds a website crawler's configuration.

type CrawlerFunc

type CrawlerFunc func(string, EventBus) error

CrawlerFunc adds possibility to use functions as a website crawler.

func (CrawlerFunc) Visit

func (fn CrawlerFunc) Visit(url string, bus EventBus) error

Visit calls itself.

type ErrorEvent

type ErrorEvent struct {
	StatusCode int
	Location   string
	Redirect   string
	Error      error
	// contains filtered or unexported fields
}

ErrorEvent contains a response' status code, its URL and an encountered error.

type EventBus

type EventBus chan<- event

EventBus is a write-only channel to communicate between a website crawler and a report builder.

type Link

type Link struct {
	Page       *Page
	Internal   bool
	StatusCode int
	Location   string
	Redirect   string
	Error      error
}

Link contains meta information about a web link.

type Page

type Page struct {
	*Link
	Links []Link
}

Page contains meta information about a website page.

type Printer

type Printer struct {
	// contains filtered or unexported fields
}

Printer represents a printer.

func NewPrinter

func NewPrinter(options ...func(*Printer)) *Printer

NewPrinter returns configured printer instance.

func (*Printer) For

func (p *Printer) For(report Reporter) *Printer

For prepares printer for passed report provider.

func (*Printer) Print

func (p *Printer) Print() error

Print prints a report into the configured output. Stdout is used as a fallback if the output is not set up.

type ProblemEvent

type ProblemEvent struct {
	Message string
	Context interface{}
	// contains filtered or unexported fields
}

ProblemEvent contains information about unexpected error.

type Report

type Report struct {
	// contains filtered or unexported fields
}

Report represents a report builder.

func NewReport

func NewReport(options ...func(*Report)) *Report

NewReport returns configured report builder.

func (*Report) Fill

func (r *Report) Fill() *Report

Fill starts to fetch sites and prepared them for reading.

func (*Report) For

func (r *Report) For(rawURLs []string) *Report

For prepares report builder for passed websites' URLs.

func (*Report) Sites

func (r *Report) Sites() <-chan Site

Sites returns a channel what will be closed when the report is available.

type Reporter

type Reporter interface {
	Sites() <-chan Site
}

Reporter defines general behavior of report providers.

type ResponseEvent

type ResponseEvent struct {
	StatusCode int
	Location   string
	// contains filtered or unexported fields
}

ResponseEvent contains a response' status code and its URL.

type Site

type Site struct {
	Name     string
	Error    error
	Pages    []*Page
	Problems []ProblemEvent
	// contains filtered or unexported fields
}

Site contains a meta information about a website.

func NewSite

func NewSite(rawURL string) *Site

NewSite returns new instance of the website.

func (*Site) Fetch

func (s *Site) Fetch(crawler Crawler) error

Fetch runs the website crawler and starts listen its events to build a website tree.

type WalkEvent

type WalkEvent struct {
	Page string
	Href string
	// contains filtered or unexported fields
}

WalkEvent contains information about a page and a link located on it.