crawler

Documentation

Index

• type Link
• type WebCrawler
  • func CreateWebCrawler(webCrawlerOpts WebCrawlerOpts) *WebCrawler
  • func (wc *WebCrawler) Crawl()
• type WebCrawlerOpts

Types

type Link

type Link struct {
	From string
	To   string
}

type WebCrawler

type WebCrawler struct {
	DomainName   string
	MaxDepth     int
	MaxPages     int
	Spider       colly.Collector
	CrawlDelayIn time.Duration
	Links        []Link
	Visited      map[string]bool
	Count        int
}

func CreateWebCrawler

func CreateWebCrawler(webCrawlerOpts WebCrawlerOpts) *WebCrawler

func (*WebCrawler) Crawl

func (wc *WebCrawler) Crawl()

type WebCrawlerOpts

type WebCrawlerOpts struct {
	DomainName     string
	MaxDepth       int
	MaxPages       int
	CrawlDelay     time.Duration
	UserAgent      string
	MaxParallelism int
}