crawl

Documentation

Index

• func MatchMIMEInExts(MIME string, ftypes []string) (string, bool)
• type Crawler
  • func New(config *conf.Configuration) *Crawler
  • func (crawler *Crawler) Crawl(urlTopo *URLTopological)
  • func (crawler *Crawler) EnableCookie()
  • func (crawler *Crawler) SetProxy(proxy func(*http.Request) (*url.URL, error))
  • func (crawler *Crawler) SetTimeout(duration time.Duration)
• type URLTopological

Functions

func MatchMIMEInExts

func MatchMIMEInExts(MIME string, ftypes []string) (string, bool)

Types

type Crawler

type Crawler struct {
	Semaphore    chan bool
	URLTopoCh    chan *URLTopological
	Logger       *log.Logger
	Server       *headless.Server
	DownloadHTML bool
	// contains filtered or unexported fields
}

Crawler ...

func New

func New(config *conf.Configuration) *Crawler

New ...

func (*Crawler) Crawl

func (crawler *Crawler) Crawl(urlTopo *URLTopological)

Crawl ...

func (*Crawler) EnableCookie

func (crawler *Crawler) EnableCookie()

EnableCookie ...

func (*Crawler) SetProxy

func (crawler *Crawler) SetProxy(proxy func(*http.Request) (*url.URL, error))

SetProxy sets proxy for crawler

func (*Crawler) SetTimeout

func (crawler *Crawler) SetTimeout(duration time.Duration)

SetTimeout ...

type URLTopological

type URLTopological struct {
	URL   *url.URL
	Depth int
}