Versions in this module Expand all Collapse all v0 v0.1.6 Nov 24, 2022 v0.1.5 Nov 24, 2022 Changes in this version + type Fetcher interface + Close func() error + Fetch func(req Request) (Response, error) + type Logger interface + Close func() error + Send func(rep Report) + type Option func(*WBot) + func SetFetcher(f Fetcher) Option + func SetFilter(allowed, disallowed []string) Option + func SetLogger(l Logger) Option + func SetMaxBodySize(size int64) Option + func SetMaxDepth(depth int32) Option + func SetParallel(parallel int) Option + func SetProxies(proxies []string) Option + func SetQueue(q Queue) Option + func SetRateLimit(rate int, interval time.Duration) Option + func SetStore(s Store) Option + func SetUserAgents(agents []string) Option + type Param struct + MaxBodySize int64 + Proxy string + Referer string + UserAgent string + type Queue interface + Close func() error + Dequeue func() (Request, error) + Enqueue func(req Request) error + Next func() bool + type Report struct + Depth int32 + Err error + RequestURL string + Status int + type Request struct + BaseDomain string + Depth int32 + Param Param + URL *url.URL + func (r *Request) AbsURL(u string) (*url.URL, error) + type Response struct + Body []byte + Depth int32 + NextURLs []string + Status int + URL *url.URL + type Store interface + Close func() error + Visited func(link string) bool + type WBot struct + func NewWBot(opts ...Option) *WBot + func (wb *WBot) Close() + func (wb *WBot) Crawl(link string) error + func (wb *WBot) SetOptions(opts ...Option) + func (wb *WBot) Stream() <-chan Response
Go to main page