Versions in this module Expand all Collapse all v1 v1.7.1 Aug 15, 2026 Changes in this version + var ErrMaxDepthReached = errors.New("max depth reached") + var ErrOutOfScope = errors.New("out of scope") + func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, ...) (*retryablehttp.Client, *fastdialer.Dialer, error) + func IsThrottled(statusCode int) bool + type CrawlSession struct + Browser *rod.Browser + CancelFunc context.CancelFunc + Ctx context.Context + Hostname string + HttpClient *retryablehttp.Client + Queue *queue.Queue + URL *url.URL + type DoRequestFunc func(crawlSession *CrawlSession, req *navigation.Request) (*navigation.Response, error) + type RedirectCallback func(resp *http.Response, depth int) + type Shared struct + DomainPageCounter sync.Map + Headers map[string]string + Jar *httputil.CookieJar + KnownFiles *files.KnownFiles + Options *types.CrawlerOptions + PathTrie *utils.PathTrie + func NewShared(options *types.CrawlerOptions) (*Shared, error) + func (s *Shared) ApplyBackoff(host string) + func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error + func (s *Shared) DomainCounter(domain string) *atomic.Int64 + func (s *Shared) Enqueue(queue *queue.Queue, navigationRequests ...*navigation.Request) + func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error) + func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, ...) + func (s *Shared) RecordSuccess(host string) + func (s *Shared) RecordThrottle(host string, statusCode int) + func (s *Shared) ValidateScope(URL string, root string) bool
Go to main page