Versions in this module Expand all Collapse all v0 v0.1.0 Aug 13, 2026 Changes in this version + const DefaultUserAgent + const MaxCrawlDelay + const MinUsableText + func DefaultDenyDomains() []string + func DomainOf(rawURL string) string + func HasStructuredData(html string) bool + func ProviderSupplied(outcomes []*store.FetchOutcome) map[string]bool + type Config struct + IgnoreRobots bool + MaxBytes int64 + MaxRedirects int + PerDomain limiter.Limit + RobotsTTL time.Duration + Timeout time.Duration + UserAgent string + type DomainDeny struct + func NewDomainDeny(domains []string) *DomainDeny + func (d *DomainDeny) Denied(host string) bool + type Fetcher interface + Fetch func(ctx context.Context, rawURL string) (*Result, error) + type Guard struct + AllowPrivateNetworks bool + AllowedPorts []int + AllowedSchemes []string + DialTimeout time.Duration + ExtraDeny []netip.Prefix + Resolver *net.Resolver + func (g *Guard) CheckAddr(addr netip.Addr) string + func (g *Guard) CheckURL(u *url.URL) error + func (g *Guard) DialContext(ctx context.Context, network, address string) (net.Conn, error) + type GuardError struct + Addr string + Host string + Reason string + URL string + func (e *GuardError) Error() string + type HTTPFetcher struct + func NewHTTP(cfg Config, opts Options) *HTTPFetcher + func (f *HTTPFetcher) Fetch(ctx context.Context, rawURL string) (*Result, error) + func (f *HTTPFetcher) Limiter() *limiter.Limiter + type Options struct + Deny *DomainDeny + Guard *Guard + Lim *limiter.Limiter + Log *slog.Logger + Transport func(base http.RoundTripper) http.RoundTripper + type Outcome string + const OutcomeBotBlock + const OutcomeConsentWall + const OutcomeExtractFailed + const OutcomeGuardDenied + const OutcomeJSRequired + const OutcomeNetworkError + const OutcomeNotFound + const OutcomeOK + const OutcomePaywall + const OutcomeProviderContent + const OutcomeRobotsDenied + const OutcomeServerError + const OutcomeStructuredOnly + const OutcomeTimeout + const OutcomeTooLarge + const OutcomeUnsupported + func ClassifyBody(html string, extractedLen int) Outcome + func ClassifyStatus(status int, body string) Outcome + func (o Outcome) Attempted() bool + func (o Outcome) CapabilityGap() bool + func (o Outcome) SystemWorking() bool + func (o Outcome) Usable() bool + type Result struct + Bytes int64 + Content []byte + ContentType string + Domain string + Duration time.Duration + Err string + FinalURL string + Outcome Outcome + StatusCode int + URL string + func (r *Result) Refine(extractedLen int) + type RobotsCache struct + Fetch func(ctx context.Context, robotsURL string) (status int, body []byte, err error) + func NewRobotsCache(ttl time.Duration, fetch func(context.Context, string) (int, []byte, error)) *RobotsCache + func (c *RobotsCache) Get(ctx context.Context, origin string) *Rules + type Rules struct + func AllowAllRules() *Rules + func DenyAllRules() *Rules + func ParseRobots(r io.Reader) *Rules + func (r *Rules) Allowed(userAgent, path string) bool + func (r *Rules) CrawlDelay(userAgent string) time.Duration
Go to main page