discover

Documentation

Index

• func CrawlSameOrigin(parent context.Context, client *http.Client, base *url.URL, userAgent string, ...) ([]string, error)
• func FetchRobots(parent context.Context, client *http.Client, base *url.URL, userAgent string, ...) ([]string, []string, error)
• func FetchSitemaps(parent context.Context, client *http.Client, base *url.URL, seeds []string, ...) ([]string, error)

Functions

func CrawlSameOrigin

func CrawlSameOrigin(parent context.Context, client *http.Client, base *url.URL, userAgent string, timeout time.Duration, maxDepth int, maxPages int, maxBytes int64) ([]string, error)

func FetchRobots

func FetchRobots(parent context.Context, client *http.Client, base *url.URL, userAgent string, timeout time.Duration, maxBytes int64) ([]string, []string, error)

func FetchSitemaps

func FetchSitemaps(parent context.Context, client *http.Client, base *url.URL, seeds []string, userAgent string, timeout time.Duration, maxBytes int64, maxFetch int) ([]string, error)