crawler

Documentation

Index

• type Crawler
  • func New(fetcher *Fetcher, store store.Store, src source.Source, workers int) *Crawler
  • func (c *Crawler) Run(ctx context.Context, opts RunOptions) error
• type Fetcher
  • func NewFetcher(requestsPerSecond int, maxRetry int, userAgent string) *Fetcher
  • func (f *Fetcher) Close()
  • func (f *Fetcher) Fetch(ctx context.Context, url string) ([]byte, error)
• type RunOptions

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func New

func New(fetcher *Fetcher, store store.Store, src source.Source, workers int) *Crawler

func (*Crawler) Run

func (c *Crawler) Run(ctx context.Context, opts RunOptions) error

type Fetcher

type Fetcher struct {
	// contains filtered or unexported fields
}

func NewFetcher

func NewFetcher(requestsPerSecond int, maxRetry int, userAgent string) *Fetcher

func (*Fetcher) Close

func (f *Fetcher) Close()

func (*Fetcher) Fetch

func (f *Fetcher) Fetch(ctx context.Context, url string) ([]byte, error)

Fetch downloads a URL with rate limiting, retries, and exponential backoff.

type RunOptions

type RunOptions struct {
	Resume      bool
	RetryFailed bool
}