crawler

Documentation

Index

• type Crawler
  • func New(filter FilterFunc, yieldTitle YieldTitleFunc, yieldURL YieldURLFunc, ...) *Crawler
  • func (cr *Crawler) Feed(ctx context.Context, depth int, links ...string)
• type FilterFunc
• type Options
• type YieldErrorFunc
• type YieldTitleFunc
• type YieldURLFunc

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func New

func New(filter FilterFunc, yieldTitle YieldTitleFunc, yieldURL YieldURLFunc, yieldError YieldErrorFunc, ops Options) *Crawler

func (*Crawler) Feed

func (cr *Crawler) Feed(ctx context.Context, depth int, links ...string)

type FilterFunc

type FilterFunc func(ctx context.Context, r io.Reader, yieldTitle func(pos int, title string) error, yieldLink func(pos int, link string) error) error

type Options

type Options struct {
	Client *http.Client

	UserAgent string

	// Zero for unlimited depth
	Depth int

	// Zero for no limit
	ParallelRequestsPerHost int
}

type YieldErrorFunc

type YieldErrorFunc func(origin, link string, pos int, err error)

type YieldTitleFunc

type YieldTitleFunc func(depth, pos int, origin string, title string)

type YieldURLFunc

type YieldURLFunc func(depth, pos int, origin string, originalLink string, link *url.URL, external bool) bool