wbot

README

WBot

A configurable, thread-safe web crawler, provides a minimal interface for crawling and downloading web pages.

Features

• Clean minimal API.
• Configurable: MaxDepth, MaxBodySize, Rate Limit, Parrallelism, User Agent & Proxy rotation.
• Memory-efficient, thread-safe.
• Provides built-in interface: Fetcher, Store, Queue & a Logger.

API

WBot provides a minimal API for crawling web pages.

 Run(links ...string) error
 OnReponse(fn func(*wbot.Response))
 Metrics() map[string]int64
 Shutdown()

Usage

package main

import (
 "fmt"
 "log"

 "github.com/rs/zerolog"

 "github.com/twiny/wbot"
 "github.com/twiny/wbot/pkg/api"
)

func main() {
 bot := wbot.New(
  wbot.WithParallel(50),
  wbot.WithMaxDepth(5),
  wbot.WithRateLimit(&api.RateLimit{
   Hostname: "*",
   Rate:     "10/1s",
  }),
  wbot.WithLogLevel(zerolog.DebugLevel),
 )
 defer bot.Shutdown()

 // read responses
 bot.OnReponse(func(resp *api.Response) {
  fmt.Printf("crawled: %s\n", resp.URL.String())
 })

 if err := bot.Run(
  "https://go.dev/",
 ); err != nil {
  log.Fatal(err)
 }

 log.Printf("finished crawling\n")
}

Wiki

More documentation can be found in the wiki.

Bugs

Bugs or suggestions? Please visit the issue tracker.

Documentation

Index

• type Crawler
  • func New(opts ...Option) *Crawler
  • func (c *Crawler) Metrics() map[string]int64
  • func (c *Crawler) OnReponse(fn func(*api.Response))
  • func (c *Crawler) Run(links ...string) error
  • func (c *Crawler) Shutdown()
• type Option
  • func WithFetcher(fetcher api.Fetcher) Option
  • func WithFilter(rules ...*api.FilterRule) Option
  • func WithLogLevel(level zerolog.Level) Option
  • func WithMaxDepth(maxDepth int32) Option
  • func WithParallel(parallel int) Option
  • func WithProxies(proxies []string) Option
  • func WithQueue(queue api.Queue) Option
  • func WithRateLimit(rates ...*api.RateLimit) Option
  • func WithStore(store api.Store) Option
  • func WithUserAgents(userAgents []string) Option

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func New

func New(opts ...Option) *Crawler

func (*Crawler) Metrics

func (c *Crawler) Metrics() map[string]int64

func (*Crawler) OnReponse

func (c *Crawler) OnReponse(fn func(*api.Response))

func (*Crawler) Run

func (c *Crawler) Run(links ...string) error

func (*Crawler) Shutdown

func (c *Crawler) Shutdown()

type Option

type Option func(c *Crawler)

func WithFetcher

func WithFetcher(fetcher api.Fetcher) Option

func WithFilter

func WithFilter(rules ...*api.FilterRule) Option

func WithLogLevel

func WithLogLevel(level zerolog.Level) Option

func WithMaxDepth

func WithMaxDepth(maxDepth int32) Option

func WithParallel

func WithParallel(parallel int) Option

func WithProxies

func WithProxies(proxies []string) Option

func WithQueue

func WithQueue(queue api.Queue) Option

func WithRateLimit

func WithRateLimit(rates ...*api.RateLimit) Option

func WithStore

func WithStore(store api.Store) Option

func WithUserAgents

func WithUserAgents(userAgents []string) Option