Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security Policy
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

fetchbot

package module

Go to main page

Versions in this module

v1

v1.4.0

May 19, 2021

v1.2.0

Sep 11, 2019

v1.1.2

Sep 4, 2017

v1.1.1

Sep 4, 2017

v1.1.0

Jul 6, 2017

Changes in this version

type Queue

+ func (q *Queue) Done() <-chan struct{}

v1.0.0

Mar 24, 2017

Changes in this version

+ const DefaultCrawlDelay

+ const DefaultUserAgent

+ const DefaultWorkerIdleTTL

+ var ErrDisallowed = errors.New("fetchbot: disallowed by robots.txt")

+ var ErrEmptyHost = errors.New("fetchbot: invalid empty host")

+ var ErrQueueClosed = errors.New("fetchbot: send on a closed queue")

+ type BasicAuthProvider interface

+ BasicAuth func() (user string, pwd string)

+ type Cmd struct

+ M string

+ U *url.URL

+ func (c *Cmd) Method() string

+ func (c *Cmd) URL() *url.URL

+ type Command interface

+ Method func() string

+ URL func() *url.URL

+ type Context struct

+ Cmd Command

+ Q *Queue

+ type CookiesProvider interface

+ Cookies func() []*http.Cookie

+ type DebugInfo struct

+ NumHosts int

+ type Doer interface

+ Do func(*http.Request) (*http.Response, error)

+ type Fetcher struct

+ AutoClose bool

+ CrawlDelay time.Duration

+ DisablePoliteness bool

+ Handler Handler

+ HttpClient Doer

+ UserAgent string

+ WorkerIdleTTL time.Duration

+ func New(h Handler) *Fetcher

+ func (f *Fetcher) Debug() <-chan *DebugInfo

+ func (f *Fetcher) Start() *Queue

+ type Handler interface

+ Handle func(*Context, *http.Response, error)

+ type HandlerCmd struct

+ func NewHandlerCmd(method, rawURL string, fn func(*Context, *http.Response, error)) (*HandlerCmd, error)

+ type HandlerFunc func(*Context, *http.Response, error)

+ func (h HandlerFunc) Handle(ctx *Context, res *http.Response, err error)

+ type HeaderProvider interface

+ Header func() http.Header

+ type Mux struct

+ DefaultHandler Handler

+ func NewMux() *Mux

+ func (mux *Mux) Handle(ctx *Context, res *http.Response, err error)

+ func (mux *Mux) HandleError(err error, h Handler)

+ func (mux *Mux) HandleErrors(h Handler)

+ func (mux *Mux) Response() *ResponseMatcher

+ type Queue struct

+ func (q *Queue) Block()

+ func (q *Queue) Cancel() error

+ func (q *Queue) Close() error

+ func (q *Queue) Send(c Command) error

+ func (q *Queue) SendString(method string, rawurl ...string) (int, error)

+ func (q *Queue) SendStringGet(rawurl ...string) (int, error)

+ func (q *Queue) SendStringHead(rawurl ...string) (int, error)

+ type ReaderProvider interface

+ Reader func() io.Reader

+ type ResponseMatcher struct

+ func (r *ResponseMatcher) ContentType(ct string) *ResponseMatcher

+ func (r *ResponseMatcher) Custom(predicate func(*http.Response) bool) *ResponseMatcher

+ func (r *ResponseMatcher) Handler(h Handler) *ResponseMatcher

+ func (r *ResponseMatcher) Host(host string) *ResponseMatcher

+ func (r *ResponseMatcher) Method(m string) *ResponseMatcher

+ func (r *ResponseMatcher) Path(p string) *ResponseMatcher

+ func (r *ResponseMatcher) Scheme(scheme string) *ResponseMatcher

+ func (r *ResponseMatcher) Status(code int) *ResponseMatcher

+ func (r *ResponseMatcher) StatusRange(min, max int) *ResponseMatcher

+ type ValuesProvider interface

+ Values func() url.Values