crawlkit

Documentation

Index

• type BasicCrawler
  • func (c *BasicCrawler) Crawl(ctx context.Context, url string) (*http.Response, error)
  • func (c *BasicCrawler) QueueSize() int
• type HostConfig
• type SiteCrawler
  • func NewSiteCrawler(userAgent string, globalMaxRPS int, siteMaxRPS int) *SiteCrawler
  • func (c *SiteCrawler) Crawl(ctx context.Context, rawUrl string) (*http.Response, error)
  • func (c *SiteCrawler) GetConfig(hostname string) *HostConfig
  • func (c *SiteCrawler) QueueSize() int
• type WebCrawler
  • func NewBasicCrawler() WebCrawler

Types

type BasicCrawler

type BasicCrawler struct {
}

func (*BasicCrawler) Crawl

func (c *BasicCrawler) Crawl(ctx context.Context, url string) (*http.Response, error)

func (*BasicCrawler) QueueSize

func (c *BasicCrawler) QueueSize() int

type HostConfig

type HostConfig struct {
	sync.RWMutex
	MaxRPS int
	// contains filtered or unexported fields
}

type SiteCrawler

type SiteCrawler struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewSiteCrawler

func NewSiteCrawler(userAgent string, globalMaxRPS int, siteMaxRPS int) *SiteCrawler

func (*SiteCrawler) Crawl

func (c *SiteCrawler) Crawl(ctx context.Context, rawUrl string) (*http.Response, error)

func (*SiteCrawler) GetConfig

func (c *SiteCrawler) GetConfig(hostname string) *HostConfig

func (*SiteCrawler) QueueSize

func (c *SiteCrawler) QueueSize() int

type WebCrawler

type WebCrawler interface {
	Crawl(ctx context.Context, url string) (*http.Response, error)
	QueueSize() int
}

func NewBasicCrawler

func NewBasicCrawler() WebCrawler