scraper

Documentation

Index

• Variables
• func ScanFile(src string, conditions ...func(string)) error
• type Client
• type Config
• type Crawler
  • func NewCrawler(params CrawlerParams) (c *Crawler, err error)
  • func (c *Crawler) Crawl(numWorkers uint8) error
  • func (c *Crawler) SetApplicationFileHandler(f ResponseHandler)
  • func (c *Crawler) SetLogger(l logger.Client)
  • func (c *Crawler) SetTextFileHandler(f ResponseHandler)
• type CrawlerParams
• type GParser
  • func (p *GParser) Parse(b []byte) (d []*obj.Document, err error)
• type GoogleClient
  • func NewGoogleClient() *GoogleClient
  • func (g *GoogleClient) Scrape(symbol, ftype string) (d []*obj.Document, err error)
• type ResponseHandler

Variables

var (
	ErrFileName = errors.New("file name not found")
)

Functions

func ScanFile

func ScanFile(src string, conditions ...func(string)) error

Types

type Client

type Client interface {
	Scrape(symbol, filetype string) ([]*obj.Document, error)
}

type Config

type Config struct {
}

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(params CrawlerParams) (c *Crawler, err error)

func (*Crawler) Crawl

func (c *Crawler) Crawl(numWorkers uint8) error

func (*Crawler) SetApplicationFileHandler

func (c *Crawler) SetApplicationFileHandler(f ResponseHandler)

func (*Crawler) SetLogger

func (c *Crawler) SetLogger(l logger.Client)

func (*Crawler) SetTextFileHandler

func (c *Crawler) SetTextFileHandler(f ResponseHandler)

type CrawlerParams

type CrawlerParams struct {
	Limit uint64
	Seeds []string
}

type GParser

type GParser struct {
}

func (*GParser) Parse

func (p *GParser) Parse(b []byte) (d []*obj.Document, err error)

type GoogleClient

type GoogleClient struct {
	// contains filtered or unexported fields
}

func NewGoogleClient

func NewGoogleClient() *GoogleClient

func (*GoogleClient) Scrape

func (g *GoogleClient) Scrape(symbol, ftype string) (d []*obj.Document, err error)

type ResponseHandler

type ResponseHandler func(*http.Response) error