crawler

Documentation

Index

• Variables
• type Crawler
  • func (self *Crawler) Close()
  • func (self *Crawler) InitEs() error
  • func (self *Crawler) InitTaskQueue(dir string) error
  • func (self *Crawler) LoadConfFromBytes(str []byte) error
  • func (self *Crawler) LoadConfFromFile(file string) error
  • func (self *Crawler) Process(task *types.Task) ([]types.Task, []map[string]interface{}, error)
  • func (self *Crawler) Save(item map[string]interface{}) error
  • func (self *Crawler) Test() (map[string]interface{}, error)

Variables

var (
	ErrEmptyCrawlerConf = errors.New("crawler/crawler.go empty CrawlerConf")
	ErrNilTask          = errors.New("crawler/crawler.go nil Task")
	ErrNilItem          = errors.New("crawler/crawler.go nil Item")
	ErrNilTaskQueue     = errors.New("crawler/crawler.go nil TaskQueue")
)

Types

type Crawler

type Crawler struct {
	Conf      *types.CrawlerConf
	TaskQueue *ds.Queue
	// contains filtered or unexported fields
}

func (*Crawler) Close

func (self *Crawler) Close()

func (*Crawler) InitEs

func (self *Crawler) InitEs() error

func (*Crawler) InitTaskQueue

func (self *Crawler) InitTaskQueue(dir string) error

func (*Crawler) LoadConfFromBytes

func (self *Crawler) LoadConfFromBytes(str []byte) error

func (*Crawler) LoadConfFromFile

func (self *Crawler) LoadConfFromFile(file string) error

func (*Crawler) Process

func (self *Crawler) Process(
	task *types.Task) ([]types.Task, []map[string]interface{}, error)

func (*Crawler) Save

func (self *Crawler) Save(item map[string]interface{}) error

func (*Crawler) Test

func (self *Crawler) Test() (map[string]interface{}, error)