engine

Documentation

Index

• func CreateWorkers(out chan ParseResult, s Scheduler, f Fetcher)
• type ConcurrentEngine
  • func (c *ConcurrentEngine) Run(seeds ...Request)
• type EtcdSave
• type Fetcher
• type GSaver
• type ParseResult
• type Request
• type RpcSaver
• type Saver
• type Scheduler
• type WriteWorker

Functions

func CreateWorkers

func CreateWorkers(out chan ParseResult, s Scheduler, f Fetcher)

CreateWorkers

Types

type ConcurrentEngine

type ConcurrentEngine struct {
	Scheduler   Scheduler
	Fetcher     Fetcher
	WorkerCount int
	Writer      WriteWorker
}

ConcurrentEngine 开启并发爬虫采集器

func (*ConcurrentEngine) Run

func (c *ConcurrentEngine) Run(seeds ...Request)

Run

type EtcdSave

type EtcdSave interface {
	EtcdWrite(host, name string, endpoint []string) chan pb.Profile
}

type Fetcher

type Fetcher interface {
	Fetch(string) ([]byte, error)
	Work(Request) (ParseResult, error)
	GetProxy() (string, error)
}

Fetcher

type GSaver

type GSaver interface {
	GrpcWrite(host string) chan pb.Profile
}

GrpcSaver

type ParseResult

type ParseResult struct {
	Requests []Request
	Items    []pb.Profile
}

ParseResult

type Request

type Request struct {
	Url        string
	ParserFunc func([]byte) ParseResult
}

Request

type RpcSaver

type RpcSaver interface {
	RpcWrite(interface{}, *string) error
}

RpcSaver

type Saver

type Saver interface {
	Write(...string) chan interface{}
}

Writer

type Scheduler

type Scheduler interface {
	Submit(Request)
	ConfigWorkerChan(chan Request)
	WorkChanFree(chan Request)
	Begin()
}

Scheduler

type WriteWorker

type WriteWorker struct {
	Payload  chan pb.Profile
	Storage  Saver
	RpcSaver RpcSaver
	GSaver   GSaver
	EtcdSave EtcdSave
}

WriteWorker