engine

package
Version: v0.0.0-...-632f8f1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 3, 2019 License: MIT Imports: 1 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func CreateWorkers

func CreateWorkers(out chan ParseResult, s Scheduler, f Fetcher)

CreateWorkers

Types

type ConcurrentEngine

type ConcurrentEngine struct {
	Scheduler   Scheduler
	Fetcher     Fetcher
	WorkerCount int
	Writer      WriteWorker
}

ConcurrentEngine 开启并发爬虫采集器

func (*ConcurrentEngine) Run

func (c *ConcurrentEngine) Run(seeds ...Request)

Run

type EtcdSave

type EtcdSave interface {
	EtcdWrite(host, name string, endpoint []string) chan pb.Profile
}

type Fetcher

type Fetcher interface {
	Fetch(string) ([]byte, error)
	Work(Request) (ParseResult, error)
	GetProxy() (string, error)
}

Fetcher

type GSaver

type GSaver interface {
	GrpcWrite(host string) chan pb.Profile
}

GrpcSaver

type ParseResult

type ParseResult struct {
	Requests []Request
	Items    []pb.Profile
}

ParseResult

type Request

type Request struct {
	Url        string
	ParserFunc func([]byte) ParseResult
}

Request

type RpcSaver

type RpcSaver interface {
	RpcWrite(interface{}, *string) error
}

RpcSaver

type Saver

type Saver interface {
	Write(...string) chan interface{}
}

Writer

type Scheduler

type Scheduler interface {
	Submit(Request)
	ConfigWorkerChan(chan Request)
	WorkChanFree(chan Request)
	Begin()
}

Scheduler

type WriteWorker

type WriteWorker struct {
	Payload  chan pb.Profile
	Storage  Saver
	RpcSaver RpcSaver
	GSaver   GSaver
	EtcdSave EtcdSave
}

WriteWorker

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL