jewel_crawler

README

jewel-crawler

Documentation

Index

• func SetLogLevel(level string)
• type Config
• type CrawlerEngine
  • func New(cfg *Config) *CrawlerEngine
  • func (p *CrawlerEngine) Close() error
  • func (p *CrawlerEngine) Push(ctx context.Context, queue string, task task.Task) error
  • func (p *CrawlerEngine) Start(ctx context.Context, maxExecuteCount int)

Functions

func SetLogLevel

func SetLogLevel(level string)

Types

type Config

type Config struct {
	Redis      *redis.Options
	Queue      string
	Concurrent int
}

type CrawlerEngine

type CrawlerEngine struct {
	Pipeline *crawler.PipeLine
	// contains filtered or unexported fields
}

func New

func New(cfg *Config) *CrawlerEngine

func (*CrawlerEngine) Close

func (p *CrawlerEngine) Close() error

func (*CrawlerEngine) Push

func (p *CrawlerEngine) Push(ctx context.Context, queue string, task task.Task) error

func (*CrawlerEngine) Start

func (p *CrawlerEngine) Start(ctx context.Context, maxExecuteCount int)

开启