scheduler

Documentation

Overview

Author: Aosen Data: 2016-01-07 QQ: 316052486 Desc: 基于队列的调度策略队列的容量可动态增加，不会产生阻塞，但无法实现分布式.

Author: Aosen Data: 2016-01-07 QQ: 316052486 Desc:最简单的调度策略,当request超过1024会发生阻塞.

Index

• type MysqlScheduler
  • func NewMysqlScheduler(name, dbinfo string) *MysqlScheduler
  • func (self *MysqlScheduler) Count() int
  • func (self *MysqlScheduler) Init(dbinfo string) *MysqlScheduler
  • func (self *MysqlScheduler) Poll() *robot.Request
  • func (self *MysqlScheduler) Push(requ *robot.Request)
• type QueueScheduler
  • func NewQueueScheduler(rmDuplicate bool) *QueueScheduler
  • func (self *QueueScheduler) Count() int
  • func (self *QueueScheduler) Poll() *robot.Request
  • func (self *QueueScheduler) Push(requ *robot.Request)
• type RedisScheduler
  • func NewRedisScheduler(options RedisSchedulerOptions) *RedisScheduler
  • func (self *RedisScheduler) Count() int
  • func (self *RedisScheduler) Init() *RedisScheduler
  • func (self *RedisScheduler) Poll() *robot.Request
  • func (self *RedisScheduler) Push(requ *robot.Request)
• type RedisSchedulerOptions
• type Requestlist
• type SimpleScheduler
  • func NewSimpleScheduler() *SimpleScheduler
  • func (this *SimpleScheduler) Count() int
  • func (this *SimpleScheduler) Poll() *robot.Request
  • func (this *SimpleScheduler) Push(requ *robot.Request)

Types

type MysqlScheduler

type MysqlScheduler struct {
	// contains filtered or unexported fields
}

func NewMysqlScheduler

func NewMysqlScheduler(name, dbinfo string) *MysqlScheduler

func (*MysqlScheduler) Count

func (self *MysqlScheduler) Count() int

func (*MysqlScheduler) Init

func (self *MysqlScheduler) Init(dbinfo string) *MysqlScheduler

func (*MysqlScheduler) Poll

func (self *MysqlScheduler) Poll() *robot.Request

func (*MysqlScheduler) Push

func (self *MysqlScheduler) Push(requ *robot.Request)

type QueueScheduler

type QueueScheduler struct {
	// contains filtered or unexported fields
}

func NewQueueScheduler

func NewQueueScheduler(rmDuplicate bool) *QueueScheduler

func (*QueueScheduler) Count

func (self *QueueScheduler) Count() int

func (*QueueScheduler) Poll

func (self *QueueScheduler) Poll() *robot.Request

func (*QueueScheduler) Push

func (self *QueueScheduler) Push(requ *robot.Request)

type RedisScheduler

type RedisScheduler struct {
	// contains filtered or unexported fields
}

func NewRedisScheduler

func NewRedisScheduler(options RedisSchedulerOptions) *RedisScheduler

func (*RedisScheduler) Count

func (self *RedisScheduler) Count() int

func (*RedisScheduler) Init

func (self *RedisScheduler) Init() *RedisScheduler

func (*RedisScheduler) Poll

func (self *RedisScheduler) Poll() *robot.Request

func (*RedisScheduler) Push

func (self *RedisScheduler) Push(requ *robot.Request)

type RedisSchedulerOptions

type RedisSchedulerOptions struct {
	//请求对流的名称
	RequestList string
	//已经抓取的url列表
	UrlList string
	//redis连接地址
	RedisAddr string
	//最大连接数
	MaxConn int
	MaxIdle int
	//是否做url去重
	ForbiddenDuplicateUrl bool
}

type Requestlist

type Requestlist struct {
	Id int64
	//爬虫名称
	Name string `orm:"size(20)"`
	//请求
	Requ       string    `orm:"type(text)"`
	Createtime time.Time `orm:"type(date)"`
}

type SimpleScheduler

type SimpleScheduler struct {
	// contains filtered or unexported fields
}

func NewSimpleScheduler

func NewSimpleScheduler() *SimpleScheduler

func (*SimpleScheduler) Count

func (this *SimpleScheduler) Count() int

func (*SimpleScheduler) Poll

func (this *SimpleScheduler) Poll() *robot.Request

func (*SimpleScheduler) Push

func (this *SimpleScheduler) Push(requ *robot.Request)