dao

Documentation

Index

• Constants
• Variables
• type RuleStatus
• type TaskDao
  • func InitTaskDao(db *sqlx.DB) *TaskDao
  • func (this *TaskDao) AddNewTasks(tasks []types.CrawlTask) (int64, []sql.Result, error)
  • func (this *TaskDao) ConvertRuleToTask(rule types.CrawlRule) types.CrawlTask
  • func (this *TaskDao) GetWaitRules() ([]types.CrawlRule, error)
  • func (this *TaskDao) GetWaitingTasks() ([]types.CrawlTask, error)
  • func (this *TaskDao) SetTasksStatus(tasks []types.CrawlTask, status TaskStatus) (int64, error)
  • func (this *TaskDao) UpdateRules(rules []types.CrawlRule, taskAddedResults []sql.Result) (int64, error)
• type TaskStatus

Constants

const (
	RuleTable = "crawl_rules"
	TaskTable = "crawl_tasks"
)

Variables

var (
	ErrNotEqual = errors.New("not equal!")
	ErrNoTasks  = errors.New("no tasks!")
)

Types

type RuleStatus

type RuleStatus int32

const (
	RULE_PAUSE RuleStatus = -1 + iota
	RULE_NORMAL
	RULE_ADDED
)

type TaskDao

type TaskDao struct {
	// contains filtered or unexported fields
}

func InitTaskDao

func InitTaskDao(db *sqlx.DB) *TaskDao

func (*TaskDao) AddNewTasks

func (this *TaskDao) AddNewTasks(tasks []types.CrawlTask) (int64, []sql.Result, error)

func (*TaskDao) ConvertRuleToTask

func (this *TaskDao) ConvertRuleToTask(rule types.CrawlRule) types.CrawlTask

规则转为任务

func (*TaskDao) GetWaitRules

func (this *TaskDao) GetWaitRules() ([]types.CrawlRule, error)

从规则库读取下一批需调度的规则

func (*TaskDao) GetWaitingTasks

func (this *TaskDao) GetWaitingTasks() ([]types.CrawlTask, error)

选取status为0, 或status=2且调度时间已到的任务

func (*TaskDao) SetTasksStatus

func (this *TaskDao) SetTasksStatus(tasks []types.CrawlTask, status TaskStatus) (int64, error)

设置任务状态

func (*TaskDao) UpdateRules

func (this *TaskDao) UpdateRules(rules []types.CrawlRule, taskAddedResults []sql.Result) (int64, error)

根据任务添加结果，修改rule的状态

type TaskStatus

type TaskStatus int32

const (
	TASK_CANCELED TaskStatus = -1 + iota
	TASK_WAITING
	TASK_CRAWLING
	TASK_FINISH
	TASK_FAILED
)