goxuiter

README

go-xuiter

A simple photo download crawler for blog xuite.

Documentation

Index

• func FolderExit(prefix, folder string) bool
• func SaveFile(client *http.Client, prefix, folder, filename string, url string) error
• type CollyController
  • func NewCollyController(qc SizeController, concurrency int) *CollyController
  • func (c *CollyController) AddRequest(r *colly.Request) error
  • func (c *CollyController) AddURL(url string) error
  • func (c *CollyController) OnQueueSizeMonitor(f QueueSizeCallback)
  • func (c *CollyController) Run()
• type NoOpController
  • func (n *NoOpController) CheckAndWait()
  • func (n *NoOpController) Pause()
  • func (n *NoOpController) Resume()
• type QueueSizeCallback
• type QueueSizeController
  • func NewQueueSizeController() *QueueSizeController
  • func (q *QueueSizeController) CheckAndWait()
  • func (q *QueueSizeController) Pause()
  • func (q *QueueSizeController) Resume()
• type SizeController

Functions

func FolderExit

func FolderExit(prefix, folder string) bool

func SaveFile

func SaveFile(client *http.Client, prefix, folder, filename string, url string) error

Types

type CollyController

type CollyController struct {
	*colly.Collector
	// contains filtered or unexported fields
}

func NewCollyController

func NewCollyController(qc SizeController, concurrency int) *CollyController

func (*CollyController) AddRequest

func (c *CollyController) AddRequest(r *colly.Request) error

func (*CollyController) AddURL

func (c *CollyController) AddURL(url string) error

func (*CollyController) OnQueueSizeMonitor

func (c *CollyController) OnQueueSizeMonitor(f QueueSizeCallback)

func (*CollyController) Run

func (c *CollyController) Run()

type NoOpController

type NoOpController struct{}

func (*NoOpController) CheckAndWait

func (n *NoOpController) CheckAndWait()

func (*NoOpController) Pause

func (n *NoOpController) Pause()

func (*NoOpController) Resume

func (n *NoOpController) Resume()

type QueueSizeCallback

type QueueSizeCallback func(size int)

type QueueSizeController

type QueueSizeController struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewQueueSizeController

func NewQueueSizeController() *QueueSizeController

func (*QueueSizeController) CheckAndWait

func (q *QueueSizeController) CheckAndWait()

func (*QueueSizeController) Pause

func (q *QueueSizeController) Pause()

func (*QueueSizeController) Resume

func (q *QueueSizeController) Resume()

type SizeController

type SizeController interface {
	Pause()
	Resume()
	CheckAndWait()
}