juemei

Documentation

Index

• Constants
• Variables
• func LoadDoc(cli *httpclient.Client, Url string) (*goquery.Document, error)
• type RedisStore
  • func NewRedisStore(cli *redis.Client) *RedisStore
  • func (r *RedisStore) Done(rets ...*ResolveResult) (*StoreResult, error)
  • func (r *RedisStore) Init() error
  • func (r *RedisStore) Jobs(count int) ([]string, error)
• type ResolveResult
  • func Resolve(cli *httpclient.Client, doc *goquery.Document) (*ResolveResult, error)
• type Store
• type StoreResult

Constants

const (
	RootURL  = "http://www.juemei.com"
	MaxWorks = 1
)

const (
	RedisKeyPrefix = "juemei_" // redis key 前缀

	KeySetToDo     = RedisKeyPrefix + "todo"      // 未爬取的页面链接
	KeySetDoing    = RedisKeyPrefix + "doing"     // 正在处理的页面
	KeySetDone     = RedisKeyPrefix + "done"      // 已爬取完成的页面
	KeySetImgs     = RedisKeyPrefix + "imgs"      // 爬取的图片链接
	KeySetOutLinks = RedisKeyPrefix + "out_links" // 外链
)

Variables

var (
	BlackPrefix = []string{
		RootURL,
	}
	IgnorePrefix = []string{
		"#",
	}
)

Functions

func LoadDoc

func LoadDoc(cli *httpclient.Client, Url string) (*goquery.Document, error)

Types

type RedisStore

type RedisStore struct {
	*redis.Client
}

func NewRedisStore

func NewRedisStore(cli *redis.Client) *RedisStore

func (*RedisStore) Done

func (r *RedisStore) Done(rets ...*ResolveResult) (*StoreResult, error)

func (*RedisStore) Init

func (r *RedisStore) Init() error

func (*RedisStore) Jobs

func (r *RedisStore) Jobs(count int) ([]string, error)

type ResolveResult

type ResolveResult struct {
	URL      string
	Imgs     []string
	Links    []string
	OutLinks []string
}

func Resolve

func Resolve(cli *httpclient.Client, doc *goquery.Document) (*ResolveResult, error)

type Store

type Store interface {
}

type StoreResult

type StoreResult struct {
	URLs                  []string
	Imgs, Links, OutLinks int64
}