goclient

Documentation

Index

• Constants
• type CrawlJob
  • func NewCrawlJob(svrHost, svrPort string) *CrawlJob
  • func (cj *CrawlJob) AddJS(typ pb.PageReqType, url, js, metaStr string) error
  • func (cj *CrawlJob) AddPage(url, metaStr string) error
  • func (cj *CrawlJob) IsAlive() bool
  • func (cj *CrawlJob) Run()
  • func (cj *CrawlJob) SetCallbackXpathMatch(mdata KVMap)
  • func (cj *CrawlJob) SetCallbackXpathRegexp(mdata KVMap)
  • func (cj *CrawlJob) SetLogin(loginUrl string, loginPayload, loginParseXpath KVMap, loginSuccessCheck KVMap)
  • func (cj *CrawlJob) SetLoginChrome(loginUrl string, loginJS string, loginSuccessCheck KVMap)
  • func (cj *CrawlJob) Start()
  • func (cj *CrawlJob) Stop()
• type KVMap
• type PageHTML

Constants

const (
	PageReqType_BUILTINJS = pb.PageReqType_BUILTINJS
	PageReqType_JSCRIPT   = pb.PageReqType_JSCRIPT
)

Types

type CrawlJob

type CrawlJob struct {
	SeedURL                 string
	MinDelay                int32
	MaxDelay                int32
	Follow                  bool
	CallbackUrlRegexp       string
	FollowUrlRegexp         string
	CallbackXpathMatch      []*pb.KVP
	CallbackXpathRegexp     []*pb.KVP
	MaxConcurrentRequests   int32
	Useragent               string
	Impolite                bool
	Depth                   int32
	Repeat                  bool
	Frequency               *google_protobuf1.Duration
	Firstrun                *google_protobuf.Timestamp
	UnsafeNormalizeURL      bool
	Login                   bool
	LoginUrl                string
	LoginJS                 string
	LoginPayload            []*pb.KVP
	LoginParseFields        bool
	LoginParseXpath         []*pb.KVP
	LoginSuccessCheck       *pb.KVP
	CheckLoginAfterEachPage bool
	Chrome                  bool
	ChromeBinary            string
	DomLoadTime             int32
	NetworkIface            string
	CancelOnDisconnect      bool
	CheckContent            bool
	Prefetch                bool

	Callback    func(*PageHTML, *CrawlJob)
	UsePageChan bool
	PageChan    chan *pb.PageHTML
	// contains filtered or unexported fields
}

func NewCrawlJob

func NewCrawlJob(svrHost, svrPort string) *CrawlJob

func (*CrawlJob) AddJS

func (cj *CrawlJob) AddJS(typ pb.PageReqType, url, js, metaStr string) error

func (*CrawlJob) AddPage

func (cj *CrawlJob) AddPage(url, metaStr string) error

func (*CrawlJob) IsAlive

func (cj *CrawlJob) IsAlive() bool

func (*CrawlJob) Run

func (cj *CrawlJob) Run()

func (*CrawlJob) SetCallbackXpathMatch

func (cj *CrawlJob) SetCallbackXpathMatch(mdata KVMap)

func (*CrawlJob) SetCallbackXpathRegexp

func (cj *CrawlJob) SetCallbackXpathRegexp(mdata KVMap)

func (*CrawlJob) SetLogin

func (cj *CrawlJob) SetLogin(loginUrl string, loginPayload, loginParseXpath KVMap, loginSuccessCheck KVMap)

func (*CrawlJob) SetLoginChrome

func (cj *CrawlJob) SetLoginChrome(loginUrl string, loginJS string, loginSuccessCheck KVMap)

func (*CrawlJob) Start

func (cj *CrawlJob) Start()

func (*CrawlJob) Stop

func (cj *CrawlJob) Stop()

type KVMap

type KVMap = map[string]string

type PageHTML

type PageHTML = pb.PageHTML