Documentation

Overview

    Package request implements request entity contains url and other relevant informaion.

    Index

    Constants

    This section is empty.

    Variables

    This section is empty.

    Functions

    This section is empty.

    Types

    type Request

    type Request struct {
    	Url string
    
    	// Responce type: html json jsonp text
    	RespType string
    
    	// GET POST
    	Method string
    
    	// POST data
    	Postdata string
    
    	// name for marking url and distinguish different urls in PageProcesser and Pipeline
    	Urltag string
    
    	// http header
    	Header http.Header
    
    	// http cookies
    	Cookies []*http.Cookie
    
    	//proxy host   example='localhost:80'
    	ProxyHost string
    
    	Meta interface{}
    	// contains filtered or unexported fields
    }

      Request represents object waiting for being crawled.

      func NewRequest

      func NewRequest(url string, respType string, urltag string, method string,
      	postdata string, header http.Header, cookies []*http.Cookie,
      	checkRedirect func(req *http.Request, via []*http.Request) error,
      	meta interface{}) *Request

      func NewRequestWithHeaderFile

      func NewRequestWithHeaderFile(url string, respType string, headerFile string) *Request

      func NewRequestWithProxy

      func NewRequestWithProxy(url string, respType string, urltag string, method string,
      	postdata string, header http.Header, cookies []*http.Cookie, proxyHost string,
      	checkRedirect func(req *http.Request, via []*http.Request) error,
      	meta interface{}) *Request

      func (*Request) AddHeaderFile

      func (this *Request) AddHeaderFile(headerFile string) *Request

        point to a json file

        xxx.json
        

        {

        "User-Agent":"curl/7.19.3 (i386-pc-win32) libcurl/7.19.3 OpenSSL/1.0.0d",
        "Referer":"http://weixin.sogou.com/gzh?openid=oIWsFt6Sb7aZmuI98AU7IXlbjJps",
        "Cookie":""
        

        }

        func (*Request) AddProxyHost

        func (this *Request) AddProxyHost(host string) *Request

          @host http://localhost:8765/

          func (*Request) GetCookies

          func (this *Request) GetCookies() []*http.Cookie

          func (*Request) GetHeader

          func (this *Request) GetHeader() http.Header

          func (*Request) GetMeta

          func (this *Request) GetMeta() interface{}

          func (*Request) GetMethod

          func (this *Request) GetMethod() string

          func (*Request) GetPostdata

          func (this *Request) GetPostdata() string

          func (*Request) GetProxyHost

          func (this *Request) GetProxyHost() string

          func (*Request) GetRedirectFunc

          func (this *Request) GetRedirectFunc() func(req *http.Request, via []*http.Request) error

          func (*Request) GetResponceType

          func (this *Request) GetResponceType() string

          func (*Request) GetUrl

          func (this *Request) GetUrl() string

          func (*Request) GetUrlTag

          func (this *Request) GetUrlTag() string

          Source Files