request

Documentation

Overview

Package request implements request entity contains url and other relevant informaion.

Index

• type Request
  • func NewRequest(url string, respType string, urltag string, method string, postdata string, ...) *Request
  • func NewRequestWithHeaderFile(url string, respType string, headerFile string) *Request
  • func NewRequestWithProxy(url string, respType string, urltag string, method string, postdata string, ...) *Request
  • func (this *Request) AddHeaderFile(headerFile string) *Request
  • func (this *Request) AddProxyHost(host string) *Request
  • func (this *Request) GetCookies() []*http.Cookie
  • func (this *Request) GetHeader() http.Header
  • func (this *Request) GetMeta() interface{}
  • func (this *Request) GetMethod() string
  • func (this *Request) GetPostdata() string
  • func (this *Request) GetProxyHost() string
  • func (this *Request) GetRedirectFunc() func(req *http.Request, via []*http.Request) error
  • func (this *Request) GetResponceType() string
  • func (this *Request) GetUrl() string
  • func (this *Request) GetUrlTag() string

Types

type Request

type Request struct {
	Url string

	// Responce type: html json jsonp text
	RespType string

	// GET POST
	Method string

	// POST data
	Postdata string

	// name for marking url and distinguish different urls in PageProcesser and Pipeline
	Urltag string

	// http header
	Header http.Header

	// http cookies
	Cookies []*http.Cookie

	//proxy host   example='localhost:80'
	ProxyHost string

	Meta interface{}
	// contains filtered or unexported fields
}

Request represents object waiting for being crawled.

func NewRequest

func NewRequest(url string, respType string, urltag string, method string,
	postdata string, header http.Header, cookies []*http.Cookie,
	checkRedirect func(req *http.Request, via []*http.Request) error,
	meta interface{}) *Request

func NewRequestWithHeaderFile

func NewRequestWithHeaderFile(url string, respType string, headerFile string) *Request

func NewRequestWithProxy

func NewRequestWithProxy(url string, respType string, urltag string, method string,
	postdata string, header http.Header, cookies []*http.Cookie, proxyHost string,
	checkRedirect func(req *http.Request, via []*http.Request) error,
	meta interface{}) *Request

func (*Request) AddHeaderFile

func (this *Request) AddHeaderFile(headerFile string) *Request

point to a json file

xxx.json

{
	"User-Agent":"curl/7.19.3 (i386-pc-win32) libcurl/7.19.3 OpenSSL/1.0.0d",
	"Referer":"http://weixin.sogou.com/gzh?openid=oIWsFt6Sb7aZmuI98AU7IXlbjJps",
	"Cookie":""
}

func (*Request) AddProxyHost

func (this *Request) AddProxyHost(host string) *Request

@host http://localhost:8765/

func (*Request) GetCookies

func (this *Request) GetCookies() []*http.Cookie

func (*Request) GetHeader

func (this *Request) GetHeader() http.Header

func (*Request) GetMeta

func (this *Request) GetMeta() interface{}

func (*Request) GetMethod

func (this *Request) GetMethod() string

func (*Request) GetPostdata

func (this *Request) GetPostdata() string

func (*Request) GetProxyHost

func (this *Request) GetProxyHost() string

func (*Request) GetRedirectFunc

func (this *Request) GetRedirectFunc() func(req *http.Request, via []*http.Request) error

func (*Request) GetResponceType

func (this *Request) GetResponceType() string

func (*Request) GetUrl

func (this *Request) GetUrl() string

func (*Request) GetUrlTag

func (this *Request) GetUrlTag() string