ghttp

Documentation

Index

• Constants
• func ChromeDriverGet(urlStr, proxy string) (string, error)
• func CommonHttpMethods() []string
• func Get(url string, proxy string, timeout time.Duration, followRedirect bool) (response *http.Response, err error)
• func GetBigFile(url string, filename string) (string, error)
• func GetBytes(url string, proxy string, timeout time.Duration) ([]byte, error)
• func GetMap(url string, proxy string, timeout time.Duration) (map[string]interface{}, error)
• func GetParamHmacMD5Sign(secret, params string) (string, error)
• func GetParamHmacSHA1Sign(secret, params string) (string, error)
• func GetParamHmacSHA256Sign(secret, params string) (string, error)
• func GetParamHmacSHA512Sign(secret, params string) (string, error)
• func GetParamMD5Sign(secret, params string) (string, error)
• func GetProxy(client *http.Client) string
• func GetSHA(text string) (string, error)
• func GetString(url string, proxy string, timeout time.Duration) (string, error)
• func HeadlessGetBow(url string) (string, error)
• func HeadlessGetPhantom(url, proxy string) (string, error)
• func IsValidMethod(method string) bool
• func ListenAndServeNoWait(addr string) error
• func NewResponse(content []byte) *http.Response
• func NewSimpleRequest(ctx context.Context, httpMethod string, baseURL, refURL string, ...) (*http.Request, error)
• func ReadBodyBytes(response *http.Response) ([]byte, error)
• func ReadBodyString(response *http.Response) (string, error)
• func ReadFinalUrl(response *http.Response) (*url.URL, error)
• func Screenshot(urlStr, proxyServer, waitVisibleByQuery string, timeout time.Duration) (snapshotImage []byte, fullHtml string, err error)
• func ScreenshotElement(urlStr, sel string, res *[]byte) chromedp.Tasks
• func ScreenshotEx(urlstr, proxyServer, waitVisibleByQuery string, timeout time.Duration, ...) error
• func SetInsecureSkipVerify(client *http.Client, skip bool)
• func SetProxy(client *http.Client, proxyUrlString string) error
• func SetProxy2(transport *http.Transport, proxyUrlString string) error
• func SetRedirect(client *http.Client, follow bool)
• func SetTLSTimeout(client *http.Client, tlsTimeout time.Duration) error
• func SetTimeout(client *http.Client, to time.Duration)
• func UploadFile(requestUrl, localFilename, remoteFilename string)
• func VerifyProxyFormat(s string) error
• type Method
• type Resp
  • func DoRequest(req *http.Request, proxy string, timeout time.Duration, output ...interface{}) (*Resp, error)
  • func (r *Resp) String() string
• type UserAgnetInfo
  • func ParseUserAgent(uaString string) (*UserAgnetInfo, error)

Constants

const (
	CloudflareSel = "div.cf-browser-verification"
)

Functions

func ChromeDriverGet

func ChromeDriverGet(urlStr, proxy string) (string, error)

func CommonHttpMethods

func CommonHttpMethods() []string

func Get

func Get(url string, proxy string, timeout time.Duration, followRedirect bool) (response *http.Response, err error)

Params: proxyAddr 支持http/https/socks5代理

NOTICE 如果url不包含http://，将返回错误 如果followRedirect==false而且确实发生了跳转，则返回值的redirectUrl将被填写真实的跳转之后的URL；否则redirectUrl返回空

func GetBigFile

func GetBigFile(url string, filename string) (string, error)

if you want auto guess filename

func GetBytes

func GetBytes(url string, proxy string, timeout time.Duration) ([]byte, error)

func GetMap

func GetMap(url string, proxy string, timeout time.Duration) (map[string]interface{}, error)

func GetParamHmacMD5Sign

func GetParamHmacMD5Sign(secret, params string) (string, error)

func GetParamHmacSHA1Sign

func GetParamHmacSHA1Sign(secret, params string) (string, error)

func GetParamHmacSHA256Sign

func GetParamHmacSHA256Sign(secret, params string) (string, error)

func GetParamHmacSHA512Sign

func GetParamHmacSHA512Sign(secret, params string) (string, error)

func GetParamMD5Sign

func GetParamMD5Sign(secret, params string) (string, error)

*

*md5签名,okcoin和huobi适用

func GetProxy

func GetProxy(client *http.Client) string

func GetSHA

func GetSHA(text string) (string, error)

func GetString

func GetString(url string, proxy string, timeout time.Duration) (string, error)

func HeadlessGetBow

func HeadlessGetBow(url string) (string, error)

bow is a headless browser with some browser behave like web browser, includes: cookie, submit forms, but JavaScript NOT supported

func HeadlessGetPhantom

func HeadlessGetPhantom(url, proxy string) (string, error)

func IsValidMethod

func IsValidMethod(method string) bool

func ListenAndServeNoWait

func ListenAndServeNoWait(addr string) error

EnableHTTPProfiling helps easy wake up built in http profiler

func NewResponse

func NewResponse(content []byte) *http.Response

if you want to treat a tcp net.Conn as http, and give dest a valid http response, please use this co create a response. some Response member is required for browser to understand it.

func NewSimpleRequest

func NewSimpleRequest(ctx context.Context, httpMethod string, baseURL, refURL string, params url.Values) (*http.Request, error)

NewRequest create new HTTP request. Relative url can be provided in refURL.

func ReadBodyBytes

func ReadBodyBytes(response *http.Response) ([]byte, error)

func ReadBodyString

func ReadBodyString(response *http.Response) (string, error)

func ReadFinalUrl

func ReadFinalUrl(response *http.Response) (*url.URL, error)

Your magic function. The Request in the Response is the last URL the client tried to access. 如果没有发生redirect，也能读到一个URL，只不过是原始请求的那个URL

func Screenshot

func Screenshot(urlStr, proxyServer, waitVisibleByQuery string, timeout time.Duration) (snapshotImage []byte, fullHtml string, err error)

fullScreenshot takes a screenshot of the entire browser viewport. Note: this will override the viewport emulation settings.

func ScreenshotElement

func ScreenshotElement(urlStr, sel string, res *[]byte) chromedp.Tasks

elementScreenshot takes a screenshot of a specific element.

func ScreenshotEx

func ScreenshotEx(urlstr, proxyServer, waitVisibleByQuery string, timeout time.Duration, savePath string) error

func SetInsecureSkipVerify

func SetInsecureSkipVerify(client *http.Client, skip bool)

func SetProxy

func SetProxy(client *http.Client, proxyUrlString string) error

func SetProxy2

func SetProxy2(transport *http.Transport, proxyUrlString string) error

func SetRedirect

func SetRedirect(client *http.Client, follow bool)

func SetTLSTimeout

func SetTLSTimeout(client *http.Client, tlsTimeout time.Duration) error

func SetTimeout

func SetTimeout(client *http.Client, to time.Duration)

func UploadFile

func UploadFile(requestUrl, localFilename, remoteFilename string)

func VerifyProxyFormat

func VerifyProxyFormat(s string) error

url.Parse对于socks5:127.0.0.1:1234这种错误的url是无法判断错误的, 如果使用socks5:127.0.0.1:1234这种缺少//的格式，url.Parse将无法识别出端口号

Types

type Method

type Method string

const (
	GET    Method = "GET"
	POST   Method = "POST"
	PUT    Method = "PUT"
	PATCH  Method = "PATCH"
	DELETE Method = "DELETE"
)

type Resp

type Resp struct {
	Response *http.Response
	Body     []byte
}

Response is wrapper for standard http.Response and provides more methods.

func DoRequest

func DoRequest(req *http.Request, proxy string, timeout time.Duration, output ...interface{}) (*Resp, error)

Do executes API request created by NewRequest method or custom *http.Request.

func (*Resp) String

func (r *Resp) String() string

String converts response body to string. An empty string will be returned if error.

type UserAgnetInfo

type UserAgnetInfo struct {
	Platform       string
	OsName         string
	OsVersion      string
	DeviceType     string
	IsMobile       bool
	IsBot          bool
	EngineName     string
	EngineVersion  string
	BrowserName    string
	BrowserVersion string
}

func ParseUserAgent

func ParseUserAgent(uaString string) (*UserAgnetInfo, error)