dl

Documentation

Index

• func CompressPage(page []byte) string
• func GetHeaders(platform string) map[string]string
• func GetProxy() (string, error)
• func GetUserAgent(platform string) string
• func GuessEncoding(content []byte) (string, error)
• func ReportProxyStatus(proxy string) error
• type HttpRequest
• type HttpResponse
  • func Download(requestInfo *HttpRequest) *HttpResponse
  • func DownloadUrl(url string) *HttpResponse
  • func DownloadUrlWithProxy(url string) *HttpResponse
• type ProxyType
  • func GetProxyType(proxy string) ProxyType

Functions

func CompressPage

func CompressPage(page []byte) string

func GetHeaders

func GetHeaders(platform string) map[string]string

func GetProxy

func GetProxy() (string, error)

func GetUserAgent

func GetUserAgent(platform string) string

func GuessEncoding

func GuessEncoding(content []byte) (string, error)

func ReportProxyStatus

func ReportProxyStatus(proxy string) error

Types

type HttpRequest

type HttpRequest struct {
	Url        string                          `json:"url"`
	Method     string                          `json:"method"`
	PostData   string                          `json:"post_data"`
	UseProxy   bool                            `json:"use_proxy"`
	Proxy      string                          `json:"proxy"`
	Timeout    int                             `json:"timeout"`
	MaxLen     int64                           `json:"max_len"`
	Platform   string                          `json:"platform"`
	Retry      int                             `json:"retry"`
	Header     map[string]string               `json:"header"`
	Username   string                          `json:"username"`
	Password   string                          `json:"password"`
	ValidFuncs []func(resp *HttpResponse) bool `json:"-"`
	Ctx        context.Context                 `json:"-"`
	Jar        http.CookieJar                  `json:"-"`
}

type HttpResponse

type HttpResponse struct {
	Url        string          `json:"url"`
	Text       string          `json:"text"`
	Content    []byte          `json:"content"`
	Encoding   string          `json:"encoding"`
	StatusCode int             `json:"status_code"`
	Proxy      string          `json:"proxy"`
	RemoteAddr string          `json:"remote_addr"`
	Error      error           `json:"error"`
	Ctx        context.Context `json:"-"`
	Jar        http.CookieJar  `json:"-"`
}

func Download

func Download(requestInfo *HttpRequest) *HttpResponse

func DownloadUrl

func DownloadUrl(url string) *HttpResponse

func DownloadUrlWithProxy

func DownloadUrlWithProxy(url string) *HttpResponse

type ProxyType

type ProxyType int

const (
	Invalid ProxyType = iota
	HTTP
	HTTPS
	Socks5
)

func GetProxyType

func GetProxyType(proxy string) ProxyType