xhttp

README

xhttp

Intro

http client for scanner

应用于扫描器场景下的http基础库。

1. client

   • 精准的http client配置：目前支持支持19项
   • 多client共享cookie
   • 跳转策略
   • 失败重试
   • 代理
   • tls
   • limiter：qps限制

2. request

   • context
   • trace
   • getbody：获取请求body
   • getRaw：获取请求报文

3. response

   • getLatency：发起请求到收到响应的整个持续时间，可用于判断时间延时场景，如盲注
   • getbody：获取响应body
   • getRaw：获取响应报文

4. requestMiddleware：请求发起之前，对请求的修饰

   • context
   • method 限制策略
   • 启用 trace
   • 根据配置修改header
   • 根据配置修改cookie

5. responseMiddleware：响应获取后，对响应的处理

   • 读body
   • 响应长度限制策略

6. debug模式：debug模式下将打印请求和响应完整信息

7. 完整的 testhttp server

Install

go get github.com/xiecat/xhttp

Demo

// 如果要继承cookie，传入cookie jar；否则填nil。
cookieJar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
ctx := context.Background()
client, err := NewDefaultClient(cookieJar)
// 构造请求
hr, _ := http.NewRequest("GET", "<TARGET URL>" , nil)
req := &Request{RawRequest: hr,}

// 发起请求
resp, err := client.Do(ctx, req)

Todo

• errorHook

Ref

• https://github.com/go-resty/resty
• https://github.com/hashicorp/go-retryablehttp

Documentation

Index

• Constants
• type Client
  • func NewClient(options *ClientOptions, jar *cookiejar.Jar) (*Client, error)
  • func NewDefaultClient(jar *cookiejar.Jar) (*Client, error)
  • func NewDefaultRedirectClient(jar *cookiejar.Jar) (*Client, error)
  • func NewRedirectClient(options *ClientOptions, jar *cookiejar.Jar) (*Client, error)
  • func NewWithHTTPClient(options *ClientOptions, hc *http.Client) (*Client, error)
  • func (c *Client) AfterResponse(fn ResponseMiddleware)
  • func (c *Client) BeforeRequest(fn RequestMiddleware)
  • func (c *Client) Do(ctx context.Context, req *Request) (*Response, error)
  • func (c *Client) SetCloseConnection(close bool) *Client
• type ClientOptions
  • func DefaultClientOptions() *ClientOptions
  • func GetHTTPOptions() *ClientOptions
  • func (o *ClientOptions) SetLimiter() *ClientOptions
  • func (o *ClientOptions) Verify() error
• type Request
  • func (r *Request) Clone() *Request
  • func (r *Request) EnableTrace() *Request
  • func (r *Request) FuzzCommonHeaders(value string)
  • func (r *Request) GetAttempt() int
  • func (r *Request) GetBody() ([]byte, error)
  • func (r *Request) GetContentType() string
  • func (r *Request) GetContext() context.Context
  • func (r *Request) GetHeaders() http.Header
  • func (r *Request) GetMethod() string
  • func (r *Request) GetRaw() ([]byte, error)
  • func (r *Request) GetUrl() *url.URL
  • func (r *Request) SetBody(body []byte) *Request
  • func (r *Request) SetContext(ctx context.Context) *Request
  • func (r *Request) SetCookie(hc *http.Cookie) *Request
  • func (r *Request) SetHeader(key, value string) *Request
  • func (r *Request) SetHeaderMulti(headers map[string]string) *Request
  • func (r *Request) SetHeaderMultiValues(headers map[string][]string) *Request
  • func (r *Request) SetHeaders(headers map[string]string) *Request
• type RequestMiddleware
• type Response
  • func (r *Response) GetBody() []byte
  • func (r *Response) GetContentType() string
  • func (r *Response) GetHeaders() http.Header
  • func (r *Response) GetLatency() (time.Duration, error)
  • func (r *Response) GetRaw() ([]byte, error)
  • func (r *Response) GetStatus() int
  • func (r *Response) GetUrl() *url.URL
• type ResponseMiddleware
• type TraceInfo

Constants

const (
	// MethodGet HTTP method
	MethodGet = "GET"

	// MethodPost HTTP method
	MethodPost = "POST"

	// MethodPut HTTP method
	MethodPut = "PUT"

	// MethodDelete HTTP method
	MethodDelete = "DELETE"

	// MethodPatch HTTP method
	MethodPatch = "PATCH"

	// MethodHead HTTP method
	MethodHead = "HEAD"

	// MethodOptions HTTP method
	MethodOptions = "OPTIONS"

	// MethodConnect HTTP method
	MethodConnect = "CONNECT"

	// MethodTrace HTTP method
	MethodTrace = "TRACE"

	// MethodMove HTTP method
	MethodMove = "MOVE"

	// MethodPURGE MethodMove HTTP method
	MethodPURGE = "PURGE"
)

Types

type Client

type Client struct {
	HTTPClient    *http.Client
	ClientOptions *ClientOptions
	Debug         bool        // if debug == true, start responseLogger middleware
	Error         interface{} // todo error handle exp
	// contains filtered or unexported fields
}

Client struct

func NewClient

func NewClient(options *ClientOptions, jar *cookiejar.Jar) (*Client, error)

NewClient xhttp.Client

func NewDefaultClient

func NewDefaultClient(jar *cookiejar.Jar) (*Client, error)

NewDefaultClient xhttp.Client not follow redirect

func NewDefaultRedirectClient

func NewDefaultRedirectClient(jar *cookiejar.Jar) (*Client, error)

NewDefaultRedirectClient follow redirect

func NewRedirectClient

func NewRedirectClient(options *ClientOptions, jar *cookiejar.Jar) (*Client, error)

NewRedirectClient xhttp.Client with Redirect

func NewWithHTTPClient

func NewWithHTTPClient(options *ClientOptions, hc *http.Client) (*Client, error)

NewWithHTTPClient with http client

func (*Client) AfterResponse

func (c *Client) AfterResponse(fn ResponseMiddleware)

func (*Client) BeforeRequest

func (c *Client) BeforeRequest(fn RequestMiddleware)

func (*Client) Do

func (c *Client) Do(ctx context.Context, req *Request) (*Response, error)

Do request

func (*Client) SetCloseConnection

func (c *Client) SetCloseConnection(close bool) *Client

type ClientOptions

type ClientOptions struct {
	Proxy string `` /* 164-byte string literal not displayed */
	//ProxyRule           []Rule       `json:"proxy_rule" yaml:"proxy_rule" #:"漏洞扫描使用多个代理的配置规则, 具体请参照文档"`
	DialTimeout         int  `json:"dial_timeout" yaml:"dial_timeout" #:"建立 tcp 连接的超时时间"`
	ReadTimeout         int  `` /* 137-byte string literal not displayed */
	MaxConnsPerHost     int  `` /* 136-byte string literal not displayed */
	EnableHTTP2         bool `` /* 141-byte string literal not displayed */
	IdleConnTimeout     int  `json:"-" yaml:"-"`
	MaxIdleConns        int  `json:"-" yaml:"-"`
	TLSHandshakeTimeout int  `json:"-" yaml:"-"`

	FailRetries       int                 `json:"fail_retries" yaml:"fail_retries" #:"请求失败的重试次数, 0 则不重试"`
	MaxRedirect       int                 `json:"max_redirect" yaml:"max_redirect" #:"单个请求最大允许的跳转数"`
	MaxRespBodySize   int64               `json:"max_resp_body_size" yaml:"max_resp_body_size" #:"最大允许的响应大小, 默认 4M"`
	MaxQPS            int                 `json:"max_qps" yaml:"max_qps" #:"每秒最大请求数"`
	AllowMethods      []string            `json:"allow_methods" yaml:"allow_methods" #:"允许的请求方法"`
	Headers           map[string]string   `json:"headers" yaml:"headers" #:"自定义 headers"`
	Cookies           map[string]string   `json:"cookies" yaml:"cookies" #:"自定义 cookies, 参考 headers 格式， key: value"`
	TlsOptions        *xtls.ClientOptions `json:"tls" yaml:"tls" #:"tls 配置"`
	Debug             bool                `json:"http_debug" yaml:"http_debug" #:"是否启用 debug 模式, 开启 request trace"`
	DisableKeepAlives bool                `json:"disable_keep_alives" yaml:"disable_keep_alives" #:"是否禁用 keepalives"`
	Limiter           *rate.Limiter       `json:"-" yaml:"-"`
}

ClientOptions http client options

var HTTPOptions *ClientOptions

func DefaultClientOptions

func DefaultClientOptions() *ClientOptions

func GetHTTPOptions

func GetHTTPOptions() *ClientOptions

func (*ClientOptions) SetLimiter

func (o *ClientOptions) SetLimiter() *ClientOptions

func (*ClientOptions) Verify

func (o *ClientOptions) Verify() error

type Request

type Request struct {
	RawRequest *http.Request
	Error      interface{}
	Body       []byte
	// contains filtered or unexported fields
}

func (*Request) Clone

func (r *Request) Clone() *Request

func (*Request) EnableTrace

func (r *Request) EnableTrace() *Request

func (*Request) FuzzCommonHeaders

func (r *Request) FuzzCommonHeaders(value string)

func (*Request) GetAttempt

func (r *Request) GetAttempt() int

GetAttempt get

func (*Request) GetBody

func (r *Request) GetBody() ([]byte, error)

func (*Request) GetContentType

func (r *Request) GetContentType() string

func (*Request) GetContext

func (r *Request) GetContext() context.Context

GetContext get

func (*Request) GetHeaders

func (r *Request) GetHeaders() http.Header

func (*Request) GetMethod

func (r *Request) GetMethod() string

func (*Request) GetRaw

func (r *Request) GetRaw() ([]byte, error)

func (*Request) GetUrl

func (r *Request) GetUrl() *url.URL

func (*Request) SetBody

func (r *Request) SetBody(body []byte) *Request

func (*Request) SetContext

func (r *Request) SetContext(ctx context.Context) *Request

SetContext set

func (*Request) SetCookie

func (r *Request) SetCookie(hc *http.Cookie) *Request

func (*Request) SetHeader

func (r *Request) SetHeader(key, value string) *Request

SetHeader set a single header field and its value in the current request

func (*Request) SetHeaderMulti

func (r *Request) SetHeaderMulti(headers map[string]string) *Request

func (*Request) SetHeaderMultiValues

func (r *Request) SetHeaderMultiValues(headers map[string][]string) *Request

SetHeaderMultiValues sets multiple headers fields and its values is list of strings For Example: To set `Accept` as `text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8`

func (*Request) SetHeaders

func (r *Request) SetHeaders(headers map[string]string) *Request

SetHeaders set multiple headers field

type RequestMiddleware

type RequestMiddleware func(*Request, *Client) error

RequestMiddleware run before request

type Response

type Response struct {
	Request     *Request
	RawResponse *http.Response
	Body        []byte
	// contains filtered or unexported fields
}

func (*Response) GetBody

func (r *Response) GetBody() []byte

func (*Response) GetContentType

func (r *Response) GetContentType() string

func (*Response) GetHeaders

func (r *Response) GetHeaders() http.Header

func (*Response) GetLatency

func (r *Response) GetLatency() (time.Duration, error)

func (*Response) GetRaw

func (r *Response) GetRaw() ([]byte, error)

func (*Response) GetStatus

func (r *Response) GetStatus() int

GetStatus method returns the HTTP status string for the executed request.

func (*Response) GetUrl

func (r *Response) GetUrl() *url.URL

type ResponseMiddleware

type ResponseMiddleware func(*Response, *Client) error

ResponseMiddleware run after receive response

type TraceInfo

type TraceInfo struct {
	// DNSLookup is a duration that transport took to perform
	DNSLookup time.Duration
	// ConnTime is a duration that took to obtain a successful connection.
	ConnTime time.Duration
	// TCPConnTime is a duration that took to obtain the TCP connection.
	TCPConnTime time.Duration
	// TLSHandshake is a duration that TLS handshake took place.
	TLSHandshake time.Duration
	// ServerTime is a duration that server took to respond first byte.
	ServerTime time.Duration
	// ResponseTime is a duration since first response byte from server to
	// request completion.
	ResponseTime time.Duration
	// TotalTime is a duration that total request took end-to-end.
	TotalTime time.Duration
	// IsConnReused is whether this connection has been previously
	// used for another HTTP request.
	IsConnReused bool
	// IsConnWasIdle is whether this connection was obtained from an
	// idle pool.
	IsConnWasIdle bool
	// ConnIdleTime is a duration how long the connection was previously
	// idle, if IsConnWasIdle is true.
	ConnIdleTime time.Duration
	// RequestAttempt is to represent the request attempt made during a Resty
	// request execution flow, including retry count.
	//RequestAttempt int
	// RemoteAddr returns the remote network address.
	RemoteAddr net.Addr
}