navigation

Documentation

Index

• type Depth
• type Form
• type Headers
  • func (h *Headers) MarshalJSON() ([]byte, error)
• type Request
  • func NewNavigationRequestURLFromDocument(path, source, tag, attribute string, resp *Response) *Request
  • func NewNavigationRequestURLFromResponse(path, source, tag, attribute string, resp *Response) *Request
  • func (n *Request) RequestURL() string
• type Response
  • func (n Response) AbsoluteURL(path string) string
  • func (n Response) AbsoluteURLFromDocument(path string) string
  • func (n Response) IsRedirect() bool

Types

type Depth

type Depth struct{}

Depth is the depth of a navigation

type Form

type Form struct {
	Method     string   `json:"method,omitempty"`
	Action     string   `json:"action,omitempty"`
	Enctype    string   `json:"enctype,omitempty"`
	Parameters []string `json:"parameters,omitempty"`
}

type Headers

type Headers map[string]string

func (*Headers) MarshalJSON

func (h *Headers) MarshalJSON() ([]byte, error)

type Request

type Request struct {
	Method         string              `json:"method,omitempty"`
	URL            string              `json:"endpoint,omitempty"`
	Body           string              `json:"body,omitempty"`
	Depth          int                 `json:"-"`
	SkipValidation bool                `json:"-"`
	Headers        map[string]string   `json:"headers,omitempty"`
	Tag            string              `json:"tag,omitempty"`
	Attribute      string              `json:"attribute,omitempty"`
	RootHostname   string              `json:"-"`
	Source         string              `json:"source,omitempty"`
	CustomFields   map[string][]string `json:"custom_fields,omitempty"`
	Raw            string              `json:"raw,omitempty"`
	DocumentURL    string              `json:"-"`
}

Request is a navigation request for the crawler

func NewNavigationRequestURLFromDocument

func NewNavigationRequestURLFromDocument(path, source, tag, attribute string, resp *Response) *Request

func NewNavigationRequestURLFromResponse

func NewNavigationRequestURLFromResponse(path, source, tag, attribute string, resp *Response) *Request

newNavigationRequestURL generates a navigation request from a relative URL

func (*Request) RequestURL

func (n *Request) RequestURL() string

RequestURL returns the request URL for the navigation

type Response

type Response struct {
	Resp               *http.Response    `json:"-"`
	Depth              int               `json:"-"`
	Reader             *goquery.Document `json:"-"`
	StatusCode         int               `json:"status_code,omitempty"`
	Headers            Headers           `json:"headers,omitempty"`
	Body               string            `json:"body,omitempty"`
	ContentLength      int64             `json:"content_length,omitempty"`
	RootHostname       string            `json:"-"`
	Technologies       []string          `json:"technologies,omitempty"`
	Raw                string            `json:"raw,omitempty"`
	Forms              []Form            `json:"forms,omitempty"`
	XhrRequests        []Request         `json:"xhr_requests,omitempty"`
	StoredResponsePath string            `json:"stored_response_path,omitempty"`
	KnowledgeBase      map[string]any    `json:"knowledgebase,omitempty"`
	DocumentURL        string            `json:"-"`
}

Response is a response generated from crawler navigation

func (Response) AbsoluteURL

func (n Response) AbsoluteURL(path string) string

func (Response) AbsoluteURLFromDocument

func (n Response) AbsoluteURLFromDocument(path string) string

func (Response) IsRedirect

func (n Response) IsRedirect() bool