fetch

Documentation

Index

• Constants
• func ProfileNames() []string
• type Config
  • func ConfigFromEnv() (Config, error)
• type ExtractQuery
• type ExtractQueryResult
• type ExtractedValue
• type Fetcher
  • func New(config Config) *Fetcher
  • func (f *Fetcher) ClearSession(sessionID string) (bool, error)
  • func (f *Fetcher) Do(ctx context.Context, input Input) (Output, error)
  • func (f *Fetcher) DoGet(ctx context.Context, input GetInput) (Output, error)
  • func (f *Fetcher) DoRequest(ctx context.Context, input RequestInput) (Output, error)
  • func (f *Fetcher) ExtractResponse(input ResponseExtractInput) (ResponseExtractOutput, error)
  • func (f *Fetcher) ReadResponse(input ResponseReadInput) (ResponseReadOutput, error)
  • func (f *Fetcher) SearchResponse(input ResponseSearchInput) (ResponseSearchOutput, error)
  • func (f *Fetcher) SessionInfo(sessionID string) (SessionInfoOutput, error)
  • func (f *Fetcher) WarmupSession(ctx context.Context, input SessionWarmupInput) (Output, error)
• type GetInput
• type Input
• type Output
• type ProfilesInput
• type ProfilesOutput
• type RequestInput
• type ResponseExtractInput
• type ResponseExtractOutput
• type ResponseReadInput
• type ResponseReadOutput
• type ResponseSearchInput
• type ResponseSearchMatch
• type ResponseSearchOutput
• type SessionClearInput
• type SessionClearOutput
• type SessionInfoInput
• type SessionInfoOutput
• type SessionWarmupInput

Constants

const DefaultProfile = "chrome_146"

Functions

func ProfileNames

func ProfileNames() []string

Types

type Config

type Config struct {
	AllowPrivate     bool
	AllowProxy       bool
	AllowedHosts     []string
	MaxResponseBytes int64
	DefaultTimeout   int
	MaxTimeout       int
	MaxSessions      int
	SessionTTL       int
	MaxResponses     int
	ResponseTTL      int
	MaxReadBytes     int64
}

Config controls the server-side security and resource limits. Tool callers may choose stricter limits, but cannot exceed these values.

func ConfigFromEnv

func ConfigFromEnv() (Config, error)

type ExtractQuery

type ExtractQuery struct {
	Name        string `json:"name" jsonschema:"Unique result name, at most 128 UTF-8 bytes"`
	Selector    string `json:"selector" jsonschema:"CSS selector for HTML or RFC 9535 JSONPath expression for JSON"`
	Mode        string `` /* 135-byte string literal not displayed */
	Attribute   string `json:"attribute,omitempty" jsonschema:"HTML attribute name when mode is attribute"`
	ResolveURLs bool   `json:"resolve_urls,omitempty" jsonschema:"Resolve relative HTML URL attributes against the stored response final URL"`
}

type ExtractQueryResult

type ExtractQueryResult struct {
	Name          string           `json:"name"`
	Selector      string           `json:"selector"`
	Mode          string           `json:"mode"`
	Values        []ExtractedValue `json:"values"`
	MatchedCount  int              `json:"matched_count"`
	ReturnedCount int              `json:"returned_count"`
	Limited       bool             `json:"limited"`
}

type ExtractedValue

type ExtractedValue struct {
	Value    string `json:"value"`
	Encoding string `json:"encoding"`
	Path     string `json:"path,omitempty"`
}

type Fetcher

type Fetcher struct {
	// contains filtered or unexported fields
}

func New

func New(config Config) *Fetcher

func (*Fetcher) ClearSession

func (f *Fetcher) ClearSession(sessionID string) (bool, error)

func (*Fetcher) Do

func (f *Fetcher) Do(ctx context.Context, input Input) (Output, error)

func (*Fetcher) DoGet

func (f *Fetcher) DoGet(ctx context.Context, input GetInput) (Output, error)

func (*Fetcher) DoRequest

func (f *Fetcher) DoRequest(ctx context.Context, input RequestInput) (Output, error)

func (*Fetcher) ExtractResponse

func (f *Fetcher) ExtractResponse(input ResponseExtractInput) (ResponseExtractOutput, error)

func (*Fetcher) ReadResponse

func (f *Fetcher) ReadResponse(input ResponseReadInput) (ResponseReadOutput, error)

func (*Fetcher) SearchResponse

func (f *Fetcher) SearchResponse(input ResponseSearchInput) (ResponseSearchOutput, error)

func (*Fetcher) SessionInfo

func (f *Fetcher) SessionInfo(sessionID string) (SessionInfoOutput, error)

func (*Fetcher) WarmupSession

func (f *Fetcher) WarmupSession(ctx context.Context, input SessionWarmupInput) (Output, error)

type GetInput

type GetInput struct {
	URL              string            `json:"url" jsonschema:"Absolute http or https URL to request"`
	Method           string            `json:"method,omitempty" jsonschema:"Read-only HTTP method: GET or HEAD; defaults to GET"`
	Headers          map[string]string `json:"headers,omitempty" jsonschema:"Request headers as name-value pairs"`
	HeaderOrder      []string          `json:"header_order,omitempty" jsonschema:"Optional lower-case HTTP header order for accurate fingerprinting"`
	Profile          string            `json:"profile,omitempty" jsonschema:"TLS browser profile; defaults to chrome_146. Call tls_profiles for all names"`
	FollowRedirects  *bool             `json:"follow_redirects,omitempty" jsonschema:"Follow redirects; defaults to true"`
	TimeoutSeconds   int               `json:"timeout_seconds,omitempty" jsonschema:"Whole-request timeout in seconds"`
	MaxResponseBytes int64             `json:"max_response_bytes,omitempty" jsonschema:"Maximum response bytes to return or store; capped by the server"`
	ProxyURL         string            `json:"proxy_url,omitempty" jsonschema:"Optional HTTP or SOCKS proxy; disabled unless the server permits proxies"`
	SessionID        string            `` /* 127-byte string literal not displayed */
	IncludeBody      *bool             `json:"include_body,omitempty" jsonschema:"Include the response body inline; defaults to true"`
	StoreResponse    bool              `` /* 143-byte string literal not displayed */
}

type Input

type Input struct {
	URL              string            `json:"url" jsonschema:"Absolute http or https URL to request"`
	Method           string            `json:"method,omitempty" jsonschema:"HTTP method: GET, HEAD, POST, PUT, PATCH, DELETE or OPTIONS; defaults to GET"`
	Headers          map[string]string `json:"headers,omitempty" jsonschema:"Request headers as name-value pairs"`
	HeaderOrder      []string          `json:"header_order,omitempty" jsonschema:"Optional lower-case HTTP header order for accurate fingerprinting"`
	Body             string            `json:"body,omitempty" jsonschema:"Raw request body"`
	Profile          string            `json:"profile,omitempty" jsonschema:"TLS browser profile; defaults to chrome_146. Call tls_profiles for all names"`
	FollowRedirects  *bool             `json:"follow_redirects,omitempty" jsonschema:"Follow redirects; defaults to true"`
	TimeoutSeconds   int               `json:"timeout_seconds,omitempty" jsonschema:"Whole-request timeout in seconds"`
	MaxResponseBytes int64             `json:"max_response_bytes,omitempty" jsonschema:"Maximum response bytes to return; capped by the server"`
	ProxyURL         string            `json:"proxy_url,omitempty" jsonschema:"Optional HTTP or SOCKS proxy; disabled unless the server permits proxies"`
	SessionID        string            `` /* 127-byte string literal not displayed */
	IncludeBody      *bool             `json:"include_body,omitempty" jsonschema:"Include the response body inline; defaults to true"`
	StoreResponse    bool              `` /* 143-byte string literal not displayed */
}

type Output

type Output struct {
	Status          int                 `json:"status"`
	StatusText      string              `json:"status_text"`
	FinalURL        string              `json:"final_url"`
	Headers         map[string][]string `json:"headers"`
	Body            string              `json:"body"`
	BodyEncoding    string              `json:"body_encoding"`
	ContentType     string              `json:"content_type,omitempty"`
	Truncated       bool                `json:"truncated"`
	ElapsedMillis   int64               `json:"elapsed_ms"`
	Profile         string              `json:"profile"`
	SessionID       string              `json:"session_id,omitempty"`
	CookiesStored   int                 `json:"cookies_stored,omitempty"`
	CookieNames     []string            `json:"cookie_names,omitempty"`
	RedactedHeaders []string            `json:"redacted_headers,omitempty"`
	RedirectHistory []string            `json:"redirect_history,omitempty"`
	HTTPVersion     string              `json:"http_version,omitempty"`
	ContentLength   int64               `json:"content_length"`
	BytesReturned   int64               `json:"bytes_returned"`
	ResponseID      string              `json:"response_id,omitempty"`
}

type ProfilesInput

type ProfilesInput struct{}

type ProfilesOutput

type ProfilesOutput struct {
	Default  string   `json:"default"`
	Profiles []string `json:"profiles"`
}

type RequestInput

type RequestInput struct {
	URL              string            `json:"url" jsonschema:"Absolute http or https URL to request"`
	Method           string            `json:"method" jsonschema:"Mutating HTTP method: POST, PUT, PATCH, DELETE or OPTIONS"`
	Headers          map[string]string `json:"headers,omitempty" jsonschema:"Request headers as name-value pairs"`
	HeaderOrder      []string          `json:"header_order,omitempty" jsonschema:"Optional lower-case HTTP header order for accurate fingerprinting"`
	Body             string            `json:"body,omitempty" jsonschema:"Raw request body"`
	Profile          string            `json:"profile,omitempty" jsonschema:"TLS browser profile; defaults to chrome_146. Call tls_profiles for all names"`
	FollowRedirects  *bool             `json:"follow_redirects,omitempty" jsonschema:"Follow redirects; defaults to true"`
	TimeoutSeconds   int               `json:"timeout_seconds,omitempty" jsonschema:"Whole-request timeout in seconds"`
	MaxResponseBytes int64             `json:"max_response_bytes,omitempty" jsonschema:"Maximum response bytes to return or store; capped by the server"`
	ProxyURL         string            `json:"proxy_url,omitempty" jsonschema:"Optional HTTP or SOCKS proxy; disabled unless the server permits proxies"`
	SessionID        string            `` /* 127-byte string literal not displayed */
	IncludeBody      *bool             `json:"include_body,omitempty" jsonschema:"Include the response body inline; defaults to true"`
	StoreResponse    bool              `` /* 143-byte string literal not displayed */
}

type ResponseExtractInput

type ResponseExtractInput struct {
	ResponseID     string         `json:"response_id" jsonschema:"Temporary response identifier returned by tls_get, tls_request or tls_fetch"`
	Format         string         `` /* 130-byte string literal not displayed */
	Queries        []ExtractQuery `json:"queries" jsonschema:"One to sixteen named CSS selector or RFC 9535 JSONPath queries"`
	MaxResults     int            `json:"max_results,omitempty" jsonschema:"Maximum values returned per query; defaults to 20 and is capped at 100"`
	MaxOutputBytes int64          `` /* 145-byte string literal not displayed */
}

type ResponseExtractOutput

type ResponseExtractOutput struct {
	ResponseID      string               `json:"response_id"`
	Format          string               `json:"format"`
	ContentType     string               `json:"content_type,omitempty"`
	FinalURL        string               `json:"final_url,omitempty"`
	SourceBytes     int64                `json:"source_bytes"`
	SourceTruncated bool                 `json:"source_truncated"`
	Results         []ExtractQueryResult `json:"results"`
	ReturnedCount   int                  `json:"returned_count"`
	Limited         bool                 `json:"limited"`
	OutputBytes     int                  `json:"output_bytes"`
}

type ResponseReadInput

type ResponseReadInput struct {
	ResponseID string `json:"response_id" jsonschema:"Temporary response identifier returned by tls_get, tls_request or tls_fetch"`
	Offset     int64  `json:"offset,omitempty" jsonschema:"Zero-based byte offset; defaults to 0"`
	MaxBytes   int64  `json:"max_bytes,omitempty" jsonschema:"Maximum raw bytes to read; defaults to 65536 and is capped by the server"`
}

type ResponseReadOutput

type ResponseReadOutput struct {
	ResponseID   string `json:"response_id"`
	Offset       int64  `json:"offset"`
	NextOffset   int64  `json:"next_offset"`
	TotalBytes   int64  `json:"total_bytes"`
	Body         string `json:"body"`
	BodyEncoding string `json:"body_encoding"`
	EOF          bool   `json:"eof"`
	ContentType  string `json:"content_type,omitempty"`
	FinalURL     string `json:"final_url,omitempty"`
	Truncated    bool   `json:"truncated"`
	CreatedAt    string `json:"created_at"`
	ExpiresAt    string `json:"expires_at"`
}

type ResponseSearchInput

type ResponseSearchInput struct {
	ResponseID    string `json:"response_id" jsonschema:"Temporary response identifier returned by tls_get, tls_request or tls_fetch"`
	Query         string `json:"query" jsonschema:"Literal UTF-8 text to find in the stored response"`
	CaseSensitive bool   `json:"case_sensitive,omitempty" jsonschema:"Use case-sensitive matching; defaults to false"`
	MaxMatches    int    `json:"max_matches,omitempty" jsonschema:"Maximum matches to return; defaults to 20 and is capped at 50"`
	ContextBytes  int    `json:"context_bytes,omitempty" jsonschema:"Context bytes around each match; defaults to 160 and is capped at 1024"`
}

type ResponseSearchMatch

type ResponseSearchMatch struct {
	Offset  int64  `json:"offset"`
	Context string `json:"context"`
}

type ResponseSearchOutput

type ResponseSearchOutput struct {
	ResponseID string                `json:"response_id"`
	Query      string                `json:"query"`
	Matches    []ResponseSearchMatch `json:"matches"`
	MatchCount int                   `json:"match_count"`
	Limited    bool                  `json:"limited"`
	TotalBytes int64                 `json:"total_bytes"`
}

type SessionClearInput

type SessionClearInput struct {
	SessionID string `json:"session_id" jsonschema:"Cookie-session identifier to clear"`
}

type SessionClearOutput

type SessionClearOutput struct {
	SessionID string `json:"session_id"`
	Cleared   bool   `json:"cleared"`
}

type SessionInfoInput

type SessionInfoInput struct {
	SessionID string `json:"session_id" jsonschema:"Cookie-session identifier to inspect"`
}

type SessionInfoOutput

type SessionInfoOutput struct {
	SessionID     string   `json:"session_id"`
	Exists        bool     `json:"exists"`
	CreatedAt     string   `json:"created_at,omitempty"`
	LastUsedAt    string   `json:"last_used_at,omitempty"`
	ExpiresAt     string   `json:"expires_at,omitempty"`
	CookiesStored int      `json:"cookies_stored,omitempty"`
	CookieNames   []string `json:"cookie_names,omitempty"`
}

type SessionWarmupInput

type SessionWarmupInput struct {
	URL              string            `json:"url" jsonschema:"Absolute http or https URL used to initialize the cookie session"`
	SessionID        string            `json:"session_id" jsonschema:"Cookie-session identifier to create or refresh"`
	Headers          map[string]string `json:"headers,omitempty" jsonschema:"Request headers as name-value pairs"`
	HeaderOrder      []string          `json:"header_order,omitempty" jsonschema:"Optional lower-case HTTP header order"`
	Profile          string            `json:"profile,omitempty" jsonschema:"TLS browser profile; defaults to chrome_146"`
	FollowRedirects  *bool             `json:"follow_redirects,omitempty" jsonschema:"Follow redirects; defaults to true"`
	TimeoutSeconds   int               `json:"timeout_seconds,omitempty" jsonschema:"Whole-request timeout in seconds"`
	MaxResponseBytes int64             `json:"max_response_bytes,omitempty" jsonschema:"Maximum bytes to consume; capped by the server"`
	ProxyURL         string            `json:"proxy_url,omitempty" jsonschema:"Optional proxy; disabled unless the server permits proxies"`
}