webfetch

Documentation

Index

• Constants
• func IsBotChallengeContent(content string) bool
• func JoinPromptAndContent(prompt, content string) string
• type Client
  • func NewClient(opts Options) *Client
  • func (c *Client) Fetch(ctx context.Context, req Request) (Result, error)
  • func (c *Client) SetExtractor(extractor Extractor)
  • func (c *Client) SetHTTPClient(httpClient *http.Client)
• type Error
  • func (e *Error) Error() string
• type Extractor
• type Options
• type RecoveryHint
• type Request
• type Result

Constants

const (
	DefaultTimeoutMS = 15000
	MaxTimeoutMS     = 60000
	MaxURLLength     = 2000
	MaxFetchBytes    = 10 * 1024 * 1024
	MaxReplayChars   = 100000
	DefaultTTL       = 15 * time.Minute
	DefaultCacheSize = 50 * 1024 * 1024
)

Functions

func IsBotChallengeContent

func IsBotChallengeContent(content string) bool

func JoinPromptAndContent

func JoinPromptAndContent(prompt, content string) string

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewClient

func NewClient(opts Options) *Client

func (*Client) Fetch

func (c *Client) Fetch(ctx context.Context, req Request) (Result, error)

func (*Client) SetExtractor

func (c *Client) SetExtractor(extractor Extractor)

func (*Client) SetHTTPClient

func (c *Client) SetHTTPClient(httpClient *http.Client)

type Error

type Error struct {
	Code    string
	Message string
	Result  Result
}

func (*Error) Error

func (e *Error) Error() string

type Extractor

type Extractor interface {
	Extract(ctx context.Context, prompt, content string) (string, error)
}

type Options

type Options struct {
	HTTPClient *http.Client
	Extractor  Extractor
	TTL        time.Duration
	MaxCache   int
}

type RecoveryHint

type RecoveryHint struct {
	Code              string   `json:"code"`
	Retryable         bool     `json:"retryable"`
	RecommendedAction string   `json:"recommended_action"`
	NextSteps         []string `json:"next_steps,omitempty"`
}

type Request

type Request struct {
	URL       string
	Prompt    string
	TimeoutMS int
}

type Result

type Result struct {
	URL              string        `json:"url"`
	FinalURL         string        `json:"final_url,omitempty"`
	StatusCode       int           `json:"status_code,omitempty"`
	CodeText         string        `json:"code_text,omitempty"`
	ContentType      string        `json:"content_type,omitempty"`
	Title            string        `json:"title,omitempty"`
	Content          string        `json:"content,omitempty"`
	ContentChars     int           `json:"content_chars"`
	Bytes            int           `json:"bytes"`
	DurationMS       int64         `json:"duration_ms"`
	FromCache        bool          `json:"from_cache"`
	Truncated        bool          `json:"truncated"`
	LowContent       bool          `json:"low_content,omitempty"`
	LowContentReason string        `json:"low_content_reason,omitempty"`
	NextSteps        string        `json:"next_steps,omitempty"`
	RedirectBlocked  bool          `json:"redirect_blocked,omitempty"`
	RedirectLocation string        `json:"redirect_location,omitempty"`
	Recovery         *RecoveryHint `json:"recovery,omitempty"`
	ExtractError     string        `json:"extract_error,omitempty"`
}