client

Documentation

Index

• Variables
• func ExtractFavicon(ctx context.Context, page string) (string, error)
• func ExtractMainContent(ctx context.Context, page string, data []byte) (string, error)
• func ExtractMainImage(ctx context.Context, page string) (string, error)
• type HeadReader
  • func NewHeadReader(r io.Reader, maxBytes int) *HeadReader
  • func (h *HeadReader) Read(page []byte) (int, error)

Variables

var (
	// UserAgent is the string which the `User-Agent` request header will be set to for underlying requests to fetch
	// feeds and content.
	UserAgent = config.AppName + "/" + config.GetVersion() + " (+https://foragd.app/policies/bot)"
	// DefaultHTTPRequestTimeout is the maximum time allowed for a background HTTP request to execute.
	DefaultHTTPRequestTimeout = 45 * time.Second
	// DefaultRequestRetries is the default number of retries for API requests.
	DefaultRequestRetries = 3
)

var Load = sync.OnceValue(func() *resty.Client {
	client = resty.New().
		SetHeader("User-Agent", UserAgent).
		SetHeader("Accept", "*/*").
		SetHeader("Accept-Encoding", "gzip, deflate")
	return client
})

Functions

func ExtractFavicon

func ExtractFavicon(ctx context.Context, page string) (string, error)

ExtractFavicon will attempt to extract the favicon link from a page.

func ExtractMainContent

func ExtractMainContent(ctx context.Context, page string, data []byte) (string, error)

ExtractMainContent extracts the main content from the page at the given URL, using the readability package.

func ExtractMainImage

func ExtractMainImage(ctx context.Context, page string) (string, error)

ExtractMainImage will attempt to extract a URL to what is likely the "main" image of a page (i.e., typically used on article/post pages).

Types

type HeadReader

type HeadReader struct {
	// contains filtered or unexported fields
}

HeadReader wraps a reader and stops after the </head> tag or a byte limit. This avoids downloading the entire page body.

func NewHeadReader

func NewHeadReader(r io.Reader, maxBytes int) *HeadReader

func (*HeadReader) Read

func (h *HeadReader) Read(page []byte) (int, error)