client

Documentation

Index

• Variables
• func ExtractMainContent(ctx context.Context, page string) (string, error)
• func ExtractMainImage(ctx context.Context, page string) (string, error)
• type HeadReader
  • func NewHeadReader(r io.Reader, maxBytes int) *HeadReader
  • func (h *HeadReader) Read(page []byte) (int, error)

Variables

var (
	// UserAgent is the string which the `User-Agent` request header will be set to for underlying requests to fetch
	// feeds and content.
	UserAgent = config.AppName + "/" + config.Version + " (+https://foragd.app/policies/bot)"
	// DefaultHTTPRequestTimeout is the maximum time allowed for a background HTTP request to execute.
	DefaultHTTPRequestTimeout = 45 * time.Second
	// DefaultRequestRetries is the default number of retries for API requests.
	DefaultRequestRetries = 3
)

var Load = sync.OnceValue(func() *resty.Client {
	client = resty.New().
		SetHeader("User-Agent", UserAgent).
		SetHeader("Accept", "*/*").
		SetHeader("Accept-Encoding", "gzip, deflate")
	return client
})

Functions

func ExtractMainContent

func ExtractMainContent(ctx context.Context, page string) (string, error)

ExtractMainContent extracts the main content from the page at the given URL, using the readability package.

func ExtractMainImage

func ExtractMainImage(ctx context.Context, page string) (string, error)

ExtractMainImage will try to find an image to represent the feed. Useful to call if the feed does not define an image itself.

Types

type HeadReader

type HeadReader struct {
	// contains filtered or unexported fields
}

HeadReader wraps a reader and stops after the </head> tag or a byte limit. This avoids downloading the entire page body.

func NewHeadReader

func NewHeadReader(r io.Reader, maxBytes int) *HeadReader

func (*HeadReader) Read

func (h *HeadReader) Read(page []byte) (int, error)