scraper

Documentation

Index

• func IsScrapeError(err error) bool
• type ScrapeError
  • func (e ScrapeError) Error() string
• type ScrapedArticle
• type Scraper
  • func NewScraper(client *http.Client) *Scraper
  • func (s *Scraper) ScrapeBlog(ctx context.Context, blogURL string, selector string) ([]ScrapedArticle, error)

Functions

func IsScrapeError

func IsScrapeError(err error) bool

Types

type ScrapeError

type ScrapeError struct {
	Message string
}

func (ScrapeError) Error

func (e ScrapeError) Error() string

type ScrapedArticle

type ScrapedArticle struct {
	Title         string
	URL           string
	PublishedDate *time.Time
}

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

Scraper scrapes HTML pages for article links.

func NewScraper

func NewScraper(client *http.Client) *Scraper

NewScraper creates a Scraper with the given HTTP client.

func (*Scraper) ScrapeBlog

func (s *Scraper) ScrapeBlog(ctx context.Context, blogURL string, selector string) ([]ScrapedArticle, error)