descrape

README

descrape

A declarative web scraping library for Go (WIP)

Documentation

Index

• type Collection
• type CollectionList
• type CollectionsMap
• type Element
• type ElementsMap
• type OutFormat
• type PageData
• type Scraper
  • func NewScraper(configData []byte) (*Scraper, error)
  • func (s *Scraper) Init() error
  • func (s *Scraper) Output(format OutFormat) (string, error)
  • func (s *Scraper) Scrape(outputFormat OutFormat) (string, error)

Types

type Collection

type Collection struct {
	Name     string
	Selector string
	Fields   []Element
}

type CollectionList

type CollectionList []ElementsMap

type CollectionsMap

type CollectionsMap map[string]CollectionList

type Element

type Element struct {
	Name     string
	Type     string
	Selector string
	Unique   bool
}

type ElementsMap

type ElementsMap map[string]interface{}

type OutFormat

type OutFormat string

const (
	JSON OutFormat = "json"
	CSV  OutFormat = "csv"
	XML  OutFormat = "xml"
)

type PageData

type PageData struct {
	PageElements ElementsMap    `json:"page"`
	Collections  CollectionsMap `json:"collections"`
}

type Scraper

type Scraper struct {
	URL          string `yaml:"url"`
	Doc          *goquery.Document
	PageData     PageData
	PageElements []Element `yaml:"page"`
	Collections  []Collection
}

func NewScraper

func NewScraper(configData []byte) (*Scraper, error)

NewScraper parses a scraper config and returns a Scraper

func (*Scraper) Init

func (s *Scraper) Init() error

func (*Scraper) Output

func (s *Scraper) Output(format OutFormat) (string, error)

func (*Scraper) Scrape

func (s *Scraper) Scrape(outputFormat OutFormat) (string, error)