crawler

Documentation

Index

• func JsonFileByHostname(host string) string
• func LoadString(file string) (string, error)
• func Marshal(data []Data) string
• func ParseTexts(dom *goquery.Selection) []string
• func Unmarshal(s string, datas *[]Data) error
• func WriteString(file string, content string) error
• type Crawler
  • func New() *Crawler
  • func (c *Crawler) Start()
• type Data

Functions

func JsonFileByHostname

func JsonFileByHostname(host string) string

func LoadString

func LoadString(file string) (string, error)

LoadString loads string from file

func Marshal

func Marshal(data []Data) string

Marshal []Data to json string

func ParseTexts

func ParseTexts(dom *goquery.Selection) []string

func Unmarshal

func Unmarshal(s string, datas *[]Data) error

Unmarshal json string to *Data[]

func WriteString

func WriteString(file string, content string) error

WriteString writes string to file

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

Crawler

func New

func New() *Crawler

func (*Crawler) Start

func (c *Crawler) Start()

Start scraping

type Data

type Data struct {
	Title string   `json:"title"`
	URL   string   `json:"url"`
	Texts []string `json:"texts"`
}