crawler

Documentation

Index

• Constants
• func StartCrawler(c Crawler) error
• type Crawler
• type Data
• type DefaultData
  • func (data *DefaultData) File() (title, filetype, content string)
  • func (data *DefaultData) IsFile() bool
  • func (data *DefaultData) String() string

Constants

const (
	CrawlerPath   = "crawler.db"
	CrawlerBucket = "crawler"
)

Functions

func StartCrawler

func StartCrawler(c Crawler) error

StartCrawler executes the crawler

Types

type Crawler

type Crawler interface {
	Init() error
	Start() error
}

Crawler interface

type Data

type Data interface {
	String() string
	File() (string, string, string)
	IsFile() bool
}

Data -

type DefaultData

type DefaultData struct {
	// Source -
	Source string
	// Date -
	Date string
	// Title -
	Title string
	// URL -
	URL string
	// Text -
	Text string
	// FileType -
	FileType string
}

DefaultData -

func (*DefaultData) File

func (data *DefaultData) File() (title, filetype, content string)

File return title, filetype and content if data is a file type.

func (*DefaultData) IsFile

func (data *DefaultData) IsFile() bool

IsFile return true if field FileType is not "".

func (*DefaultData) String

func (data *DefaultData) String() string