crawl

Documentation

Index

• func RunCrawler(db *gorm.DB)
• func RunStats(site site.Site, crawl Crawl, db *gorm.DB, wg *sync.WaitGroup, ...)
• type Crawl
• type Stats

Functions

func RunCrawler

func RunCrawler(db *gorm.DB)

func RunStats

func RunStats(site site.Site, crawl Crawl, db *gorm.DB, wg *sync.WaitGroup, maxPool chan struct{})

Types

type Crawl

type Crawl struct {
	gorm.Model
	StartDate time.Time   `json:"startDate"`
	Stats     []Stats     `gorm:"foreignKey:CrawlerRefer" json:"stats"`
	Sites     []site.Site `gorm:"foreignKey:CrawlerRefer" json:"sites"`
}

type Stats

type Stats struct {
	gorm.Model   `json:"-"`
	CrawlerRefer uint                      `json:"-"`
	SiteRefer    uint                      `json:"-"`
	Site         site.Site                 `gorm:"foreignKey:SiteRefer" json:"site"`
	Compression  compression.Compression   `json:"compression"`
	IPVersion    ipversion.IPVersion       `json:"ip_version"`
	Protocol     protocol.Protocol         `json:"protocol"`
	SSG          staticgenerator.Generator `json:"ssg"`
	Images       []image.Image             `gorm:"foreignKey:StatRefer" json:"images"`
}