crawler

Documentation

Index

• func ExtractContent(webpageURL string, processor *processor)
• func InitProcessor(params CrawlPsParams) *processor
• type CrawlPsParams
• type CrawledUrlMap
• type ProcessorLinks
• type SpendTime
• type StatisticsMap

Functions

func ExtractContent

func ExtractContent(
	webpageURL string,
	processor *processor,
)

func InitProcessor

func InitProcessor(params CrawlPsParams) *processor

Types

type CrawlPsParams

type CrawlPsParams struct {
	SiteToCrawl      string
	LaunchThreadsNum int
	VerboseMode      bool
	ReqThrottling    time.Duration
	TimeLimit        time.Duration
}

type CrawledUrlMap

type CrawledUrlMap map[string]int

type ProcessorLinks

type ProcessorLinks struct {
	Pending   int
	Requested int
	Processed int
}

type SpendTime

type SpendTime struct {
	Total      int64
	RequestAvg int64
	ParseAvg   int64
}

type StatisticsMap

type StatisticsMap struct {
	CrawledUrlMap  CrawledUrlMap
	ProcessorLinks ProcessorLinks
	SpendTime      SpendTime
}