counties

Documentation

Index

• func AddEdgesIfDoNotExist(currentNode string, neighborNodes []string) (neighborsAdded []string, err error)
• func CleanUrl(link string) string
• func FilterPage(e *colly.HTMLElement) (*colly.HTMLElement, error)
• func GetRandomNode() (string, error)
• func IsValidCrawlLink(link string) bool

Functions

func AddEdgesIfDoNotExist

func AddEdgesIfDoNotExist(
	currentNode string,
	neighborNodes []string,
) (
	neighborsAdded []string,
	err error,
)

adds edge to DB, returns new neighbors added (to crawl on)

func CleanUrl

func CleanUrl(link string) string

decodes and standaridizes URL

func FilterPage

func FilterPage(e *colly.HTMLElement) (*colly.HTMLElement, error)

filters down full page body to elements we want to focus on

func GetRandomNode

func GetRandomNode() (string, error)

func IsValidCrawlLink

func IsValidCrawlLink(link string) bool