wikipedia

Documentation

Index

• func AddEdgesIfDoNotExist(currentNode string, neighborNodes []string) (neighborsAdded []string, err error)
• func CleanUrl(link string) string
• func FilterPage(e *colly.HTMLElement) (*colly.HTMLElement, error)
• func GetRandomNode() (string, error)
• func IsValidCrawlLink(link string) bool
• type Page
• type RArticleResp
• type RQuery

Functions

func AddEdgesIfDoNotExist

func AddEdgesIfDoNotExist(
	currentNode string,
	neighborNodes []string,
) (
	neighborsAdded []string,
	err error,
)

adds edge to DB, returns new neighbors added (to crawl on)

func CleanUrl

func CleanUrl(link string) string

decodes and standaridizes URL

func FilterPage

func FilterPage(e *colly.HTMLElement) (*colly.HTMLElement, error)

filters down full page body to elements we want to focus on

func GetRandomNode

func GetRandomNode() (string, error)

gets random article from metawiki API returns article in the form "/wiki/XXXXX"

func IsValidCrawlLink

func IsValidCrawlLink(link string) bool

determines if is good link to crawl on

Types

type Page

type Page struct {
	Title string `json:"title"`
}

type RArticleResp

type RArticleResp struct {
	Query RQuery `json:"query"`
}

type RQuery

type RQuery struct {
	Pages map[string]Page `json:"pages"`
}