Crawler

Documentation

Index

• Variables
• func Crawl(StartingURL string, wg *sync.WaitGroup)
• func Go(MaxNumberOfGoRoutines int)
• func PadRight(str, pad string, lenght int) string
• func Results()
• func ResultsJSON() ([]byte, error)
• type AllLinksMap
  • func (l *AllLinksMap) Add(URL string)
  • func (l *AllLinksMap) GetLinkMap() map[string]Link
  • func (l *AllLinksMap) Update(URL string, TotalLinks int)
• type Link

Variables

var AllLinks = AllLinksMap{Links: make(map[string]Link)}

Functions

func Crawl

func Crawl(StartingURL string, wg *sync.WaitGroup)

Crawl through a URL, making a note of all External and Internal Links.

func Go

func Go(MaxNumberOfGoRoutines int)

Start the Crawler

func PadRight

func PadRight(str, pad string, lenght int) string

func Results

func Results()

func ResultsJSON

func ResultsJSON() ([]byte, error)

Types

type AllLinksMap

type AllLinksMap struct {
	Links map[string]Link
	Mux   sync.Mutex
}

func (*AllLinksMap) Add

func (l *AllLinksMap) Add(URL string)

Add a URL to the LinkMap

func (*AllLinksMap) GetLinkMap

func (l *AllLinksMap) GetLinkMap() map[string]Link

func (*AllLinksMap) Update

func (l *AllLinksMap) Update(URL string, TotalLinks int)

Update a URL already in the LinkMap

type Link

type Link struct {
	Visited    bool
	TotalLinks int
}