Versions in this module Expand all Collapse all v0 v0.1.0 Mar 21, 2021 Changes in this version + func ReplaceSQL(old, searchPattern string) string + type Link struct + FromU *url.URL + LinkText string + ToU *url.URL + type LinkBatcher struct + func NewLinkBatcher(maxBatch int, s *Storage) *LinkBatcher + func (lb *LinkBatcher) AddLink(link *Link) error + func (lb *LinkBatcher) KillWorkers() + func (lb *LinkBatcher) ResilientBatchAddLinks(links []*Link) error + func (lb *LinkBatcher) SpawnWorkers(nWorkers int) + func (lb *LinkBatcher) WaitUntilEmpty() <-chan bool + func (lb *LinkBatcher) Worker(endSignal <-chan bool, doneChan chan<- bool) + type Page struct + U *url.URL + type PageBatcher struct + Cache *lru.Cache + func NewPageBatcher(maxBatch int, s *Storage) (*PageBatcher, error) + func (pb *PageBatcher) AddPage(page *Page) bool + func (pb *PageBatcher) KillWorkers() + func (pb *PageBatcher) SpawnWorkers(nWorkers int) + func (pb *PageBatcher) WaitUntilEmpty() <-chan bool + func (pb *PageBatcher) Worker(endSignal <-chan bool, doneChan chan<- bool) + type Storage struct + LinkTable string + PageTable string + URI string + func NewStorage(uri string, pageTable string, linkTable string) (*Storage, error) + func (s *Storage) AddLink(link *Link) error + func (s *Storage) AddPage(page *Page) error + func (s *Storage) BatchAddLinks(links []*Link) error + func (s *Storage) BatchAddPages(pages []*Page) error + func (s *Storage) CheckLinkExists(fromU *url.URL, toU *url.URL) (bool, error) + func (s *Storage) CheckPageExists(u *url.URL) (bool, error) + func (s *Storage) Close() error + func (s *Storage) CountLinks() (int, error) + func (s *Storage) CountPages() (int, error) + func (s *Storage) GetLinksFrom(pageHash string, limit int) ([]string, error) + func (s *Storage) GetLinksTo(pageHash string, limit int) ([]string, error) + func (s *Storage) GetPage(pageHash string) (*Page, error) + func (s *Storage) GetPageHashesFromHost(host string, limit int) ([]string, error) + func (s *Storage) Init() error + func (s *Storage) KeepPingingOn(d time.Duration) chan<- bool