page

Documentation

Index

• func DeserializePredicate(pb []byte) (exists bool, err error)
• func SerializeJsonPage(currentPage *Page) (pb []byte, err error)
• type JsonPage
• type JsonPredicate
• type JsonResult
• type Page
  • func DeserializeJsonPage(pb []byte) (currentPage *Page, err error)
  • func DeserializeSQSPage(msg *sqs.Message) (currentPage *Page, err error)
  • func (page *Page) FetchChildPages(resp *http.Response) (childPages []*Page, err error)
  • func (page *Page) IsRelativeHtml(href string) bool
  • func (page *Page) IsRelativeUrl(href string) bool
  • func (page *Page) MaxDepth() (countDepth int)
  • func (page *Page) ParseRelativeUrl(relativeUrl string) (absoluteUrl *url.URL, err error)
• type SQSPage
  • func ConvertPageToSQSPage(currentPage *Page) *SQSPage

Functions

func DeserializePredicate

func DeserializePredicate(pb []byte) (exists bool, err error)

Checks JSON dgraph edge result to see if edge exists

func SerializeJsonPage

func SerializeJsonPage(currentPage *Page) (pb []byte, err error)

Turns a Page into a JSON string

Types

type JsonPage

type JsonPage struct {
	Uid        string      `json:"uid,omitempty"`
	Url        string      `json:"url,omitempty"`
	Depth      int         `json:"depth,omitempty"`
	Timestamp  int64       `json:"timestamp,omitempty"`
	Children   []*JsonPage `json:"links,omitempty"`
	StatusCode int         `json:"status_code,omitempty"`
}

JsonPage is used to turn Page into a dgraph compatible struct

type JsonPredicate

type JsonPredicate struct {
	Matching int `json:"matching"`
}

JsonPredicate is used to hold the Predicate result from dgraph

type JsonResult

type JsonResult struct {
	Result []*JsonPage      `json:"result,omitempty"`
	Edges  []*JsonPredicate `json:"edges,omitempty"`
}

type Page

type Page struct {
	Uid        string  `json:"-"`
	Url        string  `json:"url,omitempty"`
	Links      []*Page `json:"links,omitempty"`
	Parent     *Page   `json:"-"`
	Depth      int     `json:"-"`
	Timestamp  int64   `json:"timestamp,omitempty"`
	StartUrl   string  `json:"-"`
	StatusCode int     `json:"status_code,omitempty"`
}

Page holds page data

func DeserializeJsonPage

func DeserializeJsonPage(pb []byte) (currentPage *Page, err error)

Turns JSON dgraph result into a Page

func DeserializeSQSPage

func DeserializeSQSPage(msg *sqs.Message) (currentPage *Page, err error)

func (*Page) FetchChildPages

func (page *Page) FetchChildPages(resp *http.Response) (childPages []*Page, err error)

FetchChildPages function converts http response into child page objects

func (*Page) IsRelativeHtml

func (page *Page) IsRelativeHtml(href string) bool

IsRelativeHtml function checks to see if relative URL points to a HTML file

func (*Page) IsRelativeUrl

func (page *Page) IsRelativeUrl(href string) bool

IsRelativeUrl function checks for relative URL path

func (*Page) MaxDepth

func (page *Page) MaxDepth() (countDepth int)

MaxDepth function gets the max depth of the recursive page structure

func (*Page) ParseRelativeUrl

func (page *Page) ParseRelativeUrl(relativeUrl string) (absoluteUrl *url.URL, err error)

ParseRelativeUrl function parses a relative URL string into a URL object

type SQSPage

type SQSPage struct {
	Url       string   `json:"url,omitempty"`
	Parent    *SQSPage `json:"parent,omitempty"`
	Depth     int      `json:"depth,omitempty"`
	Timestamp int64    `json:"timestamp,omitempty"`
	StartUrl  string   `json:"start_url,omitempty"`
}

Page holds page data

func ConvertPageToSQSPage

func ConvertPageToSQSPage(currentPage *Page) *SQSPage

Converts a Page to a SQSPage