common

Documentation

Index

• func DeserializePredicate(pb []byte) (exists bool, err error)
• func InitJsonLogger(writer io.Writer, logLevel string) (logger log.Logger)
• type Config
  • func InitConfig(path string) (appConfig Config, err error)
• type Connection
• type DatabaseConfig
• type DbStore
  • func (store *DbStore) CheckOrCreatePredicate(ctx *context.Context, txn *dgo.Txn, parentUid string, childUid string) (exists bool, err error)
  • func (store *DbStore) CheckPredicate(ctx *context.Context, txn *dgo.Txn, parentUid string, childUid string) (exists bool, err error)
  • func (store *DbStore) Connect(dbConfig DatabaseConfig)
  • func (store *DbStore) DeleteAll() (err error)
  • func (store *DbStore) FindNode(ctx *context.Context, txn *dgo.Txn, Url string, depth int) (currentPage *Page, err error)
  • func (store *DbStore) FindNodeDepth(ctx *context.Context, txn *dgo.Txn, Url string) (depth int, err error)
  • func (store *DbStore) FindOrCreateNode(ctx *context.Context, txn *dgo.Txn, currentPage *Page) (uid string, err error)
  • func (store *DbStore) SetSchema() (err error)
• type GeneralConfig
• type JsonPage
• type JsonPredicate
• type Page
  • func (page *Page) FetchChildPages(resp *http.Response, logger log.Logger) (childPages []*Page, err error)
  • func (page *Page) IsRelativeHtml(href string) bool
  • func (page *Page) IsRelativeUrl(href string) bool
  • func (page *Page) MaxDepth() (countDepth int)
  • func (page *Page) ParseRelativeUrl(relativeUrl string) (absoluteUrl *url.URL, err error)
• type QueueConfig
• type Store

Functions

func DeserializePredicate

func DeserializePredicate(pb []byte) (exists bool, err error)

Checks JSON dgraph edge result to see if edge exists

func InitJsonLogger

func InitJsonLogger(writer io.Writer, logLevel string) (logger log.Logger)

InitJsonLogger function initiates a structured JSON logger, taking in the specified log level for what is displayed at runtime.

Types

type Config

type Config struct {
	General  GeneralConfig
	Database DatabaseConfig
	Queue    QueueConfig
}

Config holds General and Database config from TOML file

func InitConfig

func InitConfig(path string) (appConfig Config, err error)

InitConfig loads config in from specified TOML file.

type Connection

type Connection struct {
	Host string
	Port int
}

Connection holds the database connection data

type DatabaseConfig

type DatabaseConfig struct {
	Connections []*Connection
}

DatabaseConfig holds database section of toml config

type DbStore

type DbStore struct {
	*dgo.Dgraph
	Connection []*grpc.ClientConn
}

DbStore holds dgraph client and connections

func (*DbStore) CheckOrCreatePredicate

func (store *DbStore) CheckOrCreatePredicate(ctx *context.Context, txn *dgo.Txn, parentUid string, childUid string) (exists bool, err error)

CheckOrCreatePredicate function checks for edge, creates if doesn't exist.

func (*DbStore) CheckPredicate

func (store *DbStore) CheckPredicate(ctx *context.Context, txn *dgo.Txn, parentUid string, childUid string) (exists bool, err error)

CheckPredicate function checks to see if edge exists

func (*DbStore) Connect

func (store *DbStore) Connect(dbConfig DatabaseConfig)

Connect function initiates connections to database

func (*DbStore) DeleteAll

func (store *DbStore) DeleteAll() (err error)

DeleteAll function deletes all data in database

func (*DbStore) FindNode

func (store *DbStore) FindNode(ctx *context.Context, txn *dgo.Txn, Url string, depth int) (currentPage *Page, err error)

FindNode function finds Page by URL and depth

func (*DbStore) FindNodeDepth

func (store *DbStore) FindNodeDepth(ctx *context.Context, txn *dgo.Txn, Url string) (depth int, err error)

func (*DbStore) FindOrCreateNode

func (store *DbStore) FindOrCreateNode(ctx *context.Context, txn *dgo.Txn, currentPage *Page) (uid string, err error)

FindOrCreateNode function checks for page, creates if doesn't exist.

func (*DbStore) SetSchema

func (store *DbStore) SetSchema() (err error)

SetSchema function sets the schema for dgraph (mainly for tests)

type GeneralConfig

type GeneralConfig struct {
	MaxGoroutines       int `toml:"max_goroutines"`
	Port                int
	LogLevel            string `toml:"log_level"`
	HttpRetryAttempts   int    `toml:"http_retry_attempts"`
	HttpBackOffDuration int    `toml:"http_back_off_duration"`
	WaitCrawl           bool   `toml:"wait_crawl"`
}

GeneralConfig holds general section of toml config

type JsonPage

type JsonPage struct {
	Uid       string      `json:"uid,omitempty"`
	Url       string      `json:"url,omitempty"`
	Timestamp int64       `json:"timestamp,omitempty"`
	Children  []*JsonPage `json:"links,omitempty"`
}

JsonPage is used to turn Page into a dgraph compatible struct

type JsonPredicate

type JsonPredicate struct {
	Matching int `json:"matching"`
}

JsonPredicate is used to hold the Predicate result from dgraph

type Page

type Page struct {
	Uid       string  `json:"uid,omitempty"`
	Url       string  `json:"url,omitempty"`
	Links     []*Page `json:"links,omitempty"`
	Parent    *Page   `json:"-"`
	Depth     int     `json:"-"`
	Timestamp int64   `json:"timestamp,omitempty"`
}

Page holds page data

func (*Page) FetchChildPages

func (page *Page) FetchChildPages(resp *http.Response, logger log.Logger) (childPages []*Page, err error)

FetchChildPages function converts http response into child page objects

func (*Page) IsRelativeHtml

func (page *Page) IsRelativeHtml(href string) bool

IsRelativeHtml function checks to see if relative URL points to a HTML file

func (*Page) IsRelativeUrl

func (page *Page) IsRelativeUrl(href string) bool

IsRelativeUrl function checks for relative URL path

func (*Page) MaxDepth

func (page *Page) MaxDepth() (countDepth int)

MaxDepth function gets the max depth of the recursive page structure

func (*Page) ParseRelativeUrl

func (page *Page) ParseRelativeUrl(relativeUrl string) (absoluteUrl *url.URL, err error)

ParseRelativeUrl function parses a relative URL string into a URL object

type QueueConfig

type QueueConfig struct {
	QueueURL                      string `toml:"queue_url"`
	MaxConcurrentReceivedMessages int64  `toml:"max_concurrent_received_messages"`
	SQSWaitTimeSeconds            int64  `toml:"sqs_wait_time_seconds"`
}

type Store

type Store interface {
	Connect(dbConfig DatabaseConfig)
	SetSchema() (err error)
	DeleteAll() (err error)
	Create(currentPage *Page) (err error)
	FindNode(ctx *context.Context, txn *dgo.Txn, url string, depth int) (currentPage *Page, err error)
	FindOrCreateNode(ctx *context.Context, txn *dgo.Txn, currentPage *Page) (uid string, err error)
	CheckPredicate(ctx *context.Context, txn *dgo.Txn, parentUid string, childUid string) (exists bool, err error)
	CheckOrCreatePredicate(ctx *context.Context, txn *dgo.Txn, parentUid string, childUid string) (exists bool, err error)
}

Store is interface for database method