dev

Documentation

Index

• Constants
• Variables
• func Fetch(url string) ([]byte, error)
• func FetchAndParse(url string) (*html.Node, error)
• func GetAllTags(cb func(t []Tag)) error
• func GetUserHTML(iurl string) ([]byte, error)
• func GetUsersFromID(id int, cb func(u *User)) error
• func QuerySelector(root *html.Node, selector string) (*html.Node, error)
• func QuerySelectorAll(root *html.Node, selector string) ([]*html.Node, error)
• type Article
  • func GetAllArticles(username string) ([]Article, error)
  • func GetArticles(username string, page int, limit int) ([]Article, error)
  • func ParseArticle(b []byte) (*Article, error)
  • func ParseArticles(b []byte) ([]Article, error)
  • func (a *Article) CreatedTime() (time.Time, error)
• type Articles
  • func (a Articles) JSON() ([]byte, error)
• type Comment
  • func (c *Comment) CreatedTime() (time.Time, error)
• type Snapshot
  • func GetSnapshot(id string) (ss *Snapshot, err error)
• type Tag
  • func GetTags(page int) ([]Tag, error)
  • func ParseTag(b []byte) (*Tag, error)
  • func ParseTags(b []byte) ([]Tag, error)
• type User
  • func GetUserByID(id string) (*User, error)
  • func GetUserByUsername(username string) (*User, error)
  • func ParseUser(b []byte) (*User, error)
  • func (u *User) JSON() ([]byte, error)
  • func (u *User) JoinedAtTime() (time.Time, error)
  • func (u *User) URL() string
• type UserPage
  • func ParseUserPage(b []byte) (*UserPage, error)
  • func (u UserPage) JSON() ([]byte, error)

Constants

const (
	// UserAgent will be used in requests.
	UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
)

Variables

var (
	// ErrNotFound is returned for 404s.
	ErrNotFound = fmt.Errorf("server responded with a 404")

	// ErrNotOK is returned for non 200s.
	ErrNotOK = fmt.Errorf("server responded with a non-200")
)

var (
	// Timeout is the default HTTP timeout.
	Timeout = time.Second * 15
)

Functions

func Fetch

func Fetch(url string) ([]byte, error)

Fetch performs a simple GET request to grab html data.

func FetchAndParse

func FetchAndParse(url string) (*html.Node, error)

FetchAndParse takes care of the page fetch and root parsing.

func GetAllTags

func GetAllTags(cb func(t []Tag)) error

GetAllTags will fetch all tags returned from the Dev API. An artifical cap of 10,000 pages has been set.

func GetUserHTML

func GetUserHTML(iurl string) ([]byte, error)

GetUserHTML will fetch a page of tags from the Dev API.

func GetUsersFromID

func GetUsersFromID(id int, cb func(u *User)) error

GetUsersFromID will iterate through all users starting from the ID. An artifical cap has been set to 1m. Dev has roughly 800K users at the moment.

func QuerySelector

func QuerySelector(root *html.Node, selector string) (*html.Node, error)

QuerySelector will parse a selector and perform a query on the DOM.

func QuerySelectorAll

func QuerySelectorAll(root *html.Node, selector string) ([]*html.Node, error)

QuerySelectorAll will parse a selector and perform a query on the DOM.

Types

type Article

type Article struct {
	ID                     int      `json:"id"`
	Type                   string   `json:"type_of"`
	Title                  string   `json:"title"`
	Description            string   `json:"description"`
	ReadablePublishDate    string   `json:"readable_publish_date"`
	Slug                   string   `json:"slug"`
	Path                   string   `json:"path"`
	URL                    string   `json:"url"`
	CommentsCount          int      `json:"comments_count"`
	PublicReactionsCount   int      `json:"public_reactions_count"`
	CollectionID           int      `json:"collection_id"`
	PublishedTimestamp     string   `json:"published_timestamp"`
	PositiveReactionsCount int      `json:"positive_reactions_count"`
	CoverImage             string   `json:"cover_image"`
	SocialImage            string   `json:"social_image"`
	CanonicalURL           string   `json:"canonical_url"`
	CreatedAt              string   `json:"created_at"`
	EditedAt               string   `json:"edited_at"`
	CrosspostedAt          string   `json:"crossposted_at"`
	PublishedAt            string   `json:"published_at"`
	LastCommentAt          string   `json:"last_comment_at"`
	TagList                []string `json:"tag_list"`
	Tags                   string   `json:"tags"`
	User                   User     `json:"user"`
}

Article represents a single user article (post).

func GetAllArticles

func GetAllArticles(username string) ([]Article, error)

GetAllArticles will fetch a all pages of articles for a user from the Dev API. We set an artifical cap at 5 pages (5,000 articles).

func GetArticles

func GetArticles(username string, page int, limit int) ([]Article, error)

GetArticles will fetch a page of articles for a user from the Dev API.

func ParseArticle

func ParseArticle(b []byte) (*Article, error)

ParseArticle will parse the JSON of a single article.

func ParseArticles

func ParseArticles(b []byte) ([]Article, error)

ParseArticles will parse the JSON of a multiple articles.

func (*Article) CreatedTime

func (a *Article) CreatedTime() (time.Time, error)

CreatedTime will return a time object.

type Articles

type Articles []Article

Articles is a collection of Article.

func (Articles) JSON

func (a Articles) JSON() ([]byte, error)

JSON converts the structs to JSON.

type Comment

type Comment struct {
	Title     string `json:"title"`
	URL       string `json:"url"`
	Comment   string `json:"comment"`
	CreatedAt string `json:"created_at"`
}

Comment is a user comment.

func (*Comment) CreatedTime

func (c *Comment) CreatedTime() (time.Time, error)

CreatedTime will parse a comment's time.

type Snapshot

type Snapshot struct {
	User     User     `json:"user"`
	Articles Articles `json:"articles"`
	HTMLPage []byte   `json:"html_page"`
	UserPage UserPage `json:"user_details"`
}

Snapshot is an aggregate of all user data. This includes API data as well as data collected over via scraping.

func GetSnapshot

func GetSnapshot(id string) (ss *Snapshot, err error)

GetSnapshot will collect all info for a user in one shot.

type Tag

type Tag struct {
	ID           int    `json:"id"`
	Name         string `json:"name"`
	BGColorHex   string `json:"bg_color_hex"`
	TextColorHex string `json:"text_color_hex"`
}

Tag is an article tag.

func GetTags

func GetTags(page int) ([]Tag, error)

GetTags will fetch a page of tags from the Dev API.

func ParseTag

func ParseTag(b []byte) (*Tag, error)

ParseTag will parse the JSON of a single tag.

func ParseTags

func ParseTags(b []byte) ([]Tag, error)

ParseTags will parse the JSON of a multiple tags.

type User

type User struct {
	ID              int    `json:"id"`
	Type            string `json:"type_of"`
	Name            string `json:"name"`
	Username        string `json:"username"`
	Summary         string `json:"summary"`
	TwitterUsername string `json:"twitter_username"`
	GithubUsername  string `json:"github_username"`
	WebsiteURL      string `json:"website_url"`
	Location        string `json:"location"`
	JoinedAt        string `json:"joined_at"`
	ProfileImage    string `json:"profile_image"`
}

User represents a dev.to user record.

func GetUserByID

func GetUserByID(id string) (*User, error)

GetUserByID will fetch a single user from Dev's API.

func GetUserByUsername

func GetUserByUsername(username string) (*User, error)

GetUserByUsername will fetch a single user from Dev's API.

func ParseUser

func ParseUser(b []byte) (*User, error)

ParseUser will parse the JSON of a single user.

func (*User) JSON

func (u *User) JSON() ([]byte, error)

JSON converts the struct to JSON.

func (*User) JoinedAtTime

func (u *User) JoinedAtTime() (time.Time, error)

JoinedAtTime will return a time struct for the JoinedAt time.

func (*User) URL

func (u *User) URL() string

URL returns the fully qualified URL to the user's dev.to profile.

type UserPage

type UserPage struct {
	Education    string    `json:"education"`
	Title        string    `json:"title"`
	Company      string    `json:"company"`
	CompanyURL   string    `json:"company_url"`
	Email        string    `json:"email"`
	Links        []string  `json:"links"`
	Badges       []string  `json:"badges"`
	Comments     []Comment `json:"comments"`
	Skills       []string  `json:"skills"`
	Interests    []string  `json:"interests"`
	AvailableFor string    `json:"available_for"`

	// TODO: Not Yet Implemented
	PostCount    int `json:"post_count"`
	CommentCount int `json:"comment_count"`
	TagsFollowed int `json:"tags_followed"`
}

UserPage is a user's web page collected via scraping.

func ParseUserPage

func ParseUserPage(b []byte) (*UserPage, error)

ParseUserPage will parse the HTML of a user's wwww page.

func (UserPage) JSON

func (u UserPage) JSON() ([]byte, error)

JSON converts the UserPage to JSON.