scraper

Documentation

Index

• func GetTextLength(text string) int
• func HandleHttpStatusErr(res *http.Response) error
• func Preprocessing(text string) string
• func RemoveSimilar(arr []models.TweetArticle, threshold float64) []models.TweetArticle
• type AccessToken
• type Scraper
  • func NewScraper(token string, db *db.Database, workerId string) *Scraper
  • func (s Scraper) CreateHttpReq(url string) (string, error)
  • func (s Scraper) GetRecentSearch(keyword string, nextToken string, injectNum ...int) ([]models.TweetArticle, error)
  • func (s Scraper) MakeUniqueArticle(jobId string) error
  • func (s Scraper) MakeUniqueTweet(list []models.TweetArticle) []models.TweetArticle
  • func (s Scraper) WriteTweetOnDB(tweet models.TweetArticle) error
• type Twitter
• type Worker

Functions

func GetTextLength

func GetTextLength(text string) int

func HandleHttpStatusErr

func HandleHttpStatusErr(res *http.Response) error

func Preprocessing

func Preprocessing(text string) string

func RemoveSimilar

func RemoveSimilar(arr []models.TweetArticle, threshold float64) []models.TweetArticle

Types

type AccessToken

type AccessToken struct {
	Token string
}

type Scraper

type Scraper struct {
	AccessToken
	Twitter
	Worker
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper(token string, db *db.Database, workerId string) *Scraper

func (Scraper) CreateHttpReq

func (s Scraper) CreateHttpReq(url string) (string, error)

func (Scraper) GetRecentSearch

func (s Scraper) GetRecentSearch(keyword string, nextToken string, injectNum ...int) ([]models.TweetArticle, error)

func (Scraper) MakeUniqueArticle

func (s Scraper) MakeUniqueArticle(jobId string) error

func (Scraper) MakeUniqueTweet

func (s Scraper) MakeUniqueTweet(list []models.TweetArticle) []models.TweetArticle

func (Scraper) WriteTweetOnDB

func (s Scraper) WriteTweetOnDB(tweet models.TweetArticle) error

type Twitter

type Twitter interface {
	GetRecentSearch(keyword string) ([]models.TweetArticle, error)
	GetFullArchiveRecentSearch(keyword string) error
}

type Worker

type Worker struct {
	// contains filtered or unexported fields
}