scrapers

Documentation

Index

• Constants
• type CamCamCamScraper
  • func (s *CamCamCamScraper) AutoScrape()
  • func (s *CamCamCamScraper) CheckInit() bool
  • func (s *CamCamCamScraper) FilterByKeywords(keywords []string) []models.CamCamCamDeal
  • func (s *CamCamCamScraper) GetData() []models.CamCamCamDeal
  • func (s *CamCamCamScraper) GetDealDropString(deal *models.CamCamCamDeal) string
  • func (s *CamCamCamScraper) GetLatestDeals(count int) []models.CamCamCamDeal
  • func (s *CamCamCamScraper) IsTargetDropGreater(deal *models.CamCamCamDeal, target int) bool
  • func (s *CamCamCamScraper) Scrape() error
• type DealType
• type OzBargainScraper
  • func (s *OzBargainScraper) AutoScrape()
  • func (s *OzBargainScraper) CheckInit() bool
  • func (s *OzBargainScraper) FilterByKeywords(keywords []string) []models.OzBargainDeal
  • func (s *OzBargainScraper) GetData() []models.OzBargainDeal
  • func (s *OzBargainScraper) GetDealAge(postedOn string) time.Duration
  • func (s *OzBargainScraper) GetDealType(deal models.OzBargainDeal) int
  • func (s *OzBargainScraper) GetLatestDeals(count int) []models.OzBargainDeal
  • func (s *OzBargainScraper) Scrape() error
• type Scraper
• type ScraperID

Constants

const NUM_DEALS_TO_SEND = 5

No of latest deals to send

const URL_OZBARGAIN = "https://www.ozbargain.com.au/"

Ozbargain base url

Types

type CamCamCamScraper

type CamCamCamScraper struct {
	BaseUrl         []string               // Urls to scrape
	Logger          *zap.Logger            // Reference to main logger
	SID             ScraperID              // Scraper ID
	ScrapeInterval  int                    // Scrape interval
	MaxDealsToStore int                    // Max. no. of deals to have in memory
	Deals           []models.CamCamCamDeal // List of deals
}

Camel Camel Camel - Amazon scraper

func (*CamCamCamScraper) AutoScrape

func (s *CamCamCamScraper) AutoScrape()

go routine to auto scrape every X minutes

func (*CamCamCamScraper) CheckInit

func (s *CamCamCamScraper) CheckInit() bool

Check initialisation

func (*CamCamCamScraper) FilterByKeywords

func (s *CamCamCamScraper) FilterByKeywords(keywords []string) []models.CamCamCamDeal

Filter list of deals by keywords

func (*CamCamCamScraper) GetData

func (s *CamCamCamScraper) GetData() []models.CamCamCamDeal

Get scraper data

func (*CamCamCamScraper) GetDealDropString

func (s *CamCamCamScraper) GetDealDropString(deal *models.CamCamCamDeal) string

Get deal drop string

func (*CamCamCamScraper) GetLatestDeals

func (s *CamCamCamScraper) GetLatestDeals(count int) []models.CamCamCamDeal

Get 'count' deals from the list of deals

func (*CamCamCamScraper) IsTargetDropGreater

func (s *CamCamCamScraper) IsTargetDropGreater(deal *models.CamCamCamDeal, target int) bool

Check deal drop percent i.e. check if the price drop is greater than 'target' percent

func (*CamCamCamScraper) Scrape

func (s *CamCamCamScraper) Scrape() error

Scrape the url

type DealType

type DealType int

Deal types

const (
	UNKNOWN DealType = iota
	OZB_REG
	OZB_SUPER
	OZB_GOOD
	AMZ_DAILY
	AMZ_WEEKLY
)

type OzBargainScraper

type OzBargainScraper struct {
	BaseUrl         string
	Logger          *zap.Logger
	Deals           []models.OzBargainDeal
	SID             ScraperID // Scraper ID
	ScrapeInterval  int       // Scrape interval
	MaxDealsToStore int       // Max. no. of deals to have in memory
}

Ozbargain scraper

func (*OzBargainScraper) AutoScrape

func (s *OzBargainScraper) AutoScrape()

go routine to auto scrape every X minutes

func (*OzBargainScraper) CheckInit

func (s *OzBargainScraper) CheckInit() bool

Check initialisation

func (*OzBargainScraper) FilterByKeywords

func (s *OzBargainScraper) FilterByKeywords(keywords []string) []models.OzBargainDeal

Filter list of deals by keywords

func (*OzBargainScraper) GetData

func (s *OzBargainScraper) GetData() []models.OzBargainDeal

Get scraper data

func (*OzBargainScraper) GetDealAge

func (s *OzBargainScraper) GetDealAge(postedOn string) time.Duration

Calculate the time elapsed since the deal was posted

func (*OzBargainScraper) GetDealType

func (s *OzBargainScraper) GetDealType(deal models.OzBargainDeal) int

Check if deal is a super deal, good deal or just a regular deal

func (*OzBargainScraper) GetLatestDeals

func (s *OzBargainScraper) GetLatestDeals(count int) []models.OzBargainDeal

Get 'count' deals from the list of deals

func (*OzBargainScraper) Scrape

func (s *OzBargainScraper) Scrape() error

Scrape the url

type Scraper

type Scraper interface {
	Scrape()
	AutoScrape()
	GetData() interface{}
}

type ScraperID

type ScraperID int

Scraper type

var SID_CCC_AMAZON ScraperID = 1

var SID_OZBARGAIN ScraperID = 0