go_cbox_scraper

README

go-cbox-scraper

Handles scraping from cbox.ws

Note: this package only handles "classic" cbox pages

Note 2: the entire scraper state is saved in the cache file This means - debug level, box id, webhost id, etc. If you need to change any of those values, simply load the cache, change the values you need, and save.

Documentation

Index

• Constants
• type CBoxPage
  • func NewCBoxPage(cbxInfo ServerInfo) *CBoxPage
  • func (p *CBoxPage) FetchMain() error
  • func (p *CBoxPage) FetchNext() error
  • func (p *CBoxPage) FetchPrevious() error
  • func (p *CBoxPage) LargestID() int
  • func (p *CBoxPage) SmallestID() int
• type Message
  • func (m *Message) String() string
• type Scraper
  • func NewScraper() *Scraper
  • func NewScraperFromFile(filePath string) (*Scraper, error)
  • func (s *Scraper) Configure(config ServerInfo)
  • func (s *Scraper) Load(path string) error
  • func (s *Scraper) Save(path string) error
  • func (s *Scraper) Scrape(updatesOnly bool) error
• type ServerInfo

Constants

const DatetimeFormat = "_2 Jan 06, 03:04 PM"

const DisplayDatetimeFormat = "2006-01-02 03:04PM"

Types

type CBoxPage

type CBoxPage struct {
	Messages    map[int]*Message
	CanPaginate bool

	ServerInfo
	// contains filtered or unexported fields
}

func NewCBoxPage

func NewCBoxPage(cbxInfo ServerInfo) *CBoxPage

func (*CBoxPage) FetchMain

func (p *CBoxPage) FetchMain() error

func (*CBoxPage) FetchNext

func (p *CBoxPage) FetchNext() error

func (*CBoxPage) FetchPrevious

func (p *CBoxPage) FetchPrevious() error

func (*CBoxPage) LargestID

func (p *CBoxPage) LargestID() int

func (*CBoxPage) SmallestID

func (p *CBoxPage) SmallestID() int

type Message

type Message struct {
	MessageID int
	DateTime  time.Time
	Username  string
	Message   string
}

func (*Message) String

func (m *Message) String() string

type Scraper

type Scraper struct {
	SmallestMessageID int
	LargestMessageID  int
	Messages          map[int]*Message
	ServerInfo
}

func NewScraper

func NewScraper() *Scraper

func NewScraperFromFile

func NewScraperFromFile(filePath string) (*Scraper, error)

func (*Scraper) Configure

func (s *Scraper) Configure(config ServerInfo)

func (*Scraper) Load

func (s *Scraper) Load(path string) error

func (*Scraper) Save

func (s *Scraper) Save(path string) error

func (*Scraper) Scrape

func (s *Scraper) Scrape(updatesOnly bool) error

type ServerInfo

type ServerInfo struct {
	WebHostID int
	BoxID     int
	BoxTag    string
	Debug     bool
}