scraper

Documentation

Index

• Constants
• type Config
• type Job
• type ScrapeResult
• type Scraper
  • func NewScraper(outputDir string, workers int, skipTelegram bool) *Scraper
  • func (s *Scraper) AddSource(url string, sourceType string) error
  • func (s *Scraper) DetectSourceType(rawURL string) string
  • func (s *Scraper) LoadConfig() error
  • func (s *Scraper) ReloadConfig() error
  • func (s *Scraper) RemoveSource(urlToRemove string) error
  • func (s *Scraper) Run() error
  • func (s *Scraper) SaveConfig() error
  • func (s *Scraper) ShowConfig() error
• type SourceConfig

Constants

const (
	Reset     = "\033[0m"
	Red       = "\033[31m"
	Green     = "\033[32m"
	Yellow    = "\033[33m"
	Blue      = "\033[34m"
	Magenta   = "\033[35m"
	Cyan      = "\033[36m"
	Gray      = "\033[37m"
	White     = "\033[97m"
	Bold      = "\033[1m"
	BrightRed = "\033[91m"
)

Types

type Config

type Config struct {
	Configs      []string            `json:"configs"`
	MTProto      []string            `json:"mtproto"`
	Proxies      map[string][]string `json:"proxies"`
	Telegram     []string            `json:"telegram"`
	Custom       []SourceConfig      `json:"custom,omitempty"`
	LastRun      string              `json:"last_run"`
	SkipTelegram bool                `json:"skip_telegram"`
}

type Job

type Job struct {
	// contains filtered or unexported fields
}

type ScrapeResult

type ScrapeResult struct {
	Items      []string
	SourceType string
	Error      error
}

type Scraper

type Scraper struct {
	Config       *Config
	OutputDir    string
	Workers      int
	SkipTelegram bool
	// contains filtered or unexported fields
}

func NewScraper

func NewScraper(outputDir string, workers int, skipTelegram bool) *Scraper

func (*Scraper) AddSource

func (s *Scraper) AddSource(url string, sourceType string) error

func (*Scraper) DetectSourceType

func (s *Scraper) DetectSourceType(rawURL string) string

func (*Scraper) LoadConfig

func (s *Scraper) LoadConfig() error

func (*Scraper) ReloadConfig

func (s *Scraper) ReloadConfig() error

func (*Scraper) RemoveSource

func (s *Scraper) RemoveSource(urlToRemove string) error

func (*Scraper) Run

func (s *Scraper) Run() error

func (*Scraper) SaveConfig

func (s *Scraper) SaveConfig() error

func (*Scraper) ShowConfig

func (s *Scraper) ShowConfig() error

type SourceConfig

type SourceConfig struct {
	URLs  []string `json:"urls"`
	PType string   `json:"type,omitempty"`
}