lib

Documentation

Index

• func RunWebDownloader()
• type Config
• type CrawlMode
• type CrawlState
• type Crawler
  • func NewCrawler(cfg *Config) (*Crawler, error)
  • func (c *Crawler) Run() error
• type Dependency
• type ProgressBar
  • func NewProgressBar(total int64) *ProgressBar
  • func (pb *ProgressBar) Add(n int64)
  • func (pb *ProgressBar) Stop()
• type RateLimiter
  • func NewRateLimiter(defaultRate float64, burst int) *RateLimiter
  • func (rl *RateLimiter) Wait(ctx context.Context, domain string) error

Functions

func RunWebDownloader

func RunWebDownloader()

Types

type Config

type Config struct {
	TargetURL         string
	OutputDir         string
	Mode              CrawlMode
	MaxPages          int
	Concurrency       int
	DownloadExternal  bool
	ExternalDomains   []string
	Cookies           map[string]string
	UserAgent         string
	Timeout           time.Duration
	Retries           int
	PreserveStructure bool
	MinifyOutput      bool
	Resume            bool
	RateLimit         float64
	MaxAssetSize      int64
	CrawlIframes      bool
	CrawlHashRoutes   bool
}

type CrawlMode

type CrawlMode int

const (
	ModeFullSite CrawlMode = iota
	ModeSinglePage
)

type CrawlState

type CrawlState struct {
	Version          int       `json:"version"`
	StartURL         string    `json:"start_url"`
	OutputDir        string    `json:"output_dir"`
	VisitedPages     []string  `json:"visited_pages"`
	DownloadedAssets []string  `json:"downloaded_assets"`
	LastUpdate       time.Time `json:"last_update"`
	PagesCount       int       `json:"pages_count"`
	AssetsCount      int       `json:"assets_count"`
	TotalBytes       int64     `json:"total_bytes"`
}

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(cfg *Config) (*Crawler, error)

func (*Crawler) Run

func (c *Crawler) Run() error

type Dependency

type Dependency struct {
	URL       string
	Type      string
	LocalPath string
	IsIframe  bool
}

type ProgressBar

type ProgressBar struct {
	// contains filtered or unexported fields
}

func NewProgressBar

func NewProgressBar(total int64) *ProgressBar

func (*ProgressBar) Add

func (pb *ProgressBar) Add(n int64)

func (*ProgressBar) Stop

func (pb *ProgressBar) Stop()

type RateLimiter

type RateLimiter struct {
	// contains filtered or unexported fields
}

func NewRateLimiter

func NewRateLimiter(defaultRate float64, burst int) *RateLimiter

func (*RateLimiter) Wait

func (rl *RateLimiter) Wait(ctx context.Context, domain string) error