progress

Documentation

Index

• type Config
  • func DefaultConfig() *Config
• type ProgressReporter
  • func NewProgressReporter(config *Config) *ProgressReporter
  • func (pr *ProgressReporter) GetStats() Stats
  • func (pr *ProgressReporter) IncrementDiscovered()
  • func (pr *ProgressReporter) IncrementFailed()
  • func (pr *ProgressReporter) IncrementProcessed()
  • func (pr *ProgressReporter) IncrementSkipped()
  • func (pr *ProgressReporter) IsRateLimited() bool
  • func (pr *ProgressReporter) Start()
  • func (pr *ProgressReporter) Stop()
  • func (pr *ProgressReporter) UpdateStats(processed, discovered, failed, skipped int64, activeWorkers, queueSize int)
  • func (pr *ProgressReporter) WaitForRateLimit()
• type RateLimiter
  • func NewRateLimiter(requestsPerSecond float64) *RateLimiter
  • func (rl *RateLimiter) Stop()
  • func (rl *RateLimiter) Wait()
• type Stats

Types

type Config

type Config struct {
	UpdateInterval time.Duration // How often to update progress (default: 1s)
	ShowProgress   bool          // Whether to show progress indicators
	Output         io.Writer     // Where to write progress (default: stderr)
	Logger         *slog.Logger  // Logger instance
	RateLimit      float64       // Requests per second (0 = no limit)
}

Config holds configuration for progress reporting and rate limiting

func DefaultConfig

func DefaultConfig() *Config

DefaultConfig returns a default progress configuration

type ProgressReporter

type ProgressReporter struct {
	// contains filtered or unexported fields
}

ProgressReporter manages progress display and statistics

func NewProgressReporter

func NewProgressReporter(config *Config) *ProgressReporter

NewProgressReporter creates a new progress reporter

func (*ProgressReporter) GetStats

func (pr *ProgressReporter) GetStats() Stats

GetStats returns a copy of current statistics

func (*ProgressReporter) IncrementDiscovered

func (pr *ProgressReporter) IncrementDiscovered()

IncrementDiscovered increments the discovered URLs counter

func (*ProgressReporter) IncrementFailed

func (pr *ProgressReporter) IncrementFailed()

IncrementFailed increments the failed URLs counter

func (*ProgressReporter) IncrementProcessed

func (pr *ProgressReporter) IncrementProcessed()

IncrementProcessed increments the processed URLs counter

func (*ProgressReporter) IncrementSkipped

func (pr *ProgressReporter) IncrementSkipped()

IncrementSkipped increments the skipped URLs counter

func (*ProgressReporter) IsRateLimited

func (pr *ProgressReporter) IsRateLimited() bool

IsRateLimited returns true if rate limiting is enabled

func (*ProgressReporter) Start

func (pr *ProgressReporter) Start()

Start starts the progress reporter

func (*ProgressReporter) Stop

func (pr *ProgressReporter) Stop()

Stop stops the progress reporter and displays final statistics

func (*ProgressReporter) UpdateStats

func (pr *ProgressReporter) UpdateStats(processed, discovered, failed, skipped int64, activeWorkers, queueSize int)

UpdateStats updates the current statistics

func (*ProgressReporter) WaitForRateLimit

func (pr *ProgressReporter) WaitForRateLimit()

WaitForRateLimit waits for rate limiting if enabled

type RateLimiter

type RateLimiter struct {
	// contains filtered or unexported fields
}

RateLimiter controls the rate of requests

func NewRateLimiter

func NewRateLimiter(requestsPerSecond float64) *RateLimiter

NewRateLimiter creates a new rate limiter

func (*RateLimiter) Stop

func (rl *RateLimiter) Stop()

Stop stops the rate limiter

func (*RateLimiter) Wait

func (rl *RateLimiter) Wait()

Wait blocks until a token is available (rate limiting)

type Stats

type Stats struct {
	StartTime      time.Time     // When crawling started
	URLsProcessed  int64         // Number of URLs processed
	URLsDiscovered int64         // Total URLs discovered
	URLsFailed     int64         // Number of failed URLs
	URLsSkipped    int64         // Number of skipped URLs
	ActiveWorkers  int           // Number of active workers
	QueueSize      int           // Current queue size
	ProcessingRate float64       // URLs per second
	ElapsedTime    time.Duration // Time elapsed since start
	LastUpdateTime time.Time     // Last time stats were updated
}

Stats holds performance and progress statistics