crawler

Documentation

Index

• func Crawl(config *Config)
• type Config
• type Stats
  • func (s *Stats) String() string
• type WordMap
  • func NewWordMap() *WordMap
  • func (w *WordMap) Add(word string)
  • func (w *WordMap) Sort()

Functions

func Crawl

func Crawl(config *Config)

Types

type Config

type Config struct {
	URL               string
	Depth             int
	SkipSSL           bool
	Threads           int
	Proxy             string
	UserAgent         string
	OutputPath        string
	OutputFilename    string
	MinimumWordCount  int
	MininumWordLength int
	MaximumWordLength int
	Quiet             bool
	Domains           []string
	Offsite           bool
	ShowCount         bool
}

type Stats

type Stats struct {
	RequestCount  uint64
	ResponseCount uint64
	ErrorCount    uint64
	TotalWords    uint64
	TotalTime     float64
}

func (*Stats) String

func (s *Stats) String() string

type WordMap

type WordMap struct {
	// contains filtered or unexported fields
}

func NewWordMap

func NewWordMap() *WordMap

func (*WordMap) Add

func (w *WordMap) Add(word string)

func (*WordMap) Sort

func (w *WordMap) Sort()