gopstaticcrawler

Documentation

Index

• Variables
• func AddRessourceIfDoNotExists(ressources *[]*Ressource, ressource *Ressource) bool
• func InitCrawler() *colly.Collector
• func NewOptions(url *string, logFileOption *os.File, report *bool, recursive *bool, ...)
• func PrintBanner()
• func PrintNewRessourceFound(isInternal, ressourceType, link string)
• func PrintOptions(options *Options)
• func PrintRessourceList(ressources_string []*Ressource)
• func PrintRessourcesResume(ressourceType string, url string, ressources []*Ressource)
• func PrintStatistics(duration time.Duration, internal_ressources []*Ressource, ...)
• func RunCrawlerCmd()
• func TreatLinkHref(e *colly.HTMLElement)
• func TreatScriptSrc(e *colly.HTMLElement)
• func VisiteURL(visited *[]string, c *colly.Collector, Url string)
• func WriteRessourceListReport(ressources_string []*Ressource)
• type Options
• type Ressource
  • func CreateRessource(urlReference string, script string, kind string) (bool, *Ressource)

Variables

var (
	Red     = color.FgRed.Render
	Green   = color.FgGreen.Render
	Yellow  = color.FgYellow.Render
	Blue    = color.FgBlue.Render
	Cyan    = color.FgCyan.Render
	Magenta = color.FgMagenta.Render
)

var (
	Internal_ressources []*Ressource
	External_ressources []*Ressource
	URLVisited          []string
	ScreenshotList      []gopchromedp.Item
	ConcurrencyChan     chan struct{}
)

Functions

func AddRessourceIfDoNotExists

func AddRessourceIfDoNotExists(ressources *[]*Ressource, ressource *Ressource) bool

func InitCrawler

func InitCrawler() *colly.Collector

func NewOptions

func NewOptions(url *string, logFileOption *os.File, report *bool, recursive *bool, screenshot *bool, cookie *string, proxy *string, delay *int, concurrency *int)

func PrintBanner

func PrintBanner()

func PrintNewRessourceFound

func PrintNewRessourceFound(isInternal, ressourceType, link string)

func PrintOptions

func PrintOptions(options *Options)

func PrintRessourceList

func PrintRessourceList(ressources_string []*Ressource)

func PrintRessourcesResume

func PrintRessourcesResume(ressourceType string, url string, ressources []*Ressource)

func PrintStatistics

func PrintStatistics(duration time.Duration, internal_ressources []*Ressource, external_ressources []*Ressource)

func RunCrawlerCmd

func RunCrawlerCmd()

func TreatLinkHref

func TreatLinkHref(e *colly.HTMLElement)

func TreatScriptSrc

func TreatScriptSrc(e *colly.HTMLElement)

func VisiteURL

func VisiteURL(visited *[]string, c *colly.Collector, Url string)

func WriteRessourceListReport

func WriteRessourceListReport(ressources_string []*Ressource)

Types

type Options

type Options struct {
	UrlPtr         *string
	LogFile        *os.File
	ReportPtr      *bool
	RecursivePtr   *bool
	ScreenshotPtr  *bool
	CookiePtr      *string
	ProxyPtr       *string
	DelayPtr       *int
	ConcurrencyPtr *int
}

var (
	GoCrawlerOptions Options
)

type Ressource

type Ressource struct {
	Url      string
	Type     string
	Secure   bool
	Internal bool
	sync.RWMutex
}

func CreateRessource

func CreateRessource(urlReference string, script string, kind string) (bool, *Ressource)