crawler

Documentation

Index

• func CreateColly(delayTime int, concurrency int, cache bool, timeout int, intensive bool, ...) *colly.Collector
• func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMatched
• func ErrorsMatch(url string, body string) []scanner.ErrorMatched
• func GenerateRandomUserAgent() string
• func GetRequest(target string) (string, error)
• func HeadRequest(target string) (string, error)
• func IgnoreMatch(url string, ignoreSlice []string) bool
• func InfoMatch(url string, body string) []scanner.InfoMatched
• func New(target string, txt string, html string, delayTime int, concurrency int, ...) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched, ...)
• func PostRequest(target string, data map[string]string) (string, error)
• func RetrieveBody(target string) string
• func SecretsMatch(url string, body string, secretsFile []string) []scanner.SecretMatched

Functions

func CreateColly

func CreateColly(delayTime int, concurrency int, cache bool, timeout int,
	intensive bool, rua bool, proxy string, insecure bool, userAgent string, target string) *colly.Collector

CreateColly takes as input all the settings needed to instantiate a new Colly Collector object and it returns this object.

func EndpointsMatch

func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMatched

EndpointsMatch check if an endpoint matches a juicy parameter.

func ErrorsMatch

func ErrorsMatch(url string, body string) []scanner.ErrorMatched

ErrorsMatch checks the patterns for errors.

func GenerateRandomUserAgent

func GenerateRandomUserAgent() string

GenerateRandomUserAgent generates a random user agent (can be Chrome or Firefox).

func GetRequest

func GetRequest(target string) (string, error)

GetRequest performs a GET request and return a string (the body of the response).

func HeadRequest

func HeadRequest(target string) (string, error)

HeadRequest performs a HEAD request and return a string (the headers of the response).

func IgnoreMatch

func IgnoreMatch(url string, ignoreSlice []string) bool

IgnoreMatch checks if the URL should be ignored or not.

func InfoMatch

func InfoMatch(url string, body string) []scanner.InfoMatched

InfoMatch checks the patterns for infos.

func New

func New(target string, txt string, html string, delayTime int, concurrency int,
	ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool,
	proxy string, insecure bool, secretsFlag bool, secretsFile []string, plain bool, endpointsFlag bool,
	endpointsFile []string, fileType int, headers map[string]string, errorsFlag bool, infoFlag bool,
	debug bool, userAgent string) ([]string, []scanner.SecretMatched, []scanner.EndpointMatched,
	[]scanner.FileTypeMatched, []scanner.ErrorMatched, []scanner.InfoMatched)

New it's the actual crawler engine. It controls all the behaviours of a scan (event handlers, secrets, errors, extensions and endpoints scanning).

func PostRequest

func PostRequest(target string, data map[string]string) (string, error)

PostRequest performs a POST request and return a string (the body of the response) the map in the input should contains the data fields and values in this way for example: { email: test@example.com, password: stupid_pwd }.

func RetrieveBody

func RetrieveBody(target string) string

RetrieveBody retrieves the body (in the response) of a url.

func SecretsMatch

func SecretsMatch(url string, body string, secretsFile []string) []scanner.SecretMatched

SecretsMatch checks if a body matches some secrets.