config

Documentation

Overview

Package config - configuration for the crawler, including ignored file extensions, domains, TLDs and query strings

Index

• Variables

Variables

var FileExtensions = []string{
	".ai", ".bmp", ".css", ".csv", ".doc", ".gif", ".ico", ".jpeg", ".jpg", ".js", ".pdf", ".png", ".ppsx", ".ps", ".psd", ".svg", ".tif", ".tiff", ".txt", ".xls", ".xml", ".3g2", ".3gp", ".avi", ".flv", ".h264", ".m4v", ".mkv", ".mov", ".mp4", ".mpg", ".mpeg", ".rm", ".swf", ".vob", ".wmv", ".aif", ".cda", ".mid", ".midi", ".mp3", ".mpa", ".ogg", ".wav", ".wma", ".wpl", ".doc", ".docx", ".odt", ".pdf", ".rtf", ".tex", ".txt", ".wks", ".wps", ".wpd", ".xml", ".ods", ".xlr", ".xls", ".xlsx", ".7z", ".arj", ".deb", ".pkg", ".rar", ".rpm", ".gz", ".z", ".zip",
}

FileExtensions - list of file extensions to ignore

var IgnoreDomains = []string{}/* 178 elements not displayed */

IgnoreDomains - ignore links to these domains

var IgnoreQuery = []string{
	"lang",
	"utm_",
	"ref",
}

IgnoreQuery - ignore query starting with these strings

var IgnoreTLD = []string{
	".cn", ".blogspot.com",
}

IgnoreTLD - ignore pages and links with these domains