Versions in this module Expand all Collapse all v1 v1.7.1 Aug 15, 2026 Changes in this version + type CrawlerOptions struct + ChromeUser *user.User + ContentSimilarity *similarity.Index + Dialer *fastdialer.Dialer + DitClassifier *dit.Classifier + ExtensionsValidator *extensions.Validator + Extractors []knowledgebase.Extractor + HostRateLimit *ratelimit.AutoLimiter + Logger *slog.Logger + Options *Options + OutputWriter output.Writer + Parser *parser.Parser + RateLimit *ratelimit.Limiter + ScopeManager *scope.Manager + UniqueFilter filters.Filter + Wappalyzer *wappalyzer.Wappalyze + func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) + func (c *CrawlerOptions) BuildKnowledgeBase(body string, req *http.Request, resp *http.Response) map[string]any + func (c *CrawlerOptions) Close() error + func (c *CrawlerOptions) ValidatePath(path string) bool + func (c *CrawlerOptions) ValidateScope(absURL, rootHostname string) (bool, error) + type OnResultCallback func(output.Result) + type OnSkipURLCallback func(string) + type Options struct + AuthCredentials string + AutomaticFormFill bool + BodyReadSize int + CaptchaSolverAPIKey string + CaptchaSolverProvider string + ChromeDataDir string + ChromeWSUrl string + Concurrency int + Context context.Context + CrawlDuration time.Duration + CustomHeaders goflags.StringSlice + DOMWaitTime int + Debug bool + Delay int + DisableRedirects bool + DisableUniqueFilter bool + DisableUpdateCheck bool + DisplayOutScope bool + EnableDiagnostics bool + Endpoints bool + ErrorLogFile string + Exclude goflags.StringSlice + ExcludeOutputFields goflags.StringSlice + ExtensionFilter goflags.StringSlice + ExtensionsMatch goflags.StringSlice + FieldConfig string + FieldScope string + Fields string + FilterPageType goflags.StringSlice + FilterRegex []*regexp.Regexp + FilterSimilar bool + FilterSimilarThreshold int + FormConfig string + FormExtraction bool + Headless bool + HeadlessHybrid bool + HeadlessNoIncognito bool + HeadlessNoSandbox bool + HeadlessOptionalArguments goflags.StringSlice + HealthCheck bool + HostRateLimit int + HostRateLimitMinute int + IgnoreQueryParams bool + JSON bool + KnowledgeBase bool + KnownFiles string + ListOutputFields bool + MatchRegex []*regexp.Regexp + MaxDepth int + MaxDomainPages int + MaxFailureCount int + MaxOnclickLinks int + NoClobber bool + NoColors bool + NoDefaultExtFilter bool + NoScope bool + OmitBody bool + OmitRaw bool + OnResult OnResultCallback + OnSkipURL OnSkipURLCallback + OutOfScope goflags.StringSlice + OutputFile string + OutputFilterCondition string + OutputFilterRegex goflags.StringSlice + OutputMatchCondition string + OutputMatchRegex goflags.StringSlice + OutputTemplate string + PageContentSimilar bool + PageContentSimilarBudget int + PageContentSimilarDistance int + PageContentSimilarMode string + PageContentSimilarThresholdStr string + PageLoadStrategy string + Parallelism int + PathClimb bool + PprofServer bool + Proxy string + RateLimit int + RateLimitMinute int + Resolvers goflags.StringSlice + Resume string + Retries int + Scope goflags.StringSlice + ScrapeJSLuiceResponses bool + ScrapeJSResponses bool + Secrets bool + ShowBrowser bool + Silent bool + SimilarityDeduplication bool + StoreFieldDir string + StoreFields string + StoreResponse bool + StoreResponseDir string + Strategy string + SystemChromePath string + TechDetect bool + TimeStable int + Timeout int + TlsImpersonate bool + URLs goflags.StringSlice + UseInstalledChrome bool + ValidateSecrets bool + Verbose bool + Version bool + XhrExtraction bool + var DefaultOptions Options + func (options *Options) ConfigureOutput() + func (options *Options) ContentSimilarityEnabled() bool + func (options *Options) PageContentSimilarThreshold() float64 + func (options *Options) ParseCustomHeaders() map[string]string + func (options *Options) ParseHeadlessOptionalArguments() map[string]string + func (options *Options) ShouldResume() bool
Go to main page