engine

Documentation

Index

• Constants
• func AcceptHeaders(resp *http.Response, inc, exc []HeaderFilter) bool
• func Start(ctx context.Context, client *http.Client, fs *filter.FilterSuite, ...) <-chan Result
• func Worker(ctx context.Context, client *http.Client, fs *filter.FilterSuite, ...)
• type HeaderFilter
• type Job
• type Producer
• type Result
• type Scanner
  • func NewScanner(client *http.Client, fs *filter.FilterSuite, ...) *Scanner
  • func (s *Scanner) Err() error
  • func (s *Scanner) Scan(ctx context.Context, producer Producer, workers int) <-chan Result
  • func (s *Scanner) SetRequestManipulation(method string, body []byte, headers http.Header, cookies []*http.Cookie)
  • func (s *Scanner) SetStats(c *stats.Collector)
• type SliceProducer
  • func (p SliceProducer) Produce(ctx context.Context, jobs chan<- Job) error

Constants

const (
	OriginProfile    = "profile"
	OriginRobots     = "robots.txt"
	OriginSitemapXml = "sitemap.xml"
	OriginSitemapIdx = "sitemap index"
	OriginHTML       = "HTML"
	OriginWordlist   = "brute-force wordlist"
)

Functions

func AcceptHeaders

func AcceptHeaders(resp *http.Response, inc, exc []HeaderFilter) bool

AcceptHeaders evaluates headers matching.

func Start

func Start(
	ctx context.Context,
	client *http.Client,
	fs *filter.FilterSuite,
	incHeaders, excHeaders []HeaderFilter,
	workers int,
	delay time.Duration,
	limiter *rate.Limiter,
	method string,
	body []byte,
	headers http.Header,
	cookies []*http.Cookie,
	jobs <-chan Job,
	collector *stats.Collector,
) <-chan Result

Start launches worker goroutines and returns a results channel that is closed once every worker exits. The caller must close jobs to signal completion and must drain results to avoid blocking workers.

func Worker

func Worker(
	ctx context.Context,
	client *http.Client,
	fs *filter.FilterSuite,
	incHeaders, excHeaders []HeaderFilter,
	delay time.Duration,
	limiter *rate.Limiter,
	method string,
	body []byte,
	headers http.Header,
	cookies []*http.Cookie,
	jobs <-chan Job,
	results chan<- Result,
	collector *stats.Collector,
)

Worker executes the response pipeline for incoming jobs. Pipeline: Status -> Headers -> Content-Length -> Body

Types

type HeaderFilter

type HeaderFilter struct {
	Name  string
	Value string
}

HeaderFilter specifies an exact match rule on a case-insensitive header name.

type Job

type Job struct {
	URL    string
	Depth  uint16
	Origin string
}

Job is the unit of work dispatched to a worker. One Job produces at most one Result.

type Producer

type Producer interface {
	Produce(ctx context.Context, jobs chan<- Job) error
}

Producer generates Jobs and sends them to the provided channel. Implementations must close jobs when done and respect ctx cancellation.

type Result

type Result struct {
	URL         string
	RedirectURL string
	StatusCode  int
	Length      int64 // -1 when Content-Length is absent
	Depth       uint16
	Accepted    bool
	Origin      string
	Err         error

	// Presentation metadata
	Title   string
	Headers http.Header

	// Extracted links for smarter recursive crawling
	Links []string

	// Hash of the response body for wildcard/duplicate detection
	BodyHash uint64
}

Result carries the metadata produced by a successful pipeline pass. The body is never stored — only cheap scalar metadata survives.

type Scanner

type Scanner struct {
	// contains filtered or unexported fields
}

Scanner is the public orchestration entry point. Workers and pool management are internal details.

func NewScanner

func NewScanner(
	client *http.Client,
	fs *filter.FilterSuite,
	incHeaders, excHeaders []HeaderFilter,
	delay time.Duration,
	limiter *rate.Limiter,
) *Scanner

func (*Scanner) Err

func (s *Scanner) Err() error

Err returns the first producer error encountered during the last scan, if any. Call after draining the results channel.

func (*Scanner) Scan

func (s *Scanner) Scan(ctx context.Context, producer Producer, workers int) <-chan Result

Scan starts the producer and a worker pool, returning a results channel that is closed when the scan completes. Cancelling ctx stops job emission and aborts in-flight requests.

func (*Scanner) SetRequestManipulation

func (s *Scanner) SetRequestManipulation(method string, body []byte, headers http.Header, cookies []*http.Cookie)

SetRequestManipulation configures outbound fuzzed request templates for scanning.

func (*Scanner) SetStats

func (s *Scanner) SetStats(c *stats.Collector)

SetStats sets the statistics collector for the scanner.

type SliceProducer

type SliceProducer struct {
	URLs []string
}

SliceProducer emits one Job per URL from a fixed slice.

func (SliceProducer) Produce

func (p SliceProducer) Produce(ctx context.Context, jobs chan<- Job) error