crawler

Mar 23, 2026

Changes in this version

+ type ResponseHandlerRuntimeBinder interface

+ BindRuntime func(collector *colly.Collector, filePersister FilePersister, ...)

Mar 23, 2026

Changes in this version

type Logger

+ func EnsureLogger(logger Logger) Logger

Mar 23, 2026

Changes in this version

+ func SetPackageLogger(logger Logger)

type Config

+ CookieGenerator CookieGenerator

+ PlatformID string

+ RequestHeaders RequestHeaderProvider

+ RequestHook RequestHook

+ RuleEvaluator RuleEvaluator

+ type CookieGenerator func(domain string) []*http.Cookie

+ type NoopResponseHandler struct

+ func (NoopResponseHandler) AfterEvaluation(*colly.Response, *goquery.Document, *Result)

+ func (NoopResponseHandler) BeforeEvaluation(*colly.Response, *goquery.Document)

+ func (NoopResponseHandler) HandleBinaryResponse(*colly.Response, string, string) bool

type PlatformConfig

+ SkipRulesOnRedirect bool

type PlatformHooks

+ ExtractDOMTitle func(document *goquery.Document) string

+ InferRedirect func(productID, originalURL, finalURL, canonicalURL string) (redirected bool, redirectedProductID string)

+ IsContentComplete func(document *goquery.Document) bool

+ type Product struct

+ ID string

+ OriginalID string

+ OriginalURL string

+ Platform string

+ URL string

+ func NewProduct(id, platform, url string, opts ...ProductOption) (Product, error)

+ type ProductOption func(*Product)

+ func WithOriginalID(originalID string) ProductOption

+ func WithOriginalURL(originalURL string) ProductOption

+ type RequestHeaderProvider interface

+ Apply func(platformID string, request *colly.Request)

type ResponseHandler

+ AfterEvaluation func(resp *colly.Response, document *goquery.Document, result *Result)

+ BeforeEvaluation func(resp *colly.Response, document *goquery.Document)

+ HandleBinaryResponse func(resp *colly.Response, productID string, fileExtension string) bool

+ type ResponseProcessor interface

+ SendFinalResult func(resp *colly.Response, success bool, errorText string)

+ SetResponseHandlers func(handlers []ResponseHandler)

+ SetResultCallback func(callback func(*colly.Response))

+ Setup func(collector *colly.Collector)

type Result

+ ConfiguredVerifierCount int

+ OriginalProductID string

+ OriginalURL string

+ ProductID string

+ ProductPlatform string

+ ProductTitle string

+ ProductURL string

+ Progress int

+ RuleResults []RuleResult

+ ScoreOverride *int

+ func (result Result) CalculateScore(configuredVerifierCount int) int

+ func (result Result) IsNotFound() bool

+ func (result Result) IsNotRetryable() bool

+ type RuleEvaluation struct

+ ConfiguredVerifier int

+ Passed bool

+ RuleResults []RuleResult

+ type RuleEvaluator interface

+ ConfiguredVerifierCount func() int

+ Evaluate func(productID string, document *goquery.Document) (RuleEvaluation, error)

+ type RuleResult struct

+ Description string

+ ID string

+ Message string

+ Passed bool

+ ReportingOrder int

+ VerificationResults []VerificationResult

+ type ServiceHook interface

+ AfterInit func(collector *colly.Collector, transport http.RoundTripper)

+ AfterRun func()

+ BeforeRun func(ctx context.Context)

+ type ServiceOption func(*Service)

+ func WithResponseHandlers(handlers ...ResponseHandler) ServiceOption

+ func WithServiceHook(hook ServiceHook) ServiceOption

+ type VerificationResult struct

+ Description string

+ ID string

+ IncludeValue bool

+ Message string

+ Passed bool

+ ReportingOrder int

+ Value string

Mar 23, 2026

Changes in this version

+ const CtxCanonicalURLKey

+ const CtxErrorKey

+ const CtxEvaluationKey

+ const CtxFinalURLKey

+ const CtxHTTPStatusCodeKey

+ const CtxInitialURLKey

+ const CtxNotFoundKey

+ const CtxRedirectedKey

+ const CtxRunContextKey

+ const CtxTargetCategoryKey

+ const CtxTargetIDKey

+ const CtxTargetURLKey

+ const CtxTitleKey

+ const HTMLExtension

+ const RetriedFlagKey

+ const RetryCountKey

+ const TitleNotFound

+ const UnknownURL

+ func DescribeProxyForLog(rawProxyURL string) string

+ func ExtractCanonicalURL(doc *goquery.Document) string

+ func ExtractTitle(doc *goquery.Document) string

+ func GetContextValue(ctx *colly.Context, key, fallback string) string

+ func GetRetryAttempt(response *colly.Response) int

+ func GetTargetCategoryFromContext(resp *colly.Response) string

+ func GetTargetIDFromContext(resp *colly.Response) string

+ func GetTargetURLFromContext(resp *colly.Response) string

+ func NewContextAwareTransport(base http.RoundTripper, ctxProvider func() context.Context) http.RoundTripper

+ func NewHTTPTransport(insecureSkipVerify bool, requestTimeout time.Duration) *http.Transport

+ func NewPanicSafeTransport(base http.RoundTripper, logger Logger) http.RoundTripper

+ func NewProxyRotator(rawProxies []string, tracker ProxyHealth, logger Logger) (colly.ProxyFunc, error)

+ func ParseHTMLResponse(body []byte) (*goquery.Document, error)

+ func RecordProxyFailure(tracker ProxyHealth, resp *colly.Response)

+ func SanitizeProxyURL(rawProxyURL string) string

+ func SetupErrorHandling(collector *colly.Collector, handler ResponseHandler, retryHandler RetryHandler, ...)

type Config

+ Category string

+ FilePersister FilePersister

+ OutputDirectory string

+ Platform PlatformConfig

+ PlatformHooks PlatformHooks

+ ResponseHandler ResponseHandler

+ RunFolder string

+ Scraper ScraperConfig

+ type DefaultResponseHandler struct

+ func NewDefaultResponseHandler(cfg Config, retryHandler RetryHandler, proxyTracker ProxyHealth, ...) *DefaultResponseHandler

+ func (h *DefaultResponseHandler) SendResult(resp *colly.Response, success bool, errorMessage string)

+ func (h *DefaultResponseHandler) SetSlotReleaser(releaser func(*colly.Response))

+ func (h *DefaultResponseHandler) Setup(collector *colly.Collector)

+ type FilePersister interface

+ Close func() error

+ Save func(targetID, fileName string, content []byte) error

+ func NewBackgroundFilePersister(delegate FilePersister, workerCount, bufferSize int, logger Logger) FilePersister

+ func NewDirectoryFilePersister(rootDirectory, category, runFolder string) FilePersister

+ type PlatformConfig struct

+ AllowedDomains []string

+ CookieDomains []string

+ func (cfg PlatformConfig) Validate() error

+ type PlatformHooks interface

+ NormalizeTitle func(title string) string

+ ShouldRetry func(title string, document *goquery.Document) RetryDecision

+ type ProxyHealth interface

+ IsAvailable func(proxy string) bool

+ RecordCriticalFailure func(proxy string)

+ RecordFailure func(proxy string)

+ RecordSuccess func(proxy string)

+ func NewProxyHealthTracker(values []string, logger Logger) ProxyHealth

+ type RequestConfigurator interface

+ Configure func(collector *colly.Collector)

+ func NewRequestConfigurator(cfg Config, logger Logger) RequestConfigurator

+ type ResponseHandler interface

+ SendResult func(resp *colly.Response, success bool, errorMessage string)

+ SetSlotReleaser func(releaser func(*colly.Response))

+ Setup func(collector *colly.Collector)

type Result

+ CanonicalURL string

+ ProxyURL string

+ TargetID string

+ TargetURL string

+ type RetryDecision struct

+ ExhaustionBehavior RetryExhaustionBehavior

+ LogMessage string

+ Message string

+ Policy RetryPolicy

+ ShouldRetry bool

+ func (d RetryDecision) ResolvedLogMessage() string

+ type RetryExhaustionBehavior uint8

+ const RetryExhaustionBehaviorContinue

+ const RetryExhaustionBehaviorFail

+ type RetryHandler interface

+ Retry func(response *colly.Response, options RetryOptions) bool

+ func NewRetryHandler(scraper ScraperConfig, logger Logger) RetryHandler

+ type RetryOptions struct

+ LimitRetries bool

+ MaxRetries int

+ SkipDelay bool

+ type RetryPolicy uint8

+ const RetryPolicyDefault

+ const RetryPolicyRotateProxy

+ type ScraperConfig struct

+ HTTPTimeout time.Duration

+ InsecureSkipVerify bool

+ MaxDepth int

+ Parallelism int

+ ProxyCircuitBreakerEnabled bool

+ ProxyList []string

+ RateLimit time.Duration

+ RetryCount int

+ SaveFiles bool

+ func (cfg ScraperConfig) Validate() error

type Service

+ func (svc *Service) RetryHandler() RetryHandler

+ type Target struct

+ Category string

+ ID string

+ Metadata map[string]string

+ URL string

+ func NewTarget(id, category, url string, opts ...TargetOption) (Target, error)

+ func (t Target) MetadataValue(key string) string

+ type TargetOption func(*Target)

+ func WithMetadata(key, value string) TargetOption

Mar 22, 2026

Changes in this version

+ type Config struct

+ AllowedDomains []string

+ CookieDomains []string

+ CookieProvider CookieProvider

+ Evaluator Evaluator

+ HTTPTimeout time.Duration

+ Headers HeaderProvider

+ Hook RequestHook

+ Logger Logger

+ MaxDepth int

+ Parallelism int

+ RateLimit time.Duration

+ RetryCount int

+ func (c Config) Validate() error

+ type CookieProvider func(domain string) []*http.Cookie

+ type Evaluation struct

+ Findings []Finding

+ type Evaluator interface

+ Evaluate func(pageID string, document *goquery.Document) (Evaluation, error)

+ type Finding struct

+ Data string

+ Description string

+ ID string

+ Message string

+ Passed bool

+ type HeaderProvider interface

+ Apply func(request *colly.Request)

+ type Logger interface

+ Debug func(format string, args ...interface{})

+ Error func(format string, args ...interface{})

+ Info func(format string, args ...interface{})

+ Warning func(format string, args ...interface{})

+ type Page struct

+ Category string

+ ID string

+ URL string

+ type RequestHook interface

+ BeforeRequest func(ctx context.Context, page Page) error

+ type Result struct

+ Category string

+ Document *goquery.Document

+ ErrorMessage string

+ FinalURL string

+ Findings []Finding

+ HTTPStatusCode int

+ PageID string

+ PageURL string

+ Success bool

+ Title string

+ type Service struct

+ func NewService(cfg Config, results chan<- *Result) (*Service, error)

+ func (svc *Service) Run(ctx context.Context, pages []Page) error