config

Documentation

Index

• Constants
• func GetAllSiteKeys(appCfg *AppConfig) []string
• func GetEffectiveEnableJSONLOutput(siteCfg *SiteConfig, appCfg *AppConfig) bool
• func GetEffectiveJSONLOutputFilename(siteCfg *SiteConfig, appCfg *AppConfig) string
• func GetEffectiveMaxImageSize(siteCfg *SiteConfig, appCfg *AppConfig) int64
• func GetEffectiveMaxPageSize(appCfg *AppConfig) int64
• func GetEffectiveSkipImages(siteCfg *SiteConfig, appCfg *AppConfig) bool
• func GetEffectiveUserAgent(siteCfg *SiteConfig, appCfg *AppConfig) string
• func ValidateSiteKeys(appCfg *AppConfig, siteKeys []string) error
• type AppConfig
  • func (c *AppConfig) SiteOutputDir(siteKey string) string
  • func (c *AppConfig) Validate() (warnings []string, err error)
• type HTTPClientConfig
• type ResolvedSiteConfig
  • func NewResolvedSiteConfig(siteCfg *SiteConfig, appCfg *AppConfig) *ResolvedSiteConfig
• type SiteConfig
  • func (c *SiteConfig) Validate() (warnings []string, err error)

Constants

const (
	// DefaultMaxPageSizeBytes is the default maximum page body size (50 MB).
	DefaultMaxPageSizeBytes int64 = 50 * 1024 * 1024

	// DefaultSemaphoreAcquireTimeout is how long crawler/fetch/sitemap/image
	// code waits to acquire a global or per-host semaphore before giving up.
	// Previously exposed via the semaphore_acquire_timeout config key; that
	// key was an internal mechanic with no realistic reason to tune.
	DefaultSemaphoreAcquireTimeout = 30 * time.Second
)

Functions

func GetAllSiteKeys

func GetAllSiteKeys(appCfg *AppConfig) []string

GetAllSiteKeys returns all site keys from the config in sorted order. Sorted so callers (e.g. --all-sites crawl ordering, error messages) are deterministic across runs; Go map iteration is randomized.

func GetEffectiveEnableJSONLOutput

func GetEffectiveEnableJSONLOutput(siteCfg *SiteConfig, appCfg *AppConfig) bool

GetEffectiveEnableJSONLOutput returns the effective JSONL output setting (site overrides global).

func GetEffectiveJSONLOutputFilename

func GetEffectiveJSONLOutputFilename(siteCfg *SiteConfig, appCfg *AppConfig) string

GetEffectiveJSONLOutputFilename returns the JSONL output filename (site > global > default).

func GetEffectiveMaxImageSize

func GetEffectiveMaxImageSize(siteCfg *SiteConfig, appCfg *AppConfig) int64

GetEffectiveMaxImageSize returns the effective max image size (site overrides global).

func GetEffectiveMaxPageSize

func GetEffectiveMaxPageSize(appCfg *AppConfig) int64

GetEffectiveMaxPageSize returns the configured max page size, falling back to DefaultMaxPageSizeBytes.

func GetEffectiveSkipImages

func GetEffectiveSkipImages(siteCfg *SiteConfig, appCfg *AppConfig) bool

GetEffectiveSkipImages returns the skip_images setting. Image downloading is opt-in: when neither the site nor the global config sets skip_images, images are skipped (text-first default).

func GetEffectiveUserAgent

func GetEffectiveUserAgent(siteCfg *SiteConfig, appCfg *AppConfig) string

GetEffectiveUserAgent returns the site-specific user agent, falling back to the global default.

func ValidateSiteKeys

func ValidateSiteKeys(appCfg *AppConfig, siteKeys []string) error

ValidateSiteKeys checks that every key in siteKeys exists in appCfg.Sites. On failure the error message lists the available keys in sorted order.

Types

type AppConfig

type AppConfig struct {
	DefaultUserAgent    string                 `yaml:"default_user_agent"`
	DefaultDelayPerHost time.Duration          `yaml:"default_delay_per_host"`
	NumWorkers          int                    `yaml:"num_workers"`
	NumImageWorkers     int                    `yaml:"num_image_workers,omitempty"`
	MaxRequests         int                    `yaml:"max_requests"`
	MaxRequestsPerHost  int                    `yaml:"max_requests_per_host"`
	OutputBaseDir       string                 `yaml:"output_base_dir"`
	StateDir            string                 `yaml:"state_dir"`
	MaxRetries          int                    `yaml:"max_retries,omitempty"`
	InitialRetryDelay   time.Duration          `yaml:"initial_retry_delay,omitempty"`
	MaxRetryDelay       time.Duration          `yaml:"max_retry_delay,omitempty"`
	GlobalCrawlTimeout  time.Duration          `yaml:"global_crawl_timeout,omitempty"`
	PerPageTimeout      time.Duration          `yaml:"per_page_timeout,omitempty"` // 0 = no timeout
	SkipImages          *bool                  `yaml:"skip_images,omitempty"`      // nil = skip (text-first default)
	MaxImageSizeBytes   int64                  `yaml:"max_image_size_bytes,omitempty"`
	MaxPageSizeBytes    int64                  `yaml:"max_page_size_bytes,omitempty"` // 0 = 50 MB default
	HTTPClientSettings  HTTPClientConfig       `yaml:"http_client_settings,omitempty"`
	Sites               map[string]*SiteConfig `yaml:"sites"`
	EnableJSONLOutput   bool                   `yaml:"enable_jsonl_output,omitempty"`
	JSONLOutputFilename string                 `yaml:"jsonl_output_filename,omitempty"`
	EnableIncremental   bool                   `yaml:"enable_incremental,omitempty"`
	// CrawlHistoryRetention is the number of past crawls per site kept in the
	// SQLite history index (state_dir/index.db) that powers get_freshness and
	// diff_crawl. Anything older is pruned at write time. 0 means use the
	// default in pkg/storage/index (10).
	CrawlHistoryRetention int `yaml:"crawl_history_retention,omitempty"`
}

AppConfig holds the global application configuration.

func (*AppConfig) SiteOutputDir

func (c *AppConfig) SiteOutputDir(siteKey string) string

SiteOutputDir returns a site's output directory, keyed by site key so two site configs targeting the same domain stay isolated. Readers (MCP tools) and the crawler must derive the path here so they cannot drift.

func (*AppConfig) Validate

func (c *AppConfig) Validate() (warnings []string, err error)

Validate checks AppConfig fields, applies sensible defaults, and returns warnings and any fatal error. Modifies receiver in place.

type HTTPClientConfig

type HTTPClientConfig struct {
	Timeout             time.Duration `yaml:"timeout,omitempty"`                 // Overall request timeout (default 45s)
	MaxIdleConnsPerHost int           `yaml:"max_idle_conns_per_host,omitempty"` // Max idle connections per host (default 2)
	// AllowPrivateNetworks disables the SSRF guard that blocks outbound
	// connections to private/loopback/link-local/CGNAT/multicast addresses.
	// Default false. Set to true only if you intentionally crawl internal
	// documentation servers reachable via private IPs.
	AllowPrivateNetworks bool `yaml:"allow_private_networks,omitempty"`
}

HTTPClientConfig holds settings for the shared HTTP client. The Go-default connection-pool, dialer, and TLS timings are baked into pkg/fetch and not exposed here; only the three knobs that real users actually tune remain.

type ResolvedSiteConfig

type ResolvedSiteConfig struct {
	UserAgent           string
	JSONLOutputFilename string
	DelayPerHost        time.Duration
	MaxPageSizeBytes    int64
	MaxImageSizeBytes   int64
	SkipImages          bool
	EnableJSONLOutput   bool
}

ResolvedSiteConfig holds effective configuration for a site, resolved once at crawl start.

func NewResolvedSiteConfig

func NewResolvedSiteConfig(siteCfg *SiteConfig, appCfg *AppConfig) *ResolvedSiteConfig

NewResolvedSiteConfig resolves all effective configuration values for a site at crawl start.

type SiteConfig

type SiteConfig struct {
	StartURLs               []string      `yaml:"start_urls"`
	AllowedDomain           string        `yaml:"allowed_domain"`
	AllowedPathPrefix       string        `yaml:"allowed_path_prefix"`
	ContentSelector         string        `yaml:"content_selector"`
	LinkExtractionSelectors []string      `yaml:"link_extraction_selectors,omitempty"`
	DisallowedPathPatterns  []string      `yaml:"disallowed_path_patterns,omitempty"`
	RespectNofollow         bool          `yaml:"respect_nofollow,omitempty"`
	UserAgent               string        `yaml:"user_agent,omitempty"`
	DelayPerHost            time.Duration `yaml:"delay_per_host,omitempty"`
	// MaxDepth is the exclusive upper bound on crawl depth. Seed pages are
	// depth 0, so MaxDepth=2 fetches depth 0 and 1. 0 means unlimited.
	MaxDepth               int      `yaml:"max_depth"`
	SkipImages             *bool    `yaml:"skip_images,omitempty"`
	MaxImageSizeBytes      *int64   `yaml:"max_image_size_bytes,omitempty"`
	AllowedImageDomains    []string `yaml:"allowed_image_domains,omitempty"`
	DisallowedImageDomains []string `yaml:"disallowed_image_domains,omitempty"`
	EnableJSONLOutput      *bool    `yaml:"enable_jsonl_output,omitempty"`
	JSONLOutputFilename    string   `yaml:"jsonl_output_filename,omitempty"`
}

SiteConfig holds configuration specific to a single website crawl.

func (*SiteConfig) Validate

func (c *SiteConfig) Validate() (warnings []string, err error)

Validate checks SiteConfig fields, applies defaults, and returns warnings and any fatal error. Modifies receiver in place (e.g., path prefix normalization).