Versions in this module Expand all Collapse all v0 v0.1.2 Aug 29, 2026 v0.1.1 Aug 29, 2026 Changes in this version + const AcceptHTML + const AcceptMarkdown + const Algorithm + const AlterationFloor + const ChallengeChannelBody + const ChallengeChannelHeader + const ChallengeChannelLocation + const ChallengeStrengthStrong + const ChallengeStrengthWeak + const CrawlerToken + const CrawlerUA + const DefaultBodyCap + const DefaultBrowserRecycle + const DefaultBrowserTimeout + const DefaultConcurrency + const DefaultHeartbeatInterval + const DefaultHeartbeatStale + const DefaultHostDelay + const DefaultRedirects + const DefaultRetries + const DefaultRetryWait + const DefaultTimeout + const DirectoryCacheAge + const DirectoryLabel + const DirectoryMediaType + const DirectoryTag + const ImpersonateToken + const ImpersonateUA + const ModulePath + const PlatformCDN + const PlatformEdge + const PlatformOrigin + const ProductName + const ProductToken + const ProductUA + const ProductVersion + const RequestLabel + const RequestTag + const SignatureAlgorithm + const SignatureLifetime + const WellKnownPath + var ChallengeFamilies = []ChallengeFamily + var SignatureHeaderNames = []string + var TokenEstimateHeaders = []string + func BaselineEligible(a, b Observation) bool + func BaselineUnstable(unsigned, control Observation, diff Comparison) bool + func BlockLess(a, b Block) bool + func CanonicalURL(raw string) string + func ChallengeAsymmetry(a, b Observation) bool + func ChallengeFamilyNames(sigs []ChallengeSignal) []string + func ContentComparable(o Outcome) bool + func ContentDiffers(a, b Observation, diff Comparison) bool + func CoreCoverage(core []Block, replicas ...[]Block) float64 + func CoreTooSmall(reps []Observation, core []Block) bool + func DomainSHA256(list string) string + func EstimateTokens(s string) int + func HasStrongChallenge(sigs []ChallengeSignal) bool + func IsChallenge(status int, h http.Header, body []byte) bool + func LooksLikeHTML(contentType string) bool + func LooksLikeMarkdown(contentType string) bool + func MachineHasExtraText(diff Comparison) bool + func MarkdownFault(md, html Observation, htmlVsMarkdown Comparison) bool + func MarkdownLostBody(md, html Observation, htmlVsMarkdown Comparison) bool + func MarkdownMissingVary(md, html Observation) bool + func MarkdownVariant(md, html Observation) bool + func NormalizeText(s string) string + func PathLooksLikeMarkdown(path string) bool + func RedirectDiffers(a, b Observation) bool + func ReportedTokens(h http.Header) (n int, name string, ok bool) + func ResolveAlternate(pageURL, href string) string + func RobotsToken(k IdentityKind) string + func SameBlocks(a, b []Block) bool + func SentToOrigin(o Outcome) bool + func ShuffleSeed() uint64 + func StripFrontmatter(src []byte) (body []byte, front string) + func TokenSource(blocks []Block) string + func TrioEligible(reps []Observation) bool + func TrioUnstable(reps []Observation, core []Block) bool + func UserAgent(k IdentityKind) string + func VaryCoversAccept(h http.Header) bool + type Aggregates struct + AlternateMismatch int + CoreCoverageMean Share + CoreTooSmall Share + DuplicatePrimaryHosts int + Empty Share + InteriorFound Share + MarkdownLostBodyContent Share + MarkdownMissingVary Share + Outcomes map[IdentityKind]map[Outcome]int + PrimaryFlags int + SignedVsUnsignedChallengeNet Share + SignedVsUnsignedChallengeRaw Share + SignedVsUnsignedContentNet Share + SignedVsUnsignedContentRaw Share + SignedVsUnsignedLinkTargetNet Share + SignedVsUnsignedLinkTargetRaw Share + SignedVsUnsignedRedirectNet Share + SignedVsUnsignedRedirectRaw Share + Sites int + Uncertain Share + Unstable Share + type Alternate struct + Href string + Resolved string + Source AlternateSource + Type string + type AlternateMismatch struct + OnlyHead []AlternateRef + OnlyHeader []AlternateRef + func (m AlternateMismatch) HasMismatch() bool + type AlternateRef struct + Declared string + Resolved string + type AlternateSource string + const AlternateHead + const AlternateHeader + type Block struct + Chrome bool + Index int + Kind BlockKind + Level int + Target string + Text string + func ProjectOntoCore(seq, core []Block) []Block + func StableCore(replicas ...[]Block) []Block + func (b Block) Equal(o Block) bool + type BlockDiff struct + Confidence float64 + Evidence string + Left *Block + LeftIndex int + Op DiffOp + Right *Block + RightIndex int + func MissingBodyBlocks(diff Comparison, markdown []Block) []BlockDiff + type BlockKind string + const BlockHeading + const BlockLink + const BlockListItem + const BlockParagraph + const BlockText + type BodyEvidence struct + Body []byte + Cap int + Capped bool + Domain string + Identity IdentityKind + PageURL string + RawBytes int + RunID string + type BodyStore interface + Get func(ctx context.Context, runID, domain, pageURL string, id IdentityKind) (BodyEvidence, bool, error) + Put func(ctx context.Context, rec BodyEvidence) error + type CacheState struct + Age string + CFCacheStatus string + CacheControl string + Expires string + XCache string + XCacheHits string + func CacheStateFromHeaders(h http.Header) CacheState + func (c CacheState) Present() bool + type ChallengeFamily struct + Body []familyBody + Headers []familyHeader + Locations []familyLocation + Name string + Phrases []string + type ChallengeSignal struct + Channel string + Evidence string + Family string + Name string + Offset int + Strength string + func ChallengeSignals(status int, h http.Header, body []byte) []ChallengeSignal + func (s ChallengeSignal) String() string + func (s ChallengeSignal) Strong() bool + type Comparer interface + Compare func(left, right []Block) Comparison + type Comparison struct + Diffs []BlockDiff + LeftCount int + LeftIdentity IdentityKind + LinkTargets []LinkTargetChange + MatchRatio float64 + Matched int + RightCount int + RightIdentity IdentityKind + func (c Comparison) HasDiffs() bool + func (c Comparison) HasLinkTargetChanges() bool + type ContextProbe interface + FetchOrigin func(ctx context.Context, pageURL string, pause func(context.Context) error) (SiteContext, error) + Probe func(ctx context.Context, pageURL string, html []byte, headers http.Header) (SiteContext, error) + WithPage func(sc SiteContext, pageURL string, html []byte, headers http.Header) SiteContext + type DiffOp string + const DiffAdded + const DiffAltered + const DiffMissing + type DirectoryChecker interface + CheckPublished func(ctx context.Context) error + type DirectoryServer interface + ServeDirectory func(http.ResponseWriter, *http.Request) + type Extracted struct + Blocks []Block + Frontmatter string + TokenCount int + type Extractor interface + Extract func(contentType string, body []byte) (Extracted, error) + type FetchRequest struct + Identity IdentityKind + URL string + type FetchResult struct + Body []byte + ByteSize int64 + Cache CacheState + Challenge []ChallengeSignal + ContentType string + Duration time.Duration + Err string + FinalURL string + Headers http.Header + Identity IdentityKind + Outcome Outcome + Platform Platform + StatusCode int + URL string + type Fetcher interface + Available func(kind IdentityKind) (ready bool, reason string) + Fetch func(ctx context.Context, req FetchRequest) FetchResult + FetchAll func(ctx context.Context, rawURL string) []FetchResult + type IdentityKind string + const IdentityBrowser + const IdentityControl + const IdentityImpersonate + const IdentityMarkdown + const IdentitySigned + const IdentityUnsigned + const IdentityUnsignedB + const IdentityUnsignedC + func AllIdentities() []IdentityKind + func ShuffledIdentities(seed uint64) []IdentityKind + func UnsignedReplicas() []IdentityKind + func (k IdentityKind) Machine() bool + func (k IdentityKind) UnsignedLike() bool + func (k IdentityKind) UnsignedReplica() bool + func (k IdentityKind) Valid() bool + type LLMSLink struct + Href string + IsMarkdown bool + type LLMSTxt struct + Links []LLMSLink + Present bool + Status int + URL string + type LinkTargetChange struct + Evidence string + LeftIndex int + LeftTarget string + RightIndex int + RightTarget string + Text string + type Observation struct + Blocks []Block + ByteSize int64 + Cache CacheState + Challenge []ChallengeSignal + ContentType string + Domain string + Duration time.Duration + Err string + FetchedAt time.Time + FinalURL string + Headers http.Header + Identity IdentityKind + Outcome Outcome + PageURL string + Platform Platform + RunID string + StatusCode int + TokenCount int + TokenCountHeader string + TokenCountReported int + type Outcome string + const OutcomeBlocked + const OutcomeChallenged + const OutcomeDisallowed + const OutcomeEmpty + const OutcomeRateLimited + const OutcomeRedirectLoop + const OutcomeServed + const OutcomeTransport + const OutcomeUnavailable + const OutcomeUncertain + func AllOutcomes() []Outcome + func ClassifyHTTP(status int, h http.Header, body []byte, redirectLoop bool) Outcome + func (o Outcome) Valid() bool + type PageRole string + const PageHome + const PageInterior + type PageSelection struct + Evidence string + Href string + Primary bool + Role PageRole + Rule string + Source string + type Platform struct + Class string + MarkdownEdge bool + Name string + func PlatformFromHeaders(h http.Header) Platform + func (p Platform) Key() string + type RequestSigner interface + Sign func(req *http.Request) error + type RobotsRule struct + Agent string + Allowed bool + CrawlDelay time.Duration + Identity IdentityKind + Pattern string + type Run struct + DomainList string + FinishedAt *time.Time + HeartbeatAt *time.Time + HeartbeatPID int + ID string + LastHost string + Settings RunSettings + StartedAt time.Time + Status RunStatus + StopReason string + func (r Run) HeartbeatLive(staleAfter time.Duration, alive func(int) bool) bool + type RunSettings struct + AgentOrigin string + Concurrency int + Delay string + DomainSHA256 string + KeyID string + Redirects int + Retries int + Timeout string + func ParseRunSettings(raw string) RunSettings + func (s RunSettings) Empty() bool + func (s RunSettings) Equal(o RunSettings) bool + func (s RunSettings) Marshal() string + func (s RunSettings) Mismatch(o RunSettings) string + type RunStatus string + const RunComplete + const RunInterrupted + const RunRunning + type Share struct + ByPlatform map[string]Share + N int + Rate float64 + type SiteContext struct + Alternates []Alternate + LLMSTxt LLMSTxt + Mismatch AlternateMismatch + Robots []RobotsRule + RobotsRaw []byte + Selection PageSelection + type SiteContextRecord struct + Context SiteContext + Domain string + PageURL string + RunID string + type Store interface + CreateRun func(ctx context.Context, domainList string, settings RunSettings) (Run, error) + FinishRun func(ctx context.Context, id string) error + HasObservation func(ctx context.Context, runID, domain, pageURL string, id IdentityKind) (bool, error) + LatestRunning func(ctx context.Context) (Run, bool, error) + LatestUnfinished func(ctx context.Context) (Run, bool, error) + Observation func(ctx context.Context, runID, domain, pageURL string, id IdentityKind) (Observation, bool, error) + ObservationsByDomain func(ctx context.Context, runID, domain string) ([]Observation, error) + ObservationsByRun func(ctx context.Context, runID string) ([]Observation, error) + ResumeRun func(ctx context.Context, id string, settings RunSettings) error + Run func(ctx context.Context, id string) (Run, error) + SaveObservation func(ctx context.Context, o Observation) error + SaveSiteContext func(ctx context.Context, rec SiteContextRecord) error + SetRunSettings func(ctx context.Context, id string, settings RunSettings) error + SiteContext func(ctx context.Context, runID, domain, pageURL string) (SiteContextRecord, bool, error) + SiteContextsByRun func(ctx context.Context, runID string) ([]SiteContextRecord, error) + StopRun func(ctx context.Context, id, reason string) error + TouchHeartbeat func(ctx context.Context, id string, pid int, lastHost string) error v0.1.0 Aug 29, 2026
Go to main page