Versions in this module Expand all Collapse all v0 v0.2.0 Jul 2, 2026 v0.1.0 Jun 30, 2026 Changes in this version + const HostFlagDeadHost + const HostFlagGroupingOverride + const HostFlagRobotsMissing + const HostFlagTrapSuspect + var Commit = "none" + var Date = "unknown" + var Version = "dev" + func HostKeyOf(grouping string) uint64 + func PathKeyOf(pathAndQuery string) uint64 + type AnchorHint uint8 + const AnchorDescriptive + const AnchorEmpty + const AnchorGeneric + const AnchorSpammy + const AnchorUnknown + type Discovery struct + AnchorHint AnchorHint + CanonicalURL string + Depth uint16 + DiscoverySource DiscoverySource + LinkWeight float32 + ObservedAt uint32 + SrcHostKey uint64 + URLKey URLKey + type DiscoverySource uint8 + const SourceLink + const SourceManual + const SourceRedirect + const SourceSeed + const SourceSitemap + type HostGrouping uint8 + const GroupFullHost + const GroupRegistrableDomain + type HostRecord struct + AvgLatency uint16 + CrawlDelay uint16 + CrawlTotal uint32 + DepthCap uint16 + ErrorTotal uint32 + Flags uint16 + Grouping HostGrouping + HostKey uint64 + HostNextEligible uint32 + HostRef uint64 + HostScore float32 + IPExpiry uint32 + IPNextEligible uint32 + RegistrableRef uint64 + ResolvedIP [16]byte + RobotsExpiry uint32 + RobotsFetched uint32 + RobotsRef uint64 + URLBudget uint32 + URLCount uint32 + type HostSignal struct + HostKey uint64 + HostScore float32 + type Outcome struct + ContentFP uint64 + ContentLen uint32 + ETag string + FetchedAt uint32 + HTTPStatus uint16 + LastModified uint32 + LatencyMS uint16 + Links []Discovery + NotModified bool + RedirectTarget string + RetryAfter uint16 + Retryable bool + RobotsBody []byte + RobotsCrawlDelay uint16 + Simhash uint64 + URLKey URLKey + type Signal struct + Epoch uint64 + Hosts []HostSignal + URLs []URLSignal + type URLKey struct + HostKey uint64 + PathKey uint64 + func MakeURLKey(grouping, pathAndQuery string) URLKey + func URLKeyFromBytes(b [16]byte) URLKey + func (a URLKey) Compare(b URLKey) int + func (a URLKey) Less(b URLKey) bool + func (k URLKey) Bytes() [16]byte + type URLRecord struct + ChangeCount uint32 + ContentFP uint64 + CrawlCount uint32 + Depth uint16 + DiscoverySource DiscoverySource + ETagRef uint64 + ErrorCount uint16 + FirstSeen uint32 + HTTPStatus uint16 + HostKey uint64 + Lambda float32 + LastChanged uint32 + LastCrawled uint32 + LastModified uint32 + NextDue uint32 + NoChangeStreak uint16 + Priority float32 + RedirectRef uint64 + RetryCount uint8 + Simhash uint64 + Status URLStatus + URLKey URLKey + URLRef uint64 + type URLSignal struct + PageRank float32 + URLKey URLKey + type URLStatus uint8 + const StatusCrawled + const StatusDiscovered + const StatusDueRecrawl + const StatusExcludedRobots + const StatusGone + const StatusInFlight + const StatusReady + const StatusScheduled + const StatusTrapped + func (s URLStatus) String() string
Go to main page