googlebot

Documentation

Index

• Constants
• type Doc
  • func ProcessData(data []byte) (doc Doc, err error)
• type Googlebot
  • func New() Googlebot
  • func (gc *Googlebot) Fetch() (doc Doc, err error)
  • func (gc *Googlebot) FetchData() (data []byte, headers http.Header, status int, err error)
• type IPv4Entry
• type IPv6Entry
• type RawDoc
• type RawIPv4Entry
• type RawIPv6Entry

Constants

const (
	ShortName   = "googlebot"
	FullName    = "Google Crawler Bots"
	HostType    = "crawlers"
	SourceURL   = "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
	DownloadURL = "https://developers.google.com/static/search/apis/ipranges/googlebot.json"
)

Types

type Doc

type Doc struct {
	CreationTime time.Time
	IPv4Prefixes []IPv4Entry
	IPv6Prefixes []IPv6Entry
}

func ProcessData

func ProcessData(data []byte) (doc Doc, err error)

type Googlebot

type Googlebot struct {
	Client      *retryablehttp.Client
	DownloadURL string
}

func New

func New() Googlebot

func (*Googlebot) Fetch

func (gc *Googlebot) Fetch() (doc Doc, err error)

func (*Googlebot) FetchData

func (gc *Googlebot) FetchData() (data []byte, headers http.Header, status int, err error)

type IPv4Entry

type IPv4Entry struct {
	IPv4Prefix netip.Prefix `json:"ipv4Prefix"`
}

type IPv6Entry

type IPv6Entry struct {
	IPv6Prefix netip.Prefix `json:"ipv6Prefix"`
}

type RawDoc

type RawDoc struct {
	CreationTime  string `json:"creationTime"`
	LastRequested time.Time
	Entries       []json.RawMessage `json:"prefixes"`
}

type RawIPv4Entry

type RawIPv4Entry struct {
	IPv4Prefix string `json:"ipv4Prefix"`
}

type RawIPv6Entry

type RawIPv6Entry struct {
	IPv6Prefix string `json:"ipv6Prefix"`
}