robots

Documentation

Overview

Package robots provides support for parsing robots.txt files.

Index

• type RobotsData
  • func GetRobotsForURL(target *url.URL, factory client.ClientFactory) (*RobotsData, error)
  • func ParseRobotsTxt(text []byte) (*RobotsData, error)
  • func (data *RobotsData) GetAllPaths() []string
  • func (data *RobotsData) GetForUserAgent(targetAgent string) []string
• type RobotsGroup

Types

type RobotsData

type RobotsData struct {
	Groups []RobotsGroup
}

func GetRobotsForURL

func GetRobotsForURL(target *url.URL, factory client.ClientFactory) (*RobotsData, error)

func ParseRobotsTxt

func ParseRobotsTxt(text []byte) (*RobotsData, error)

func (*RobotsData) GetAllPaths

func (data *RobotsData) GetAllPaths() []string

func (*RobotsData) GetForUserAgent

func (data *RobotsData) GetForUserAgent(targetAgent string) []string

type RobotsGroup

type RobotsGroup struct {
	UserAgents []string
	Disallow   []string
}