robots

Documentation

Index

• Constants
• type RoboTester
  • func NewRoboTester(fullURL string) *RoboTester
  • func (rt *RoboTester) GetDelay() time.Duration
  • func (rt *RoboTester) GetRobots(path string) (robots *robotstxt.RobotsData, err error)
  • func (rt *RoboTester) InitWithUA(ua string)
  • func (rt *RoboTester) Test(path string) bool

Constants

const (
	RobotsTimeout = 10 * time.Second
)

Types

type RoboTester

type RoboTester struct {
	// contains filtered or unexported fields
}

func NewRoboTester

func NewRoboTester(fullURL string) *RoboTester

func (*RoboTester) GetDelay

func (rt *RoboTester) GetDelay() time.Duration

GetDelay - be as careful as possible, if there are two definitions - sum them up and use both

func (*RoboTester) GetRobots

func (rt *RoboTester) GetRobots(path string) (robots *robotstxt.RobotsData, err error)

func (*RoboTester) InitWithUA

func (rt *RoboTester) InitWithUA(ua string)

func (*RoboTester) Test

func (rt *RoboTester) Test(path string) bool