config

Documentation

Index

• type Changeable
• type Config
  • func FromFile(file string) (*Config, error)
  • func FromFiles(config string, proxies string) (*Config, error)
• type Filter
• type ParamsFilter
• type ParsingConfig
• type Proxy
• type Regex
  • func (r *Regex) MarshalJSON() ([]byte, error)
  • func (r *Regex) UnmarshalJSON(b []byte) error

Types

type Changeable

type Changeable struct {
	ChangeFrequency string  `json:"changeFrequency,omitempty" xml:"changefreq,omitempty"`
	Priority        float64 `json:"priority,omitempty" xml:"priority,omitempty"`
}

type Config

type Config struct {
	URL     string         `json:"url"`
	Parsing *ParsingConfig `json:"parsing"`
	Output  []*Filter      `json:"output"`
}

Config describes the whole process of generating sitemap

func FromFile

func FromFile(file string) (*Config, error)

FromFile parses file into a Config

func FromFiles

func FromFiles(config string, proxies string) (*Config, error)

FromFiles parses regular config file for website setting and proxy config file for proxies

type Filter

type Filter struct {
	Regex                   *Regex     `json:"regex"`
	PerFile                 int        `json:"perFile"`
	FilePrefix              string     `json:"filePrefix"`
	Modifiers               Changeable `json:"modifiers,omitempty"`
	IncludeModificationDate bool       `json:"modificationDate,omitempty"`
}

Filter defines how urls are being put into separated files sitemap

type ParamsFilter

type ParamsFilter struct {
	Regex   *Regex   `json:"regex"`
	Params  []string `json:"params"`
	Include bool     `json:"include"`
}

type ParsingConfig

type ParsingConfig struct {
	Workers int `json:"workers"`
	//ParseExclusions regex array defines what sites content is not parsed but included in sitemap
	ParseExclusions []*Regex `json:"parseExclusions,omitempty"`
	RespectRobots   bool     `json:"respectRobots,omitempty"`
	//ResultExclusions regex array defines what sites are being excluded from the sitemap
	Params            []*ParamsFilter `json:"params,omitempty"`
	CutProtocol       bool            `json:"cutProtocol,omitempty"`
	Proxies           []Proxy         `json:"proxies,omitempty"`
	UserAgent         string          `json:"userAgent,omitempty"`
	RequestsPerSecond int64           `json:"requestsPerSec,omitempty"`
	Burst             int             `json:"burst,omitempty"`
	StripQueryString  bool            `json:"stripQueryString,omitempty"`
	StripWWW          bool            `json:"stripWWW,omitempty"`
	NoProxyClient     bool            `json:"noProxyClient,omitempty"`
}

ParsingConfig describes site parsing process

type Proxy

type Proxy struct {
	Address  string `json:"address"`
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
}

Proxy represents a proxy configuration to be used for parsing

type Regex

type Regex struct {
	*regexp.Regexp
}

Regex is a wrapper for regexp.Regexp but implements json marshaler interface

func (*Regex) MarshalJSON

func (r *Regex) MarshalJSON() ([]byte, error)

MarshalJSON turns regex into string/json

func (*Regex) UnmarshalJSON

func (r *Regex) UnmarshalJSON(b []byte) error

UnmarshalJSON turns a string into proper regex