useragent

README

useragent

useragent is a go package that generates a random combination of millions of user-agents strings. Currently used in production at DataHen to crawl/scrape through billions of pages.

Documentation

Index

• func Desktop() (ua string, err error)
• func GoogleBot2() (string, error)
• func LoadUAConfig(path string) (err error)
• func Mobile() (string, error)
• type Browser
  • func (bwr *Browser) Init()
  • func (bwr *Browser) RandomVariant() *BrowserVariant
  • func (bwr *Browser) SignError(msg string) error
• type BrowserVariant
  • func (bwrv *BrowserVariant) AddVars(data map[string]string) (err error)
  • func (bwrv *BrowserVariant) SignError(msg string) error
• type Device
  • func (dv *Device) BrowsersInit()
  • func (dv *Device) BuildUserAgent() (ua string, err error)
  • func (dv *Device) Init()
  • func (dv *Device) OSesInit()
  • func (dv *Device) RandomBrowser(bwrIDs []string) *Browser
  • func (dv *Device) RandomOS() *OS
  • func (dv *Device) SignError(msg string) error
• type OS
  • func (os *OS) Init()
  • func (os *OS) RandomVariant() *OSVariant
  • func (os *OS) SignError(msg string) error
• type OSVariant
  • func (osv *OSVariant) AddVars(data map[string]string) (err error)
  • func (osv *OSVariant) RandomSignature() (string, error)
  • func (osv *OSVariant) SignError(msg string) error
• type UAConfig
  • func LoadUAConfigFromBytes(bs []byte) (c *UAConfig, err error)
  • func LoadUAConfigFromFile(path string) (*UAConfig, error)
  • func LoadUAConfigFromJSON(raw string) (*UAConfig, error)
  • func (c *UAConfig) BuildDesktopUA() (string, error)
  • func (c *UAConfig) BuildMobileUA() (string, error)
  • func (c *UAConfig) BuildTabletUA() (string, error)
  • func (c *UAConfig) Init()
• type Variant
  • func (vrt *Variant) AddDataVars(prefix string, data map[string]string) error

Functions

func Desktop

func Desktop() (ua string, err error)

Desktop returns a random generated UA for desktop browsers like so: Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8.1.4) Chrome/ Safari/530.6 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.0.4) Gecko/2008092417 Firefox/3.0.3

func GoogleBot2

func GoogleBot2() (string, error)

GoogleBot2 returns UA for google bot2

func LoadUAConfig

func LoadUAConfig(path string) (err error)

LoadUAConfig loads UserAgent configuration file

func Mobile

func Mobile() (string, error)

Mobile returns a random generated UA for mobile browsers To do: make proper implementation

Types

type Browser

type Browser struct {
	types.WeightedElement

	ID              string            `json:"id"`
	UserAgentFormat string            `json:"ua_format"`
	Variants        []*BrowserVariant `json:"variants"`

	VariantProbabilityLimit float64
}

Browser browser's user agent configuration

func (*Browser) Init

func (bwr *Browser) Init()

Init recusively init browser

func (*Browser) RandomVariant

func (bwr *Browser) RandomVariant() *BrowserVariant

RandomVariant gets a random variant

func (*Browser) SignError

func (bwr *Browser) SignError(msg string) error

SignError add a text signature to an error

type BrowserVariant

type BrowserVariant struct {
	Variant
	types.WeightedElement

	ID string `json:"id"`
}

BrowserVariant represents an user agent configuration for a specific browser variant

func (*BrowserVariant) AddVars

func (bwrv *BrowserVariant) AddVars(data map[string]string) (err error)

AddVars adds the browser data variables to a map

func (*BrowserVariant) SignError

func (bwrv *BrowserVariant) SignError(msg string) error

SignError add a text signature to an error

type Device

type Device struct {
	ID       string              `json:"id"`
	OSes     []*OS               `json:"oses"`
	Browsers map[string]*Browser `json:"browsers"`

	OSProbabilityLimit float64
}

Device device's user agent configuration

func (*Device) BrowsersInit

func (dv *Device) BrowsersInit()

BrowsersInit recusively init all browsers

func (*Device) BuildUserAgent

func (dv *Device) BuildUserAgent() (ua string, err error)

BuildUserAgent builds a random desktop user agent

func (*Device) Init

func (dv *Device) Init()

Init init the device

func (*Device) OSesInit

func (dv *Device) OSesInit()

OSesInit recusively init all oses

func (*Device) RandomBrowser

func (dv *Device) RandomBrowser(bwrIDs []string) *Browser

RandomBrowser gets a random browser

func (*Device) RandomOS

func (dv *Device) RandomOS() *OS

RandomOS gets a random os

func (*Device) SignError

func (dv *Device) SignError(msg string) error

SignError add a text signature to an error

type OS

type OS struct {
	types.WeightedElement

	ID       string       `json:"id"`
	Variants []*OSVariant `json:"variants"`

	VariantProbabilityLimit float64
}

OS operating system's user agent configuration

func (*OS) Init

func (os *OS) Init()

Init init os

func (*OS) RandomVariant

func (os *OS) RandomVariant() *OSVariant

RandomVariant gets a random variant

func (*OS) SignError

func (os *OS) SignError(msg string) error

SignError add a text signature to an error

type OSVariant

type OSVariant struct {
	Variant
	types.WeightedElement

	ID         string   `json:"id"`
	Signatures []string `json:"signatures"`
	BrowserIDs []string `json:"browser_ids"`
}

OSVariant represents an user agent configuration for a specific operating system variant

func (*OSVariant) AddVars

func (osv *OSVariant) AddVars(data map[string]string) (err error)

AddVars adds the os data variables to a map

func (*OSVariant) RandomSignature

func (osv *OSVariant) RandomSignature() (string, error)

RandomSignature gets a random signature

func (*OSVariant) SignError

func (osv *OSVariant) SignError(msg string) error

SignError add a text signature to an error

type UAConfig

type UAConfig struct {
	Desktop Device `json:"desktop"`
	Tablet  Device `json:"tablet"`
	Mobile  Device `json:"mobile"`
}

UAConfig user agents configuration

func LoadUAConfigFromBytes

func LoadUAConfigFromBytes(bs []byte) (c *UAConfig, err error)

LoadUAConfigFromBytes loads config from a byte slice

func LoadUAConfigFromFile

func LoadUAConfigFromFile(path string) (*UAConfig, error)

LoadUAConfigFromFile loads config from a file

func LoadUAConfigFromJSON

func LoadUAConfigFromJSON(raw string) (*UAConfig, error)

LoadUAConfigFromJSON loads config from a JSON string

func (*UAConfig) BuildDesktopUA

func (c *UAConfig) BuildDesktopUA() (string, error)

BuildDesktopUA builds a random desktop user agent

func (*UAConfig) BuildMobileUA

func (c *UAConfig) BuildMobileUA() (string, error)

BuildMobileUA builds a random mobile user agent

func (*UAConfig) BuildTabletUA

func (c *UAConfig) BuildTabletUA() (string, error)

BuildTabletUA builds a random tablet user agent

func (*UAConfig) Init

func (c *UAConfig) Init()

Init init user agent config and recursively init all devices

type Variant

type Variant struct {
	Data map[string][]string `json:"data"`
}

Variant represents a variant

func (*Variant) AddDataVars

func (vrt *Variant) AddDataVars(prefix string, data map[string]string) error

AddDataVars adds the data variables to a map