proxyinabox

README

Proxy-in-a-Box

Proxy-in-a-Box helps programmers quickly and easily develop powerful crawler services. one-script, easy-to-use: proxies in a box.

Usage:
  proxy-in-a-box [flags]

Flags:
  -c, --conf string   config file (default "./pb.yaml")
  -p, --ha string     http proxy server addr (default "127.0.0.1:8080")
  -h, --help          help for proxy-in-a-box
  -s, --sa string     https proxy server addr (default "127.0.0.1:8081")

Usage

1. get lastest Proxy-in-a-Box

   go get -u -v github.com/naiba/proxyinabox/cmd/proxy-in-a-box/...
   

2. enter the application directory

   cd $GOPATH/bin
   

3. write config file #Config
4. run it

   ./proxy-in-a-box
   

5. configured in your code

   HTTP proxy: `http://[IP]:8080`
   HTTPS proxy: `https://[IP]:8081`
   * Please set http header when requesting: "Naiba: lifelonglearning" ref:https://github.com/naiba/proxyinabox/blob/master/cmd/proxy-in-a-box/test_server.sh
   

   Set in the code, and then grab it, the Proxy-in-a-Box will automatically assign the proxy.

Config

# run in debug mode
debug: true
# mysql config
mysql:
  host: 127.0.0.1
  port: 3306
  user: root
  pass: 123456
  dbname: proxy
# system config
sys:
  name: Naiba
  # verify proxy's worker num
  proxy_verify_worker: 20
  # how many domains can request per ip in 30min
  domains_per_ip: 30
  # how many request can do per ip in 1s
  request_limit_per_ip: 10
  # verify interval of the proxy stored in the database
  verify_duration: 30

Benchmark

ab -H 'Naiba: lifelonglearning' -v4  -n100 -c10 -X 127.0.0.1:8080 http://api.ip.la/cn

Documentation

Index

• Variables
• func Init()
• type Cache
• type Conf
• type Proxy
  • func (p Proxy) String() string
  • func (p Proxy) URI() string
• type ProxyCrawler
• type ProxyService

Variables

var DB *gorm.DB

DB instance

Functions

func Init

func Init()

Init init system

Types

type Cache

type Cache interface {
	PickProxy(req *http.Request) (string, error)
	IPLimiter(req *http.Request) bool
	HostLimiter(req *http.Request) bool
	HasProxy(p string) bool
	SaveProxy(p Proxy) error
	DeleteProxy(p Proxy)
}

Cache rt

var CI Cache

CI cache instance

type Conf

type Conf struct {
	Debug bool
	MySQL struct {
		Host   string
		Port   string
		User   string
		Pass   string
		Dbname string
	} `mapstructure:"mysql"`
	Redis struct {
		Host string
		Port string
		Pass string
		Db   int
	}
	Sys struct {
		Name              string
		ProxyVerifyWorker int   `mapstructure:"proxy_verify_worker"`
		DomainsPerIP      int   `mapstructure:"domains_per_ip"`
		RequestLimitPerIP int64 `mapstructure:"request_limit_per_ip"`
		VerifyDuration    int   `mapstructure:"verify_duration"`
	}
}

Conf config struct

var Config Conf

Config system config

type Proxy

type Proxy struct {
	gorm.Model
	IP         string `gorm:"type:varchar(15);unique_index"`
	Port       string `gorm:"type:varchar(5)"`
	Country    string `gorm:"type:varchar(15)"`
	Provence   string `gorm:"type:varchar(15)"`
	Platform   int
	HTTPS      bool
	Delay      int64
	LastVerify time.Time
}

Proxy proxy model

func (Proxy) String

func (p Proxy) String() string

func (Proxy) URI

func (p Proxy) URI() string

URI get uri

type ProxyCrawler

type ProxyCrawler interface {
	Fetch() error
}

ProxyCrawler proxy crawler

type ProxyService

type ProxyService interface {
	GetUnVerified() ([]Proxy, error)
}

ProxyService proxy service