internal

Documentation

Index

• Constants
• Variables
• func CheckProxyIP(protocol string, ip string, port int, validUrl string) (string, int, error)
• func CheckProxyUrl(proxyUrlStr, validUrl string) (string, int, error)
• func CreateRouter(pool *proxyPool) *gin.Engine
• func HandleGetConfigurations(pool *proxyPool) gin.HandlerFunc
• func HandleGetIP(pool *proxyPool) gin.HandlerFunc
• func HandleGetIPIndexPage() gin.HandlerFunc
• func HandleGetIPList(pool *proxyPool) gin.HandlerFunc
• func NewProxyPool(config *ProxyPoolConfig) *proxyPool
• func RenderConfigTable(writer io.Writer, caption string, config ProxyPoolConfig)
• func RenderProxyIPTable(writer io.Writer, caption string, data []*HttpProxyIP)
• type Crawler
• type DefaultGetProxyIPStrategy
  • func (s *DefaultGetProxyIPStrategy) Get(proxies []*HttpProxyIP) (*HttpProxyIP, error)
  • func (s *DefaultGetProxyIPStrategy) Name() string
• type GetProxyIPStrategy
• type HttpProxyIP
  • func (ip *HttpProxyIP) ToTableRow() []string
• type PollingGetProxyIPStrategy
  • func (s *PollingGetProxyIPStrategy) Get(proxies []*HttpProxyIP) (*HttpProxyIP, error)
  • func (s *PollingGetProxyIPStrategy) Name() string
• type ProxyIPStatus
• type ProxyPoolConfig

Constants

const (
	ProxyIPStatusOk    = 200
	ProxyIPStatusError = 201
)

const DefaultIPCheckUrl = "http://icanhazip.com"

const TimeLayout = "15:04:05.000"

Variables

var PoolEmptyError = errors.New("proxy pool is empty")

Functions

func CheckProxyIP

func CheckProxyIP(protocol string, ip string, port int, validUrl string) (string, int, error)

CheckProxyIP check proxy ip status

func CheckProxyUrl

func CheckProxyUrl(proxyUrlStr, validUrl string) (string, int, error)

func CreateRouter

func CreateRouter(pool *proxyPool) *gin.Engine

func HandleGetConfigurations

func HandleGetConfigurations(pool *proxyPool) gin.HandlerFunc

HandleGetConfigurations get proxy pool configurations

func HandleGetIP

func HandleGetIP(pool *proxyPool) gin.HandlerFunc

func HandleGetIPIndexPage

func HandleGetIPIndexPage() gin.HandlerFunc

func HandleGetIPList

func HandleGetIPList(pool *proxyPool) gin.HandlerFunc

func NewProxyPool

func NewProxyPool(config *ProxyPoolConfig) *proxyPool

NewProxyPool create proxy pool with configuration

func RenderConfigTable

func RenderConfigTable(writer io.Writer, caption string, config ProxyPoolConfig)

func RenderProxyIPTable

func RenderProxyIPTable(writer io.Writer, caption string, data []*HttpProxyIP)

RenderProxyIPTable render proxy ip table

Types

type Crawler

type Crawler interface {
	GetProxies() <-chan *HttpProxyIP
	Crawl() error
	Close()
}

type DefaultGetProxyIPStrategy

type DefaultGetProxyIPStrategy struct{}

func (*DefaultGetProxyIPStrategy) Get

func (s *DefaultGetProxyIPStrategy) Get(proxies []*HttpProxyIP) (*HttpProxyIP, error)

func (*DefaultGetProxyIPStrategy) Name

func (s *DefaultGetProxyIPStrategy) Name() string

type GetProxyIPStrategy

type GetProxyIPStrategy interface {
	Get([]*HttpProxyIP) (*HttpProxyIP, error)
	Name() string
}

type HttpProxyIP

type HttpProxyIP struct {
	Port             int
	IP               string
	Anonymity        string
	HttpProtocol     string
	Location         string
	ISP              string
	ResponseSpeed    string
	TTL              string
	LastValidateTime string
	Metadata         map[string]string
	LastCheckedTime  time.Time
	LastCheckedState ProxyIPStatus
}

HttpProxyIP proxy ip item

func (*HttpProxyIP) ToTableRow

func (ip *HttpProxyIP) ToTableRow() []string

ToTableRow map to a table row

type PollingGetProxyIPStrategy

type PollingGetProxyIPStrategy struct {
	// contains filtered or unexported fields
}

func (*PollingGetProxyIPStrategy) Get

func (s *PollingGetProxyIPStrategy) Get(proxies []*HttpProxyIP) (*HttpProxyIP, error)

Get get proxy IP by polling strategy

func (*PollingGetProxyIPStrategy) Name

func (s *PollingGetProxyIPStrategy) Name() string

type ProxyIPStatus

type ProxyIPStatus uint8

type ProxyPoolConfig

type ProxyPoolConfig struct {
	// collector will pull the latest Proxy IP when pool size less than this value
	AlarmValue int
	// delay of checker
	DelayOfChecker time.Duration
	// delay of collector
	DelayOfCollector time.Duration
	GetProxyStrategy GetProxyIPStrategy
}