proxy_pool

Documentation

Index

• type AutoProxyGetter
• type DefaultAutoProxyGetter
• type DefaultProxyPool
  • func (p *DefaultProxyPool) CheckExist(proxyAddr string) bool
  • func (p *DefaultProxyPool) CheckProxy(proxyAddr string) bool
  • func (p *DefaultProxyPool) EraseProxy(proxy string) int
  • func (p *DefaultProxyPool) GetProxy() (string, error)
  • func (p *DefaultProxyPool) LenOfProxies() int
  • func (p *DefaultProxyPool) PushProxy(proxy ...string)
• type ProxyPool
• type WithThresholdDecorator
  • func WrapWithThresholdDecorator(a AutoProxyGetter, threshold int) *WithThresholdDecorator
• type WithTimeDecorator
  • func WrapWithTimeDecorator(a AutoProxyGetter, interval int) *WithTimeDecorator

Types

type AutoProxyGetter

type AutoProxyGetter interface {
	ProxyPool
	proxy_crawler.Crawler
}

type DefaultAutoProxyGetter

type DefaultAutoProxyGetter struct {
	ProxyPool
	proxy_crawler.Crawler
}

type DefaultProxyPool

type DefaultProxyPool struct {
	CheckBeforeGetProxy bool // if true, proxy will be checked for availability before returning; otherwise, it will be returned directly
	// contains filtered or unexported fields
}

func (*DefaultProxyPool) CheckExist

func (p *DefaultProxyPool) CheckExist(proxyAddr string) bool

func (*DefaultProxyPool) CheckProxy

func (p *DefaultProxyPool) CheckProxy(proxyAddr string) bool

func (*DefaultProxyPool) EraseProxy

func (p *DefaultProxyPool) EraseProxy(proxy string) int

The efficiency is not high when the number of proxies is large

func (*DefaultProxyPool) GetProxy

func (p *DefaultProxyPool) GetProxy() (string, error)

func (*DefaultProxyPool) LenOfProxies

func (p *DefaultProxyPool) LenOfProxies() int

func (*DefaultProxyPool) PushProxy

func (p *DefaultProxyPool) PushProxy(proxy ...string)

type ProxyPool

type ProxyPool interface {
	GetProxy() (string, error)    // return an usable proxy. if there is not an usable proxy, return "", error
	CheckProxy(proxy string) bool // check if the proxy is usable
	EraseProxy(proxy string) int  // erase the proxy from the proxy list
	PushProxy(proxy ...string)    // push the proxy into the proxy list
	LenOfProxies() int
}

type WithThresholdDecorator

type WithThresholdDecorator struct {
	AutoProxyGetter
	// contains filtered or unexported fields
}

func WrapWithThresholdDecorator

func WrapWithThresholdDecorator(a AutoProxyGetter, threshold int) *WithThresholdDecorator

WrapWithThresholdDecorator 达到阈值时爬取代理ip

type WithTimeDecorator

type WithTimeDecorator struct {
	AutoProxyGetter
	// contains filtered or unexported fields
}

func WrapWithTimeDecorator

func WrapWithTimeDecorator(a AutoProxyGetter, interval int) *WithTimeDecorator

WrapWithTimeDecorator 周期性爬取代理ip