downloader

Documentation

Index

• type ContentType
  • func (c ContentType) String() string
• type Downloader
• type HttpClientDownloader
  • func CreateDownloader() *HttpClientDownloader
  • func (downloader *HttpClientDownloader) Download(request model.Request, site *model.Site) (*model.Page, error)
  • func (downloader HttpClientDownloader) GetClient(site model.Site) *http.Client
  • func (downloader *HttpClientDownloader) SetProxyProvider(proxyProvider proxy.ProxyProvider)
• type HttpClientGenerator
  • func (generator *HttpClientGenerator) CreateClient(site model.Site) *http.Client
  • func (generator *HttpClientGenerator) CreateProxyClient(site model.Site, proxy *proxy.Proxy) (*http.Client, error)

Types

type ContentType

type ContentType int

const (
	JSON ContentType = iota
	XML
	FORM
	MULTIPART
)

func (ContentType) String

func (c ContentType) String() string

type Downloader

type Downloader interface {
	Download(request model.Request, site *model.Site) (*model.Page, error)
}

Downloader 下载器接口

type HttpClientDownloader

type HttpClientDownloader struct {
	HttpClientGenerator *HttpClientGenerator
	HttpClients         sync.Map
	ProxyProvider       proxy.ProxyProvider
}

func CreateDownloader

func CreateDownloader() *HttpClientDownloader

func (*HttpClientDownloader) Download

func (downloader *HttpClientDownloader) Download(request model.Request, site *model.Site) (*model.Page, error)

Download 下载 实现接口方法

func (HttpClientDownloader) GetClient

func (downloader HttpClientDownloader) GetClient(site model.Site) *http.Client

func (*HttpClientDownloader) SetProxyProvider

func (downloader *HttpClientDownloader) SetProxyProvider(proxyProvider proxy.ProxyProvider)

type HttpClientGenerator

type HttpClientGenerator struct {
	MaxIdleConnections int32
	RequestTimeout     int32
}

func (*HttpClientGenerator) CreateClient

func (generator *HttpClientGenerator) CreateClient(site model.Site) *http.Client

func (*HttpClientGenerator) CreateProxyClient

func (generator *HttpClientGenerator) CreateProxyClient(site model.Site, proxy *proxy.Proxy) (*http.Client, error)