httpfetch

Documentation

Index

• Constants
• Variables
• type FetchRequest
  • func (req *FetchRequest) Host() (string, error)
  • func (req *FetchRequest) String() string
• type FetchResult
• type Fetcher
  • func New() *Fetcher
  • func (h *Fetcher) FetchFile(req *FetchRequest, complete chan<- *FetchResult)
  • func (h *Fetcher) FileGetResponse(url string, headers Headers) (*http.Response, error)
  • func (h *Fetcher) GetConcurrentRequestCount(count int) int
  • func (h *Fetcher) NewFileDownload(req *FetchRequest, complete chan<- *FetchResult)
  • func (h *Fetcher) QueueFetch(req []*FetchRequest)
  • func (h *Fetcher) ResumeFileDownload(req *FetchRequest, complete chan<- *FetchResult)
  • func (h *Fetcher) Shutdown()
• type HTTPError
  • func (err *HTTPError) Error() string
• type Headers
  • func HeadersWith(headers Headers, newHeader, newValue string) Headers
  • func (headers Headers) AddHeaders(h *http.Header)
  • func (headers Headers) Copy() Headers

Constants

const DefaultUserAgent = "Sequell httpfetch/1.0"

DefaultUserAgent is the HTTP user agent string.

Variables

var (
	// DefaultConnectTimeout is how long to wait for a connection to timeout.
	DefaultConnectTimeout = 3 * time.Second

	// DefaultReadTimeout is how long to wait for a HTTP read to timeout.
	DefaultReadTimeout = 20 * time.Second

	// DefaultHTTPTransport is the default transport to use for HTTP requests
	DefaultHTTPTransport = http.Transport{
		Dial:                  dialer(DefaultConnectTimeout, DefaultReadTimeout),
		ResponseHeaderTimeout: DefaultConnectTimeout,
	}

	// DefaultHTTPClient is the default HTTP client to use for requests.
	DefaultHTTPClient = &http.Client{
		Transport: &DefaultHTTPTransport,
	}
)

Types

type FetchRequest

type FetchRequest struct {
	URL      string
	Filename string

	// Don't try to resume downloads if this is set.
	FullDownload   bool
	RequestHeaders Headers
}

A FetchRequest is a request to download URL to Filename

func (*FetchRequest) Host

func (req *FetchRequest) Host() (string, error)

Host gets the HTTP host to make the request to

func (*FetchRequest) String

func (req *FetchRequest) String() string

type FetchResult

type FetchResult struct {
	Req          *FetchRequest
	Err          error
	DownloadSize int64
}

A FetchResult is the result of a fetch

type Fetcher

type Fetcher struct {
	HTTPClient                   *http.Client
	ConnectTimeout               time.Duration
	ReadTimeout                  time.Duration
	UserAgent                    string
	MaxConcurrentRequestsPerHost int
	// contains filtered or unexported fields
}

A Fetcher downloads files from remote servers.

func New

func New() *Fetcher

New returns a new Fetcher for parallel downloads. Fetcher methods are not threadsafe.

func (*Fetcher) FetchFile

func (h *Fetcher) FetchFile(req *FetchRequest, complete chan<- *FetchResult)

FetchFile downloads a file as specified in req, writing a completion FetchResult to complete.

func (*Fetcher) FileGetResponse

func (h *Fetcher) FileGetResponse(url string, headers Headers) (*http.Response, error)

FileGetResponse makes a HTTP GET request to url and returns the response object.

func (*Fetcher) GetConcurrentRequestCount

func (h *Fetcher) GetConcurrentRequestCount(count int) int

GetConcurrentRequestCount returns the maximum concurrent requests to make when processing count requests.

func (*Fetcher) NewFileDownload

func (h *Fetcher) NewFileDownload(req *FetchRequest, complete chan<- *FetchResult)

NewFileDownload downloads a file as specified in req, writing a fetch result to the complete chan when done. File downloads are not resumed, so any existing file will be overwritten.

func (*Fetcher) QueueFetch

func (h *Fetcher) QueueFetch(req []*FetchRequest)

QueueFetch enqueues the given download requests for asynchronous download.

func (*Fetcher) ResumeFileDownload

func (h *Fetcher) ResumeFileDownload(req *FetchRequest, complete chan<- *FetchResult)

ResumeFileDownload downloads req and attempts to resume the download into req.Filename. On completion, a FetchResult is written to the complete chan.

func (*Fetcher) Shutdown

func (h *Fetcher) Shutdown()

Shutdown gracefully shuts down the fetcher, cleaning up all background goroutines, and waiting for all outstanding downloads to end.

type HTTPError

type HTTPError struct {
	StatusCode int
	Response   *http.Response
}

An HTTPError represents a HTTP status code and the server's response.

func (*HTTPError) Error

func (err *HTTPError) Error() string

type Headers

type Headers map[string]string

Headers is a map of HTTP headers.

func HeadersWith

func HeadersWith(headers Headers, newHeader, newValue string) Headers

HeadersWith creates a copy of headers with newHeader=newValue

func (Headers) AddHeaders

func (headers Headers) AddHeaders(h *http.Header)

AddHeaders adds all headers in h to headers.

func (Headers) Copy

func (headers Headers) Copy() Headers

Copy returns a copy of headers