mdown

README

mdown

Multiple downloads library for Golang.

You can limit the total max num of simultaneous downloads and the max num of simultaneous downloads from the same host.

Usage

Import

import "github.com/shiolier/mdown"

Usage

// New
md := mdown.New(mdown.Config{
    // Max num of simultaneous downloads
    DownloadProc: runtime.NumCPU(),
    // Max num of simultaneous downloads from the same host
    DownloadProcPerHost: 2,
    // HTTP Client
    HttpClient: http.DefaultClient,
})

// Get
ch1 := make(chan mdown.Result, 1)
md.Add(mdown.NewDownloadURL("http://host1.example.com/", ch1))
go func() {
    for r := range ch1 {
        // check
        // r.Err == nil
        // r.StatusCode == http.StatusOK
        // r.ResponseHeader contains what you need
        // etc

        // r.Data is response body
        // Save it locally, parse it as HTML, etc.

        // If you want to download something new, add it.
        // For example, you want to access the link in the response(HTML).
        ch2 := make(chan mdown.Result, 1)
        md.Add(mdown.NewDownloadURL("http://host2.example.com/", ch2))
        go func() {
            for r := range ch2 {
                // Same as the above example, so omitted
            }
        }()
    }
}()

// Post
ch3 := make(chan mdown.Result, 2)
md.Add(mdown.NewDownload(
    // Request Method
    http.MethodPost,
    // URL
    "http://host3.example.com/",
    // Request Body
    bytes.NewReader([]byte(`{"msg":"This is a message"}`)),
    // Channel for receiving result
    ch3,
    mdown.DownloadConfig{
        // Timeout
        Timeout: 5 * time.Second,
        // Request Header
        RequestHeader: map[string]string{
            "User-Agent":   "mdown",
            "Content-Type": "application/json",
            // etc
        },
        // If true, send to the channel at download start
        SendToChWhenStart: true,
    },
))
go func() {
    for r := range ch3 {
        if r.IsStartEvent() {
            fmt.Printf("Started: %s\n", r.Url)
            continue
        }

        // Same as the above example, so omitted
    }
}()

// Start downloads
md.Start(false)
// Cancel downloads
defer md.Cancel()

// Wait for all downloads to done
md.Wait()

Also there is a singleton version.

import (
    "github.com/shiolier/mdown"
    md "github.com/shiolier/mdown/singleton"
)

// SetConfig
md.SetConfig(mdown.Config{DownloadProc: runtime.NumCPU(), DownloadProcPerHost: 2})

// Get
ch1 := make(chan mdown.Result, 1)
md.AddDownloadURL("http://host1.example.com/", ch1)
go func() {
    for r := range ch1 {
        // Same as the above example, so omitted
    }
}()

// Post
ch2 := make(chan mdown.Result, 2)
md.Add(mdown.NewDownload(
    http.MethodPost,
    "http://host2.example.com/",
    bytes.NewReader([]byte(`{"msg":"This is a message"}`)),
    ch2,
    mdown.DownloadConfig{
        Timeout: 5 * time.Second,
        RequestHeader: map[string]string{
            "User-Agent":   "mdown",
            "Content-Type": "application/json",
        },
        SendToChWhenStart: true,
    },
))
go func() {
    for r := range ch2 {
        // Same as the above example, so omitted
    }
}()

md.Start()
defer md.Cancel()

md.Wait()

Documentation

Index

• type Config
• type Download
  • func NewDownload(method, url string, reqBody io.Reader, ch chan<- Result, conf DownloadConfig) Download
  • func NewDownloadURL(url string, ch chan<- Result) Download
• type DownloadConfig
• type MDown
  • func New(conf Config) *MDown
  • func (m *MDown) Add(d Download)
  • func (m *MDown) Cancel()
  • func (m *MDown) Config() Config
  • func (m *MDown) SetConfig(conf Config)
  • func (m *MDown) Start(panicIfStarted bool)
  • func (m *MDown) Wait()
• type Result
  • func (r Result) IsStartEvent() bool

Types

type Config

type Config struct {
	// Max num of simultaneous downloads.
	DownloadProc int
	// Max num of simultaneous downloads from same host
	DownloadProcPerHost int
	// HTTP Client
	HttpClient *http.Client
}

Config

type Download

type Download interface {
	Hostname() string
	Download(ctx context.Context, client *http.Client)
}

Download is one download to manage with mdown

func NewDownload

func NewDownload(method, url string, reqBody io.Reader, ch chan<- Result, conf DownloadConfig) Download

NewDownload returns Download

ch capacity must be 1 or higher, and if DownloadConfig.SendToChWhenStart is true then 2 or higher.

func NewDownloadURL

func NewDownloadURL(url string, ch chan<- Result) Download

NewDownloadURL returns Download

Request Method is GET

type DownloadConfig

type DownloadConfig struct {
	// Request Timeout
	Timeout time.Duration
	// Request Header
	RequestHeader map[string]string
	// If true, send to the channel at download start
	SendToChWhenStart bool
	// If true, not close the channel when the download is done
	NotCloseChWhenDone bool
}

DownloadConfig is configration used for download

type MDown

type MDown struct {
	// contains filtered or unexported fields
}

MDown

func New

func New(conf Config) *MDown

New returns MDown

func (*MDown) Add

func (m *MDown) Add(d Download)

Add Download to the waiting list

func (*MDown) Cancel

func (m *MDown) Cancel()

Cancel cancels downloads

func (*MDown) Config

func (m *MDown) Config() Config

Config is a thread-safe getter

func (*MDown) SetConfig

func (m *MDown) SetConfig(conf Config)

SetConfig is a thread-safe setter

func (*MDown) Start

func (m *MDown) Start(panicIfStarted bool)

Start will start downloads.

If panicIfStarted is true, this func will cause panic if it has already started.

func (*MDown) Wait

func (m *MDown) Wait()

Wait waits for all downloads to done

type Result

type Result struct {
	Url            string
	Method         string
	StatusCode     int
	ResponseHeader http.Header
	Data           []byte
	Err            error
}

Result is download result

func (Result) IsStartEvent

func (r Result) IsStartEvent() bool

IsStartEvent returns true if this Result represents the start of a download