pget

README

Pget - parallel file download client

Description

Download using a parallel requests

Installation

Homebrew

brew tap Code-Hex/pget
brew install pget

go get

Install

$ go get github.com/Code-Hex/pget/cmd/pget

Update

$ go get -u github.com/Code-Hex/pget/cmd/pget

Synopsis

% pget -p 6 URL 
% pget -p 6 MIRROR1 MIRROR2 MIRROR3

If you have created such as this file

cat list.txt
MIRROR1
MIRROR2
MIRROR3

You can do this

cat list.txt | pget -p 6

Options

  Options:
  -h,  --help                   print usage and exit
  -v,  --version                display the version of pget and exit
  -p,  --procs <num>            split ratio to download file
  -o,  --output <filename>      output file to <filename>
  -d,  --target-dir <path>      path to the directory to save the downloaded file, filename will be taken from url
  -t,  --timeout <seconds>      timeout of checking request in seconds
  -u,  --user-agent <agent>     identify as <agent>
  -r,  --referer <referer>      identify as <referer>
  --check-update                check if there is update available
  --trace                       display detail error messages

Pget vs Wget

URL: http://ubuntutym2.u-toyama.ac.jp/ubuntu/16.04/ubuntu-16.04-desktop-amd64.iso

Using

time wget http://ubuntutym2.u-toyama.ac.jp/ubuntu/16.04/ubuntu-16.04-desktop-amd64.iso
time pget -p 6 http://ubuntutym2.u-toyama.ac.jp/ubuntu/16.04/ubuntu-16.04-desktop-amd64.iso

Results

wget   3.92s user 23.52s system 3% cpu 13:35.24 total
pget -p 6   10.54s user 34.52s system 25% cpu 2:56.93 total

Binary

You can download from here

Author

codehex

Documentation

Index

• type Ch
  • func MakeCh() *Ch
  • func (ch *Ch) CheckingListen(ctx context.Context, cancelAll context.CancelFunc, totalActiveProcs int) (size uint, e error)
  • func (ch *Ch) Close()
• type Data
  • func (d *Data) BindwithFiles(procs int) error
  • func (d Data) DirName() string
  • func (d Data) FileName() string
  • func (d Data) FileSize() uint
  • func (d Data) FullFileName() string
  • func (d Data) IsFree(split uint) error
  • func (d *Data) MakeRange(i, split, procs uint) Range
  • func (d Data) Progress(dirname string) (int64, error)
  • func (d Data) ProgressBar(ctx context.Context) error
  • func (d *Data) SetDirName(path, filename string, procs int)
  • func (d *Data) SetFileName(filename string)
  • func (d *Data) SetFileSize(size uint)
  • func (d *Data) SetFullFileName(directory, filename string)
  • func (d *Data) URLFileName(targetDir, url string) string
• type Options
• type Pget
  • func New() *Pget
  • func (p Pget) Assignment(grp *errgroup.Group, procs, split uint)
  • func (p *Pget) CheckMirrors(ctx context.Context, url string, ch *Ch)
  • func (p *Pget) Checking() error
  • func (p *Pget) Download() error
  • func (pget Pget) ErrTop(err error) error
  • func (p Pget) MakeResponse(r Range, url string) (*http.Response, error)
  • func (pget *Pget) Ready() error
  • func (p Pget) Requests(r Range, filename, dirname, url string) error
  • func (pget *Pget) Run() error
• type Range
• type Utils

Types

type Ch

type Ch struct {
	Err  chan error
	Size chan uint
	Done chan bool
}

Ch struct for request

func MakeCh

func MakeCh() *Ch

MakeCh instead of init

func (*Ch) CheckingListen

func (ch *Ch) CheckingListen(ctx context.Context, cancelAll context.CancelFunc, totalActiveProcs int) (size uint, e error)

CheckingListen method wait all channels for Checking method in requests

func (*Ch) Close

func (ch *Ch) Close()

Close method will close channel in Ch struct

type Data

type Data struct {
	// contains filtered or unexported fields
}

Data struct has file of relational data

func (*Data) BindwithFiles

func (d *Data) BindwithFiles(procs int) error

BindwithFiles function for file binding after split download

func (Data) DirName

func (d Data) DirName() string

DirName get from Data structs member

func (Data) FileName

func (d Data) FileName() string

FileName get from Data structs member

func (Data) FileSize

func (d Data) FileSize() uint

FileSize get from Data structs member

func (Data) FullFileName

func (d Data) FullFileName() string

FullFileName get from Data structs member

func (Data) IsFree

func (d Data) IsFree(split uint) error

IsFree is check your disk space for size needed to download

func (*Data) MakeRange

func (d *Data) MakeRange(i, split, procs uint) Range

MakeRange will return Range struct to download function

func (Data) Progress

func (d Data) Progress(dirname string) (int64, error)

Progress In order to confirm the degree of progress

func (Data) ProgressBar

func (d Data) ProgressBar(ctx context.Context) error

ProgressBar is to show progressbar

func (*Data) SetDirName

func (d *Data) SetDirName(path, filename string, procs int)

SetDirName set to Data structs member

func (*Data) SetFileName

func (d *Data) SetFileName(filename string)

SetFileName set to Data structs member

func (*Data) SetFileSize

func (d *Data) SetFileSize(size uint)

SetFileSize set to Data structs member

func (*Data) SetFullFileName

func (d *Data) SetFullFileName(directory, filename string)

SetFullFileName set to Data structs member

func (*Data) URLFileName

func (d *Data) URLFileName(targetDir, url string) string

URLFileName set to Data structs member using url

type Options

type Options struct {
	Help      bool   `short:"h" long:"help"`
	Version   bool   `short:"v" long:"version"`
	Procs     int    `short:"p" long:"procs"`
	Output    string `short:"o" long:"output"`
	TargetDir string `short:"d" long:"target-dir"`
	Timeout   int    `short:"t" long:"timeout"`
	UserAgent string `short:"u" long:"user-agent"`
	Referer   string `short:"r" long:"referer"`
	Update    bool   `long:"check-update"`
	Trace     bool   `long:"trace"`
}

Options struct for parse command line arguments

type Pget

type Pget struct {
	Trace bool
	Utils
	TargetDir  string
	Procs      int
	URLs       []string
	TargetURLs []string
	// contains filtered or unexported fields
}

Pget structs

func New

func New() *Pget

New for pget package

func (Pget) Assignment

func (p Pget) Assignment(grp *errgroup.Group, procs, split uint)

Assignment method that to each goroutine gives the task

func (*Pget) CheckMirrors

func (p *Pget) CheckMirrors(ctx context.Context, url string, ch *Ch)

CheckMirrors method check be able to range access. also get redirected url.

func (*Pget) Checking

func (p *Pget) Checking() error

Checking is check to can request

func (*Pget) Download

func (p *Pget) Download() error

Download method distributes the task to each goroutine for each URL

func (Pget) ErrTop

func (pget Pget) ErrTop(err error) error

ErrTop get important message from wrapped error message

func (Pget) MakeResponse

func (p Pget) MakeResponse(r Range, url string) (*http.Response, error)

MakeResponse return *http.Response include context and range header

func (*Pget) Ready

func (pget *Pget) Ready() error

Ready method define the variables required to Download.

func (Pget) Requests

func (p Pget) Requests(r Range, filename, dirname, url string) error

Requests method will download the file

func (*Pget) Run

func (pget *Pget) Run() error

Run execute methods in pget package

type Range

type Range struct {
	// contains filtered or unexported fields
}

Range struct for range access

type Utils

type Utils interface {
	ProgressBar(context.Context) error
	BindwithFiles(int) error
	IsFree(uint) error
	Progress(string) (int64, error)
	MakeRange(uint, uint, uint) Range
	URLFileName(string, string) string

	// like setter
	SetFileName(string)
	SetFullFileName(string, string)
	SetDirName(string, string, int)
	SetFileSize(uint)

	// like getter
	FileName() string
	FullFileName() string
	FileSize() uint
	DirName() string
}

Utils interface indicate function