rapidshorthair

README

rapidshorthair

A high-performance concurrent downloader.

Overview

In order to speed up the download speed of large files, a very simple idea is to open multiple parallel download instances. In the process of downloading, you also need to consider the allocation of buffers, the process of writing files, and so on. In order to facilitate the process of speeding up file downloads in various projects, the process of downloading files in parallel is implemented as a common library.

This library implements the following features:

• A generic work queue.
• Parallel Download Framework.
• A simple memory pool.
• File Parallel Write Framework.

Example

A simple use case is as follow.

package main

import (
	"github.com/caplyso/rapidshorthair"
)

func main() {
    f, err := NewFetcher("https://go.dev/dl/go1.20.5.darwin-amd64.tar.gz",
                         3, 2, 0,
                         "/home/go1.20.5.darwin-amd64.tar.gz",
                         false)
    if err != nil {
        return
    }

    f.Start()
}

Contributing

• Fork it
• Create your feature branch (git checkout -b my-new-feature)
• Commit your changes (git commit -am 'Add some feature')
• Push to the branch (git push origin my-new-feature)
• Create new Pull Request

Documentation

Index

• func GetFileName(download_url string) (string, error)
• func GetRangeBody(url string, start int64, end int64) (io.ReadCloser, int64, error)
• func GetSizeAndCheckRangeSupport(url string) (size int64, err error)
• func NewBufferPair(channels int, sz int64, init bool) *bufferPair
• type BatchCrawler
  • func NewBatchCrawler(f *os.File, recvBufferChan chan *segment, releaseBufferChan chan *segment, ...) (*BatchCrawler, error)
  • func (c *BatchCrawler) Downcall(_ interface{}) (interface{}, error)
  • func (c *BatchCrawler) Identity() string
  • func (c *BatchCrawler) Run(ctx context.Context) error
  • func (c *BatchCrawler) Status() bool
• type Fetcher
  • func NewFetcher(url string, concurrency int, batches int, bufferSz int64, filePath string, ...) (*Fetcher, error)
  • func (f *Fetcher) CurrentLength() int64
  • func (f *Fetcher) Run() error
  • func (f *Fetcher) Start() error
  • func (f *Fetcher) TotalLength() int64
• type SegmentCrawler
  • func NewSegmentCrawler(index int, f *os.File, url string, batches int, bufferSz int64, start int64, ...) (*SegmentCrawler, error)
  • func (c *SegmentCrawler) Downcall(_ interface{}) (interface{}, error)
  • func (c *SegmentCrawler) Identity() string
  • func (c *SegmentCrawler) Run(ctx context.Context) error
  • func (c *SegmentCrawler) Status() bool
• type Worker
• type WorkerManager
  • func NewWorkerManager(name string, count int, f func() (Worker, error)) (*WorkerManager, error)
  • func (m *WorkerManager) Add(w Worker)
  • func (m *WorkerManager) Count() int32
  • func (m *WorkerManager) GetWorker(id int32) (Worker, error)
  • func (m *WorkerManager) Run(ctx context.Context, notifier chan error) error

Functions

func GetFileName

func GetFileName(download_url string) (string, error)

func GetRangeBody

func GetRangeBody(url string, start int64, end int64) (io.ReadCloser, int64, error)

func GetSizeAndCheckRangeSupport

func GetSizeAndCheckRangeSupport(url string) (size int64, err error)

func NewBufferPair

func NewBufferPair(channels int, sz int64, init bool) *bufferPair

Types

type BatchCrawler

type BatchCrawler struct {
	File *os.File
	// contains filtered or unexported fields
}

func NewBatchCrawler

func NewBatchCrawler(f *os.File,
	recvBufferChan chan *segment,
	releaseBufferChan chan *segment, update func(int64)) (*BatchCrawler, error)

func (*BatchCrawler) Downcall

func (c *BatchCrawler) Downcall(_ interface{}) (interface{}, error)

func (*BatchCrawler) Identity

func (c *BatchCrawler) Identity() string

func (*BatchCrawler) Run

func (c *BatchCrawler) Run(ctx context.Context) error

func (*BatchCrawler) Status

func (c *BatchCrawler) Status() bool

type Fetcher

type Fetcher struct {
	File *os.File
	// contains filtered or unexported fields
}

func NewFetcher

func NewFetcher(url string, concurrency int, batches int, bufferSz int64,
	filePath string, withDateTime bool) (*Fetcher, error)

func (*Fetcher) CurrentLength

func (f *Fetcher) CurrentLength() int64

func (*Fetcher) Run

func (f *Fetcher) Run() error

func (*Fetcher) Start

func (f *Fetcher) Start() error

func (*Fetcher) TotalLength

func (f *Fetcher) TotalLength() int64

type SegmentCrawler

type SegmentCrawler struct {
	RangeBatch *WorkerManager
	// contains filtered or unexported fields
}

func NewSegmentCrawler

func NewSegmentCrawler(index int, f *os.File, url string,
	batches int, bufferSz int64,
	start int64, end int64) (*SegmentCrawler, error)

func (*SegmentCrawler) Downcall

func (c *SegmentCrawler) Downcall(_ interface{}) (interface{}, error)

func (*SegmentCrawler) Identity

func (c *SegmentCrawler) Identity() string

func (*SegmentCrawler) Run

func (c *SegmentCrawler) Run(ctx context.Context) error

func (*SegmentCrawler) Status

func (c *SegmentCrawler) Status() bool

type Worker

type Worker interface {
	Identity() string
	Status() bool
	Run(context.Context) error
	Downcall(interface{}) (interface{}, error)
}

type WorkerManager

type WorkerManager struct {
	// contains filtered or unexported fields
}

func NewWorkerManager

func NewWorkerManager(name string, count int, f func() (Worker, error)) (*WorkerManager, error)

func (*WorkerManager) Add

func (m *WorkerManager) Add(w Worker)

func (*WorkerManager) Count

func (m *WorkerManager) Count() int32

func (*WorkerManager) GetWorker

func (m *WorkerManager) GetWorker(id int32) (Worker, error)

func (*WorkerManager) Run

func (m *WorkerManager) Run(ctx context.Context, notifier chan error) error