gokhttp_download

README

gOkHttp-download

Addon for gOkHttp for downloading files and streams

Documentation

Index

• Variables
• func Censor(in string, pad string, chars int, front bool) string
• func GetCurrentIPAddress(hClient *http.Client) (ip string)
• func Truncate(in string, chars, mode int) string
• type DownloadTask
• type DownloadType
  • func ParseDownloadType(name string) (DownloadType, error)
  • func (x DownloadType) MarshalText() ([]byte, error)
  • func (x DownloadType) Ptr() *DownloadType
  • func (x DownloadType) String() string
  • func (x *DownloadType) UnmarshalText(text []byte) error
• type DownloadVersion
  • func ParseDownloadVersion(name string) (DownloadVersion, error)
  • func (x DownloadVersion) MarshalText() ([]byte, error)
  • func (x DownloadVersion) Ptr() *DownloadVersion
  • func (x DownloadVersion) String() string
  • func (x *DownloadVersion) UnmarshalText(text []byte) error
• type GlobalDownloadTracker
  • func NewGlobalDownloadTracker(idleTimeout time.Duration) *GlobalDownloadTracker
  • func (global *GlobalDownloadTracker) IdleTimeoutExceeded() bool
  • func (global *GlobalDownloadTracker) PollIP(hClient *http.Client)
  • func (global *GlobalDownloadTracker) Stop()
  • func (global *GlobalDownloadTracker) Tick(humanReadable bool) string
• type StreamHLSTask
  • func NewStreamHLSTask(global *GlobalDownloadTracker, hClient *http.Client, ...) (*StreamHLSTask, error)
  • func (st *StreamHLSTask) Download(ctx context.Context) error
  • func (st *StreamHLSTask) Progress(sb *strings.Builder) error
  • func (st *StreamHLSTask) ResetDelta()
  • func (st *StreamHLSTask) Type() DownloadType
• type StreamMuxer
  • func NewStreamMuxer(f *os.File) *StreamMuxer
  • func (m *StreamMuxer) AddStreams(buf *bytes.Buffer) error
  • func (m *StreamMuxer) Demux(cid mpeg2.TS_STREAM_TYPE, frame []byte, pts uint64, dts uint64)
  • func (m *StreamMuxer) Mux(pkg []byte)
• type StreamStats
• type ThreadedChunk
• type ThreadedDownloadTask
  • func NewThreadedDownloadTask(ctx context.Context, hClient *http.Client, global *GlobalDownloadTracker, ...) (*ThreadedDownloadTask, error)
  • func (tt *ThreadedDownloadTask) Download(ctx context.Context) error
  • func (tt *ThreadedDownloadTask) Progress(sb *strings.Builder) error
  • func (tt *ThreadedDownloadTask) ResetDelta()
  • func (tt *ThreadedDownloadTask) Type() DownloadType

Variables

var ErrInvalidDownloadType = errors.New("not a valid DownloadType")

var ErrInvalidDownloadVersion = errors.New("not a valid DownloadVersion")

Functions

func Censor

func Censor(in string, pad string, chars int, front bool) string

func GetCurrentIPAddress

func GetCurrentIPAddress(hClient *http.Client) (ip string)

func Truncate

func Truncate(in string, chars, mode int) string

Types

type DownloadTask

type DownloadTask interface {
	Download(ctx context.Context) error
	Type() DownloadType
	Progress(sb *strings.Builder) error
	ResetDelta()
}

type DownloadType

type DownloadType int

DownloadType x ENUM(Threaded, LiveHLS)

const (
	// DownloadTypeThreaded is a DownloadType of type Threaded.
	DownloadTypeThreaded DownloadType = iota
	// DownloadTypeLiveHLS is a DownloadType of type LiveHLS.
	DownloadTypeLiveHLS
)

func ParseDownloadType

func ParseDownloadType(name string) (DownloadType, error)

ParseDownloadType attempts to convert a string to a DownloadType.

func (DownloadType) MarshalText

func (x DownloadType) MarshalText() ([]byte, error)

MarshalText implements the text marshaller method.

func (DownloadType) Ptr

func (x DownloadType) Ptr() *DownloadType

func (DownloadType) String

func (x DownloadType) String() string

String implements the Stringer interface.

func (*DownloadType) UnmarshalText

func (x *DownloadType) UnmarshalText(text []byte) error

UnmarshalText implements the text unmarshaller method.

type DownloadVersion

type DownloadVersion int

DownloadVersion x ENUM(v1)

const (
	// DownloadVersionV1 is a DownloadVersion of type V1.
	DownloadVersionV1 DownloadVersion = iota
)

func ParseDownloadVersion

func ParseDownloadVersion(name string) (DownloadVersion, error)

ParseDownloadVersion attempts to convert a string to a DownloadVersion.

func (DownloadVersion) MarshalText

func (x DownloadVersion) MarshalText() ([]byte, error)

MarshalText implements the text marshaller method.

func (DownloadVersion) Ptr

func (x DownloadVersion) Ptr() *DownloadVersion

func (DownloadVersion) String

func (x DownloadVersion) String() string

String implements the Stringer interface.

func (*DownloadVersion) UnmarshalText

func (x *DownloadVersion) UnmarshalText(text []byte) error

UnmarshalText implements the text unmarshaller method.

type GlobalDownloadTracker

type GlobalDownloadTracker struct {
	// Synchronization
	sync.WaitGroup
	GraceFulStop *atomic.Bool `json:"-"`
	// Idle stats
	IdleSince   *atomic.Time   `json:"idleSince"`
	IdleValue   *atomic.Uint64 `json:"-"`
	IdleTimeout *atomic.Int64  `json:"-"`
	// MetaData
	LastTick     *atomic.Time                       `json:"lastTick"`
	CurrentIP    *atomic.String                     `json:"currentIP"`
	TotalThreads *atomic.Uint64                     `json:"totalThreads"`
	Tasks        *hashmap.Map[string, DownloadTask] `json:"tasks"`
	// Total stats
	TotalFiles *atomic.Uint64 `json:"totalFiles"`
	TotalBytes *atomic.Uint64 `json:"totalBytes"`
	// Download stats
	DownloadedFiles *atomic.Uint64 `json:"downloadedFiles"`
	DownloadedBytes *atomic.Uint64 `json:"downloadedBytes"`
	// Per tick stats
	DeltaBytes *atomic.Uint64 `json:"deltaBytes"`
}

func NewGlobalDownloadTracker

func NewGlobalDownloadTracker(idleTimeout time.Duration) *GlobalDownloadTracker

func (*GlobalDownloadTracker) IdleTimeoutExceeded

func (global *GlobalDownloadTracker) IdleTimeoutExceeded() bool

func (*GlobalDownloadTracker) PollIP

func (global *GlobalDownloadTracker) PollIP(hClient *http.Client)

func (*GlobalDownloadTracker) Stop

func (global *GlobalDownloadTracker) Stop()

func (*GlobalDownloadTracker) Tick

func (global *GlobalDownloadTracker) Tick(humanReadable bool) string

type StreamHLSTask

type StreamHLSTask struct {
	// Tracking ref
	Global  *GlobalDownloadTracker    `json:"-"`
	HClient *http.Client              `json:"-"`
	ReqOpts []gokhttp_requests.Option `json:"-"`

	Muxer     *StreamMuxer `json:"-"`
	TaskStats *StreamStats `json:"taskStats"`

	TaskType     DownloadType                    `json:"taskType"`
	TaskVersion  DownloadVersion                 `json:"taskVersion"`
	FileName     *atomic.String                  `json:"fileName"`
	FileLocation *atomic.String                  `json:"fileLocation"`
	PlayListUrl  *atomic.String                  `json:"playListUrl"`
	BaseUrl      *url.URL                        `json:"-"`
	Opts         []gokhttp_requests.Option       `json:"-"`
	SegmentChan  chan *m3u8.SegmentItem          `json:"-"`
	BufferChan   chan *bytes.Buffer              `json:"-"`
	SegmentCache *hashmap.Map[string, time.Time] `json:"-"`
	SaveSegments bool                            `json:"-"`
}

func NewStreamHLSTask

func NewStreamHLSTask(global *GlobalDownloadTracker, hClient *http.Client, playlistUrl, fileLocation string, saveSegments bool, opts ...gokhttp_requests.Option) (*StreamHLSTask, error)

func (*StreamHLSTask) Download

func (st *StreamHLSTask) Download(ctx context.Context) error

func (*StreamHLSTask) Progress

func (st *StreamHLSTask) Progress(sb *strings.Builder) error

func (*StreamHLSTask) ResetDelta

func (st *StreamHLSTask) ResetDelta()

func (*StreamHLSTask) Type

func (st *StreamHLSTask) Type() DownloadType

type StreamMuxer

type StreamMuxer struct {
	F       *os.File
	Muxer   *mpeg2.TSMuxer
	Demuxer *mpeg2.TSDemuxer
	Streams map[mpeg2.TS_STREAM_TYPE]uint16
}

func NewStreamMuxer

func NewStreamMuxer(f *os.File) *StreamMuxer

func (*StreamMuxer) AddStreams

func (m *StreamMuxer) AddStreams(buf *bytes.Buffer) error

func (*StreamMuxer) Demux

func (m *StreamMuxer) Demux(cid mpeg2.TS_STREAM_TYPE, frame []byte, pts uint64, dts uint64)

func (*StreamMuxer) Mux

func (m *StreamMuxer) Mux(pkg []byte)

type StreamStats

type StreamStats struct {
	DownloadedSegments *atomic.Uint64 `json:"downloadedSegments"`
	DownloadedDuration *atomic.Int64  `json:"downloadedDuration"`
	DownloadedBytes    *atomic.Uint64 `json:"downloadedBytes"`
}

type ThreadedChunk

type ThreadedChunk struct {
	// Metadata
	ChunkID    *atomic.Uint64 `json:"chunkID"`
	ChunkStart uint64         `json:"-"`
	ChunkStop  uint64         `json:"-"`
	F          *os.File       `json:"-"`
	// Totals
	FileSize *atomic.Uint64 `json:"fileSize"`
	// Download stats
	DownloadedBytes *atomic.Uint64 `json:"downloadedBytes"`
	DeltaBytes      *atomic.Uint64 `json:"deltaBytes"`
}

type ThreadedDownloadTask

type ThreadedDownloadTask struct {
	// Tracking ref
	Global  *GlobalDownloadTracker    `json:"-"`
	HClient *http.Client              `json:"-"`
	ReqOpts []gokhttp_requests.Option `json:"-"`
	// Metadata
	TaskType     DownloadType    `json:"taskType"`
	TaskVersion  DownloadVersion `json:"taskVersion"`
	FileName     *atomic.String  `json:"fileName"`
	FileLocation *atomic.String  `json:"fileLocation"`
	FileURL      *atomic.String  `json:"fileURL"`
	Resumable    *atomic.Bool    `json:"resumable"`
	// Totals
	TaskStats *ThreadedChunk `json:"taskStats"`
	// Chunks
	Chunks     *hashmap.Map[string, *ThreadedChunk] `json:"chunks"`
	ChunkCount *atomic.Uint64                       `json:"chunkCount"`
}

func NewThreadedDownloadTask

func NewThreadedDownloadTask(ctx context.Context, hClient *http.Client, global *GlobalDownloadTracker, fileLocation, fileURL string, threads, expectedSize uint64, opts ...gokhttp_requests.Option) (*ThreadedDownloadTask, error)

func (*ThreadedDownloadTask) Download

func (tt *ThreadedDownloadTask) Download(ctx context.Context) error

func (*ThreadedDownloadTask) Progress

func (tt *ThreadedDownloadTask) Progress(sb *strings.Builder) error

func (*ThreadedDownloadTask) ResetDelta

func (tt *ThreadedDownloadTask) ResetDelta()

func (*ThreadedDownloadTask) Type

func (tt *ThreadedDownloadTask) Type() DownloadType