storclient

package
Version: v0.0.0-...-d2b18e3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 20, 2019 License: MIT Imports: 17 Imported by: 0

README

storclient

-- import "github.com/avast/stor-client/client"

Package storclient to download samples from stor service

SYNOPSIS
client := storclient.New(storageUrl, storclient.StorClientOpts{})

client.Start()

for _, sha := range shaList {
	client.Download(sha)
}

downloadStatus := client.Wait()

Usage

const (
	DefaultMax           = 4
	DefaultTimeout       = 30 * time.Second
	DefaultRetryAttempts = 10
	DefaultRetryDelay    = 1e5 * time.Microsecond
	DefaultS3Template    = "{{.FirstShaByte}}/{{.SecondShaByte}}/{{.ThirdShaByte}}/{{.Sha}}"
)
type DownPool
type DownPool struct {
}
type DownStat
type DownStat struct {
	Size     int64
	Duration time.Duration
	Status   DownloadStatus
}
type DownloadStatus
type DownloadStatus int
const (
	// DOWN_FAIL - downlad fail (default)
	DOWN_FAIL DownloadStatus = iota
	// DOWN_SKIP - downlad skipped because file is downlad
	DOWN_SKIP
	// DOWN_OK - downlad ok
	DOWN_OK
)
type StorClient
type StorClient struct {
	StorClientOpts
}
func New
func New(storUrl url.URL, downloadDir string, opts StorClientOpts) (*StorClient, error)

Create new instance of stor client

func (*StorClient) Download
func (client *StorClient) Download(sha hashutil.Hash)

add sha to douwnload queue

func (*StorClient) Start
func (client *StorClient) Start()

start stor downloading process

func (*StorClient) Wait
func (client *StorClient) Wait() TotalStat

wait to all downloads return download stats

type StorClientOpts
type StorClientOpts struct {
	//	max size of download pool
	Max int
	//	write to devnull instead of file
	Devnull bool
	//	connection timeout
	//
	//	-1 means no limit (no timeout)
	Timeout time.Duration
	// exponential retry - start delay time
	// default is 10e5 microseconds
	RetryDelay time.Duration
	// count of tries of retry
	// default is 10
	RetryAttempts uint
	// downladed file suffix
	// e.g. .dat => SHA.dat file
	// default ("") means without suffix
	Suffix string
	// name of file will be upper case (not applied to extension)
	UpperCase bool
	// host to s3 endpoint with bucket e.g. https://bucket.s3.eu-central-1.amazonaws.com, if is s3url set, first will be use S3, then fallback to stor
	S3URL *url.URL
	// template to S3 path
	S3Template string
}

StorClientOpts is base struct

type TotalStat
type TotalStat struct {
	Size     int64
	Duration time.Duration
	// Count of downloaded files
	Count int
	// Count of skipped files
	Skip int
}

Size and Duration is duplicate, becuse embedding not works, because

https://stackoverflow.com/questions/41686692/embedding-structs-in-golang-gives-error-unknown-field
func (TotalStat) Print
func (total TotalStat) Print(startTime time.Time)

format and log total stats

func (TotalStat) Status
func (total TotalStat) Status() bool

Status return true if all files are downloaded

Documentation

Overview

Package storclient to download samples from stor service

SYNOPSIS

client := storclient.New(storageUrl, storclient.StorClientOpts{})

client.Start()

for _, sha := range shaList {
	client.Download(sha)
}

downloadStatus := client.Wait()

Index

Constants

View Source
const (
	DefaultMax           = 4
	DefaultTimeout       = 30 * time.Second
	DefaultRetryAttempts = 10
	DefaultRetryDelay    = 1e5 * time.Microsecond
	DefaultS3Template    = "{{.FirstShaByte}}/{{.SecondShaByte}}/{{.ThirdShaByte}}/{{.Sha}}"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type DownPool

type DownPool struct {
	// contains filtered or unexported fields
}

type DownStat

type DownStat struct {
	Size     int64
	Duration time.Duration
	Status   DownloadStatus
}

type DownloadStatus

type DownloadStatus int
const (
	// DOWN_FAIL - downlad fail (default)
	DOWN_FAIL DownloadStatus = iota
	// DOWN_SKIP - downlad skipped because file is downlad
	DOWN_SKIP
	// DOWN_OK - downlad ok
	DOWN_OK
)

type StorClient

type StorClient struct {
	StorClientOpts
	// contains filtered or unexported fields
}

func New

func New(storUrl url.URL, downloadDir string, opts StorClientOpts) (*StorClient, error)

Create new instance of stor client

func (*StorClient) Download

func (client *StorClient) Download(sha hashutil.Hash)

add sha to douwnload queue

func (*StorClient) Start

func (client *StorClient) Start()

start stor downloading process

func (*StorClient) Wait

func (client *StorClient) Wait() TotalStat

wait to all downloads return download stats

type StorClientOpts

type StorClientOpts struct {
	//	max size of download pool
	Max int
	//	write to devnull instead of file
	Devnull bool
	//	connection timeout
	//
	//	-1 means no limit (no timeout)
	Timeout time.Duration
	// exponential retry - start delay time
	// default is 10e5 microseconds
	RetryDelay time.Duration
	// count of tries of retry
	// default is 10
	RetryAttempts uint
	// downladed file suffix
	// e.g. .dat => SHA.dat file
	// default ("") means without suffix
	Suffix string
	// name of file will be upper case (not applied to extension)
	UpperCase bool
	// host to s3 endpoint with bucket e.g. https://bucket.s3.eu-central-1.amazonaws.com, if is s3url set, first will be use S3, then fallback to stor
	S3URL *url.URL
	// template to S3 path
	S3Template string
}

StorClientOpts is base struct

type TotalStat

type TotalStat struct {
	Size     int64
	Duration time.Duration
	// Count of downloaded files
	Count int
	// Count of skipped files
	Skip int
	// contains filtered or unexported fields
}

Size and Duration is duplicate, becuse embedding not works, because

https://stackoverflow.com/questions/41686692/embedding-structs-in-golang-gives-error-unknown-field

func (TotalStat) Print

func (total TotalStat) Print(startTime time.Time)

format and log total stats

func (TotalStat) Status

func (total TotalStat) Status() bool

Status return true if all files are downloaded

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
t or T : Toggle theme light dark auto
y or Y : Canonical URL