loader

Documentation

Index

• type Chunk
• type FileLoader
  • func NewFileLoader(outputFile, zoneURL string, numWorkers int) FileLoader
  • func (fl FileLoader) Download(ctx context.Context, accessToken string) (io.Reader, error)
  • func (fl FileLoader) DownloadZone(ctx context.Context, accessToken string) error
  • func (fl FileLoader) Save(ctx context.Context, r io.Reader) error
• type S3Loader
  • func NewS3Loader(outputFile, zoneURL string, numWorkers int) (*S3Loader, error)
  • func (sl S3Loader) Download(ctx context.Context, accessToken string) (io.Reader, error)
  • func (sl S3Loader) DownloadZone(ctx context.Context, accessToken string) error
  • func (sl S3Loader) Save(ctx context.Context, r io.Reader) error

Types

type Chunk

type Chunk struct {
	Index int
	Data  []byte
}

type FileLoader

type FileLoader struct {
	File string

	NumWorkers int
	ZoneURL    string
	// contains filtered or unexported fields
}

func NewFileLoader

func NewFileLoader(outputFile, zoneURL string, numWorkers int) FileLoader

func (FileLoader) Download

func (fl FileLoader) Download(ctx context.Context, accessToken string) (io.Reader, error)

Downloads the zone data concurrently and returns the resulting file as an `io.Reader`.

func (FileLoader) DownloadZone

func (fl FileLoader) DownloadZone(ctx context.Context, accessToken string) error

Simply combines the functionality of FileLoader's `Download` and `Save` functions.

func (FileLoader) Save

func (fl FileLoader) Save(ctx context.Context, r io.Reader) error

Saves the data in the `io.Reader` out to the FileLoader's File.

type S3Loader

type S3Loader struct {
	Bucket string
	Key    string

	NumWorkers int
	Uploader   manager.Uploader
	ZoneURL    string
	// contains filtered or unexported fields
}

func NewS3Loader

func NewS3Loader(outputFile, zoneURL string, numWorkers int) (*S3Loader, error)

func (S3Loader) Download

func (sl S3Loader) Download(ctx context.Context, accessToken string) (io.Reader, error)

Downloads the zone data concurrently and returns the resulting file as an `io.Reader`.

func (S3Loader) DownloadZone

func (sl S3Loader) DownloadZone(ctx context.Context, accessToken string) error

Simply combines the functionality of FileLoader's `Download` and `Save` functions.

func (S3Loader) Save

func (sl S3Loader) Save(ctx context.Context, r io.Reader) error

Saves the data in the `io.Reader` out to the FileLoader's File.