data

Documentation

Overview

This module contains Flyte CoPilot related code. Currently it only has 2 utilities - downloader and an uploader. Usage Downloader:

downloader := NewDownloader(...)
downloader.DownloadInputs(...) // will recursively download all inputs

Usage uploader: uploader := NewUploader(...) uploader.RecursiveUpload(...) // Will recursively upload all the data from the given path depending on the output interface All errors are bubbled up.

Both the uploader and downloader accept context.Context variables. These should be used to control timeouts etc. TODO: Currently retries are not automatically handled.

Index

• func DownloadFileFromHTTP(ctx context.Context, ref storage.DataReference) (io.ReadCloser, error)
• func DownloadFileFromStorage(ctx context.Context, ref storage.DataReference, store *storage.DataStore) (io.ReadCloser, error)
• func IsFileReadable(fpath string, ignoreExtension bool) (string, os.FileInfo, error)
• func UploadFileToStorage(ctx context.Context, filePath string, toPath storage.DataReference, size int64, ...) error
• type Downloader
  • func NewDownloader(_ context.Context, store *storage.DataStore, ...) Downloader
  • func (d Downloader) DownloadInputs(ctx context.Context, inputRef storage.DataReference, outputDir string) error
  • func (d Downloader) RecursiveDownload(ctx context.Context, inputs *core.LiteralMap, dir string, ...) (VarMap, *core.LiteralMap, error)
• type FutureMap
• type Unmarshal
• type Uploader
  • func NewUploader(_ context.Context, store *storage.DataStore, ...) Uploader
  • func (u Uploader) RecursiveUpload(ctx context.Context, vars *core.VariableMap, fromPath string, ...) error
• type VarMap

Functions

func DownloadFileFromHTTP

func DownloadFileFromHTTP(ctx context.Context, ref storage.DataReference) (io.ReadCloser, error)

Downloads data from the given HTTP URL. If context is canceled then the request will be canceled.

func DownloadFileFromStorage

func DownloadFileFromStorage(ctx context.Context, ref storage.DataReference, store *storage.DataStore) (io.ReadCloser, error)

func IsFileReadable

func IsFileReadable(fpath string, ignoreExtension bool) (string, os.FileInfo, error)

Checks if the given filepath is a valid and existing file path. If ignoreExtension is true, then the dir + basepath is checked for existence ignoring the extension. In the return the first return value is the actual path that exists (with the extension), second argument is the file info and finally the error

func UploadFileToStorage

func UploadFileToStorage(ctx context.Context, filePath string, toPath storage.DataReference, size int64, store *storage.DataStore) error

Uploads a file to the data store.

Types

type Downloader

type Downloader struct {
	// contains filtered or unexported fields
}

func NewDownloader

func NewDownloader(_ context.Context, store *storage.DataStore, format core.DataLoadingConfig_LiteralMapFormat, mode core.IOStrategy_DownloadMode) Downloader

func (Downloader) DownloadInputs

func (d Downloader) DownloadInputs(ctx context.Context, inputRef storage.DataReference, outputDir string) error

func (Downloader) RecursiveDownload

func (d Downloader) RecursiveDownload(ctx context.Context, inputs *core.LiteralMap, dir string, writePrimitiveToFile bool) (VarMap, *core.LiteralMap, error)

type FutureMap

type FutureMap map[string]futures.Future

type Unmarshal

type Unmarshal func(r io.Reader, msg proto.Message) error

type Uploader

type Uploader struct {
	// contains filtered or unexported fields
}

func NewUploader

func NewUploader(_ context.Context, store *storage.DataStore, format core.DataLoadingConfig_LiteralMapFormat, mode core.IOStrategy_UploadMode, errorFileName string) Uploader

func (Uploader) RecursiveUpload

func (u Uploader) RecursiveUpload(ctx context.Context, vars *core.VariableMap, fromPath string, metaOutputPath, dataRawPath storage.DataReference) error

type VarMap

type VarMap map[string]interface{}