skbn

Documentation

Index

• func Copy(src, dst string, parallel int, bufferSize float64) error
• func Download(ctx context.Context, srcClient interface{}, srcPrefix, srcPath string, ...) error
• func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error
• func DownloadFromGcs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error
• func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error
• func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error
• func DownloadUpload(srcClient, dstClient interface{}, ...)
• func Exec(client K8sClient, namespace, podName, containerName string, command []string, ...) ([]byte, error)
• func GetClientToAbs(ctx context.Context, path string) (pipeline.Pipeline, error)
• func GetClientToGcs(ctx context.Context, path string) (*storage.Client, error)
• func GetClientToS3(path string) (*session.Session, error)
• func GetClients(srcPrefix, dstPrefix, srcPath, dstPath string) (interface{}, interface{}, error)
• func GetListOfFiles(client interface{}, prefix, path string) ([]string, error)
• func GetListOfFilesFromAbs(ctx context.Context, iClient interface{}, path string) ([]string, error)
• func GetListOfFilesFromGcs(ctx context.Context, iClient interface{}, path string) ([]string, error)
• func GetListOfFilesFromK8s(iClient interface{}, path, findType, findName string) ([]string, error)
• func GetListOfFilesFromS3(iClient interface{}, path string) ([]string, error)
• func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, ...) error
• func TestImplementationsExist(srcPrefix, dstPrefix string) error
• func Upload(ctx context.Context, dstClient interface{}, dstPrefix, dstPath, srcPath string, ...) error
• func UploadToAbs(ctx context.Context, iClient interface{}, toPath, fromPath string, ...) error
• func UploadToGcs(ctx context.Context, iClient interface{}, toPath, fromPath string, ...) error
• func UploadToK8s(iClient interface{}, toPath, fromPath string, reader io.Reader) error
• func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) error
• type FromToPair
  • func GetFromToPaths(srcClient interface{}, srcPrefix, srcPath, dstPath string) ([]FromToPair, error)
• type K8sClient
  • func GetClientToK8s() (*K8sClient, error)

Functions

func Copy

func Copy(src, dst string, parallel int, bufferSize float64) error

Copy copies files from src to dst

func Download

func Download(ctx context.Context, srcClient interface{}, srcPrefix, srcPath string, writer io.Writer) error

Download downloads a single file from path into an io.Writer

func DownloadFromAbs

func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error

DownloadFromAbs downloads a single file from azure blob storage

func DownloadFromGcs

func DownloadFromGcs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error

DownloadFromGcs downloads a single file from GCS

func DownloadFromK8s

func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error

DownloadFromK8s downloads a single file from Kubernetes

func DownloadFromS3

func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error

DownloadFromS3 downloads a single file from S3

func DownloadUpload

func DownloadUpload(srcClient, dstClient interface{}, srcPrefix, fromPath, dstPrefix, toPath, currentLinePadded string, totalFiles int, bufferSize float64, failureCount *int, bwg *utils.BoundedWaitGroup)

DownloadUpload concurrently downloads a file and uploads them to its destination

func Exec

func Exec(client K8sClient, namespace, podName, containerName string, command []string, stdin io.Reader, stdout io.Writer) ([]byte, error)

Exec executes a command in a given container

func GetClientToAbs

func GetClientToAbs(ctx context.Context, path string) (pipeline.Pipeline, error)

GetClientToAbs checks the connection to azure blob storage and returns the tested client (pipeline)

func GetClientToGcs

func GetClientToGcs(ctx context.Context, path string) (*storage.Client, error)

GetClientToGcs checks the connection to GCS and returns the tested client

func GetClientToS3

func GetClientToS3(path string) (*session.Session, error)

GetClientToS3 checks the connection to S3 and returns the tested client

func GetClients

func GetClients(srcPrefix, dstPrefix, srcPath, dstPath string) (interface{}, interface{}, error)

GetClients gets the clients for the source and destination

func GetListOfFiles

func GetListOfFiles(client interface{}, prefix, path string) ([]string, error)

GetListOfFiles gets relative paths from the provided path

func GetListOfFilesFromAbs

func GetListOfFilesFromAbs(ctx context.Context, iClient interface{}, path string) ([]string, error)

GetListOfFilesFromAbs gets list of files in path from azure blob storage (recursive)

func GetListOfFilesFromGcs

func GetListOfFilesFromGcs(ctx context.Context, iClient interface{}, path string) ([]string, error)

GetListOfFilesFromGcs gets list of files in path from GCS (recursive)

func GetListOfFilesFromK8s

func GetListOfFilesFromK8s(iClient interface{}, path, findType, findName string) ([]string, error)

GetListOfFilesFromK8s gets list of files in path from Kubernetes (recursive)

func GetListOfFilesFromS3

func GetListOfFilesFromS3(iClient interface{}, path string) ([]string, error)

GetListOfFilesFromS3 gets list of files in path from S3 (recursive)

func PerformCopy

func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, fromToPaths []FromToPair, parallel int, bufferSize float64) error

PerformCopy performs the actual copy action

func TestImplementationsExist

func TestImplementationsExist(srcPrefix, dstPrefix string) error

TestImplementationsExist checks that implementations exist for the desired action

func Upload

func Upload(ctx context.Context, dstClient interface{}, dstPrefix, dstPath, srcPath string, reader io.Reader) error

Upload uploads a single file provided as an io.Reader array to path

func UploadToAbs

func UploadToAbs(ctx context.Context, iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToAbs uploads a single file to azure blob storage

func UploadToGcs

func UploadToGcs(ctx context.Context, iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToGCS uploads a single file to GCS

func UploadToK8s

func UploadToK8s(iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToK8s uploads a single file to Kubernetes

func UploadToS3

func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToS3 uploads a single file to S3

Types

type FromToPair

type FromToPair struct {
	FromPath string
	ToPath   string
}

FromToPair is a pair of FromPath and ToPath

func GetFromToPaths

func GetFromToPaths(srcClient interface{}, srcPrefix, srcPath, dstPath string) ([]FromToPair, error)

GetFromToPaths gets from and to paths to perform the copy on

type K8sClient

type K8sClient struct {
	ClientSet *kubernetes.Clientset
	Config    *rest.Config
}

K8sClient holds a clientset and a config

func GetClientToK8s

func GetClientToK8s() (*K8sClient, error)

GetClientToK8s returns a k8sClient