skbn

Documentation

Index

• Constants
• func Copy(src, dst string, parallel int, bufferSize float64) error
• func Delete(path string, parallel int) error
• func DeleteFile(client interface{}, prefix, path string) error
• func DeleteFromK8s(iClient interface{}, path string) error
• func Download(srcClient interface{}, srcPrefix, srcPath string, writer io.Writer) error
• func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error
• func DownloadFromGcs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error
• func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error
• func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error
• func Exec(client K8sClient, namespace, podName, containerName string, command []string, ...) ([]byte, error)
• func GetClient(prefix, path string) (interface{}, error)
• func GetClientToAbs(ctx context.Context, path string) (pipeline.Pipeline, error)
• func GetClientToGcs(ctx context.Context, path string) (*storage.Client, error)
• func GetClientToS3(path string) (*session.Session, error)
• func GetClients(srcPrefix, dstPrefix, srcPath, dstPath string) (interface{}, interface{}, error)
• func GetFileNames(client interface{}, prefix, path string) ([]string, error)
• func GetListOfFiles(client interface{}, prefix, path string) ([]string, error)
• func GetListOfFilesFromAbs(ctx context.Context, iClient interface{}, path string) ([]string, error)
• func GetListOfFilesFromGcs(ctx context.Context, iClient interface{}, path string) ([]string, error)
• func GetListOfFilesFromK8s(iClient interface{}, path, findType, findName string) ([]string, error)
• func GetListOfFilesFromK8sV2(iClient interface{}, path, findType, findName string) (map[string]*K8sFile, error)
• func GetListOfFilesFromS3(iClient interface{}, path string) ([]string, error)
• func GetListOfFilesFromS3V2(iClient interface{}, path string) (map[string]*S3File, error)
• func GetPaths(path string, fileNames []string) []string
• func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, ...) error
• func PerformDelete(client interface{}, prefix string, paths []string, parallel int) error
• func PerformSync(srcClient, dstClient interface{}, ...) error
• func SetFileETag(iClient interface{}, path string, files map[string]*K8sFile) error
• func Sync(src, dst string, parallel int, bufferSize float64) error
• func TestImplementationsExist(srcPrefix, dstPrefix string) error
• func TestImplementationsExistForDelete(prefix string) error
• func TestImplementationsExistForSync(srcPrefix, dstPrefix string) error
• func Upload(dstClient interface{}, dstPrefix, dstPath, srcPath string, reader io.Reader) error
• func UploadToAbs(ctx context.Context, iClient interface{}, toPath, fromPath string, ...) error
• func UploadToGcs(ctx context.Context, iClient interface{}, toPath, fromPath string, ...) error
• func UploadToK8s(iClient interface{}, toPath, fromPath string, reader io.Reader) error
• func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) error
• type FromToPair
  • func GetFromToPaths(srcPath, dstPath string, relativePaths []string) []FromToPair
  • func K8sToS3(srcClient, dstClient interface{}, srcPath, dstPath string, parallel int, ...) ([]FromToPair, []string, error)
  • func S3ToK8s(srcClient, dstClient interface{}, srcPath, dstPath string, parallel int, ...) ([]FromToPair, []string, error)
• type K8sClient
  • func GetClientToK8s() (*K8sClient, error)
• type K8sFile
• type S3File

Constants

const (
	S3  string = "s3"
	K8S string = "k8s"
)

Functions

func Copy

func Copy(src, dst string, parallel int, bufferSize float64) error

Copy copies files from src to dst

func Delete

func Delete(path string, parallel int) error

Delete files from path

func DeleteFile

func DeleteFile(client interface{}, prefix, path string) error

DeleteFile deletes a single file from path

func DeleteFromK8s

func DeleteFromK8s(iClient interface{}, path string) error

DeleteFromK8s deletes a single file from Kubernetes

func Download

func Download(srcClient interface{}, srcPrefix, srcPath string, writer io.Writer) error

Download downloads a single file from path into an io.Writer

func DownloadFromAbs

func DownloadFromAbs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error

DownloadFromAbs downloads a single file from azure blob storage

func DownloadFromGcs

func DownloadFromGcs(ctx context.Context, iClient interface{}, path string, writer io.Writer) error

DownloadFromGcs downloads a single file from GCS

func DownloadFromK8s

func DownloadFromK8s(iClient interface{}, path string, writer io.Writer) error

DownloadFromK8s downloads a single file from Kubernetes

func DownloadFromS3

func DownloadFromS3(iClient interface{}, path string, writer io.Writer) error

DownloadFromS3 downloads a single file from S3

func Exec

func Exec(client K8sClient, namespace, podName, containerName string, command []string, stdin io.Reader, stdout io.Writer) ([]byte, error)

Exec executes a command in a given container

func GetClient

func GetClient(prefix, path string) (interface{}, error)

GetClient gets the client for cloud service provider

func GetClientToAbs

func GetClientToAbs(ctx context.Context, path string) (pipeline.Pipeline, error)

GetClientToAbs checks the connection to azure blob storage and returns the tested client (pipeline)

func GetClientToGcs

func GetClientToGcs(ctx context.Context, path string) (*storage.Client, error)

GetClientToGcs checks the connection to GCS and returns the tested client

func GetClientToS3

func GetClientToS3(path string) (*session.Session, error)

GetClientToS3 checks the connection to S3 and returns the tested client

func GetClients

func GetClients(srcPrefix, dstPrefix, srcPath, dstPath string) (interface{}, interface{}, error)

GetClients gets the clients for the source and destination

func GetFileNames

func GetFileNames(client interface{}, prefix, path string) ([]string, error)

GetFileNames returns all file nams present at path

func GetListOfFiles

func GetListOfFiles(client interface{}, prefix, path string) ([]string, error)

GetListOfFiles gets relative paths from the provided path

func GetListOfFilesFromAbs

func GetListOfFilesFromAbs(ctx context.Context, iClient interface{}, path string) ([]string, error)

GetListOfFilesFromAbs gets list of files in path from azure blob storage (recursive)

func GetListOfFilesFromGcs

func GetListOfFilesFromGcs(ctx context.Context, iClient interface{}, path string) ([]string, error)

GetListOfFilesFromGcs gets list of files in path from GCS (recursive)

func GetListOfFilesFromK8s

func GetListOfFilesFromK8s(iClient interface{}, path, findType, findName string) ([]string, error)

GetListOfFilesFromK8s gets list of files in path from Kubernetes (recursive)

func GetListOfFilesFromK8sV2

func GetListOfFilesFromK8sV2(iClient interface{}, path, findType, findName string) (map[string]*K8sFile, error)

GetListOfFilesFromK8sV2 gets list of files in path from Kubernetes (recursive)

func GetListOfFilesFromS3

func GetListOfFilesFromS3(iClient interface{}, path string) ([]string, error)

GetListOfFilesFromS3 gets list of files in path from S3 (recursive)

func GetListOfFilesFromS3V2

func GetListOfFilesFromS3V2(iClient interface{}, path string) (map[string]*S3File, error)

GetListOfFilesFromS3 gets list of files in path from S3 (recursive) TODO: GetListOfFilesFromS3 and GetListOfFilesFromS3V2 are same functions with some minor change GetListOfFilesFromS3 is needed for copy and GetListOfFilesFromS3V2 is needed for sync remove GetListOfFilesFromS3 and use GetListOfFilesFromS3V2 for copy also.

func GetPaths

func GetPaths(path string, fileNames []string) []string

GetPaths returns absolute path for each file name

func PerformCopy

func PerformCopy(srcClient, dstClient interface{}, srcPrefix, dstPrefix string, fromToPaths []FromToPair, parallel int, bufferSize float64) error

PerformCopy performs the actual copy action

func PerformDelete

func PerformDelete(client interface{}, prefix string, paths []string, parallel int) error

PerformDelete performs the actual delete action

func PerformSync

func PerformSync(srcClient, dstClient interface{}, srcPrefix, dstPrefix, srcPath, dstPath string,
	parallel int, bufferSize float64) error

PerformSync performs actual sync operation

func SetFileETag

func SetFileETag(iClient interface{}, path string, files map[string]*K8sFile) error

func Sync

func Sync(src, dst string, parallel int, bufferSize float64) error

Sync copies files from src to dst whick are not present at sest and delete extra files from dest

func TestImplementationsExist

func TestImplementationsExist(srcPrefix, dstPrefix string) error

TestImplementationsExist checks that implementations exist for the desired action

func TestImplementationsExistForDelete

func TestImplementationsExistForDelete(prefix string) error

TestImplementationsExistForDelete checks that implementations exist for the desired action

func TestImplementationsExistForSync

func TestImplementationsExistForSync(srcPrefix, dstPrefix string) error

TestImplementationsExistForSync checks that implementations exist for the desired action

func Upload

func Upload(dstClient interface{}, dstPrefix, dstPath, srcPath string, reader io.Reader) error

Upload uploads a single file provided as an io.Reader array to path

func UploadToAbs

func UploadToAbs(ctx context.Context, iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToAbs uploads a single file to azure blob storage

func UploadToGcs

func UploadToGcs(ctx context.Context, iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToGCS uploads a single file to GCS

func UploadToK8s

func UploadToK8s(iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToK8s uploads a single file to Kubernetes

func UploadToS3

func UploadToS3(iClient interface{}, toPath, fromPath string, reader io.Reader) error

UploadToS3 uploads a single file to S3

Types

type FromToPair

type FromToPair struct {
	FromPath string
	ToPath   string
}

FromToPair is a pair of FromPath and ToPath

func GetFromToPaths

func GetFromToPaths(srcPath, dstPath string, relativePaths []string) []FromToPair

GetFromToPaths returns list of pair of source and destination files to copy

func K8sToS3

func K8sToS3(srcClient, dstClient interface{}, srcPath, dstPath string,
	parallel int, bufferSize float64) ([]FromToPair, []string, error)

func S3ToK8s

func S3ToK8s(srcClient, dstClient interface{}, srcPath, dstPath string,
	parallel int, bufferSize float64) ([]FromToPair, []string, error)

type K8sClient

type K8sClient struct {
	ClientSet *kubernetes.Clientset
	Config    *rest.Config
}

K8sClient holds a clientset and a config

func GetClientToK8s

func GetClientToK8s() (*K8sClient, error)

GetClientToK8s returns a k8sClient

type K8sFile

type K8sFile struct {
	// contains filtered or unexported fields
}

type S3File

type S3File struct {
	// contains filtered or unexported fields
}