utils

Documentation

Index

• Variables
• func DownloadGCS(remoteSource string, localTarget string) error
• func DownloadS3(remoteSource string, localTarget string) error
• func GetByteHash(bt []byte) string
• func GetFileHash(path string) string
• func GetStringHash(plainString string) string
• func IsDir(path string) bool
• func IsGCS(source string) bool
• func IsS3(source string) bool
• func ShortGCSExtract(source string) (string, string, error)
• func ShortS3Extract(source string) (string, string, string, error)
• func StringInSlice(a string, list []string) bool
• func UploadGCS(localSource string, remoteTarget string, permissionName string, ...) error
• func UploadS3(localSource string, remoteTarget string, permissionName string, ...) error
• func WriteReaderToFile(r io.Reader, dest string) error
• type FileWalk
  • func (f FileWalk) IterateUpload(localSource string, callback func(path, rel string) error)
  • func (f FileWalk) Walk(path string, info os.FileInfo, err error) error
• type ParamArg

Variables

var GCSCredentialSession context.Context

GCSCredentialSession the GCS credential session

var GCSCredentialSessionDefined bool

GCSCredentialSessionDefined when true that means the credential for GCS is defined

var S3Region string

S3Region global default region

var S3Regions = []string{"us-east-1", "us-west-1", "us-west-2", "eu-west-1", "eu-central-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "sa-east-1"}

S3Regions is list of s3 regions

Functions

func DownloadGCS

func DownloadGCS(remoteSource string, localTarget string) error

DownloadGCS to download GCS file from remote source to local file target

func DownloadS3

func DownloadS3(remoteSource string, localTarget string) error

DownloadS3 to download s3 file from remote source to local file target

func GetByteHash

func GetByteHash(bt []byte) string

GetByteHash will return hex sha264 of specified byte sequence

func GetFileHash

func GetFileHash(path string) string

GetFileHash will return string hash of specified file

func GetStringHash

func GetStringHash(plainString string) string

GetStringHash will return string hash of specified string

func IsDir

func IsDir(path string) bool

IsDir return true if the path is a directory

func IsGCS

func IsGCS(source string) bool

IsGCS will return true if the path is a GCS path gs://bucket/path

func IsS3

func IsS3(source string) bool

IsS3 extract the path and identify whether it's s3 short path or not

func ShortGCSExtract

func ShortGCSExtract(source string) (string, string, error)

ShortGCSExtract will extract GCS short path into bucket, path string

func ShortS3Extract

func ShortS3Extract(source string) (string, string, string, error)

ShortS3Extract to extract our short s3 path to region, bucket, path components

func StringInSlice

func StringInSlice(a string, list []string) bool

StringInSlice Return true if a is in the list slice

func UploadGCS

func UploadGCS(localSource string, remoteTarget string, permissionName string, storageClass string) error

UploadGCS to be used to upload local-file to target s3 path in our short notation gs://bucket/path

func UploadS3

func UploadS3(localSource string, remoteTarget string, permissionName string, storageClass string) error

UploadS3 to be used to upload local-file to target s3 path in our short notation s3:region:bucket:path or s3:bucket:path

func WriteReaderToFile

func WriteReaderToFile(r io.Reader, dest string) error

WriteReaderToFile will write an io.Reader to file dest

Types

type FileWalk

type FileWalk chan string

FileWalk type channel that represent interation trough a path

func (FileWalk) IterateUpload

func (f FileWalk) IterateUpload(localSource string, callback func(path, rel string) error)

IterateUpload will walk the files in the walker and call callback to upload it one by one

func (FileWalk) Walk

func (f FileWalk) Walk(path string, info os.FileInfo, err error) error

Walk do the walk

type ParamArg

type ParamArg struct {
	StorageClass   string
	PermissionName string

	IsVerbose bool
	IsPublic  bool
}

ParamArg struct that keep the param flags

var GlobalParamArg ParamArg

GlobalParamArg global struct that keep the param flags