core

package
v0.6.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 24, 2020 License: MIT Imports: 32 Imported by: 0

Documentation

Overview

Package core is the core package for s5cmd.

Index

Constants

View Source 
const CancelFuncKey = contextKey("cancelFunc")

CancelFuncKey is the key name of the cancel function in context

View Source 
const ExitFuncKey = contextKey("exitFunc")

ExitFuncKey is the key name of the exit function in context

View Source 
const (
	// GlobCharacters is valid glob characters for local files
	GlobCharacters string = "?*["
)

Variables

View Source 
var (
	// ErrObjectExistsButOk is used when a destination object already exists and opt.IfNotExists is set.
	ErrObjectExistsButOk = NewAcceptableError("Object already exists")
	// ErrObjectIsNewerButOk is used when a destination object is newer than the source and opt.IfSourceNewer is set.
	ErrObjectIsNewerButOk = NewAcceptableError("Object is newer or same age")
	// ErrObjectSizesMatchButOk is used when a destination object size matches the source and opt.IfSizeDiffers is set.
	ErrObjectSizesMatchButOk = NewAcceptableError("Object size matches")
	// ErrDisplayedHelp is used when a command is invoked with "-h"
	ErrDisplayedHelp = NewAcceptableError("Displayed help for command")
)
View Source 
var (
	// ErrInterrupted is the error used when the main context is canceled
	ErrInterrupted = errors.New("Operation interrupted")

	// ErrNilResult is returned if a nil result in encountered
	ErrNilResult = errors.New("nil result")
)
View Source 
var Commands = []CommandMap{
	{"exit", op.Abort, []opt.ParamType{}, opt.OptionList{}},
	{"exit", op.Abort, []opt.ParamType{opt.Unchecked}, opt.OptionList{}},

	{"cp", op.LocalCopy, []opt.ParamType{opt.FileObj, opt.FileOrDir}, opt.OptionList{}},
	{"cp", op.BatchLocalCopy, []opt.ParamType{opt.Glob, opt.Dir}, opt.OptionList{}},
	{"cp", op.BatchLocalCopy, []opt.ParamType{opt.Dir, opt.Dir}, opt.OptionList{}},

	{"cp", op.Copy, []opt.ParamType{opt.S3SimpleObj, opt.S3ObjOrDir}, opt.OptionList{}},
	{"cp", op.BatchCopy, []opt.ParamType{opt.S3WildObj, opt.S3Dir}, opt.OptionList{}},

	{"cp", op.Upload, []opt.ParamType{opt.FileObj, opt.S3ObjOrDir}, opt.OptionList{}},
	{"cp", op.BatchUpload, []opt.ParamType{opt.Glob, opt.S3Dir}, opt.OptionList{}},
	{"cp", op.BatchUpload, []opt.ParamType{opt.Dir, opt.S3Dir}, opt.OptionList{}},

	{"cp", op.Download, []opt.ParamType{opt.S3SimpleObj, opt.FileOrDir}, opt.OptionList{}},
	{"get", op.AliasGet, []opt.ParamType{opt.S3SimpleObj, opt.OptionalFileOrDir}, opt.OptionList{}},
	{"cp", op.BatchDownload, []opt.ParamType{opt.S3WildObj, opt.Dir}, opt.OptionList{}},
	{"get", op.AliasBatchGet, []opt.ParamType{opt.S3WildObj, opt.OptionalDir}, opt.OptionList{}},

	{"mv", op.LocalCopy, []opt.ParamType{opt.FileObj, opt.FileOrDir}, opt.OptionList{opt.DeleteSource}},
	{"mv", op.BatchLocalCopy, []opt.ParamType{opt.Glob, opt.Dir}, opt.OptionList{opt.DeleteSource}},
	{"mv", op.BatchLocalCopy, []opt.ParamType{opt.Dir, opt.Dir}, opt.OptionList{opt.DeleteSource}},

	{"mv", op.Copy, []opt.ParamType{opt.S3SimpleObj, opt.S3ObjOrDir}, opt.OptionList{opt.DeleteSource}},
	{"mv", op.BatchCopy, []opt.ParamType{opt.S3WildObj, opt.S3Dir}, opt.OptionList{opt.DeleteSource}},

	{"mv", op.Upload, []opt.ParamType{opt.FileObj, opt.S3ObjOrDir}, opt.OptionList{opt.DeleteSource}},
	{"mv", op.BatchUpload, []opt.ParamType{opt.Glob, opt.S3Dir}, opt.OptionList{opt.DeleteSource}},
	{"mv", op.BatchUpload, []opt.ParamType{opt.Dir, opt.S3Dir}, opt.OptionList{opt.DeleteSource}},

	{"mv", op.Download, []opt.ParamType{opt.S3SimpleObj, opt.FileOrDir}, opt.OptionList{opt.DeleteSource}},
	{"mv", op.BatchDownload, []opt.ParamType{opt.S3WildObj, opt.Dir}, opt.OptionList{opt.DeleteSource}},

	{"rm", op.LocalDelete, []opt.ParamType{opt.FileObj}, opt.OptionList{}},

	{"rm", op.Delete, []opt.ParamType{opt.S3SimpleObj}, opt.OptionList{}},
	{"rm", op.BatchDelete, []opt.ParamType{opt.S3WildObj}, opt.OptionList{}},
	{"batch-rm", op.BatchDeleteActual, []opt.ParamType{opt.S3Obj, opt.UncheckedOneOrMore}, opt.OptionList{}},

	{"ls", op.ListBuckets, []opt.ParamType{}, opt.OptionList{}},
	{"ls", op.List, []opt.ParamType{opt.S3ObjOrDir}, opt.OptionList{}},
	{"ls", op.List, []opt.ParamType{opt.S3WildObj}, opt.OptionList{}},

	{"du", op.Size, []opt.ParamType{opt.S3ObjOrDir}, opt.OptionList{}},
	{"du", op.Size, []opt.ParamType{opt.S3WildObj}, opt.OptionList{}},

	{"!", op.ShellExec, []opt.ParamType{opt.UncheckedOneOrMore}, opt.OptionList{}},
}

Commands is a list of registered commands

View Source 
var Verbose bool

Verbose is an ugly global variable for verbose output, mainly for debugging

Functions

func CleanupError 

func CleanupError(err error) (s string)

CleanupError converts multiline error messages generated by aws-sdk-go into a single line

func CommandHelps added in v0.6.0 

func CommandHelps(filter string) (string, []opt.OptionType, int)

CommandHelps returns a text of accepted Commands with their options and arguments, list of accepted options, and a count of command alternates

func CommandList added in v0.6.0 

func CommandList() []string

CommandList returns a list of accepted Commands

func GetSessionForBucket added in v0.5.8 

func GetSessionForBucket(svc *s3.S3, bucket string) (*session.Session, error)

func HumanizeBytes 

func HumanizeBytes(b int64) string

HumanizeBytes takes a byte-size and returns a human-readable string

func IsRetryableError 

func IsRetryableError(err error) (code string, retryable bool)

IsRetryableError returns if an error (probably awserr) is retryable, along with an error code

func NewAwsSession added in v0.5.4 

func NewAwsSession(maxRetries int, endpointURL string, region string, noVerifySSL bool) (*session.Session, error)

NewAwsSession initializes a new AWS session with region fallback and custom options

func UsageLine added in v0.6.0 

func UsageLine() string

UsageLine returns the generic usage line for s5cmd

Types

type AcceptableError 

type AcceptableError interface {
	error
	Acceptable() bool
}

AcceptableError interface defines an error which is OK-to-have, for things like "cp -n" etc. It should not be treated as an error (regarding the exit code etc)

func IsAcceptableError 

func IsAcceptableError(err error) AcceptableError

IsAcceptableError determines if the error is an AcceptableError, and if so, returns the error as such

type AcceptableErrorType 

type AcceptableErrorType struct {
	// contains filtered or unexported fields
}

AcceptableErrorType embeds the stdlib error interface so that we can have more receivers on it

func NewAcceptableError 

func NewAcceptableError(s string) AcceptableErrorType

NewAcceptableError creates a new AcceptableError

func (AcceptableErrorType) Acceptable 

func (e AcceptableErrorType) Acceptable() bool

Acceptable is always true for errors of AcceptableError type

type CancelableScanner 

type CancelableScanner struct {
	*bufio.Scanner
	// contains filtered or unexported fields
}

CancelableScanner is a scanner which also listens for the context cancellations and optionally pumps data between channels

func NewCancelableScanner 

func NewCancelableScanner(ctx context.Context, r io.Reader) *CancelableScanner

NewCancelableScanner creates a new CancelableScanner

func (*CancelableScanner) ReadOne 

func (s *CancelableScanner) ReadOne(from <-chan *Job, to chan<- *Job) (string, error)

ReadOne reads one line from the started CancelableScanner, as well as listening to ctx.Done messages and optionally/continuously pumping *Jobs from the from<- chan to the to<- chan

func (*CancelableScanner) Start 

func (s *CancelableScanner) Start() *CancelableScanner

Start stats the goroutine on the CancelableScanner and returns itself

type CommandMap added in v0.5.4 

type CommandMap struct {
	// Keyword is the command's invocation name
	Keyword string
	// Operation is the operation to invoke when this command runs
	Operation op.Operation
	// Params are the accepted parameter types
	Params []opt.ParamType
	// Opts are the options to invoke the operation with, when this command is run
	Opts opt.OptionList
}

CommandMap describes each command

func (*CommandMap) String added in v0.5.4 

func (c *CommandMap) String(optsOverride ...opt.OptionType) (s string)

String formats the CommandMap using its Operation and ParamTypes

type Job 

type Job struct {
	// contains filtered or unexported fields
}

Job is our basic job type.

func ParseJob 

func ParseJob(jobdesc string) (*Job, error)

ParseJob parses a job description and returns a *Job type, possibly with other *Job types in successCommand/failCommand

func (Job) MakeSubJob 

func (j Job) MakeSubJob(command string, operation op.Operation, args []*JobArgument, opts opt.OptionList) *Job

MakeSubJob creates a sub-job linked to the original. sourceDesc is copied, numSuccess/numFails are linked. Returns a pointer to the new job.

func (*Job) Notify 

func (j *Job) Notify(success bool)

Notify informs the parent/issuer job if the job succeeded or failed.

func (*Job) PrintErr 

func (j *Job) PrintErr(err error)

PrintErr prints the error response from a Job

func (*Job) PrintOK 

func (j *Job) PrintOK(err AcceptableError)

PrintOK notifies the user about the positive outcome of the job. Internal operations are not shown, sub-jobs use short syntax.

func (*Job) Run 

func (j *Job) Run(wp *WorkerParams) error

Run runs the Job and returns error

func (Job) String 

func (j Job) String() (s string)

String formats the job using its command and arguments.

type JobArgument 

type JobArgument struct {
	// contains filtered or unexported fields
}

JobArgument is an argument of the job. Can be a file/directory, an s3 url ("s3" is set in this case) or an arbitrary string.

func NewJobArgument added in v0.5.7 

func NewJobArgument(arg string, s3 *url.S3Url) *JobArgument

func (*JobArgument) Append 

func (a *JobArgument) Append(s string, isS3path bool) *JobArgument

Append appends a string to a JobArgument and returns itself

func (*JobArgument) CheckConditionals added in v0.5.7 

func (a *JobArgument) CheckConditionals(wp *WorkerParams, src *JobArgument, opts opt.OptionList) (ret error)

func (*JobArgument) Clone 

func (a *JobArgument) Clone() *JobArgument

Clone duplicates a JobArgument and returns a pointer to a new one

func (*JobArgument) Exists added in v0.5.7 

func (a *JobArgument) Exists(wp *WorkerParams) (bool, error)

func (*JobArgument) ModTime added in v0.5.7 

func (a *JobArgument) ModTime(wp *WorkerParams) (time.Time, error)

func (*JobArgument) Size added in v0.5.7 

func (a *JobArgument) Size(wp *WorkerParams) (int64, error)

func (*JobArgument) StripS3 added in v0.5.4 

func (a *JobArgument) StripS3() *JobArgument

StripS3 strips the S3 data from JobArgument and returns a new one

type WorkerParams 

type WorkerParams struct {
	// contains filtered or unexported fields
}

WorkerParams is the params/state of a single worker.

type WorkerPool 

type WorkerPool struct {
	// contains filtered or unexported fields
}

WorkerPool is the state of our worker pool.

func NewWorkerPool 

func NewWorkerPool(ctx context.Context, params *WorkerPoolParams, st *stats.Stats) *WorkerPool

NewWorkerPool creates a new worker pool and start the workers.

func (*WorkerPool) Run 

func (p *WorkerPool) Run(filename string)

Run runs the commands in filename in the worker pool, on EOF it will wait for all commands to finish, clean up and return.

func (*WorkerPool) RunCmd 

func (p *WorkerPool) RunCmd(commandLine string)

RunCmd will run a single command (and subsequent sub-commands) in the worker pool, wait for it to finish, clean up and return.

type WorkerPoolParams 

type WorkerPoolParams struct {
	NumWorkers             int
	UploadChunkSizeBytes   int64
	UploadConcurrency      int
	DownloadChunkSizeBytes int64
	DownloadConcurrency    int
	Retries                int
	EndpointURL            string
	NoVerifySSL            bool
}

WorkerPoolParams is the common parameters of all worker pools.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.