common

package
v10.24.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 26, 2024 License: MIT Imports: 65 Imported by: 2

Documentation

Overview

Copyright © 2023 microsoft <wastore@microsoft.com>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Index

Constants

View Source 
const (
	AutologinTypeSPN    = "spn"
	AutologinTypeMSI    = "msi"
	AutologinTypeDevice = "device"
	AutologinTypeAzCLI  = "azcli"
	AutologinTypePsCred = "pscred"
)
View Source 
const (
	AZCOPY_PATH_SEPARATOR_STRING = "/"
	AZCOPY_PATH_SEPARATOR_CHAR   = '/'
	OS_PATH_SEPARATOR            = string(os.PathSeparator)
	EXTENDED_PATH_PREFIX         = `\\?\`
	EXTENDED_UNC_PATH_PREFIX     = `\\?\UNC`
	Dev_Null                     = os.DevNull

	//  this is the perm that AzCopy has used throughout its preview.  So, while we considered relaxing it to 0666
	//  we decided that the best option was to leave it as is, and only relax it if user feedback so requires.
	DEFAULT_FILE_PERM = 0644 // the os package will handle base-10 for us.

	// Since we haven't updated the Go SDKs to handle CPK just yet, we need to detect CPK related errors
	// and inform the user that we don't support CPK yet.
	CPK_ERROR_SERVICE_CODE = "BlobUsesCustomerSpecifiedEncryption"
	BLOB_NOT_FOUND         = "BlobNotFound"
	FILE_NOT_FOUND         = "The specified file was not found."
)
View Source 
const (
	DefaultBlockBlobBlockSize      = 8 * 1024 * 1024
	MaxBlockBlobBlockSize          = 4000 * 1024 * 1024
	MaxPutBlobSize                 = 5000 * 1024 * 1024
	MaxAppendBlobBlockSize         = 100 * 1024 * 1024
	DefaultPageBlobChunkSize       = 4 * 1024 * 1024
	DefaultAzureFileChunkSize      = 4 * 1024 * 1024
	MaxRangeGetSize                = 4 * 1024 * 1024
	MaxNumberOfBlocksPerBlob       = 50000
	BlockSizeThreshold             = 256 * 1024 * 1024
	MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */
	MegaByte                       = 1024 * 1024
)

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View Source 
const (
	POSIXNlinkMeta         = "posix_nlink"
	POSIXINodeMeta         = "posix_ino"
	POSIXCTimeMeta         = "posix_ctime"
	LINUXBTimeMeta         = "linux_btime"
	POSIXBlockDeviceMeta   = "is_block_dev" // todo: read & use these
	POSIXCharDeviceMeta    = "is_char_dev"
	POSIXSocketMeta        = "is_socket"
	POSIXFIFOMeta          = "is_fifo"
	POSIXDevMeta           = "posix_dev"
	POSIXRDevMeta          = "posix_rdev"
	POSIXATimeMeta         = "posix_atime"
	POSIXFolderMeta        = "hdi_isfolder" // todo: read & use these
	POSIXSymlinkMeta       = "is_symlink"
	POSIXOwnerMeta         = "posix_owner"
	POSIXGroupMeta         = "posix_group"
	POSIXModeMeta          = "permissions"
	POSIXModTimeMeta       = "modtime"
	LINUXAttributeMeta     = "linux_attribute"
	LINUXAttributeMaskMeta = "linux_attribute_mask"
	LINUXStatxMaskMeta     = "linux_statx_mask"
)
View Source 
const (
	STATX_ALL             = 0xfff
	STATX_ATIME           = 0x20
	STATX_ATTR_APPEND     = 0x20
	STATX_ATTR_AUTOMOUNT  = 0x1000
	STATX_ATTR_COMPRESSED = 0x4
	STATX_ATTR_DAX        = 0x200000
	STATX_ATTR_ENCRYPTED  = 0x800
	STATX_ATTR_IMMUTABLE  = 0x10
	STATX_ATTR_MOUNT_ROOT = 0x2000
	STATX_ATTR_NODUMP     = 0x40
	STATX_ATTR_VERITY     = 0x100000
	STATX_BASIC_STATS     = 0x7ff
	STATX_BLOCKS          = 0x400
	STATX_BTIME           = 0x800
	STATX_CTIME           = 0x80
	STATX_GID             = 0x10
	STATX_INO             = 0x100
	STATX_MNT_ID          = 0x1000
	STATX_MODE            = 0x2
	STATX_MTIME           = 0x40
	STATX_NLINK           = 0x4
	STATX_SIZE            = 0x200
	STATX_TYPE            = 0x1
	STATX_UID             = 0x8

	S_IFSOCK = 0xc000
	S_IFBLK  = 0x6000
	S_IFCHR  = 0x2000
	S_IFDIR  = 0x4000
	S_IFIFO  = 0x1000
	S_IFLNK  = 0xa000

	S_IRUSR = 0x400
	S_IWUSR = 0x200
	S_IXUSR = 0x100
	S_IRGRP = 0x040
	S_IWGRP = 0x020
	S_IXGRP = 0x010
	S_IROTH = 0x004
	S_IWOTH = 0x002
	S_IXOTH = 0x001

	S_ALLPERM = 0x777
)
View Source 
const (
	CIFS_XATTR_CREATETIME     = "user.cifs.creationtime" // File creation time.
	CIFS_XATTR_ATTRIB         = "user.cifs.dosattrib"    // FileAttributes.
	CIFS_XATTR_CIFS_ACL       = "system.cifs_acl"        // DACL only.
	CIFS_XATTR_CIFS_NTSD      = "system.cifs_ntsd"       // Owner, Group, DACL.
	CIFS_XATTR_CIFS_NTSD_FULL = "system.cifs_ntsd_full"  // Owner, Group, DACL, SACL.
)

Extended Attribute (xattr) keys for fetching various information from Linux cifs client.

View Source 
const AZCOPY_BLOCKNAME_LENGTH = 48

Current size of block names in AzCopy is 48B. To be consistent with this, we have to generate a 36B string and then base64-encode this to retain the same size. Block Names of blobs are of format noted below. <5B empty placeholder> <16B GUID of AzCopy re-interpreted as string><5B PartNum><5B Index in the jobPart><5B blockNum>

View Source 
const ApplicationID = "579a7132-0e58-4d80-b1e1-7a1e2d337859"

ApplicationID represents 1st party ApplicationID for AzCopy. const ApplicationID = "a45c21f4-7066-40b4-97d8-14f4313c3caa" // 3rd party test ApplicationID for AzCopy.

View Source 
const AzCopyHashDataStream = `.azcopysyncmeta`

AzCopyHashDataStream is used as both the name of a data stream, xattr key, and the suffix of os-agnostic hash data files. The local traverser intentionally skips over files with this suffix.

View Source 
const AzcopyVersion = "10.24.0"
View Source 
const BackupModeFlagName = "backup" // original name, backup mode, matches the name used for the same thing in Robocopy
View Source 
const BenchmarkFinalDisclaimer = `` /* 226-byte string literal not displayed */
View Source 
const BenchmarkLinuxExtraDisclaimer = `` /* 159-byte string literal not displayed */
View Source 
const BenchmarkPreviewNotice = "The benchmark feature is currently in Preview status."
View Source 
const BenchmarkUserAgent = "Benchmark " + UserAgent
View Source 
const DefaultActiveDirectoryEndpoint = "https://login.microsoftonline.com"
View Source 
const DefaultTenantID = "common"
View Source 
const EnvVarOAuthTokenInfo = "AZCOPY_OAUTH_TOKEN_INFO"

EnvVarOAuthTokenInfo passes oauth token info into AzCopy through environment variable. Note: this is only used for testing, and not encouraged to be used in production environments.

View Source 
const ErrorCodeEnvVarOAuthTokenInfoNotSet = "environment variable AZCOPY_OAUTH_TOKEN_INFO is not set"

ErrorCodeEnvVarOAuthTokenInfoNotSet defines error code when environment variable AZCOPY_OAUTH_TOKEN_INFO is not set.

View Source 
const FileCountDefault = 100
View Source 
const FileCountParam = "file-count"
View Source 
const GCPImportUserAgent = "GCPImport " + UserAgent
View Source 
const IncludeAfterFlagName = "include-after"
View Source 
const IncludeBeforeFlagName = "include-before"
View Source 
const MDResource = "https://disk.azure.com/" // There must be a trailing slash-- The service checks explicitly for "https://disk.azure.com/"
View Source 
const ManagedDiskScope = "https://disk.azure.com//.default" // There must be a trailing slash-- The service checks explicitly for "https://disk.azure.com/"
View Source 
const MetadataAndBlobTagsClearFlag = "clear" // clear flag used for metadata and tags

Metadata used in AzCopy.

View Source 
const PreserveOwnerDefault = true
View Source 
const PreserveOwnerFlagName = "preserve-owner"
View Source 
const PreserveSymlinkFlagName = "preserve-symlinks"
View Source 
const Resource = "https://storage.azure.com"

Resource used in azure storage OAuth authentication

View Source 
const S3ImportUserAgent = "S3Import " + UserAgent
View Source 
const SigAzure = "sig"
View Source 
const SigXAmzForAws = "x-amz-signature"
View Source 
const SizePerFileParam = "size-per-file"
View Source 
const StorageScope = "https://storage.azure.com/.default"
View Source 
const (
	TICKS_FROM_WINDOWS_EPOCH_TO_UNIX_EPOCH = 116444736000000000
)

100-nanosecond intervals from Windows Epoch (January 1, 1601) to Unix Epoch (January 1, 1970).

View Source 
const TokenRefreshSourceTokenStore = "tokenstore"

TokenRefreshSourceTokenStore indicates enabling azcopy oauth integration through tokenstore. Note: This should be only used for internal integrations.

View Source 
const TryEquals string = "Try=" // TODO: refactor so that this can be used by the retry policies too?  So that when you search the logs for Try= you are guaranteed to find both types of retry (i.e. request send retries, and body read retries)
View Source 
const UserAgent = "AzCopy/" + AzcopyVersion

Variables

View Source 
var AllLinuxProperties = []string{
	POSIXNlinkMeta,
	POSIXINodeMeta,
	LINUXBTimeMeta,
	POSIXBlockDeviceMeta,
	POSIXCharDeviceMeta,
	POSIXSocketMeta,
	POSIXFIFOMeta,
	POSIXDevMeta,
	POSIXRDevMeta,
	POSIXATimeMeta,
	POSIXFolderMeta,
	POSIXSymlinkMeta,
	POSIXOwnerMeta,
	POSIXGroupMeta,
	POSIXModeMeta,
	LINUXStatxMaskMeta,
	LINUXAttributeMaskMeta,
	POSIXCTimeMeta,
	POSIXModTimeMeta,
	LINUXAttributeMeta,
}
View Source 
var AzcopyJobPlanFolder string
View Source 
var BenchmarkLmt = time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC)
View Source 
var ChunkWriterAlreadyFailed = errors.New("chunk Writer already failed")
View Source 
var DefaultHashValidationOption = EHashValidationOption.FailIfDifferent()
View Source 
var DefaultInvalidMetadataHandleOption = EInvalidMetadataHandleOption.ExcludeIfInvalid()
View Source 
var EBenchMarkMode = BenchMarkMode(0)
View Source 
var EBlobType = BlobType(0)
View Source 
var EBlockBlobTier = BlockBlobTier(0)
View Source 
var ECompressionType = CompressionType(0)
View Source 
var ECredentialType = CredentialType(0)
View Source 
var EDeleteDestination = DeleteDestination(0)
View Source 
var EDeleteSnapshotsOption = DeleteSnapshotsOption(0)

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View Source 
var EEntityType = EntityType(0)
View Source 
var EEnvironmentVariable = EnvironmentVariable{}
View Source 
var EExitCode = ExitCode(0)
View Source 
var EFolderPropertiesOption = FolderPropertyOption(0)
View Source 
var EFromTo = FromTo(0)
View Source 
var EHashStorageMode = HashStorageMode(0)
View Source 
var EHashValidationOption = HashValidationOption(0)
View Source 
var EInvalidMetadataHandleOption = InvalidMetadataHandleOption(0)
View Source 
var EJobPriority = JobPriority(0)

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View Source 
var EJobStatus = JobStatus(0)
View Source 
var ELocation = Location(0)
View Source 
var ELogLevel = LogLevel(LogNone)
View Source 
var EOutputFormat = OutputFormat(0)
View Source 
var EOutputVerbosity = OutputVerbosity(0)
View Source 
var EOverwriteOption = OverwriteOption(0)
View Source 
var EPageBlobTier = PageBlobTier(0)
View Source 
var EPerfConstraint = PerfConstraint(0)
View Source 
var EPermanentDeleteOption = PermanentDeleteOption(3) // Default to "None"

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View Source 
var EPreservePermissionsOption = PreservePermissionsOption(0)
View Source 
var EPromptType = PromptType("")
View Source 
var ERehydratePriorityType = RehydratePriorityType(0) // setting default as none
View Source 
var EResponseOption = ResponseOption{ResponseType: "", UserFriendlyResponseType: "", ResponseString: ""}

represents one possible response

View Source 
var ERpcCmd = RpcCmd("")
View Source 
var ESetPropertiesFlags = SetPropertiesFlags(0)
View Source 
var ESymlinkHandlingType = eSymlinkHandlingType(0)
View Source 
var ETrailingDotOption = TrailingDotOption(0)

//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View Source 
var ETransferDirection = TransferDirection(0)
View Source 
var ETransferStatus = TransferStatus(0)
View Source 
var EWaitReason = WaitReason{0, ""}
View Source 
var GlobalTestOAuthInjection = TestOAuthInjection{
	DoTokenRefreshInjection: false,
	TokenRefreshDuration:    time.Second * 10,
}

GlobalTestOAuthInjection is the global setting for OAuth testing injection control

View Source 
var LocalHashDir = ""
View Source 
var LocalHashStorageMode = EHashStorageMode.Default()

LocalHashStorageMode & LocalHashDir are temporary global variables pending some level of refactor on parameters

View Source 
var NewHashDataAdapter = func(hashPath, dataPath string, mode HashStorageMode) (HashDataAdapter, error) {
	return &HiddenFileDataAdapter{hashPath, dataPath}, nil
}

NewHashDataAdapter is a function that creates a new HiddenFileDataAdapter on systems that do not override the default functionality.

View Source 
var ProxyLookupTimeoutError = errors.New("proxy lookup timed out")
View Source 
var RootDriveRegex = regexp.MustCompile(`(?i)(^[A-Z]:\/?$)`)

The regex doesn't require a / on the ending, it just requires something similar to the following C: C:/ //myShare //myShare/ demonstrated at: https://regexr.com/4mf6l

View Source 
var RootShareRegex = regexp.MustCompile(`(^\/\/[^\/]*\/?$)`)
View Source 
var VisibleEnvironmentVariables = []EnvironmentVariable{
	EEnvironmentVariable.LogLocation(),
	EEnvironmentVariable.JobPlanLocation(),
	EEnvironmentVariable.ConcurrencyValue(),
	EEnvironmentVariable.TransferInitiationPoolSize(),
	EEnvironmentVariable.EnumerationPoolSize(),
	EEnvironmentVariable.DisableHierarchicalScanning(),
	EEnvironmentVariable.ParallelStatFiles(),
	EEnvironmentVariable.BufferGB(),
	EEnvironmentVariable.AWSAccessKeyID(),
	EEnvironmentVariable.AWSSecretAccessKey(),
	EEnvironmentVariable.GoogleAppCredentials(),
	EEnvironmentVariable.ShowPerfStates(),
	EEnvironmentVariable.PacePageBlobs(),
	EEnvironmentVariable.AutoTuneToCpu(),
	EEnvironmentVariable.CacheProxyLookup(),
	EEnvironmentVariable.DefaultServiceApiVersion(),
	EEnvironmentVariable.UserAgentPrefix(),
	EEnvironmentVariable.AWSAccessKeyID(),
	EEnvironmentVariable.AWSSecretAccessKey(),
	EEnvironmentVariable.ClientSecret(),
	EEnvironmentVariable.CertificatePassword(),
	EEnvironmentVariable.AutoLoginType(),
	EEnvironmentVariable.TenantID(),
	EEnvironmentVariable.AADEndpoint(),
	EEnvironmentVariable.ApplicationID(),
	EEnvironmentVariable.CertificatePath(),
	EEnvironmentVariable.ManagedIdentityClientID(),
	EEnvironmentVariable.ManagedIdentityObjectID(),
	EEnvironmentVariable.ManagedIdentityResourceString(),
	EEnvironmentVariable.RequestTryTimeout(),
	EEnvironmentVariable.CPKEncryptionKey(),
	EEnvironmentVariable.CPKEncryptionKeySHA256(),
	EEnvironmentVariable.DisableSyslog(),
	EEnvironmentVariable.MimeMapping(),
	EEnvironmentVariable.DownloadToTempPath(),
}

This array needs to be updated when a new public environment variable is added Things are here, rather than in command line parameters for one of two reasons: 1. They are optional and obscure (e.g. performance tuning parameters) or 2. They are authentication secrets, which we do not accept on the command line

Functions

func AddStatToBlobMetadata added in v10.16.0 

func AddStatToBlobMetadata(s UnixStatAdapter, metadata Metadata)

func AtomicMorphInt32 

func AtomicMorphInt32(target *int32, morpher AtomicMorpherInt32) interface{}

AtomicMorph atomically morphs target in to new value (and result) as indicated by the AtomicMorpher callback function.

func AtomicMorphInt64 

func AtomicMorphInt64(target *int64, morpher AtomicMorpherInt64) interface{}

AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function.

func AtomicMorphUint32 

func AtomicMorphUint32(target *uint32, morpher AtomicMorpherUint32) interface{}

AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function.

func AtomicMorphUint64 

func AtomicMorphUint64(target *uint64, morpher AtomicMorpherUint64) interface{}

AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function.

func Cause added in v10.22.0 

func Cause(err error) error

Cause walks all the preceding errors and return the originating error.

func ClearStatFromBlobMetadata added in v10.16.0 

func ClearStatFromBlobMetadata(metadata Metadata)

func ConsolidatePathSeparators 

func ConsolidatePathSeparators(path string) string

Replace azcopy path separators (/) with the OS path separator

func CreateDirectoryIfNotExist 

func CreateDirectoryIfNotExist(directory string, tracker FolderCreationTracker) error

func CreateFileOfSizeWithWriteThroughOption 

func CreateFileOfSizeWithWriteThroughOption(destinationPath string, fileSize int64, writeThrough bool, t FolderCreationTracker, forceIfReadOnly bool) (*os.File, error)

func CreateGCPClient 

func CreateGCPClient(ctx context.Context) (*gcpUtils.Client, error)

==================================================================== GCP credential factory related methods ====================================================================

func CreateParentDirectoryIfNotExist 

func CreateParentDirectoryIfNotExist(destinationPath string, tracker FolderCreationTracker) error

func CreateS3Client 

func CreateS3Client(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions, logger ILogger) (*minio.Client, error)

============================================================================================== S3 credential related factory methods ==============================================================================================

func CreateS3Credential 

func CreateS3Credential(ctx context.Context, credInfo CredentialInfo, options CredentialOpOptions) (*credentials.Credentials, error)

CreateS3Credential creates AWS S3 credential according to credential info.

func DeterminePathSeparator 

func DeterminePathSeparator(path string) string

func DoWithOverrideReadOnlyOnAzureFiles added in v10.23.0 

func DoWithOverrideReadOnlyOnAzureFiles(ctx context.Context, action func() (interface{}, error), targetFileOrDir FileClientStub, enableForcing bool) error

DoWithOverrideReadOnlyOnAzureFiles performs the given action, and forces it to happen even if the target is read only. NOTE that all SMB attributes (and other headers?) on the target will be lost, so only use this if you don't need them any more (e.g. you are about to delete the resource, or you are going to reset the attributes/headers)

func DocumentationForDependencyOnChangeDetection 

func DocumentationForDependencyOnChangeDetection()

func EnvVarOAuthTokenInfoExists 

func EnvVarOAuthTokenInfoExists() bool

EnvVarOAuthTokenInfoExists verifies if environment variable for OAuthTokenInfo is specified. The method returns true if the environment variable is set. Note: This is useful for only checking whether the env var exists, please use getTokenInfoFromEnvVar directly in the case getting token info is necessary.

func ForceLog added in v10.22.0 

func ForceLog(level LogLevel, msg string)

ForceLog should rarely be used. It forceable logs an entry to the Windows Event Log (on Windows) or to the SysLog (on Linux)

func GenerateBlockBlobBlockID added in v10.18.0 

func GenerateBlockBlobBlockID(blockNamePrefix string, index int32) string

func GenerateFullPath 

func GenerateFullPath(rootPath, childPath string) string

it's possible that enumerators didn't form rootPath and childPath correctly for them to be combined plainly so we must behave defensively and make sure the full path is correct

func GenerateFullPathWithQuery 

func GenerateFullPathWithQuery(rootPath, childPath, extraQuery string) string

func GetBlobSharedKeyCredential added in v10.22.0 

func GetBlobSharedKeyCredential() (*blob.SharedKeyCredential, error)

func GetBlocksRoundedUp 

func GetBlocksRoundedUp(size uint64, blockSize uint64) uint16

TODO : Remove this? GetBlocksRoundedUp returns the number of blocks given size, rounded up

func GetCpkInfo 

func GetCpkInfo(cpkInfo bool) *blob.CPKInfo

func GetCpkScopeInfo 

func GetCpkScopeInfo(cpkScopeInfo string) *blob.CPKScopeInfo

func GetDatalakeSharedKeyCredential added in v10.22.0 

func GetDatalakeSharedKeyCredential() (*azdatalake.SharedKeyCredential, error)

func GetJsonStringFromTemplate 

func GetJsonStringFromTemplate(template interface{}) string

func GetProxyFunc added in v10.17.0 

func GetProxyFunc() func(*http.Request) (*url.URL, error)

GetProxyFunc is a forwarder for the OS-Exclusive proxyMiddleman_os.go files

func GetTokenStoreCredential added in v10.22.1 

func GetTokenStoreCredential(accessToken string, expiresOn time.Time) azcore.TokenCredential

GetNewTokenFromTokenStore gets token from token store. (Credential Manager in Windows, keyring in Linux and keychain in MacOS.) Note: This approach should only be used in internal integrations.

func Iff added in v10.21.0 

func Iff[T any](test bool, trueVal, falseVal T) T

func IffNotEmpty added in v10.22.0 

func IffNotEmpty(wanted string) *string

func IffNotNil added in v10.21.0 

func IffNotNil[T any](wanted *T, instead T) T

func IsErrorEnvVarOAuthTokenInfoNotSet 

func IsErrorEnvVarOAuthTokenInfoNotSet(err error) bool

IsErrorEnvVarOAuthTokenInfoNotSet verifies if an error indicates environment variable AZCOPY_OAUTH_TOKEN_INFO is not set.

func IsForceLoggingDisabled 

func IsForceLoggingDisabled() bool

func IsGCPURL 

func IsGCPURL(u url.URL) bool

IsGCPURL validates whether a given URL is a valid GCP Object/Service/Bucket URL

func IsS3URL 

func IsS3URL(u url.URL) bool

IsS3URL verifies if a given URL points to S3 URL supported by AzCopy-v10

func IsShortPath 

func IsShortPath(s string) bool

func IsSourcePublicBlob added in v10.13.0 

func IsSourcePublicBlob(sourceURI string, ctx context.Context) bool

func LogHashStorageFailure added in v10.19.0 

func LogHashStorageFailure()

func NewBlobReadLogFunc added in v10.21.0 

func NewBlobReadLogFunc(logger ILogger, fullUrl string) func(int32, error, blob.HTTPRange, bool)

func NewDatalakeReadLogFunc added in v10.22.0 

func NewDatalakeReadLogFunc(logger ILogger, fullUrl string) func(int32, error, datalakefile.HTTPRange, bool)

func NewDecompressingWriter 

func NewDecompressingWriter(destination io.WriteCloser, ct CompressionType) io.WriteCloser

NewDecompressingWriter returns a WriteCloser which decompresses the data that is written to it, before passing the decompressed data on to a final destination. This decompressor is intended to work with compressed data wrapped in either the ZLib headers or the slightly larger Gzip headers. Both of those formats compress a single file (often a .tar archive in the case of Gzip). So there is no need to to expand the decompressed info out into multiple files (as we would have to do, if we were to support "zip" compression). See https://stackoverflow.com/a/20765054

func NewFileReadLogFunc added in v10.21.0 

func NewFileReadLogFunc(logger ILogger, fullUrl string) func(int32, error, sharefile.HTTPRange, bool)

func NewNullHasher 

func NewNullHasher() hash.Hash

func NewRandomDataGenerator 

func NewRandomDataGenerator(length int64) *randomDataGenerator

func NewRecursivePolicy added in v10.22.0 

func NewRecursivePolicy() policy.Policy

NewRecursivePolicy creates a policy that applies the recursive parameter to the request.

func NewRotatingWriter added in v10.22.0 

func NewRotatingWriter(filePath string, size uint64) (io.WriteCloser, error)

func OSOpenFile 

func OSOpenFile(name string, flag int, perm os.FileMode) (*os.File, error)

NOTE: OSOpenFile not safe to use on directories on Windows. See comment on the Windows version of this routine

func OSStat 

func OSStat(name string) (os.FileInfo, error)

func PanicIfErr 

func PanicIfErr(err error)

captures the common logic of exiting if there's an expected error

func RedactSecretQueryParam 

func RedactSecretQueryParam(rawQuery, queryKeyNeedRedact string) (bool, string)

func ScopedCredential1 added in v10.23.0 

func ScopedCredential1(cred azcore.TokenCredential, scopes []string) func(context.Context) (*string, error)

ScopedCredential1 takes in a azcore.TokenCredential object & a list of scopes and returns a function object. This function object on invocation returns a bearer token with specified scope and is of format "Bearer + <Token>". TODO: Token should be cached.

func SetBackupMode 

func SetBackupMode(enable bool, fromTo FromTo) error

func SplitContainerNameFromPath added in v10.22.0 

func SplitContainerNameFromPath(u string) (container string, filepath string, err error)

SplitContainerNameFromPath returns blob/file/dir path excluding container. Ex. For input https://account1.blob.core.windows.net/container1/a/b/c/d a/b/c/d is returned.

func StatXReturned added in v10.16.0 

func StatXReturned(mask uint32, want uint32) bool

func ToExtendedPath 

func ToExtendedPath(short string) string

ToExtendedPath converts short paths to an extended path.

func ToShortPath 

func ToShortPath(long string) string

ToShortPath converts an extended path to a short path.

func TryAddMetadata added in v10.23.0 

func TryAddMetadata(metadata Metadata, key, value string)

Metadata utility functions to work around GoLang's metadata capitalization

func TryReadMetadata added in v10.23.0 

func TryReadMetadata(metadata Metadata, key string) (*string, bool)

func UnixNanoToWindowsTicks added in v10.17.0 

func UnixNanoToWindowsTicks(nsec int64) int64

UnixNanoToWindowsTicks converts nanoseconds since Unix Epoch to ticks since Windows Epoch.

func VerifyIsURLResolvable added in v10.18.0 

func VerifyIsURLResolvable(url_string string) error

func WindowsTicksToUnixNano added in v10.17.0 

func WindowsTicksToUnixNano(ticks int64) int64

WindowsTicksToUnixNano converts ticks (100-ns intervals) since Windows Epoch to nanoseconds since Unix Epoch.

func WithRecursive added in v10.22.0 

func WithRecursive(parent context.Context, recursive bool) context.Context

WithRecursive applies the recursive parameter to the request.

Types

type AtomicMorpherInt32 

type AtomicMorpherInt32 func(startVal int32) (val int32, morphResult interface{})

AtomicMorpherInt32 identifies a method passed to and invoked by the AtomicMorphInt32 function. The AtomicMorpher callback is passed a startValue and based on this value it returns what the new value should be and the result that AtomicMorph should return to its caller.

type AtomicMorpherInt64 

type AtomicMorpherInt64 func(startVal int64) (val int64, morphResult interface{})

AtomicMorpherUint64 identifies a method passed to and invoked by the AtomicMorphUint64 function. The AtomicMorpher callback is passed a startValue and based on this value it returns what the new value should be and the result that AtomicMorph should return to its caller.

type AtomicMorpherUint32 

type AtomicMorpherUint32 func(startVal uint32) (val uint32, morphResult interface{})

AtomicMorpherUint32 identifies a method passed to and invoked by the AtomicMorph function. The AtomicMorpher callback is passed a startValue and based on this value it returns what the new value should be and the result that AtomicMorph should return to its caller.

type AtomicMorpherUint64 

type AtomicMorpherUint64 func(startVal uint64) (val uint64, morphResult interface{})

AtomicMorpherUint64 identifies a method passed to and invoked by the AtomicMorphUint64 function. The AtomicMorpher callback is passed a startValue and based on this value it returns what the new value should be and the result that AtomicMorph should return to its caller.

type AzError 

type AzError struct {
	// contains filtered or unexported fields
}

AzError is to handle AzCopy internal errors in a fine way 

var EAzError AzError

func ErrInvalidClient added in v10.22.0 

func ErrInvalidClient(msg string) AzError

func NewAzError 

func NewAzError(base AzError, additionalInfo string) AzError

NewAzError composes an AzError with given code and message

func (AzError) Equals 

func (lhs AzError) Equals(rhs AzError) bool

func (AzError) Error 

func (err AzError) Error() string

func (AzError) ErrorCode 

func (err AzError) ErrorCode() uint64

func (AzError) InvalidBlobName added in v10.12.0 

func (err AzError) InvalidBlobName() AzError

func (AzError) InvalidBlobOrWindowsName added in v10.20.0 

func (err AzError) InvalidBlobOrWindowsName() AzError

func (AzError) InvalidServiceClient added in v10.22.0 

func (err AzError) InvalidServiceClient() AzError

func (AzError) LoginCredMissing 

func (err AzError) LoginCredMissing() AzError

type BenchMarkMode 

type BenchMarkMode uint8

BenchMarkMode enumerates values for Azcopy bench command. Valid values Upload or Download

func (BenchMarkMode) Download 

func (BenchMarkMode) Download() BenchMarkMode

func (*BenchMarkMode) Parse 

func (bm *BenchMarkMode) Parse(s string) error

func (BenchMarkMode) String 

func (bm BenchMarkMode) String() string

func (BenchMarkMode) Upload 

func (BenchMarkMode) Upload() BenchMarkMode

type BlobTags 

type BlobTags map[string]string

BlobTags is a map of key-value pair

func ToCommonBlobTagsMap 

func ToCommonBlobTagsMap(blobTagsString string) BlobTags

func (BlobTags) ToString 

func (bt BlobTags) ToString() string

type BlobTransferAttributes 

type BlobTransferAttributes struct {
	BlobType                         BlobType              // The type of a blob - BlockBlob, PageBlob, AppendBlob
	ContentType                      string                // The content type specified for the blob.
	ContentEncoding                  string                // Specifies which content encodings have been applied to the blob.
	ContentLanguage                  string                // Specifies the language of the content
	ContentDisposition               string                // Specifies the content disposition
	CacheControl                     string                // Specifies the cache control header
	BlockBlobTier                    BlockBlobTier         // Specifies the tier to set on the block blobs.
	PageBlobTier                     PageBlobTier          // Specifies the tier to set on the page blobs.
	Metadata                         string                // User-defined Name-value pairs associated with the blob
	NoGuessMimeType                  bool                  // represents user decision to interpret the content-encoding from source file
	PreserveLastModifiedTime         bool                  // when downloading, tell engine to set file's timestamp to timestamp of blob
	PutMd5                           bool                  // when uploading, should we create and PUT Content-MD5 hashes
	MD5ValidationOption              HashValidationOption  // when downloading, how strictly should we validate MD5 hashes?
	BlockSizeInBytes                 int64                 // when uploading/downloading/copying, specify the size of each chunk
	PutBlobSizeInBytes               int64                 // when uploading, specify the threshold to determine if the blob should be uploaded in a single PUT request
	DeleteSnapshotsOption            DeleteSnapshotsOption // when deleting, specify what to do with the snapshots
	BlobTagsString                   string                // when user explicitly provides blob tags
	PermanentDeleteOption            PermanentDeleteOption // Permanently deletes soft-deleted snapshots when indicated by user
	RehydratePriority                RehydratePriorityType // rehydrate priority of blob
	DeleteDestinationFileIfNecessary bool                  // deletes the dst blob if indicated
}

This struct represents the optional attribute for blob request header

type BlobType 

type BlobType uint8

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Enumerates the values for blob type.

func FromBlobType added in v10.21.0 

func FromBlobType(bt blob.BlobType) BlobType

func (BlobType) AppendBlob 

func (BlobType) AppendBlob() BlobType

func (BlobType) BlockBlob 

func (BlobType) BlockBlob() BlobType

func (BlobType) Detect 

func (BlobType) Detect() BlobType

func (BlobType) PageBlob 

func (BlobType) PageBlob() BlobType

func (*BlobType) Parse 

func (bt *BlobType) Parse(s string) error

func (BlobType) String 

func (bt BlobType) String() string

func (*BlobType) ToBlobType added in v10.21.0 

func (bt *BlobType) ToBlobType() blob.BlobType

ToBlobType returns the equivalent blob.BlobType for given string.

type BlockBlobTier 

type BlockBlobTier uint8

func (BlockBlobTier) Archive 

func (BlockBlobTier) Archive() BlockBlobTier

func (BlockBlobTier) Cold added in v10.18.0 

func (BlockBlobTier) Cold() BlockBlobTier

func (BlockBlobTier) Cool 

func (BlockBlobTier) Cool() BlockBlobTier

func (BlockBlobTier) Hot 

func (BlockBlobTier) Hot() BlockBlobTier

func (BlockBlobTier) MarshalJSON 

func (bbt BlockBlobTier) MarshalJSON() ([]byte, error)

func (BlockBlobTier) None 

func (BlockBlobTier) None() BlockBlobTier

func (*BlockBlobTier) Parse 

func (bbt *BlockBlobTier) Parse(s string) error

func (BlockBlobTier) String 

func (bbt BlockBlobTier) String() string

func (BlockBlobTier) ToAccessTierType 

func (bbt BlockBlobTier) ToAccessTierType() blob.AccessTier

func (*BlockBlobTier) UnmarshalJSON 

func (bbt *BlockBlobTier) UnmarshalJSON(b []byte) error

Implementing UnmarshalJSON() method for type BlockBlobTier.

type ByHandleFileInformation added in v10.17.0 

type ByHandleFileInformation struct {
	FileAttributes     uint32
	CreationTime       Filetime
	LastAccessTime     Filetime
	LastWriteTime      Filetime
	VolumeSerialNumber uint32
	FileSizeHigh       uint32
	FileSizeLow        uint32
	NumberOfLinks      uint32
	FileIndexHigh      uint32
	FileIndexLow       uint32
}

windows.ByHandleFileInformation

func GetFileInformation added in v10.17.0 

func GetFileInformation(path string) (ByHandleFileInformation, error)

type ByteSlice 

type ByteSlice []byte

///////////////////////////////////////////////////////////////////////////////////////////////

type ByteSliceExtension 

type ByteSliceExtension struct {
	ByteSlice
}

func (ByteSliceExtension) RemoveBOM 

func (bs ByteSliceExtension) RemoveBOM() []byte

RemoveBOM removes any BOM from the byte slice

type ByteSlicePooler 

type ByteSlicePooler interface {
	RentSlice(desiredLength int64) []byte
	ReturnSlice(slice []byte)
	Prune()
}

A pool of byte slices Like sync.Pool, but strongly-typed to byte slices

func NewMultiSizeSlicePool 

func NewMultiSizeSlicePool(maxSliceLength int64) ByteSlicePooler

Create new slice pool capable of pooling slices up to maxSliceLength in size

type CPUMonitor 

type CPUMonitor interface {
	CPUContentionExists() bool
}

func NewCalibratedCpuUsageMonitor 

func NewCalibratedCpuUsageMonitor() CPUMonitor

NewCalibratedCpuUsageMonitor should be called early in the app's life cycle, before we are creating any significant CPU load so that it's self-calibration will be accurate

func NewNullCpuMonitor 

func NewNullCpuMonitor() CPUMonitor

type CacheLimiter 

type CacheLimiter interface {
	TryAdd(count int64, useRelaxedLimit bool) (added bool)
	WaitUntilAdd(ctx context.Context, count int64, useRelaxedLimit Predicate) error
	Remove(count int64)
	Limit() int64
	StrictLimit() int64
}

Used to limit the amounts of things. E.g. amount of in-flight data in RAM, to keep it an an acceptable level. Also used for number of open files (since that's limited on Linux). In the case of RAM usage, for downloads, network is producer and disk is consumer, while for uploads the roles are reversed. In either case, if the producer is faster than the consumer, this CacheLimiter is necessary prevent unbounded RAM usage.

func NewCacheLimiter 

func NewCacheLimiter(limit int64) CacheLimiter

type CancelPauseResumeResponse 

type CancelPauseResumeResponse struct {
	ErrorMsg              string
	CancelledPauseResumed bool
	JobStatus             JobStatus
}

type ChunkID 

type ChunkID struct {
	Name string
	// contains filtered or unexported fields
}

Identifies a chunk. Always create with NewChunkID

func NewChunkID 

func NewChunkID(name string, offsetInFile int64, length int64) ChunkID

func NewPseudoChunkIDForWholeFile 

func NewPseudoChunkIDForWholeFile(name string) ChunkID

func (ChunkID) IsPseudoChunk 

func (id ChunkID) IsPseudoChunk() bool

func (ChunkID) Length 

func (id ChunkID) Length() int64

func (ChunkID) OffsetInFile 

func (id ChunkID) OffsetInFile() int64

func (ChunkID) SetCompletionNotificationSent 

func (id ChunkID) SetCompletionNotificationSent()

type ChunkReaderSourceFactory 

type ChunkReaderSourceFactory func() (CloseableReaderAt, error)

Factory method for data source for singleChunkReader

type ChunkStatusLogger 

type ChunkStatusLogger interface {
	LogChunkStatus(id ChunkID, reason WaitReason)
	IsWaitingOnFinalBodyReads() bool
}

type ChunkStatusLoggerCloser 

type ChunkStatusLoggerCloser interface {
	ChunkStatusLogger
	GetCounts(td TransferDirection) []chunkStatusCount
	GetPrimaryPerfConstraint(td TransferDirection, rc RetryCounter) PerfConstraint
	FlushLog() // not close, because we had issues with writes coming in after this // TODO: see if that issue still exists
	CloseLogger()
}

func NewChunkStatusLogger 

func NewChunkStatusLogger(jobID JobID, cpuMon CPUMonitor, logFileFolder string, enableOutput bool) ChunkStatusLoggerCloser

type ChunkedFileWriter 

type ChunkedFileWriter interface {

	// WaitToScheduleChunk blocks until enough RAM is available to handle the given chunk, then it
	// "reserves" that amount of RAM in the CacheLimiter and returns.
	WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error

	// EnqueueChunk hands the given chunkContents over to the ChunkedFileWriter, to be written to disk.
	// Because ChunkedFileWriter writes sequentially, the actual time of writing is not known to the caller.
	// All the caller knows, is that responsibility for writing the chunk has been passed to the ChunkedFileWriter.
	// While any error may be returned immediately, errors are more likely to be returned later, on either a subsequent
	// call to this routine or on the final return to Flush.
	// After the chunk is written to disk, its reserved memory byte allocation is automatically subtracted from the CacheLimiter.
	EnqueueChunk(ctx context.Context, id ChunkID, chunkSize int64, chunkContents io.Reader, retryable bool) error

	// Flush will block until all the chunks have been written to disk.  err will be non-nil if and only in any chunk failed to write.
	// Flush must be called exactly once, after all chunks have been enqueued with EnqueueChunk.
	Flush(ctx context.Context) (md5HashOfFileAsWritten []byte, err error)

	// MaxRetryPerDownloadBody returns the maximum number of retries that will be done for the download of a single chunk body
	MaxRetryPerDownloadBody() int
}

Used to write all the chunks to a disk file

func NewChunkedFileWriter 

func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheLimiter CacheLimiter, chunkLogger ChunkStatusLogger, file io.WriteCloser, numChunks uint32, maxBodyRetries int, md5ValidationOption HashValidationOption, sourceMd5Exists bool) ChunkedFileWriter

type CloseableReaderAt 

type CloseableReaderAt interface {
	io.ReaderAt
	io.Closer
}

Simple aggregation of existing io interfaces

type CompressionType 

type CompressionType uint8

func GetCompressionType 

func GetCompressionType(contentEncoding string) (CompressionType, error)

func (CompressionType) GZip 

func (CompressionType) GZip() CompressionType

func (CompressionType) None 

func (CompressionType) None() CompressionType

func (CompressionType) String 

func (ct CompressionType) String() string

func (CompressionType) Unsupported 

func (CompressionType) Unsupported() CompressionType

func (CompressionType) ZLib 

func (CompressionType) ZLib() CompressionType

type CopyJobPartOrderErrorType 

type CopyJobPartOrderErrorType string
var ECopyJobPartOrderErrorType CopyJobPartOrderErrorType

func (CopyJobPartOrderErrorType) NoTransfersScheduledErr 

func (CopyJobPartOrderErrorType) NoTransfersScheduledErr() CopyJobPartOrderErrorType

type CopyJobPartOrderRequest 

type CopyJobPartOrderRequest struct {
	Version             Version         // version of azcopy
	JobID               JobID           // Guid - job identifier
	PartNum             PartNumber      // part number of the job
	IsFinalPart         bool            // to determine the final part for a specific job
	ForceWrite          OverwriteOption // to determine if the existing needs to be overwritten or not. If set to true, existing blobs are overwritten
	ForceIfReadOnly     bool            // Supplements ForceWrite with addition setting for Azure Files objects with read-only attribute
	AutoDecompress      bool            // if true, source data with encodings that represent compression are automatically decompressed when downloading
	Priority            JobPriority     // priority of the task
	FromTo              FromTo
	Fpo                 FolderPropertyOption // passed in from front-end to ensure that front-end and STE agree on the desired behaviour for the job
	SymlinkHandlingType SymlinkHandlingType
	// list of blobTypes to exclude.
	ExcludeBlobType []blob.BlobType

	SourceRoot       ResourceString
	DestinationRoot  ResourceString
	SrcServiceClient *ServiceClient
	DstServiceClient *ServiceClient

	//These clients are required only in S2S transfers from/to datalake
	SrcDatalakeClient *datalake.Client
	DstDatalakeClient *datalake.Client

	Transfers      Transfers
	LogLevel       LogLevel
	BlobAttributes BlobTransferAttributes
	CommandString  string // commandString hold the user given command which is logged to the Job log file
	CredentialInfo CredentialInfo

	PreserveSMBPermissions         PreservePermissionsOption
	PreserveSMBInfo                bool
	PreservePOSIXProperties        bool
	S2SGetPropertiesInBackend      bool
	S2SSourceChangeValidation      bool
	DestLengthValidation           bool
	S2SInvalidMetadataHandleOption InvalidMetadataHandleOption
	S2SPreserveBlobTags            bool
	CpkOptions                     CpkOptions
	SetPropertiesFlags             SetPropertiesFlags
	BlobFSRecursiveDelete          bool

	// S2SSourceCredentialType will override CredentialInfo.CredentialType for use on the source.
	// As a result, CredentialInfo.OAuthTokenInfo may end up being fulfilled even _if_ CredentialInfo.CredentialType is _not_ OAuth.
	// This may not always be the case (for instance, if we opt to use multiple OAuth tokens). At that point, this will likely be it's own CredentialInfo.
	S2SSourceCredentialType CredentialType // Only Anonymous and OAuth will really be used in response to this, but S3 and GCP will come along too...
	FileAttributes          FileTransferAttributes
}

This struct represents the job info (a single part) to be sent to the storage engine

type CopyJobPartOrderResponse 

type CopyJobPartOrderResponse struct {
	ErrorMsg   CopyJobPartOrderErrorType
	JobStarted bool
}

type CopyTransfer 

type CopyTransfer struct {
	Source           string
	Destination      string
	EntityType       EntityType
	LastModifiedTime time.Time // represents the last modified time of source which ensures that source hasn't changed while transferring
	SourceSize       int64     // size of the source entity in bytes.

	// Properties for service to service copy (some also used in upload or download too)
	ContentType        string
	ContentEncoding    string
	ContentDisposition string
	ContentLanguage    string
	CacheControl       string
	ContentMD5         []byte
	Metadata           Metadata

	// Properties for S2S blob copy
	BlobType      blob.BlobType
	BlobTier      blob.AccessTier
	BlobVersionID string
	// Blob index tags categorize data in your storage account utilizing key-value tag attributes
	BlobTags BlobTags

	BlobSnapshotID string
}

This struct represent a single transfer entry with source and destination details ** DO NOT construct directly. Use cmd.storedObject.ToNewCopyTransfer **

type CountPerSecond 

type CountPerSecond interface {
	// Add atomically adds delta to *addr and returns the new value.
	// To subtract a signed positive constant value c, do Add(^uint64(c-1)).
	Add(delta uint64) uint64 // Pass 0 to get the current count value
	LatestRate() float64
	Reset()
}

CountPerSecond ...

func NewCountPerSecond 

func NewCountPerSecond() CountPerSecond

type CpkOptions 

type CpkOptions struct {
	// Optional flag to encrypt user data with user provided key.
	// Key is provide in the REST request itself
	// Provided key (EncryptionKey and EncryptionKeySHA256) and its hash will be fetched from environment variables
	// Set EncryptionAlgorithm = "AES256" by default.
	CpkInfo bool
	// Key is present in AzureKeyVault and Azure KeyVault is linked with storage account.
	// Provided key name will be fetched from Azure Key Vault and will be used to encrypt the data
	CpkScopeInfo string
	// flag to check if the source is encrypted by user provided key or not.
	// True only if user wishes to download source encrypted by user provided key
	IsSourceEncrypted bool
}

func (CpkOptions) GetCPKInfo added in v10.21.0 

func (options CpkOptions) GetCPKInfo() *blob.CPKInfo

func (CpkOptions) GetCPKScopeInfo added in v10.21.0 

func (options CpkOptions) GetCPKScopeInfo() *blob.CPKScopeInfo

type CredCache 

type CredCache struct {
	// contains filtered or unexported fields
}

CredCache manages credential caches. Use keyring in Linux OS. Session keyring is chosen, the session hooks key should be created since user first login (i.e. by pam). So the session is inherited by processes created from login session. When user logout, the session keyring is recycled.

func NewCredCache 

func NewCredCache(options CredCacheOptions) *CredCache

NewCredCache creates a cred cache.

func (*CredCache) HasCachedToken 

func (c *CredCache) HasCachedToken() (bool, error)

HasCachedToken returns if there is cached token for current executing user.

func (*CredCache) LoadToken 

func (c *CredCache) LoadToken() (*OAuthTokenInfo, error)

LoadToken gets the cached oauth token.

func (*CredCache) RemoveCachedToken 

func (c *CredCache) RemoveCachedToken() error

RemoveCachedToken deletes the cached token.

func (*CredCache) SaveToken 

func (c *CredCache) SaveToken(token OAuthTokenInfo) error

SaveToken saves an oauth token.

type CredCacheInternalIntegration 

type CredCacheInternalIntegration struct {
	// contains filtered or unexported fields
}

CredCacheInternalIntegration manages credential caches with Gnome keyring. Note: This should be only used for internal integration.

func NewCredCacheInternalIntegration 

func NewCredCacheInternalIntegration(options CredCacheOptions) *CredCacheInternalIntegration

NewCredCacheInternalIntegration creates a cred cache.

func (*CredCacheInternalIntegration) HasCachedToken 

func (c *CredCacheInternalIntegration) HasCachedToken() (bool, error)

HasCachedToken returns if there is cached token for current executing user.

func (*CredCacheInternalIntegration) LoadToken 

func (c *CredCacheInternalIntegration) LoadToken() (*OAuthTokenInfo, error)

LoadToken gets the cached oauth token.

func (*CredCacheInternalIntegration) RemoveCachedToken 

func (c *CredCacheInternalIntegration) RemoveCachedToken() error

RemoveCachedToken deletes the cached token.

func (*CredCacheInternalIntegration) SaveToken 

func (c *CredCacheInternalIntegration) SaveToken(token OAuthTokenInfo) error

SaveToken saves an oauth token.

type CredCacheOptions 

type CredCacheOptions struct {
	// Used by credCache in Windows.
	DPAPIFilePath string

	// Used by credCacheSegmented in Windows, and keyring in Linux.
	KeyName string
	// Used by keychain in Mac OS, and gnome keyring in Linux.
	ServiceName string
	AccountName string
}

CredCacheOptions contains options could be used in different kinds of cred caches in different platform.

type CredentialInfo 

type CredentialInfo struct {
	CredentialType    CredentialType
	OAuthTokenInfo    OAuthTokenInfo
	S3CredentialInfo  S3CredentialInfo
	GCPCredentialInfo GCPCredentialInfo
}

CredentialInfo contains essential credential info which need be transited between modules, and used during creating Azure storage client Credential.

func (CredentialInfo) WithType added in v10.15.0 

func (c CredentialInfo) WithType(credentialType CredentialType) CredentialInfo

type CredentialOpOptions 

type CredentialOpOptions struct {
	LogInfo  func(string)
	LogError func(string)
	Panic    func(error)
	CallerID string

	// Used to cancel operations, if fatal error happened during operation.
	Cancel context.CancelFunc
}

CredentialOpOptions contains credential operations' parameters.

type CredentialType 

type CredentialType uint8

CredentialType defines the different types of credentials

func (CredentialType) Anonymous 

func (CredentialType) Anonymous() CredentialType

func (CredentialType) GoogleAppCredentials 

func (CredentialType) GoogleAppCredentials() CredentialType

func (CredentialType) IsAzureOAuth added in v10.16.0 

func (ct CredentialType) IsAzureOAuth() bool

func (CredentialType) IsSharedKey added in v10.22.0 

func (ct CredentialType) IsSharedKey() bool

func (CredentialType) MDOAuthToken added in v10.16.0 

func (CredentialType) MDOAuthToken() CredentialType

func (CredentialType) OAuthToken 

func (CredentialType) OAuthToken() CredentialType

func (*CredentialType) Parse 

func (ct *CredentialType) Parse(s string) error

func (CredentialType) S3AccessKey 

func (CredentialType) S3AccessKey() CredentialType

func (CredentialType) S3PublicBucket added in v10.12.0 

func (CredentialType) S3PublicBucket() CredentialType

func (CredentialType) SharedKey 

func (CredentialType) SharedKey() CredentialType

func (CredentialType) String 

func (ct CredentialType) String() string

func (CredentialType) Unknown 

func (CredentialType) Unknown() CredentialType

type CtxRecursiveKey added in v10.22.0 

type CtxRecursiveKey struct{}

CtxRecursiveKey is used as a context key to apply the recursive query parameter.

type DeleteDestination 

type DeleteDestination uint32

func (DeleteDestination) False 

func (DeleteDestination) False() DeleteDestination

func (*DeleteDestination) Parse 

func (dd *DeleteDestination) Parse(s string) error

func (DeleteDestination) Prompt 

func (DeleteDestination) Prompt() DeleteDestination

func (DeleteDestination) String 

func (dd DeleteDestination) String() string

func (DeleteDestination) True 

func (DeleteDestination) True() DeleteDestination

type DeleteSnapshotsOption 

type DeleteSnapshotsOption uint8

func (DeleteSnapshotsOption) Include 

func (DeleteSnapshotsOption) Include() DeleteSnapshotsOption

func (DeleteSnapshotsOption) None 

func (DeleteSnapshotsOption) None() DeleteSnapshotsOption

func (DeleteSnapshotsOption) Only 

func (DeleteSnapshotsOption) Only() DeleteSnapshotsOption

func (*DeleteSnapshotsOption) Parse 

func (d *DeleteSnapshotsOption) Parse(s string) error

func (DeleteSnapshotsOption) String 

func (d DeleteSnapshotsOption) String() string

func (DeleteSnapshotsOption) ToDeleteSnapshotsOptionType 

func (d DeleteSnapshotsOption) ToDeleteSnapshotsOptionType() *blob.DeleteSnapshotsOptionType

type DeviceCodeCredential added in v10.21.0 

type DeviceCodeCredential struct {
	// contains filtered or unexported fields
}

func (*DeviceCodeCredential) GetToken added in v10.21.0 

func (dcc *DeviceCodeCredential) GetToken(ctx context.Context, options policy.TokenRequestOptions) (azcore.AccessToken, error)

func (*DeviceCodeCredential) RefreshTokenWithUserCredential added in v10.21.0 

func (dcc *DeviceCodeCredential) RefreshTokenWithUserCredential(ctx context.Context, resource string) (*adal.Token, error)

RefreshTokenWithUserCredential gets new token with user credential through refresh.

type EntityType 

type EntityType uint8

func (EntityType) File 

func (EntityType) File() EntityType

func (EntityType) FileProperties added in v10.18.0 

func (EntityType) FileProperties() EntityType

func (EntityType) Folder 

func (EntityType) Folder() EntityType

func (EntityType) String 

func (e EntityType) String() string
func (EntityType) Symlink() EntityType

type EnvironmentVariable 

type EnvironmentVariable struct {
	Name         string
	DefaultValue string
	Description  string
	Hidden       bool
}

func (EnvironmentVariable) AADEndpoint 

func (EnvironmentVariable) AADEndpoint() EnvironmentVariable

func (EnvironmentVariable) AWSAccessKeyID 

func (EnvironmentVariable) AWSAccessKeyID() EnvironmentVariable

func (EnvironmentVariable) AWSSecretAccessKey 

func (EnvironmentVariable) AWSSecretAccessKey() EnvironmentVariable

func (EnvironmentVariable) AccountKey 

func (EnvironmentVariable) AccountKey() EnvironmentVariable

func (EnvironmentVariable) AccountName 

func (EnvironmentVariable) AccountName() EnvironmentVariable

func (EnvironmentVariable) ApplicationID 

func (EnvironmentVariable) ApplicationID() EnvironmentVariable

func (EnvironmentVariable) AutoLoginType 

func (EnvironmentVariable) AutoLoginType() EnvironmentVariable

func (EnvironmentVariable) AutoTuneToCpu 

func (EnvironmentVariable) AutoTuneToCpu() EnvironmentVariable

added in so that CPU usage detection can be disabled if advanced users feel it is causing tuning to be too conservative (i.e. not enough concurrency, due to detected CPU usage)

func (EnvironmentVariable) AwsSessionToken 

func (EnvironmentVariable) AwsSessionToken() EnvironmentVariable

AwsSessionToken is temporarily internally reserved, and not exposed to users.

func (EnvironmentVariable) BufferGB 

func (EnvironmentVariable) BufferGB() EnvironmentVariable

func (EnvironmentVariable) CPKEncryptionKey 

func (EnvironmentVariable) CPKEncryptionKey() EnvironmentVariable

func (EnvironmentVariable) CPKEncryptionKeySHA256 

func (EnvironmentVariable) CPKEncryptionKeySHA256() EnvironmentVariable

func (EnvironmentVariable) CacheProxyLookup 

func (EnvironmentVariable) CacheProxyLookup() EnvironmentVariable

func (EnvironmentVariable) CertificatePassword 

func (EnvironmentVariable) CertificatePassword() EnvironmentVariable

func (EnvironmentVariable) CertificatePath 

func (EnvironmentVariable) CertificatePath() EnvironmentVariable

func (EnvironmentVariable) ClientSecret 

func (EnvironmentVariable) ClientSecret() EnvironmentVariable

func (EnvironmentVariable) ConcurrencyValue 

func (EnvironmentVariable) ConcurrencyValue() EnvironmentVariable

func (EnvironmentVariable) CredentialType 

func (EnvironmentVariable) CredentialType() EnvironmentVariable

CredentialType is only used for internal integration.

func (EnvironmentVariable) DefaultServiceApiVersion 

func (EnvironmentVariable) DefaultServiceApiVersion() EnvironmentVariable

func (EnvironmentVariable) DisableBlobTransferResume added in v10.18.0 

func (EnvironmentVariable) DisableBlobTransferResume() EnvironmentVariable

func (EnvironmentVariable) DisableHierarchicalScanning 

func (EnvironmentVariable) DisableHierarchicalScanning() EnvironmentVariable

func (EnvironmentVariable) DisableSyslog 

func (EnvironmentVariable) DisableSyslog() EnvironmentVariable

func (EnvironmentVariable) DownloadToTempPath added in v10.15.0 

func (EnvironmentVariable) DownloadToTempPath() EnvironmentVariable

func (EnvironmentVariable) EnumerationPoolSize 

func (EnvironmentVariable) EnumerationPoolSize() EnvironmentVariable

func (EnvironmentVariable) GoogleAppCredentials 

func (EnvironmentVariable) GoogleAppCredentials() EnvironmentVariable

func (EnvironmentVariable) GoogleCloudProject 

func (EnvironmentVariable) GoogleCloudProject() EnvironmentVariable

func (EnvironmentVariable) JobPlanLocation 

func (EnvironmentVariable) JobPlanLocation() EnvironmentVariable

func (EnvironmentVariable) LogLocation 

func (EnvironmentVariable) LogLocation() EnvironmentVariable

func (EnvironmentVariable) ManagedIdentityClientID 

func (EnvironmentVariable) ManagedIdentityClientID() EnvironmentVariable

For MSI login

func (EnvironmentVariable) ManagedIdentityObjectID 

func (EnvironmentVariable) ManagedIdentityObjectID() EnvironmentVariable

func (EnvironmentVariable) ManagedIdentityResourceString 

func (EnvironmentVariable) ManagedIdentityResourceString() EnvironmentVariable

func (EnvironmentVariable) MimeMapping added in v10.11.0 

func (EnvironmentVariable) MimeMapping() EnvironmentVariable

func (EnvironmentVariable) OAuthTokenInfo 

func (EnvironmentVariable) OAuthTokenInfo() EnvironmentVariable

OAuthTokenInfo is only used for internal integration.

func (EnvironmentVariable) OptimizeSparsePageBlobTransfers 

func (EnvironmentVariable) OptimizeSparsePageBlobTransfers() EnvironmentVariable

func (EnvironmentVariable) PacePageBlobs 

func (EnvironmentVariable) PacePageBlobs() EnvironmentVariable

func (EnvironmentVariable) ParallelStatFiles 

func (EnvironmentVariable) ParallelStatFiles() EnvironmentVariable

func (EnvironmentVariable) ProfileCPU 

func (EnvironmentVariable) ProfileCPU() EnvironmentVariable

func (EnvironmentVariable) ProfileMemory 

func (EnvironmentVariable) ProfileMemory() EnvironmentVariable

func (EnvironmentVariable) RequestTryTimeout 

func (EnvironmentVariable) RequestTryTimeout() EnvironmentVariable

func (EnvironmentVariable) ShowPerfStates 

func (EnvironmentVariable) ShowPerfStates() EnvironmentVariable

func (EnvironmentVariable) TenantID 

func (EnvironmentVariable) TenantID() EnvironmentVariable

func (EnvironmentVariable) TransferInitiationPoolSize 

func (EnvironmentVariable) TransferInitiationPoolSize() EnvironmentVariable

func (EnvironmentVariable) UserAgentPrefix 

func (EnvironmentVariable) UserAgentPrefix() EnvironmentVariable

func (EnvironmentVariable) UserDir 

func (EnvironmentVariable) UserDir() EnvironmentVariable

type ExclusiveStringMap 

type ExclusiveStringMap struct {
	// contains filtered or unexported fields
}

func NewExclusiveStringMap 

func NewExclusiveStringMap(fromTo FromTo, goos string) *ExclusiveStringMap

func (*ExclusiveStringMap) Add 

func (e *ExclusiveStringMap) Add(key string) error

Add succeeds if and only if key is not currently in the map

func (*ExclusiveStringMap) Remove 

func (e *ExclusiveStringMap) Remove(key string)

type ExitCode 

type ExitCode uint32

func (ExitCode) Error 

func (ExitCode) Error() ExitCode

func (ExitCode) NoExit 

func (ExitCode) NoExit() ExitCode

NoExit is used as a marker, to suppress the normal exit behaviour

func (ExitCode) Success 

func (ExitCode) Success() ExitCode

type FileClientOptions added in v10.22.0 

type FileClientOptions struct {
	AllowTrailingDot       bool
	AllowSourceTrailingDot bool
}

type FileClientStub added in v10.23.0 

type FileClientStub interface {
	URL() string
}

type FileTransferAttributes added in v10.19.0 

type FileTransferAttributes struct {
	TrailingDot TrailingDotOption
}

This struct represents the optional attribute for file request header

type Filetime added in v10.17.0 

type Filetime struct {
	LowDateTime  uint32
	HighDateTime uint32
}

windows.Filetime.

func NsecToFiletime added in v10.17.0 

func NsecToFiletime(nsec int64) Filetime

Convert nanoseconds since Unix Epoch (January 1, 1970) to Filetime since Windows Epoch (January 1, 1601).

func StatxTimestampToFiletime added in v10.17.0 

func StatxTimestampToFiletime(ts unix.StatxTimestamp) Filetime

StatxTimestampToFiletime converts the unix StatxTimestamp (sec, nsec) to the Windows' Filetime. Note that StatxTimestamp is from Unix Epoch while Filetime holds time from Windows Epoch.

func (*Filetime) Nanoseconds added in v10.17.0 

func (ft *Filetime) Nanoseconds() int64

Nanoseconds converts Filetime (as ticks since Windows Epoch) to nanoseconds since Unix Epoch (January 1, 1970).

type FolderCreationTracker 

type FolderCreationTracker interface {
	CreateFolder(folder string, doCreation func() error) error
	ShouldSetProperties(folder string, overwrite OverwriteOption, prompter Prompter) bool
	StopTracking(folder string)
}

folderCreationTracker is used to ensure than in an overwrite=false situation we only set folder properties on folders which were created by the current job. (To be consistent with the fact that when overwrite == false, we only set file properties on files created by the current job)

type FolderDeletionFunc 

type FolderDeletionFunc func(context.Context, ILogger) bool

folderDeletionFunc should delete the folder IF IT IS EMPTY, and return true. If it is not empty, false must be returned. FolderDeletionManager is allowed to call this on a folder that is not yet empty. In that case, FolderDeletionManager may call it again later. Errors are not returned because of the delay to when deletion might happen, so it's up to the func to do its own logging

type FolderDeletionManager 

type FolderDeletionManager interface {

	// RecordChildExists takes a child name and counts it against the child's immediate parent
	// Should be called for both types of child: folders and files.
	// Only counts it against the immediate parent (that's all that's necessary, because we recurse in tryDeletion)
	RecordChildExists(childFileOrFolder *url.URL)

	// RecordChildDelete records that a file, previously passed to RecordChildExists, has now been deleted
	// Only call for files, not folders
	RecordChildDeleted(childFile *url.URL)

	// RequestDeletion registers a function that will be called to delete the given folder, when that
	// folder has no more known children.  May be called before, after or during the time that
	// the folder's children are being passed to RecordChildExists and RecordChildDeleted
	//
	// Warning: only pass in deletionFuncs that will do nothing and return FALSE if the
	// folder is not yet empty. If they return false, they may be called again later.
	RequestDeletion(folder *url.URL, deletionFunc FolderDeletionFunc)
}

FolderDeletionManager handles the fact that (in most locations) we can't delete folders that still contain files. So it allows us to request deletion of a folder, and have that be attempted after the last file is removed. Note that maybe the apparent last file isn't the last (e.g. there are other files, still to be deleted, in future job parts), in which case any failed deletion will be retried if there's a new "candidate last child" removed. Takes URLs rather than strings because that ensures correct (un)escaping, and makes it clear that we don't support Windows & MacOS local paths (which have cases insensitivity that we don't support here).

func NewFolderDeletionManager 

func NewFolderDeletionManager(ctx context.Context, fpo FolderPropertyOption, logger ILogger) FolderDeletionManager

type FolderPropertyOption 

type FolderPropertyOption uint8

FolderPropertyOption controls which folders get their properties recorded in the Plan file

func (FolderPropertyOption) AllFolders 

func (FolderPropertyOption) AllFolders() FolderPropertyOption

func (FolderPropertyOption) AllFoldersExceptRoot 

func (FolderPropertyOption) AllFoldersExceptRoot() FolderPropertyOption

func (FolderPropertyOption) NoFolders 

func (FolderPropertyOption) NoFolders() FolderPropertyOption

func (FolderPropertyOption) Unspecified 

func (FolderPropertyOption) Unspecified() FolderPropertyOption

no FPO has been selected. Make sure the zero-like value is "unspecified" so that we detect any code paths that that do not nominate any FPO

type FromTo 

type FromTo uint16

FromTo defines the different types of sources/destination location combinations FromTo is 16 bit where first 8 bit represents the from location and other 8 bits represents the to location

func (FromTo) AreBothFolderAware 

func (ft FromTo) AreBothFolderAware() bool

func (FromTo) BenchmarkBlob 

func (FromTo) BenchmarkBlob() FromTo

todo: to we really want these? Starts to look like a bit of a combinatorial explosion

func (FromTo) BenchmarkBlobFS 

func (FromTo) BenchmarkBlobFS() FromTo

func (FromTo) BenchmarkFile 

func (FromTo) BenchmarkFile() FromTo

func (FromTo) BlobBlob 

func (FromTo) BlobBlob() FromTo

func (FromTo) BlobBlobFS added in v10.19.0 

func (FromTo) BlobBlobFS() FromTo

func (FromTo) BlobFSBlob added in v10.19.0 

func (FromTo) BlobFSBlob() FromTo

func (FromTo) BlobFSBlobFS added in v10.19.0 

func (FromTo) BlobFSBlobFS() FromTo

func (FromTo) BlobFSFile added in v10.19.0 

func (FromTo) BlobFSFile() FromTo

func (FromTo) BlobFSLocal 

func (FromTo) BlobFSLocal() FromTo

func (FromTo) BlobFSNone added in v10.16.0 

func (FromTo) BlobFSNone() FromTo

func (FromTo) BlobFSTrash 

func (FromTo) BlobFSTrash() FromTo

func (FromTo) BlobFile 

func (FromTo) BlobFile() FromTo

func (FromTo) BlobLocal 

func (FromTo) BlobLocal() FromTo

func (FromTo) BlobNone added in v10.16.0 

func (FromTo) BlobNone() FromTo

func (FromTo) BlobPipe 

func (FromTo) BlobPipe() FromTo

func (FromTo) BlobTrash 

func (FromTo) BlobTrash() FromTo

func (FromTo) BothSupportTrailingDot added in v10.22.0 

func (ft FromTo) BothSupportTrailingDot() bool

func (FromTo) FileBlob 

func (FromTo) FileBlob() FromTo

func (FromTo) FileBlobFS added in v10.19.0 

func (FromTo) FileBlobFS() FromTo

func (FromTo) FileFile 

func (FromTo) FileFile() FromTo

func (FromTo) FileLocal 

func (FromTo) FileLocal() FromTo

func (FromTo) FileNone added in v10.16.0 

func (FromTo) FileNone() FromTo

func (FromTo) FilePipe 

func (FromTo) FilePipe() FromTo

func (FromTo) FileTrash 

func (FromTo) FileTrash() FromTo

func (FromTo) From 

func (ft FromTo) From() Location

func (FromTo) FromAndTo 

func (ft FromTo) FromAndTo(s string) (srcLocation, dstLocation Location, err error)

func (FromTo) GCPBlob 

func (FromTo) GCPBlob() FromTo

func (FromTo) IsDelete added in v10.19.0 

func (ft FromTo) IsDelete() bool

func (FromTo) IsDownload 

func (ft FromTo) IsDownload() bool

func (FromTo) IsPropertyOnlyTransfer added in v10.16.0 

func (ft FromTo) IsPropertyOnlyTransfer() bool

func (FromTo) IsS2S 

func (ft FromTo) IsS2S() bool

func (FromTo) IsSetProperties added in v10.19.0 

func (ft FromTo) IsSetProperties() bool

func (FromTo) IsUpload 

func (ft FromTo) IsUpload() bool

func (FromTo) LocalBlob 

func (FromTo) LocalBlob() FromTo

func (FromTo) LocalBlobFS 

func (FromTo) LocalBlobFS() FromTo

func (FromTo) LocalFile 

func (FromTo) LocalFile() FromTo

func (*FromTo) Parse 

func (ft *FromTo) Parse(s string) error

func (FromTo) PipeBlob 

func (FromTo) PipeBlob() FromTo

func (FromTo) PipeFile 

func (FromTo) PipeFile() FromTo

func (FromTo) S3Blob 

func (FromTo) S3Blob() FromTo

func (FromTo) String 

func (ft FromTo) String() string

func (FromTo) To 

func (ft FromTo) To() Location

func (FromTo) Unknown 

func (FromTo) Unknown() FromTo

type GCPClientFactory 

type GCPClientFactory struct {
	// contains filtered or unexported fields
}

func NewGCPClientFactory 

func NewGCPClientFactory() GCPClientFactory

func (*GCPClientFactory) GetGCPClient 

func (f *GCPClientFactory) GetGCPClient(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions) (*gcpUtils.Client, error)

type GCPCredentialInfo 

type GCPCredentialInfo struct {
}

type GCPObjectInfoExtension 

type GCPObjectInfoExtension struct {
	ObjectInfo gcpUtils.ObjectAttrs
}

func (*GCPObjectInfoExtension) CacheControl 

func (gie *GCPObjectInfoExtension) CacheControl() string

func (*GCPObjectInfoExtension) ContentDisposition 

func (gie *GCPObjectInfoExtension) ContentDisposition() string

func (*GCPObjectInfoExtension) ContentEncoding 

func (gie *GCPObjectInfoExtension) ContentEncoding() string

func (*GCPObjectInfoExtension) ContentLanguage 

func (gie *GCPObjectInfoExtension) ContentLanguage() string

func (*GCPObjectInfoExtension) ContentMD5 

func (gie *GCPObjectInfoExtension) ContentMD5() []byte

func (*GCPObjectInfoExtension) ContentType 

func (gie *GCPObjectInfoExtension) ContentType() string

func (*GCPObjectInfoExtension) NewCommonMetadata 

func (gie *GCPObjectInfoExtension) NewCommonMetadata() Metadata

NewCommonMetadata returns a map of user-defined key/value pairs

type GCPURLParts 

type GCPURLParts struct {
	Scheme         string
	Host           string
	BucketName     string
	ObjectKey      string
	UnparsedParams string
}

GCPURLParts structure is used to parse and hold the different components of GCP Object/Service/Bucket URL

func NewGCPURLParts 

func NewGCPURLParts(u url.URL) (GCPURLParts, error)

NewGCPURLParts processes the given URL and returns a valid GCPURLParts structure that contains all the necessary components.

func (*GCPURLParts) IsBucketSyntactically 

func (gUrl *GCPURLParts) IsBucketSyntactically() bool

func (*GCPURLParts) IsDirectorySyntactically 

func (gUrl *GCPURLParts) IsDirectorySyntactically() bool

IsDirectorySyntactically returns true if the given GCPURLParts points to a directory or not based on the path.

func (*GCPURLParts) IsObjectSyntactically 

func (gUrl *GCPURLParts) IsObjectSyntactically() bool

func (*GCPURLParts) IsServiceSyntactically 

func (gUrl *GCPURLParts) IsServiceSyntactically() bool

func (*GCPURLParts) String 

func (gUrl *GCPURLParts) String() string

func (*GCPURLParts) URL 

func (gUrl *GCPURLParts) URL() url.URL

URL returns a valid net/url.URL object initialised from the components of GCP URL

type GenericResourceURLParts 

type GenericResourceURLParts struct {
	// contains filtered or unexported fields
}

GenericResourceURLParts is intended to be a generic solution to code duplication when using *URLParts TODO: Use this to reduce code dupe in the cca.Source and jobPartOrder.Source setups Currently this just contains generic functions for what we *need*. This isn't an overarching, perfect implementation. The above suggestion would be preferable to continuing to expand this (due to 4x code dupe for every function)-- it's just a bridge over a LARGE gap for now.

func NewGenericResourceURLParts 

func NewGenericResourceURLParts(resourceURL url.URL, location Location) GenericResourceURLParts

func (*GenericResourceURLParts) GetContainerName 

func (g *GenericResourceURLParts) GetContainerName() string

func (*GenericResourceURLParts) GetObjectName 

func (g *GenericResourceURLParts) GetObjectName() string

func (*GenericResourceURLParts) SetObjectName 

func (g *GenericResourceURLParts) SetObjectName(objectName string)

func (*GenericResourceURLParts) String 

func (g *GenericResourceURLParts) String() string

func (*GenericResourceURLParts) URL 

func (g *GenericResourceURLParts) URL() url.URL

type GetJobFromToRequest 

type GetJobFromToRequest struct {
	JobID JobID
}

GetJobFromToRequest indicates request to get job's FromTo info from job part plan header

type GetJobFromToResponse 

type GetJobFromToResponse struct {
	ErrorMsg    string
	FromTo      FromTo
	Source      string
	Destination string
}

GetJobFromToResponse indicates response to get job's FromTo info.

type HTTPResponseExtension 

type HTTPResponseExtension struct {
	*http.Response
}

///////////////////////////////////////////////////////////////////////////////////////////////

func (HTTPResponseExtension) IsSuccessStatusCode 

func (r HTTPResponseExtension) IsSuccessStatusCode(successStatusCodes ...int) bool

IsSuccessStatusCode checks if response's status code is contained in specified success status codes.

type HashDataAdapter added in v10.19.0 

type HashDataAdapter interface {
	GetHashData(relativePath string) (*SyncHashData, error)
	SetHashData(relativePath string, data *SyncHashData) error
	GetMode() HashStorageMode // for testing purposes primarily, should be a static value.
}

HashDataAdapter implements an interface to pull and set hash data on files based upon a relative path

type HashStorageMode added in v10.19.0 

type HashStorageMode uint8

func (*HashStorageMode) Default added in v10.19.0 

func (e *HashStorageMode) Default() HashStorageMode

func (HashStorageMode) HiddenFiles added in v10.19.0 

func (HashStorageMode) HiddenFiles() HashStorageMode

func (HashStorageMode) IsOSAgnostic added in v10.19.0 

func (mode HashStorageMode) IsOSAgnostic() bool

func (*HashStorageMode) Parse added in v10.19.0 

func (mode *HashStorageMode) Parse(s string) error

func (HashStorageMode) String added in v10.19.0 

func (mode HashStorageMode) String() string

func (HashStorageMode) XAttr added in v10.19.0 

func (HashStorageMode) XAttr() HashStorageMode

type HashValidationOption 

type HashValidationOption uint8

func (HashValidationOption) FailIfDifferent 

func (HashValidationOption) FailIfDifferent() HashValidationOption

FailIfDifferent says fail if hashes different, but NOT fail if saved hash is totally missing. This is a balance of convenience (for cases where no hash is saved) vs strictness (to validate strictly when one is present)

func (HashValidationOption) FailIfDifferentOrMissing 

func (HashValidationOption) FailIfDifferentOrMissing() HashValidationOption

FailIfDifferentOrMissing is the strictest option, and useful for testing or validation in cases when we _know_ there should be a hash

func (HashValidationOption) LogOnly 

func (HashValidationOption) LogOnly() HashValidationOption

LogOnly means only log if missing or different, don't fail the transfer

func (HashValidationOption) MarshalJSON 

func (hvo HashValidationOption) MarshalJSON() ([]byte, error)

func (HashValidationOption) NoCheck 

func (HashValidationOption) NoCheck() HashValidationOption

Do not check hashes at download time at all

func (*HashValidationOption) Parse 

func (hvo *HashValidationOption) Parse(s string) error

func (HashValidationOption) String 

func (hvo HashValidationOption) String() string

func (*HashValidationOption) UnmarshalJSON 

func (hvo *HashValidationOption) UnmarshalJSON(b []byte) error

type HiddenFileDataAdapter added in v10.19.0 

type HiddenFileDataAdapter struct {
	// contains filtered or unexported fields
}

func (*HiddenFileDataAdapter) GetHashData added in v10.19.0 

func (a *HiddenFileDataAdapter) GetHashData(relativePath string) (*SyncHashData, error)

func (*HiddenFileDataAdapter) GetMode added in v10.19.0 

func (a *HiddenFileDataAdapter) GetMode() HashStorageMode

func (*HiddenFileDataAdapter) SetHashData added in v10.19.0 

func (a *HiddenFileDataAdapter) SetHashData(relativePath string, data *SyncHashData) error

type ILogger 

type ILogger interface {
	ShouldLog(level LogLevel) bool
	Log(level LogLevel, msg string)
	Panic(err error)
}

type ILoggerCloser 

type ILoggerCloser interface {
	ILogger
	CloseLog()
}

type ILoggerResetable 

type ILoggerResetable interface {
	OpenLog()
	MinimumLogLevel() LogLevel
	ILoggerCloser
}
var AzcopyCurrentJobLogger ILoggerResetable

func NewJobLogger 

func NewJobLogger(jobID JobID, minimumLevelToLog LogLevel, logFileFolder string, logFileNameSuffix string) ILoggerResetable

func NewSysLogger added in v10.15.0 

func NewSysLogger(jobID JobID, minimumLevelToLog LogLevel, logSuffix string) ILoggerResetable

type IdentityInfo 

type IdentityInfo struct {
	ClientID string `json:"_identity_client_id"`
	ObjectID string `json:"_identity_object_id"`
	MSIResID string `json:"_identity_msi_res_id"`
}

IdentityInfo contains info for MSI.

func (*IdentityInfo) Validate 

func (identityInfo *IdentityInfo) Validate() error

Validate validates identity info, at most only one of clientID, objectID or MSI resource ID could be set.

type InitMsgJsonTemplate 

type InitMsgJsonTemplate struct {
	LogFileLocation string
	JobID           string
	IsCleanupJob    bool
}

type InvalidMetadataHandleOption 

type InvalidMetadataHandleOption uint8

func (InvalidMetadataHandleOption) ExcludeIfInvalid 

func (InvalidMetadataHandleOption) ExcludeIfInvalid() InvalidMetadataHandleOption

ExcludeIfInvalid indicates whenever invalid metadata key is found, exclude the specific metadata with WARNING logged.

func (InvalidMetadataHandleOption) FailIfInvalid 

func (InvalidMetadataHandleOption) FailIfInvalid() InvalidMetadataHandleOption

FailIfInvalid indicates whenever invalid metadata key is found, directly fail the transfer.

func (InvalidMetadataHandleOption) MarshalJSON 

func (i InvalidMetadataHandleOption) MarshalJSON() ([]byte, error)

func (*InvalidMetadataHandleOption) Parse 

func (i *InvalidMetadataHandleOption) Parse(s string) error

func (InvalidMetadataHandleOption) RenameIfInvalid 

func (InvalidMetadataHandleOption) RenameIfInvalid() InvalidMetadataHandleOption

RenameIfInvalid indicates whenever invalid metadata key is found, rename the metadata key and save the metadata with renamed key.

func (InvalidMetadataHandleOption) String 

func (i InvalidMetadataHandleOption) String() string

func (*InvalidMetadataHandleOption) UnmarshalJSON 

func (i *InvalidMetadataHandleOption) UnmarshalJSON(b []byte) error

type JobID 

type JobID UUID

func NewJobID 

func NewJobID() JobID

func ParseJobID 

func ParseJobID(jobID string) (JobID, error)

func (JobID) IsEmpty 

func (j JobID) IsEmpty() bool

var EmptyJobId JobID = JobID{}

func (JobID) MarshalJSON 

func (j JobID) MarshalJSON() ([]byte, error)

Implementing MarshalJSON() method for type JobID

func (JobID) String 

func (j JobID) String() string

func (*JobID) UnmarshalJSON 

func (j *JobID) UnmarshalJSON(b []byte) error

Implementing UnmarshalJSON() method for type JobID

type JobIDDetails 

type JobIDDetails struct {
	JobId         JobID
	CommandString string
	StartTime     int64
	JobStatus     JobStatus
}

type JobPriority 

type JobPriority uint8

JobPriority defines the transfer priorities supported by the Storage Transfer Engine's channels The default priority is Normal

func (JobPriority) Low 

func (JobPriority) Low() JobPriority

func (JobPriority) Normal 

func (JobPriority) Normal() JobPriority

func (JobPriority) String 

func (jp JobPriority) String() string

type JobStatus 

type JobStatus uint32 // Must be 32-bit for atomic operations

JobStatus indicates the status of a Job; the default is InProgress.

func (JobStatus) All 

func (JobStatus) All() JobStatus

func (*JobStatus) AtomicLoad 

func (j *JobStatus) AtomicLoad() JobStatus

func (*JobStatus) AtomicStore 

func (j *JobStatus) AtomicStore(newJobStatus JobStatus)

func (JobStatus) Cancelled 

func (JobStatus) Cancelled() JobStatus

func (JobStatus) Cancelling 

func (JobStatus) Cancelling() JobStatus

func (JobStatus) Completed 

func (JobStatus) Completed() JobStatus

func (JobStatus) CompletedWithErrors 

func (JobStatus) CompletedWithErrors() JobStatus

func (JobStatus) CompletedWithErrorsAndSkipped 

func (JobStatus) CompletedWithErrorsAndSkipped() JobStatus

func (JobStatus) CompletedWithSkipped 

func (JobStatus) CompletedWithSkipped() JobStatus

func (*JobStatus) EnhanceJobStatusInfo 

func (j *JobStatus) EnhanceJobStatusInfo(skippedTransfers, failedTransfers, successfulTransfers bool) JobStatus

func (JobStatus) Failed 

func (JobStatus) Failed() JobStatus

func (JobStatus) InProgress 

func (JobStatus) InProgress() JobStatus

func (*JobStatus) IsJobDone 

func (j *JobStatus) IsJobDone() bool

func (JobStatus) MarshalJSON 

func (j JobStatus) MarshalJSON() ([]byte, error)

Implementing MarshalJSON() method for type JobStatus

func (*JobStatus) Parse 

func (j *JobStatus) Parse(s string) error

func (JobStatus) Paused 

func (JobStatus) Paused() JobStatus

func (JobStatus) String 

func (js JobStatus) String() string

func (*JobStatus) UnmarshalJSON 

func (j *JobStatus) UnmarshalJSON(b []byte) error

Implementing UnmarshalJSON() method for type JobStatus

type JsonOutputTemplate 

type JsonOutputTemplate struct {
	TimeStamp      time.Time
	MessageType    string
	MessageContent string // a simple string for INFO and ERROR, a serialized JSON for INIT, PROGRESS, EXIT
	PromptDetails  PromptDetails
}

defines the general output template when the format is set to json

type LCMMsg added in v10.15.0 

type LCMMsg struct {
	Req  *LCMMsgReq
	Resp *LCMMsgResp
	// contains filtered or unexported fields
}

func NewLCMMsg added in v10.15.0 

func NewLCMMsg() *LCMMsg

func (*LCMMsg) Reply added in v10.15.0 

func (m *LCMMsg) Reply()

func (*LCMMsg) SetRequest added in v10.15.0 

func (m *LCMMsg) SetRequest(req *LCMMsgReq)

func (*LCMMsg) SetResponse added in v10.15.0 

func (m *LCMMsg) SetResponse(resp *LCMMsgResp)

type LCMMsgReq added in v10.15.0 

type LCMMsgReq struct {
	TimeStamp time.Time `json:"TimeStamp"`
	MsgType   string    `json:"RequestType"`
	Value     string    `json:"Value"`
}

type LCMMsgResp added in v10.15.0 

type LCMMsgResp struct {
	TimeStamp time.Time    `json:"TimeStamp"`
	MsgType   string       `json:"ResponseType"`
	Value     fmt.Stringer `json:"Value"`
	Err       error        `json:"-"`
}

type LCMMsgType added in v10.15.0 

type LCMMsgType uint16

////////////////////////////////////////////////////////////// 

var ELCMMsgType LCMMsgType

func (LCMMsgType) CancelJob added in v10.15.0 

func (LCMMsgType) CancelJob() LCMMsgType

func (LCMMsgType) E2EInterrupts added in v10.15.0 

func (LCMMsgType) E2EInterrupts() LCMMsgType

func (LCMMsgType) Invalid added in v10.15.0 

func (LCMMsgType) Invalid() LCMMsgType

func (*LCMMsgType) Parse added in v10.15.0 

func (m *LCMMsgType) Parse(s string) error

func (LCMMsgType) PerformanceAdjustment added in v10.15.0 

func (LCMMsgType) PerformanceAdjustment() LCMMsgType

func (*LCMMsgType) String added in v10.15.0 

func (m *LCMMsgType) String() string

type LifecycleMgr 

type LifecycleMgr interface {
	Init(OutputBuilder)                                          // let the user know the job has started and initial information like log location
	Progress(OutputBuilder)                                      // print on the same line over and over again, not allowed to float up
	Exit(OutputBuilder, ExitCode)                                // indicates successful execution exit after printing, allow user to specify exit code
	Info(string)                                                 // simple print, allowed to float up
	Warn(string)                                                 // simple print, allowed to float up
	Dryrun(OutputBuilder)                                        // print files for dry run mode
	Error(string)                                                // indicates fatal error, exit after printing, exit code is always Failed (1)
	Prompt(message string, details PromptDetails) ResponseOption // ask the user a question(after erasing the progress), then return the response
	SurrenderControl()                                           // give up control, this should never return
	InitiateProgressReporting(WorkController)                    // start writing progress with another routine
	AllowReinitiateProgressReporting()                           // allow re-initiation of progress reporting for followup job
	GetEnvironmentVariable(EnvironmentVariable) string           // get the environment variable or its default value
	ClearEnvironmentVariable(EnvironmentVariable)                // clears the environment variable
	SetOutputFormat(OutputFormat)                                // change the output format of the entire application
	EnableInputWatcher()                                         // depending on the command, we may allow user to give input through Stdin
	EnableCancelFromStdIn()                                      // allow user to send in `cancel` to stop the job
	AddUserAgentPrefix(string) string                            // append the global user agent prefix, if applicable
	E2EAwaitContinue()                                           // used by E2E tests
	E2EAwaitAllowOpenFiles()                                     // used by E2E tests
	E2EEnableAwaitAllowOpenFiles(enable bool)                    // used by E2E tests
	RegisterCloseFunc(func())
	SetForceLogging()
	IsForceLoggingDisabled() bool
	DownloadToTempPath() bool
	MsgHandlerChannel() <-chan *LCMMsg
	ReportAllJobPartsDone()
	SetOutputVerbosity(mode OutputVerbosity)
}

create a public interface so that consumers outside of this package can refer to the lifecycle manager but they would not be able to instantiate one

func GetLifecycleMgr 

func GetLifecycleMgr() LifecycleMgr

type ListContainerResponse 

type ListContainerResponse struct {
	Blobs []string
}

ListContainerResponse represents the list of blobs within the container.

type ListJobSummaryResponse 

type ListJobSummaryResponse struct {
	ErrorMsg  string
	Timestamp time.Time `json:"-"`
	JobID     JobID
	// TODO: added for debugging purpose. remove later
	ActiveConnections int64 `json:",string"`
	// CompleteJobOrdered determines whether the Job has been completely ordered or not
	CompleteJobOrdered bool
	JobStatus          JobStatus

	TotalTransfers uint32 `json:",string"` // = FileTransfers + FolderPropertyTransfers. It also = TransfersCompleted + TransfersFailed + TransfersSkipped
	// FileTransfers and FolderPropertyTransfers just break the total down into the two types.
	// The name FolderPropertyTransfers is used to emphasise that is is only counting transferring the properties and existence of
	// folders. A "folder property transfer" does not include any files that may be in the folder. Those are counted as
	// FileTransfers.
	FileTransfers           uint32 `json:",string"`
	FolderPropertyTransfers uint32 `json:",string"`
	SymlinkTransfers        uint32 `json:",string"`

	FoldersCompleted   uint32 `json:",string"` // Files can be figured out by TransfersCompleted - FoldersCompleted
	TransfersCompleted uint32 `json:",string"`
	FoldersFailed      uint32 `json:",string"`
	TransfersFailed    uint32 `json:",string"`
	FoldersSkipped     uint32 `json:",string"`
	TransfersSkipped   uint32 `json:",string"`

	// includes bytes sent in retries (i.e. has double counting, if there are retries) and in failed transfers
	BytesOverWire uint64 `json:",string"`

	// does not include failed transfers or bytes sent in retries (i.e. no double counting). Includes successful transfers and transfers in progress
	TotalBytesTransferred uint64 `json:",string"`

	// sum of the total transfer enumerated so far.
	TotalBytesEnumerated uint64 `json:",string"`
	// sum of total bytes expected in the job (i.e. based on our current expectation of which files will be successful)
	TotalBytesExpected uint64 `json:",string"`

	PercentComplete float32 `json:",string"`

	// Stats measured from the network pipeline
	// Values are all-time values, for the duration of the job.
	// Will be zero if read outside the process running the job (e.g. with 'jobs show' command)
	AverageIOPS            int     `json:",string"`
	AverageE2EMilliseconds int     `json:",string"`
	ServerBusyPercentage   float32 `json:",string"`
	NetworkErrorPercentage float32 `json:",string"`

	FailedTransfers  []TransferDetail
	SkippedTransfers []TransferDetail
	PerfConstraint   PerfConstraint
	PerfStrings      []string `json:"-"`

	PerformanceAdvice []PerformanceAdvice
	IsCleanupJob      bool
}

represents the JobProgressPercentage Summary response for list command when requested the Job Progress Summary for given JobId

type ListJobTransfersRequest 

type ListJobTransfersRequest struct {
	JobID    JobID
	OfStatus TransferStatus
}

type ListJobTransfersResponse 

type ListJobTransfersResponse struct {
	ErrorMsg string
	JobID    JobID
	Details  []TransferDetail
}

represents the list of Details and details of number of transfers

type ListJobsResponse 

type ListJobsResponse struct {
	ErrorMessage string
	JobIDDetails []JobIDDetails
}

ListJobsResponse represent the Job with JobId and

type ListOfFiles 

type ListOfFiles struct {
	Files []string
}

this struct is used to parse the contents of file passed with list-of-files flag.

type ListRequest 

type ListRequest struct {
	JobID    JobID
	OfStatus string // TODO: OfStatus with string type sounds not good, change it to enum
	Output   OutputFormat
}

represents the raw list command input from the user when requested the list of transfer with given status for given JobId

type ListSyncJobSummaryResponse 

type ListSyncJobSummaryResponse struct {
	ListJobSummaryResponse
	DeleteTotalTransfers     uint32 `json:",string"`
	DeleteTransfersCompleted uint32 `json:",string"`
}

wraps the standard ListJobSummaryResponse with sync-specific stats

type Location 

type Location uint8

Location indicates the type of Location

func (Location) AllStandardLocations 

func (Location) AllStandardLocations() []Location

AllStandardLocations returns all locations that are "normal" for testing purposes. Excludes the likes of Unknown, Benchmark and Pipe

func (Location) AzureAccount added in v10.24.0 

func (Location) AzureAccount() Location

func (Location) Benchmark 

func (Location) Benchmark() Location

func (Location) Blob 

func (Location) Blob() Location

func (Location) BlobFS 

func (Location) BlobFS() Location

func (Location) CanForwardOAuthTokens added in v10.19.0 

func (l Location) CanForwardOAuthTokens() bool

func (Location) File 

func (Location) File() Location

func (Location) GCP 

func (Location) GCP() Location

func (Location) IsFolderAware 

func (l Location) IsFolderAware() bool

IsFolderAware returns true if the location has real folders (e.g. there's such a thing as an empty folder, and folders may have properties). Folders are only virtual, and so not real, in Blob Storage.

func (Location) IsLocal 

func (l Location) IsLocal() bool

func (Location) IsRemote 

func (l Location) IsRemote() bool

func (Location) Local 

func (Location) Local() Location

func (Location) None added in v10.16.0 

func (Location) None() Location

func (Location) Pipe 

func (Location) Pipe() Location

func (Location) S3 

func (Location) S3() Location

func (Location) String 

func (l Location) String() string

func (Location) SupportsHnsACLs added in v10.21.0 

func (l Location) SupportsHnsACLs() bool

func (Location) SupportsTrailingDot added in v10.22.0 

func (l Location) SupportsTrailingDot() bool

func (Location) Unknown 

func (Location) Unknown() Location

type LogLevel 

type LogLevel uint8

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// 

const (
	// LogNone tells a logger not to log any entries passed to it.
	LogNone LogLevel = iota

	// LogFatal tells a logger to log all LogFatal entries passed to it.
	LogFatal

	// LogPanic tells a logger to log all LogPanic and LogFatal entries passed to it.
	LogPanic

	// LogError tells a logger to log all LogError, LogPanic and LogFatal entries passed to it.
	LogError

	// LogWarning tells a logger to log all LogWarning, LogError, LogPanic and LogFatal entries passed to it.
	LogWarning

	// LogInfo tells a logger to log all LogInfo, LogWarning, LogError, LogPanic and LogFatal entries passed to it.
	LogInfo

	// LogDebug tells a logger to log all LogDebug, LogInfo, LogWarning, LogError, LogPanic and LogFatal entries passed to it.
	LogDebug
)

func (LogLevel) Debug 

func (LogLevel) Debug() LogLevel

func (LogLevel) Error 

func (LogLevel) Error() LogLevel

func (LogLevel) Fatal 

func (LogLevel) Fatal() LogLevel

func (LogLevel) Info 

func (LogLevel) Info() LogLevel

func (LogLevel) None 

func (LogLevel) None() LogLevel

func (LogLevel) Panic 

func (LogLevel) Panic() LogLevel

func (*LogLevel) Parse 

func (ll *LogLevel) Parse(s string) error

func (LogLevel) String 

func (ll LogLevel) String() string

func (LogLevel) Warning 

func (LogLevel) Warning() LogLevel

type LogSanitizer added in v10.21.0 

type LogSanitizer interface {
	SanitizeLogMessage(raw string) string
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// LogSanitizer can be implemented to clean secrets from lines logged by ForceLog By default no implementation is provided here, because pipeline may be used in many different contexts, so the correct implementation is context-dependent

func NewAzCopyLogSanitizer 

func NewAzCopyLogSanitizer() LogSanitizer

type MMF 

type MMF struct {
	// contains filtered or unexported fields
}

func NewMMF 

func NewMMF(file *os.File, writable bool, offset int64, length int64) (*MMF, error)

func (*MMF) Slice 

func (m *MMF) Slice() []byte

Slice() returns the memory mapped byte slice

func (*MMF) Unmap 

func (m *MMF) Unmap()

To unmap, we need exclusive (write) access to the MMF and then we set isMapped to false so that future readers know the MMF is unusable.

func (*MMF) UnuseMMF 

func (m *MMF) UnuseMMF()

RUnlock unlocks the held lock

func (*MMF) UseMMF 

func (m *MMF) UseMMF() bool

type Metadata 

type Metadata map[string]*string

func StringToMetadata added in v10.16.0 

func StringToMetadata(metadataString string) (Metadata, error)

func UnMarshalToCommonMetadata 

func UnMarshalToCommonMetadata(metadataString string) (Metadata, error)

UnMarshalToCommonMetadata unmarshals string to common metadata.

func (Metadata) Clone added in v10.16.0 

func (m Metadata) Clone() Metadata

func (Metadata) ConcatenatedKeys 

func (m Metadata) ConcatenatedKeys() string

func (Metadata) ExcludeInvalidKey 

func (m Metadata) ExcludeInvalidKey() (retainedMetadata Metadata, excludedMetadata Metadata, invalidKeyExists bool)

func (Metadata) Marshal 

func (m Metadata) Marshal() (string, error)

Marshal marshals metadata to string.

func (Metadata) ResolveInvalidKey 

func (m Metadata) ResolveInvalidKey() (resolvedMetadata Metadata, err error)

ResolveInvalidKey resolves invalid metadata key with following steps: 1. replace all invalid char(i.e. ASCII chars expect [0-9A-Za-z_]) with '_' 2. add 'rename_' as prefix for the new valid key, this key will be used to save original metadata's value. 3. add 'rename_key_' as prefix for the new valid key, this key will be used to save original metadata's invalid key. Example, given invalid metadata for Azure: '123-invalid':'content', it will be resolved as two new k:v pairs: 'rename_123_invalid':'content' 'rename_key_123_invalid':'123-invalid' So user can try to recover the metadata in Azure side. Note: To keep first version simple, whenever collision is found during key resolving, error will be returned. This can be further improved once any user feedback get.

type NoCopy 

type NoCopy struct {
	// contains filtered or unexported fields
}

The NoCopy struct is used as a field inside another struct that should not be copied by value. After embedded this field, the out struct's members can call the Check method which will panic if it detects the out struct has been copied by value.

func (*NoCopy) Check 

func (nc *NoCopy) Check()

Check panics if the struct embedded this NoCopy field has been copied by value.

type OAuthTokenInfo 

type OAuthTokenInfo struct {
	azcore.TokenCredential `json:"-"`
	adal.Token
	Tenant                  string `json:"_tenant"`
	ActiveDirectoryEndpoint string `json:"_ad_endpoint"`
	TokenRefreshSource      string `json:"_token_refresh_source"`
	ApplicationID           string `json:"_application_id"`
	Identity                bool   `json:"_identity"`
	IdentityInfo            IdentityInfo
	ServicePrincipalName    bool `json:"_spn"`
	SPNInfo                 SPNInfo
	AzCLICred               bool
	PSCred                  bool
	// Note: ClientID should be only used for internal integrations through env var with refresh token.
	// It indicates the Application ID assigned to your app when you registered it with Azure AD.
	// In this case AzCopy refresh token on behalf of caller.
	// For more details, please refer to
	// https://docs.microsoft.com/en-us/azure/active-directory/develop/v1-protocols-oauth-code#refreshing-the-access-tokens
	ClientID string `json:"_client_id"`
}

OAuthTokenInfo contains info necessary for refresh OAuth credentials.

func (*OAuthTokenInfo) GetAzCliCredential added in v10.22.0 

func (credInfo *OAuthTokenInfo) GetAzCliCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetClientCertificateCredential added in v10.21.0 

func (credInfo *OAuthTokenInfo) GetClientCertificateCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetClientSecretCredential added in v10.21.0 

func (credInfo *OAuthTokenInfo) GetClientSecretCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetDeviceCodeCredential added in v10.21.0 

func (credInfo *OAuthTokenInfo) GetDeviceCodeCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetManagedIdentityCredential added in v10.21.0 

func (credInfo *OAuthTokenInfo) GetManagedIdentityCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetPSContextCredential added in v10.22.0 

func (credInfo *OAuthTokenInfo) GetPSContextCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetTokenCredential added in v10.21.0 

func (credInfo *OAuthTokenInfo) GetTokenCredential() (azcore.TokenCredential, error)

func (*OAuthTokenInfo) GetTokenStoreCredential added in v10.21.0 

func (credInfo *OAuthTokenInfo) GetTokenStoreCredential() (azcore.TokenCredential, error)

func (OAuthTokenInfo) IsEmpty 

func (credInfo OAuthTokenInfo) IsEmpty() bool

IsEmpty returns if current OAuthTokenInfo is empty and doesn't contain any useful info.

func (*OAuthTokenInfo) Refresh 

func (credInfo *OAuthTokenInfo) Refresh(ctx context.Context) (*adal.Token, error)

Refresh gets new token with token info.

type ObjectInfoExtension 

type ObjectInfoExtension struct {
	ObjectInfo minio.ObjectInfo
}

func (*ObjectInfoExtension) CacheControl 

func (oie *ObjectInfoExtension) CacheControl() string

CacheControl returns the value for header Cache-Control.

func (*ObjectInfoExtension) ContentDisposition 

func (oie *ObjectInfoExtension) ContentDisposition() string

ContentDisposition returns the value for header Content-Disposition.

func (*ObjectInfoExtension) ContentEncoding 

func (oie *ObjectInfoExtension) ContentEncoding() string

ContentEncoding returns the value for header Content-Encoding.

func (*ObjectInfoExtension) ContentLanguage 

func (oie *ObjectInfoExtension) ContentLanguage() string

ContentLanguage returns the value for header Content-Language.

func (*ObjectInfoExtension) ContentMD5 

func (oie *ObjectInfoExtension) ContentMD5() []byte

ContentMD5 returns the value for header Content-MD5.

func (*ObjectInfoExtension) ContentType 

func (oie *ObjectInfoExtension) ContentType() string

func (*ObjectInfoExtension) NewCommonMetadata 

func (oie *ObjectInfoExtension) NewCommonMetadata() Metadata

NewMetadata returns user-defined key/value pairs.

type OutputBuilder 

type OutputBuilder func(OutputFormat) string

used for output types that are not simple strings, such as progress and init a given format(text,json) is passed in, and the appropriate string is returned

func GetStandardInitOutputBuilder 

func GetStandardInitOutputBuilder(jobID string, logFileLocation string, isCleanupJob bool, cleanupMessage string) OutputBuilder

type OutputFormat 

type OutputFormat uint32

func (OutputFormat) Json 

func (OutputFormat) Json() OutputFormat

func (OutputFormat) None 

func (OutputFormat) None() OutputFormat

func (*OutputFormat) Parse 

func (of *OutputFormat) Parse(s string) error

func (OutputFormat) String 

func (of OutputFormat) String() string

func (OutputFormat) Text 

func (OutputFormat) Text() OutputFormat

type OutputVerbosity added in v10.16.0 

type OutputVerbosity uint8

func (OutputVerbosity) Default added in v10.16.0 

func (OutputVerbosity) Default() OutputVerbosity

func (OutputVerbosity) Essential added in v10.16.0 

func (OutputVerbosity) Essential() OutputVerbosity

func (*OutputVerbosity) Parse added in v10.16.0 

func (qm *OutputVerbosity) Parse(s string) error

func (OutputVerbosity) Quiet added in v10.16.0 

func (OutputVerbosity) Quiet() OutputVerbosity

func (OutputVerbosity) String added in v10.16.0 

func (qm OutputVerbosity) String() string

type OverwriteOption 

type OverwriteOption uint8

func (OverwriteOption) False 

func (OverwriteOption) False() OverwriteOption

func (OverwriteOption) IfSourceNewer 

func (OverwriteOption) IfSourceNewer() OverwriteOption

func (*OverwriteOption) Parse 

func (o *OverwriteOption) Parse(s string) error

func (OverwriteOption) PosixProperties added in v10.17.0 

func (OverwriteOption) PosixProperties() OverwriteOption

func (OverwriteOption) Prompt 

func (OverwriteOption) Prompt() OverwriteOption

func (OverwriteOption) String 

func (o OverwriteOption) String() string

func (OverwriteOption) True 

func (OverwriteOption) True() OverwriteOption

type PSTokenProvider added in v10.22.0 

type PSTokenProvider func(ctx context.Context, resource string, tenant string) ([]byte, error)

type PageBlobTier 

type PageBlobTier uint8

func (PageBlobTier) MarshalJSON 

func (pbt PageBlobTier) MarshalJSON() ([]byte, error)

func (PageBlobTier) None 

func (PageBlobTier) None() PageBlobTier

func (PageBlobTier) P10 

func (PageBlobTier) P10() PageBlobTier

func (PageBlobTier) P15 

func (PageBlobTier) P15() PageBlobTier

func (PageBlobTier) P20 

func (PageBlobTier) P20() PageBlobTier

func (PageBlobTier) P30 

func (PageBlobTier) P30() PageBlobTier

func (PageBlobTier) P4 

func (PageBlobTier) P4() PageBlobTier

func (PageBlobTier) P40 

func (PageBlobTier) P40() PageBlobTier

func (PageBlobTier) P50 

func (PageBlobTier) P50() PageBlobTier

func (PageBlobTier) P6 

func (PageBlobTier) P6() PageBlobTier

func (*PageBlobTier) Parse 

func (pbt *PageBlobTier) Parse(s string) error

func (PageBlobTier) String 

func (pbt PageBlobTier) String() string

func (PageBlobTier) ToAccessTierType 

func (pbt PageBlobTier) ToAccessTierType() blob.AccessTier

func (*PageBlobTier) UnmarshalJSON 

func (pbt *PageBlobTier) UnmarshalJSON(b []byte) error

Implementing UnmarshalJSON() method for type BlockBlobTier.

type PartNumber 

type PartNumber uint32

type PerfAdjustmentReq added in v10.15.0 

type PerfAdjustmentReq struct {
	Throughput int64 `json:"cap-mbps,string"`
}

PerfAdjustment message.

type PerfAdjustmentResp added in v10.15.0 

type PerfAdjustmentResp struct {
	Status              bool      `json:"status"`
	AdjustedThroughPut  int64     `json:"cap-mbps"`
	NextAdjustmentAfter time.Time `json:"NextAdjustmentAfter"`
	Err                 string    `json:"error"`
}

func (PerfAdjustmentResp) Json added in v10.15.0 

func (p PerfAdjustmentResp) Json() string

func (PerfAdjustmentResp) String added in v10.15.0 

func (p PerfAdjustmentResp) String() string

type PerfConstraint 

type PerfConstraint int32

func (PerfConstraint) CPU 

func (PerfConstraint) CPU() PerfConstraint

func (PerfConstraint) Disk 

func (PerfConstraint) Disk() PerfConstraint

func (PerfConstraint) PageBlobService 

func (PerfConstraint) PageBlobService() PerfConstraint

func (*PerfConstraint) Parse 

func (pc *PerfConstraint) Parse(s string) error

func (PerfConstraint) Service 

func (PerfConstraint) Service() PerfConstraint

func (PerfConstraint) String 

func (pc PerfConstraint) String() string

func (PerfConstraint) Unknown 

func (PerfConstraint) Unknown() PerfConstraint

type PerformanceAdvice 

type PerformanceAdvice struct {

	// Code representing the type of the advice
	Code string `json:"Code"` // reminder that PerformanceAdvice may be serialized in JSON output

	// Human-friendly title (directly corresponds to Code, but more readable)
	Title string

	// Reason why this advice has been given
	Reason string

	// Is this the primary advice (used to distinguish most important advice in cases where multiple advice objects are returned)
	PriorityAdvice bool
}

type PermanentDeleteOption added in v10.14.0 

type PermanentDeleteOption uint8

func (PermanentDeleteOption) None added in v10.14.0 

func (PermanentDeleteOption) None() PermanentDeleteOption

func (*PermanentDeleteOption) Parse added in v10.14.0 

func (p *PermanentDeleteOption) Parse(s string) error

func (PermanentDeleteOption) Snapshots added in v10.14.0 

func (PermanentDeleteOption) Snapshots() PermanentDeleteOption

func (PermanentDeleteOption) SnapshotsAndVersions added in v10.14.0 

func (PermanentDeleteOption) SnapshotsAndVersions() PermanentDeleteOption

func (PermanentDeleteOption) String added in v10.14.0 

func (p PermanentDeleteOption) String() string

func (PermanentDeleteOption) ToPermanentDeleteOptionType added in v10.14.0 

func (p PermanentDeleteOption) ToPermanentDeleteOptionType() *blob.DeleteType

func (PermanentDeleteOption) Versions added in v10.14.0 

func (PermanentDeleteOption) Versions() PermanentDeleteOption

type PowershellContextCredential added in v10.22.0 

type PowershellContextCredential struct {
	// contains filtered or unexported fields
}

PowershellContextCredential authenticates as the identity logged in to the Azure Developer CLI.

func NewPowershellContextCredential added in v10.22.0 

func NewPowershellContextCredential(options *PowershellContextCredentialOptions) (*PowershellContextCredential, error)

NewPowershellContextCredential constructs an AzureDeveloperCLICredential. Pass nil to accept default options.

func (*PowershellContextCredential) GetToken added in v10.22.0 

func (c *PowershellContextCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error)

GetToken requests a token from the Azure Developer CLI. This credential doesn't cache tokens, so every call invokes azd. This method is called automatically by Azure SDK clients.

type PowershellContextCredentialOptions added in v10.22.0 

type PowershellContextCredentialOptions struct {
	// TenantID identifies the tenant the credential should authenticate in. Defaults to the azd environment,
	// which is the tenant of the selected Azure subscription.
	TenantID string
	// contains filtered or unexported fields
}

PowershellContextCredentialOptions contains optional parameters for AzureDeveloperCLICredential.

type Predicate 

type Predicate func() bool

type PreservePermissionsOption 

type PreservePermissionsOption uint8

func NewPreservePermissionsOption 

func NewPreservePermissionsOption(preserve, includeOwnership bool, fromTo FromTo) PreservePermissionsOption

func (PreservePermissionsOption) ACLsOnly 

func (PreservePermissionsOption) ACLsOnly() PreservePermissionsOption

func (PreservePermissionsOption) IsTruthy 

func (p PreservePermissionsOption) IsTruthy() bool

func (PreservePermissionsOption) None 

func (PreservePermissionsOption) None() PreservePermissionsOption

func (PreservePermissionsOption) OwnershipAndACLs 

func (PreservePermissionsOption) OwnershipAndACLs() PreservePermissionsOption

type PrologueState 

type PrologueState struct {
	// Leading bytes are the early bytes of the file, to be used
	// for mime-type detection (or nil if file is empty or the bytes code
	// not be read).
	LeadingBytes []byte
}

PrologueState contains info necessary for different sending operations' prologue.

func (PrologueState) GetInferredContentType 

func (ps PrologueState) GetInferredContentType(jptm cutdownJptm) *string

type PromptDetails 

type PromptDetails struct {
	PromptType      PromptType
	ResponseOptions []ResponseOption // used from prompt messages where we expect a response
	PromptTarget    string           // used when prompt message is targeting a specific resource, ease partner team integration
}

type PromptType 

type PromptType string

func (PromptType) Cancel 

func (PromptType) Cancel() PromptType

func (PromptType) DeleteDestination 

func (PromptType) DeleteDestination() PromptType

func (PromptType) Overwrite 

func (PromptType) Overwrite() PromptType

type Prompter added in v10.12.0 

type Prompter interface {
	ShouldOverwrite(objectPath string, objectType EntityType) bool
}

type ProxyLookupFunc 

type ProxyLookupFunc func(req *http.Request) (*url.URL, error) // signature of normal Transport.Proxy lookup

var GlobalProxyLookup ProxyLookupFunc

type RehydratePriorityType added in v10.16.0 

type RehydratePriorityType uint8

//////////////////////////////////////////////////////////////////////////////

func (RehydratePriorityType) High added in v10.16.0 

func (RehydratePriorityType) High() RehydratePriorityType

func (RehydratePriorityType) None added in v10.16.0 

func (RehydratePriorityType) None() RehydratePriorityType

func (*RehydratePriorityType) Parse added in v10.16.0 

func (rpt *RehydratePriorityType) Parse(s string) error

func (RehydratePriorityType) Standard added in v10.16.0 

func (RehydratePriorityType) Standard() RehydratePriorityType

func (RehydratePriorityType) String added in v10.16.0 

func (rpt RehydratePriorityType) String() string

func (RehydratePriorityType) ToRehydratePriorityType added in v10.16.0 

func (rpt RehydratePriorityType) ToRehydratePriorityType() blob.RehydratePriority

type ResourceHTTPHeaders 

type ResourceHTTPHeaders struct {
	ContentType        string
	ContentMD5         []byte
	ContentEncoding    string
	ContentLanguage    string
	ContentDisposition string
	CacheControl       string
}

Common resource's HTTP headers stands for properties used in AzCopy.

func (ResourceHTTPHeaders) ToBlobFSHTTPHeaders 

func (h ResourceHTTPHeaders) ToBlobFSHTTPHeaders() datalakefile.HTTPHeaders

ToBlobFSHTTPHeaders converts ResourceHTTPHeaders to BlobFS Headers.

func (ResourceHTTPHeaders) ToBlobHTTPHeaders added in v10.21.0 

func (h ResourceHTTPHeaders) ToBlobHTTPHeaders() blob.HTTPHeaders

ToBlobHTTPHeaders converts ResourceHTTPHeaders to blob's HTTPHeaders.

func (ResourceHTTPHeaders) ToFileHTTPHeaders added in v10.21.0 

func (h ResourceHTTPHeaders) ToFileHTTPHeaders() sharefile.HTTPHeaders

ToFileHTTPHeaders converts ResourceHTTPHeaders to sharefile's HTTPHeaders.

type ResourceString 

type ResourceString struct {
	Value      string
	SAS        string // SAS should NOT be persisted in the plan files (both for security reasons, and because, at the time of any resume, it may be stale anyway. Resume requests fresh SAS on command line)
	ExtraQuery string
}

ResourceString represents a source or dest string, that can have three parts: the main part, a sas, and extra query parameters that are not part of the sas.

func (ResourceString) Clone 

func (r ResourceString) Clone() ResourceString

func (ResourceString) CloneWithConsolidatedSeparators 

func (r ResourceString) CloneWithConsolidatedSeparators() ResourceString

func (ResourceString) CloneWithValue 

func (r ResourceString) CloneWithValue(newValue string) ResourceString

func (ResourceString) FullURL 

func (r ResourceString) FullURL() (*url.URL, error)

func (ResourceString) String added in v10.21.0 

func (r ResourceString) String() (string, error)

func (ResourceString) ValueLocal 

func (r ResourceString) ValueLocal() string

to be used when the value is assumed to be a local path Using this signals "Yes, I really am ignoring the SAS and ExtraQuery on purpose", and will result in a panic in the case of programmer error of calling this method when those fields have values

type ResponseOption 

type ResponseOption struct {
	ResponseType             string // helps us clarify the user's intent and support partner team's localization
	UserFriendlyResponseType string // text to print in interactive mode
	ResponseString           string // short (abbreviation) string that gets sent back by the user to indicate that this response is chosen
}

func (ResponseOption) Default 

func (ResponseOption) Default() ResponseOption

func (ResponseOption) No 

func (ResponseOption) No() ResponseOption

func (ResponseOption) NoForAll 

func (ResponseOption) NoForAll() ResponseOption

func (*ResponseOption) Parse 

func (o *ResponseOption) Parse(s string) error

func (ResponseOption) Yes 

func (ResponseOption) Yes() ResponseOption

NOTE: these enums are shared with StgExp, so the text is spelled out explicitly (for easy json marshalling)

func (ResponseOption) YesForAll 

func (ResponseOption) YesForAll() ResponseOption

type ResumeJobRequest 

type ResumeJobRequest struct {
	JobID            JobID
	SourceSAS        string
	DestinationSAS   string
	SrcServiceClient *ServiceClient
	DstServiceClient *ServiceClient
	IncludeTransfer  map[string]int
	ExcludeTransfer  map[string]int
	CredentialInfo   CredentialInfo
}

type RetryCounter 

type RetryCounter interface {
	GetTotalRetries() int64
}

type RpcCmd 

type RpcCmd string

JobStatus indicates the status of a Job; the default is InProgress.

func (RpcCmd) CancelJob 

func (RpcCmd) CancelJob() RpcCmd

func (RpcCmd) CopyJobPartOrder 

func (RpcCmd) CopyJobPartOrder() RpcCmd

func (RpcCmd) GetJobFromTo 

func (RpcCmd) GetJobFromTo() RpcCmd

func (RpcCmd) GetJobLCMWrapper 

func (RpcCmd) GetJobLCMWrapper() RpcCmd

func (RpcCmd) ListJobSummary 

func (RpcCmd) ListJobSummary() RpcCmd

func (RpcCmd) ListJobTransfers 

func (RpcCmd) ListJobTransfers() RpcCmd

func (RpcCmd) ListJobs 

func (RpcCmd) ListJobs() RpcCmd

func (RpcCmd) ListSyncJobSummary 

func (RpcCmd) ListSyncJobSummary() RpcCmd

func (RpcCmd) None 

func (RpcCmd) None() RpcCmd

func (*RpcCmd) Parse 

func (c *RpcCmd) Parse(s string) error

func (RpcCmd) Pattern 

func (c RpcCmd) Pattern() string

func (RpcCmd) PauseJob 

func (RpcCmd) PauseJob() RpcCmd

func (RpcCmd) ResumeJob 

func (RpcCmd) ResumeJob() RpcCmd

func (RpcCmd) String 

func (c RpcCmd) String() string

type S3ClientFactory 

type S3ClientFactory struct {
	// contains filtered or unexported fields
}

func NewS3ClientFactory 

func NewS3ClientFactory() S3ClientFactory

NewS3ClientFactory creates new S3 client factory.

func (*S3ClientFactory) GetS3Client 

func (f *S3ClientFactory) GetS3Client(ctx context.Context, credInfo CredentialInfo, option CredentialOpOptions, logger ILogger) (*minio.Client, error)

GetS3Client gets S3 client from pool, or create a new S3 client if no client created for specific credInfo.

type S3CredentialInfo 

type S3CredentialInfo struct {
	Endpoint string
	Region   string
}

S3CredentialInfo contains essential credential info which need to build up S3 client.

type S3HTTPTraceLogger added in v10.15.0 

type S3HTTPTraceLogger struct {
	// contains filtered or unexported fields
}

func NewS3HTTPTraceLogger added in v10.15.0 

func NewS3HTTPTraceLogger(logger ILogger, level LogLevel) S3HTTPTraceLogger

func (S3HTTPTraceLogger) Write added in v10.15.0 

func (e S3HTTPTraceLogger) Write(msg []byte) (n int, err error)

type S3URLParts 

type S3URLParts struct {
	Scheme         string // Ex: "https://", "s3://"
	Host           string // Ex: "s3.amazonaws.com", "s3-eu-west-1.amazonaws.com", "bucket.s3-eu-west-1.amazonaws.com"
	Endpoint       string // Ex: "s3.amazonaws.com", "s3-eu-west-1.amazonaws.com"
	BucketName     string // Ex: "MyBucket"
	ObjectKey      string // Ex: "hello.txt", "foo/bar"
	Version        string
	Region         string // Ex: endpoint region, e.g. "eu-west-1"
	UnparsedParams string
	// contains filtered or unexported fields
}

S3URLParts represents the components that make up AWS S3 Service/Bucket/Object URL. You parse an existing URL into its parts by calling NewS3URLParts(). According to http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro and https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region, S3URLParts supports virtual-hosted-style and path-style URL: Ex, virtual-hosted-style(the bucket name is part of the domain name in the URL) : a. http://bucket.s3.amazonaws.com b. http://bucket.s3-aws-region.amazonaws.com Ex, path-style URL(the bucket name is not part of the domain (unless you use a Region-specific endpoint)): a. http://s3.amazonaws.com/bucket (US East (N. Virginia) Region endpoint) b. http://s3-aws-region.amazonaws.com/bucket (Region-specific endpoint) Dual stack endpoint(IPv6&IPv4) is also supported (https://docs.aws.amazon.com/AmazonS3/latest/dev/dual-stack-endpoints.html#dual-stack-endpoints-description) i.e. the endpoint in http://bucketname.s3.dualstack.aws-region.amazonaws.com or http://s3.dualstack.aws-region.amazonaws.com/bucketname

func NewS3URLParts 

func NewS3URLParts(u url.URL) (S3URLParts, error)

NewS3URLParts parses a URL initializing S3URLParts' fields. This method overwrites all fields in the S3URLParts object.

func (*S3URLParts) IsBucketSyntactically 

func (p *S3URLParts) IsBucketSyntactically() bool

func (*S3URLParts) IsDirectorySyntactically 

func (p *S3URLParts) IsDirectorySyntactically() bool

IsDirectorySyntactically validates if the S3URLParts is indicating a directory. Note: directory in S3 is a virtual abstract, and a object as well.

func (*S3URLParts) IsObjectSyntactically 

func (p *S3URLParts) IsObjectSyntactically() bool

func (*S3URLParts) IsServiceSyntactically 

func (p *S3URLParts) IsServiceSyntactically() bool

func (*S3URLParts) String 

func (p *S3URLParts) String() string

func (*S3URLParts) URL 

func (p *S3URLParts) URL() url.URL

URL returns a URL object whose fields are initialized from the S3URLParts fields.

type SPNInfo 

type SPNInfo struct {
	// Secret is used for two purposes: The certificate secret, and a client secret.
	// The secret is persisted to the JSON file because AAD does not issue a refresh token.
	// Thus, the original secret is needed to refresh.
	Secret   string `json:"_spn_secret"`
	CertPath string `json:"_spn_cert_path"`
}

SPNInfo contains info for authenticating with Service Principal Names

type ScopedCredential added in v10.22.0 

type ScopedCredential struct {
	// contains filtered or unexported fields
}

func NewScopedCredential added in v10.23.0 

func NewScopedCredential(cred azcore.TokenCredential, credType CredentialType) *ScopedCredential

ScopedCredential takes in a credInfo object and returns ScopedCredential if credentialType is either MDOAuth or oAuth. For anything else, nil is returned

func (*ScopedCredential) GetToken added in v10.23.0 

func (s *ScopedCredential) GetToken(ctx context.Context,
	_ policy.TokenRequestOptions) (
	azcore.AccessToken, error)

type ServiceClient added in v10.22.0 

type ServiceClient struct {
	// contains filtered or unexported fields
}

func GetServiceClientForLocation added in v10.22.0 

func GetServiceClientForLocation(loc Location,
	resource ResourceString,
	credType CredentialType,
	cred azcore.TokenCredential,
	policyOptions *azcore.ClientOptions,
	locationSpecificOptions any,
) (*ServiceClient, error)

GetServiceClientForLocation returns service client for the resourceURL. It strips the container and file related details before creating the client. locationSpecificOptions are required currently only for files.

func NewServiceClient added in v10.23.0 

func NewServiceClient(bsc *blobservice.Client,
	fsc *fileservice.Client,
	dsc *datalake.Client) *ServiceClient

This is currently used only in testcases

func (*ServiceClient) BlobServiceClient added in v10.22.0 

func (s *ServiceClient) BlobServiceClient() (*blobservice.Client, error)

func (*ServiceClient) DatalakeServiceClient added in v10.22.0 

func (s *ServiceClient) DatalakeServiceClient() (*datalake.Client, error)

func (*ServiceClient) FileServiceClient added in v10.22.0 

func (s *ServiceClient) FileServiceClient() (*fileservice.Client, error)

type SetPropertiesFlags added in v10.16.0 

type SetPropertiesFlags uint32 // [0000000000...32 times]

//////////////////////////////////////////////////////////////////////////////

func (SetPropertiesFlags) None added in v10.16.0 

func (SetPropertiesFlags) None() SetPropertiesFlags

functions to set values

func (SetPropertiesFlags) SetBlobTags added in v10.16.0 

func (SetPropertiesFlags) SetBlobTags() SetPropertiesFlags

func (SetPropertiesFlags) SetMetadata added in v10.16.0 

func (SetPropertiesFlags) SetMetadata() SetPropertiesFlags

func (SetPropertiesFlags) SetTier added in v10.16.0 

func (SetPropertiesFlags) SetTier() SetPropertiesFlags

func (*SetPropertiesFlags) ShouldTransferBlobTags added in v10.16.0 

func (op *SetPropertiesFlags) ShouldTransferBlobTags() bool

func (*SetPropertiesFlags) ShouldTransferMetaData added in v10.16.0 

func (op *SetPropertiesFlags) ShouldTransferMetaData() bool

func (*SetPropertiesFlags) ShouldTransferTier added in v10.16.0 

func (op *SetPropertiesFlags) ShouldTransferTier() bool

functions to get values (to be used in sde) If Y is inside X then X & Y == Y

type SingleChunkReader 

type SingleChunkReader interface {

	// ReadSeeker is used to read the contents of the chunk, and because the sending pipeline seeks at various times
	io.ReadSeeker

	// Closer is needed to clean up resources
	io.Closer

	// BlockingPrefetch tries to read the full contents of the chunk into RAM.
	BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error

	// GetPrologueState is used to grab enough of the initial bytes to do MIME-type detection.  Expected to be called only
	// on the first chunk in each file (since there's no point in calling it on others)
	// There is deliberately no error return value from the Prologue.
	// If it failed, the Prologue itself must call jptm.FailActiveSend.
	GetPrologueState() PrologueState

	// Length is the number of bytes in the chunk
	Length() int64

	// HasPrefectchedEntirelyZeros gives an indication of whether this chunk is entirely zeros.  If it returns true
	// then the chunk content has been prefetched AND it was all zeroes. For some remote destinations, that support "sparse file"
	// semantics, it is safe and correct to skip the upload of those chunks where this returns true.
	// In the rare edge case where this returns false due to the prefetch having failed (rather than the contents being non-zero),
	// we'll just treat it as a non-zero chunk. That's simpler (to code, to review and to test) than having this code force a prefetch.
	HasPrefetchedEntirelyZeros() bool

	// WriteBufferTo writes the entire contents of the prefetched buffer to h
	// Panics if the internal buffer has not been prefetched (or if its been discarded after a complete Read)
	WriteBufferTo(h hash.Hash)
}

Reader of ONE chunk of a file. Maybe used to re-read multiple times (e.g. if we must retry the sending of the chunk). A instance of this type cannot be used by multiple threads (since it's Read/Seek are inherently stateful) The reader can throw away the data after each successful read, and then re-read it from disk if there is a need to retry the transmission of the chunk. That saves us the RAM cost of from having to keep every transmitted chunk in RAM until acknowledged by the service. We just re-read if the service says we need to retry. Although there's a time (performance) cost in the re-read, that's fine in a retry situation because the retry indicates we were going too fast for the service anyway.

func NewSingleChunkReader 

func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFactory, chunkId ChunkID, length int64, chunkLogger ChunkStatusLogger, generalLogger ILogger, slicePool ByteSlicePooler, cacheLimiter CacheLimiter) SingleChunkReader

type Status 

type Status uint32

type SymlinkHandlingType added in v10.18.0 

type SymlinkHandlingType uint8 // SymlinkHandlingType is only utilized internally to avoid having to carry around two contradictory flags. Thus, it doesn't have a parse method.

//////////////////////////////////////////////////////////////////////////////

func (*SymlinkHandlingType) Determine added in v10.18.0 

func (sht *SymlinkHandlingType) Determine(Follow, Preserve bool) error

func (SymlinkHandlingType) Follow added in v10.18.0 

func (sht SymlinkHandlingType) Follow() bool

func (SymlinkHandlingType) None added in v10.18.0 

func (sht SymlinkHandlingType) None() bool

func (SymlinkHandlingType) Preserve added in v10.18.0 

func (sht SymlinkHandlingType) Preserve() bool

type SyncHashData added in v10.17.0 

type SyncHashData struct {
	Mode SyncHashType
	Data string // base64 encoded
	LMT  time.Time
}

type SyncHashType added in v10.17.0 

type SyncHashType uint8

////////////////////////////////////////////////////////////////////////////// 

var ESyncHashType SyncHashType = 0

func (SyncHashType) MD5 added in v10.17.0 

func (SyncHashType) MD5() SyncHashType

func (SyncHashType) None added in v10.17.0 

func (SyncHashType) None() SyncHashType

func (*SyncHashType) Parse added in v10.17.0 

func (ht *SyncHashType) Parse(s string) error

func (SyncHashType) String added in v10.17.0 

func (ht SyncHashType) String() string

type TestOAuthInjection 

type TestOAuthInjection struct {
	DoTokenRefreshInjection bool
	TokenRefreshDuration    time.Duration
}

TestOAuthInjection controls variables for OAuth testing injections

type TokenStoreCredential added in v10.21.0 

type TokenStoreCredential struct {
	// contains filtered or unexported fields
}

func (*TokenStoreCredential) GetToken added in v10.21.0 

func (tsc *TokenStoreCredential) GetToken(_ context.Context, _ policy.TokenRequestOptions) (azcore.AccessToken, error)

type TrailingDotOption added in v10.19.0 

type TrailingDotOption uint8

func (TrailingDotOption) Disable added in v10.19.0 

func (TrailingDotOption) Disable() TrailingDotOption

func (TrailingDotOption) Enable added in v10.19.0 

func (TrailingDotOption) Enable() TrailingDotOption

func (*TrailingDotOption) Parse added in v10.19.0 

func (d *TrailingDotOption) Parse(s string) error

func (TrailingDotOption) String added in v10.19.0 

func (d TrailingDotOption) String() string

type TransferDetail 

type TransferDetail struct {
	Src                string
	Dst                string
	IsFolderProperties bool
	TransferStatus     TransferStatus
	TransferSize       uint64
	ErrorCode          int32 `json:",string"`
}

represents the Details and details of a single transfer

type TransferDirection 

type TransferDirection int32

func (*TransferDirection) AtomicLoad 

func (td *TransferDirection) AtomicLoad() TransferDirection

func (*TransferDirection) AtomicStore 

func (td *TransferDirection) AtomicStore(newTransferDirection TransferDirection)

func (TransferDirection) Download 

func (TransferDirection) Download() TransferDirection

func (*TransferDirection) Parse 

func (td *TransferDirection) Parse(s string) error

func (TransferDirection) S2SCopy 

func (TransferDirection) S2SCopy() TransferDirection

func (TransferDirection) String 

func (td TransferDirection) String() string

func (TransferDirection) UnKnown 

func (TransferDirection) UnKnown() TransferDirection

func (TransferDirection) Upload 

func (TransferDirection) Upload() TransferDirection

type TransferStatus 

type TransferStatus int32 // Must be 32-bit for atomic operations; negative #s represent a specific failure code

func (TransferStatus) All 

func (TransferStatus) All() TransferStatus

Transfer is any of the three possible state (InProgress, Completer or Failed)

func (*TransferStatus) AtomicLoad 

func (ts *TransferStatus) AtomicLoad() TransferStatus

func (*TransferStatus) AtomicStore 

func (ts *TransferStatus) AtomicStore(newTransferStatus TransferStatus)

func (TransferStatus) BlobTierFailure 

func (TransferStatus) BlobTierFailure() TransferStatus

Transfer failed due to failure while Setting blob tier.

func (TransferStatus) Cancelled 

func (TransferStatus) Cancelled() TransferStatus

func (TransferStatus) Failed 

func (TransferStatus) Failed() TransferStatus

Transfer failed due to some error.

func (TransferStatus) FolderCreated added in v10.12.0 

func (TransferStatus) FolderCreated() TransferStatus

Folder was created, but properties have not been persisted yet. Equivalent to Started, but never intended to be set on anything BUT folders.

func (TransferStatus) MarshalJSON 

func (ts TransferStatus) MarshalJSON() ([]byte, error)

Implementing MarshalJSON() method for type Transfer Status

func (TransferStatus) NotStarted 

func (TransferStatus) NotStarted() TransferStatus

Transfer is ready to transfer and not started transferring yet

func (*TransferStatus) Parse 

func (ts *TransferStatus) Parse(s string) error

func (TransferStatus) Restarted added in v10.18.0 

func (TransferStatus) Restarted() TransferStatus

func (TransferStatus) SkippedBlobHasSnapshots 

func (TransferStatus) SkippedBlobHasSnapshots() TransferStatus

func (TransferStatus) SkippedEntityAlreadyExists 

func (TransferStatus) SkippedEntityAlreadyExists() TransferStatus

func (TransferStatus) Started 

func (TransferStatus) Started() TransferStatus

TODO confirm whether this is actually needed 

Outdated:
  Transfer started & at least 1 chunk has successfully been transferred.
  Used to resume a transfer that started to avoid transferring all chunks thereby improving performance

Update(Jul 2020): This represents the state of transfer as soon as the file is scheduled.

func (TransferStatus) StatusLocked added in v10.17.0 

func (t TransferStatus) StatusLocked() bool

func (TransferStatus) String 

func (ts TransferStatus) String() string

func (TransferStatus) Success 

func (TransferStatus) Success() TransferStatus

Transfer successfully completed

func (TransferStatus) TierAvailabilityCheckFailure 

func (TransferStatus) TierAvailabilityCheckFailure() TransferStatus

func (*TransferStatus) UnmarshalJSON 

func (ts *TransferStatus) UnmarshalJSON(b []byte) error

Implementing UnmarshalJSON() method for type Transfer Status

type Transfers 

type Transfers struct {
	List                 []CopyTransfer
	TotalSizeInBytes     uint64
	FileTransferCount    uint32
	FolderTransferCount  uint32
	SymlinkTransferCount uint32
}

Transfers describes each file/folder being transferred in a given JobPartOrder, and other auxiliary details of this order.

type URLExtension 

type URLExtension struct {
	url.URL
}

///////////////////////////////////////////////////////////////////////////////////////////////

func (URLExtension) RedactSecretQueryParamForLogging 

func (u URLExtension) RedactSecretQueryParamForLogging() string

func (URLExtension) URLWithPlusDecodedInPath 

func (u URLExtension) URLWithPlusDecodedInPath() url.URL

URLWithPlusDecodedInPath returns a URL with '+' in path decoded as ' '(space). This is useful for the cases, e.g: S3 management console encode ' '(space) as '+', which is not supported by Azure resources.

type URLStringExtension 

type URLStringExtension string

///////////////////////////////////////////////////////////////////////////////////////////////

func (URLStringExtension) RedactSecretQueryParamForLogging 

func (s URLStringExtension) RedactSecretQueryParamForLogging() string

type UUID 

type UUID struct {
	D1 uint32
	D2 uint16
	D3 uint16
	D4 [8]uint8
}

A UUID representation compliant with specification in RFC 4122 document.

func NewUUID 

func NewUUID() (u UUID)

NewUUID returns a new UUID using RFC 4122 algorithm.

func ParseUUID 

func ParseUUID(uuidStr string) (UUID, error)

ParseUUID parses a string formatted as "003020100-0504-0706-0809-0a0b0c0d0e0f" or "{03020100-0504-0706-0809-0a0b0c0d0e0f}" into a UUID.

func (UUID) MarshalJSON 

func (u UUID) MarshalJSON() ([]byte, error)

Implementing MarshalJSON() method for type UUID

func (UUID) String 

func (u UUID) String() string

String returns an unparsed version of the generated UUID sequence.

func (*UUID) UnmarshalJSON 

func (u *UUID) UnmarshalJSON(b []byte) error

Implementing UnmarshalJSON() method for type UUID

type UnixStatAdapter added in v10.16.0 

type UnixStatAdapter interface {
	Extended() bool // Did this call come from StatX?

	// Statx properties
	StatxMask() uint32 // Mask determines the availability of all stat/stax properties (except the device ID!). It is only used in statx though.
	Attribute() uint64 // Attribute is masked by AttributeMask.
	AttributeMask() uint64
	BTime() time.Time // BTime may not always be available on every filesystem. It's important to check Mask first!

	// Base Stat properties
	NLink() uint64
	Owner() uint32
	Group() uint32
	FileMode() uint32 // Mode may not always be available to check in a Statx call (though it should be, since we requested it.) Best safe than sorry; check Mask!
	INode() uint64
	Device() uint64
	RDevice() uint64 // RDevice is ONLY useful when Mode has S_IFCHR or S_IFBLK; as those determine if the file is a representitive of a block or character device.
	ATime() time.Time
	MTime() time.Time
	CTime() time.Time
}

func ReadStatFromMetadata added in v10.16.0 

func ReadStatFromMetadata(metadata Metadata, contentLength int64) (UnixStatAdapter, error)

ReadStatFromMetadata is not fault-tolerant. If any given article does not parse, it will throw an error instead of continuing on, as it may be considered incorrect to attempt to persist the rest of the data. despite this function being used only in Downloads at the current moment, it still attempts to re-create as complete of a UnixStatAdapter as possible.

type UnixStatContainer added in v10.16.0 

type UnixStatContainer struct {
	// contains filtered or unexported fields
}

func (UnixStatContainer) ATime added in v10.16.0 

func (u UnixStatContainer) ATime() time.Time

func (UnixStatContainer) Attribute added in v10.16.0 

func (u UnixStatContainer) Attribute() uint64

func (UnixStatContainer) AttributeMask added in v10.16.0 

func (u UnixStatContainer) AttributeMask() uint64

func (UnixStatContainer) BTime added in v10.16.0 

func (u UnixStatContainer) BTime() time.Time

func (UnixStatContainer) CTime added in v10.16.0 

func (u UnixStatContainer) CTime() time.Time

func (UnixStatContainer) Device added in v10.16.0 

func (u UnixStatContainer) Device() uint64

func (UnixStatContainer) Extended added in v10.16.0 

func (u UnixStatContainer) Extended() bool

func (UnixStatContainer) FileMode added in v10.16.0 

func (u UnixStatContainer) FileMode() uint32

func (UnixStatContainer) Group added in v10.16.0 

func (u UnixStatContainer) Group() uint32

func (UnixStatContainer) INode added in v10.16.0 

func (u UnixStatContainer) INode() uint64

func (UnixStatContainer) MTime added in v10.16.0 

func (u UnixStatContainer) MTime() time.Time
func (u UnixStatContainer) NLink() uint64

func (UnixStatContainer) Owner added in v10.16.0 

func (u UnixStatContainer) Owner() uint32

func (UnixStatContainer) RDevice added in v10.16.0 

func (u UnixStatContainer) RDevice() uint64

func (UnixStatContainer) StatxMask added in v10.16.0 

func (u UnixStatContainer) StatxMask() uint32

type UserOAuthTokenManager 

type UserOAuthTokenManager struct {
	// contains filtered or unexported fields
}

UserOAuthTokenManager for token management.

func NewUserOAuthTokenManagerInstance 

func NewUserOAuthTokenManagerInstance(credCacheOptions CredCacheOptions) *UserOAuthTokenManager

NewUserOAuthTokenManagerInstance creates a token manager instance.

func (*UserOAuthTokenManager) AzCliLogin added in v10.22.0 

func (uotm *UserOAuthTokenManager) AzCliLogin(tenantID string) error

func (*UserOAuthTokenManager) CertLogin 

func (uotm *UserOAuthTokenManager) CertLogin(tenantID, activeDirectoryEndpoint, certPath, certPass, applicationID string, persist bool) error

CertLogin non-interactively logs in using a specified certificate, certificate password, and activedirectory endpoint.

func (*UserOAuthTokenManager) GetTokenInfo 

func (uotm *UserOAuthTokenManager) GetTokenInfo(ctx context.Context) (*OAuthTokenInfo, error)

GetTokenInfo gets token info, it follows rule:

  1. If there is token passed from environment variable(note this is only for testing purpose), use token passed from environment variable.
  2. Otherwise, try to get token from cache.

This method either successfully return token, or return error.

func (*UserOAuthTokenManager) HasCachedToken 

func (uotm *UserOAuthTokenManager) HasCachedToken() (bool, error)

HasCachedToken returns if there is cached token in token manager.

func (*UserOAuthTokenManager) MSILogin 

func (uotm *UserOAuthTokenManager) MSILogin(identityInfo IdentityInfo, persist bool) error

MSILogin tries to get token from MSI, persist indicates whether to cache the token on local disk.

func (*UserOAuthTokenManager) PSContextToken added in v10.22.0 

func (uotm *UserOAuthTokenManager) PSContextToken(tenantID string) error

func (*UserOAuthTokenManager) RemoveCachedToken 

func (uotm *UserOAuthTokenManager) RemoveCachedToken() error

RemoveCachedToken delete all the cached token.

func (*UserOAuthTokenManager) SecretLogin 

func (uotm *UserOAuthTokenManager) SecretLogin(tenantID, activeDirectoryEndpoint, secret, applicationID string, persist bool) error

SecretLogin is a UOTM shell for secretLoginNoUOTM.

func (*UserOAuthTokenManager) UserLogin 

func (uotm *UserOAuthTokenManager) UserLogin(tenantID, activeDirectoryEndpoint string, persist bool) error

UserLogin interactively logins in with specified tenantID and activeDirectoryEndpoint, persist indicates whether to cache the token on local disk.

type Version 

type Version uint32

type WaitReason 

type WaitReason struct {
	Name string
	// contains filtered or unexported fields
}

WaitReason identifies the one thing that a given chunk is waiting on, at a given moment. Basically = state, phrased in terms of "the thing I'm waiting for"

func (WaitReason) Body 

func (WaitReason) Body() WaitReason

func (WaitReason) BodyReReadDueToMem 

func (WaitReason) BodyReReadDueToMem() WaitReason

func (WaitReason) BodyReReadDueToSpeed 

func (WaitReason) BodyReReadDueToSpeed() WaitReason

func (WaitReason) Cancelled 

func (WaitReason) Cancelled() WaitReason

NOTE: when adding new statuses please renumber to make Cancelled numerically the last, to avoid the need to also change numWaitReasons()

func (WaitReason) ChunkDone 

func (WaitReason) ChunkDone() WaitReason

func (WaitReason) CreateLocalFile 

func (WaitReason) CreateLocalFile() WaitReason

func (WaitReason) DiskIO 

func (WaitReason) DiskIO() WaitReason

func (WaitReason) Epilogue 

func (WaitReason) Epilogue() WaitReason

func (WaitReason) FilePacer 

func (WaitReason) FilePacer() WaitReason

func (WaitReason) HeaderResponse 

func (WaitReason) HeaderResponse() WaitReason

func (WaitReason) LockDestination 

func (WaitReason) LockDestination() WaitReason

func (WaitReason) ModifiedTimeRefresh 

func (WaitReason) ModifiedTimeRefresh() WaitReason

func (WaitReason) Nothing 

func (WaitReason) Nothing() WaitReason

Head (below) has index between GB and Body, just so the ordering is numerical ascending during typical chunk lifetime for both upload and download We use just the first letters of these when displaying perf states as we run (if enabled) so try to keep the first letters unique (except for Done and Cancelled, which are not displayed, and so may duplicate the first letter of something else)

func (WaitReason) OpenLocalSource 

func (WaitReason) OpenLocalSource() WaitReason

func (WaitReason) PriorChunk 

func (WaitReason) PriorChunk() WaitReason

func (WaitReason) QueueToWrite 

func (WaitReason) QueueToWrite() WaitReason

func (WaitReason) RAMToSchedule 

func (WaitReason) RAMToSchedule() WaitReason

func (WaitReason) S2SCopyOnWire 

func (WaitReason) S2SCopyOnWire() WaitReason

func (WaitReason) Sorting 

func (WaitReason) Sorting() WaitReason

func (WaitReason) String 

func (wr WaitReason) String() string

func (WaitReason) WorkerGR 

func (WaitReason) WorkerGR() WaitReason

func (WaitReason) XferStart 

func (WaitReason) XferStart() WaitReason

extra ones for start of uploads (prior to chunk scheduling)

type WorkController 

type WorkController interface {
	Cancel(mgr LifecycleMgr)                                        // handle to cancel the work
	ReportProgressOrExit(mgr LifecycleMgr) (totalKnownCount uint32) // print the progress status, optionally exit the application if work is done
}

for the lifecycleMgr to babysit a job, it must be given a controller to get information about the job

type XAttrHashDataAdapter added in v10.19.0 

type XAttrHashDataAdapter struct {
	// contains filtered or unexported fields
}

XAttrHashDataAdapter stores hash data in the Xattr fields

func (*XAttrHashDataAdapter) GetHashData added in v10.19.0 

func (a *XAttrHashDataAdapter) GetHashData(relativePath string) (*SyncHashData, error)

func (*XAttrHashDataAdapter) GetMode added in v10.19.0 

func (a *XAttrHashDataAdapter) GetMode() HashStorageMode

func (*XAttrHashDataAdapter) SetHashData added in v10.19.0 

func (a *XAttrHashDataAdapter) SetHashData(relativePath string, data *SyncHashData) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.