common

Documentation

Index

• Constants
• Variables
• func CanDownloadFile(signedURL string) error
• func GetAbsolutePath(filePath string) (string, error)
• func GetOid(ctx context.Context) string
• func IsHidden(filePath string) (bool, error)
• func LoadFailedLog(path string) (map[string]RetryObject, error)
• func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error)
• func ParseRootPath(filePath string) (string, error)
• func ProjectToResource(project string) (string, error)
• func ToJSONReader(payload any) (io.Reader, error)
• func WithOid(ctx context.Context, oid string) context.Context
• func WithProgress(ctx context.Context, cb ProgressCallback) context.Context
• type AccessTokenStruct
• type FileDownloadResponseObject
• type FileMetadata
• type FileUploadRequestObject
• type ManifestObject
• type ProgressCallback
  • func GetProgress(ctx context.Context) ProgressCallback
• type ProgressEvent
• type RetryObject

Constants

const (
	// B is bytes
	B int64 = iota
	// KB is kilobytes
	KB int64 = 1 << (10 * iota)
	// MB is megabytes
	MB
	// GB is gigabytes
	GB
	// TB is terabytes
	TB
)

const (
	// DefaultUseShepherd sets whether gen3client will attempt to use the Shepherd / Object Management API
	// endpoints if available.
	// The user can override this default using the `data-client configure` command.
	DefaultUseShepherd = false

	// DefaultMinShepherdVersion is the minimum version of Shepherd that the gen3client will use.
	// Before attempting to use Shepherd, the client will check for Shepherd's version, and if the version is
	// below this number the gen3client will instead warn the user and fall back to fence/indexd.
	// The user can override this default using the `data-client configure` command.
	DefaultMinShepherdVersion = "2.0.0"

	// ShepherdEndpoint is the endpoint postfix for SHEPHERD / the Object Management API
	ShepherdEndpoint = "/mds"

	// ShepherdVersionEndpoint is the endpoint used to check what version of Shepherd a commons has deployed
	ShepherdVersionEndpoint = "/mds/version"

	// IndexdIndexEndpoint is the endpoint postfix for INDEXD index
	IndexdIndexEndpoint = "/index/index"

	// FenceUserEndpoint is the endpoint postfix for FENCE user
	FenceUserEndpoint = "/user/user"

	// FenceDataEndpoint is the endpoint postfix for FENCE data
	FenceDataEndpoint = "/user/data"

	// FenceAccessTokenEndpoint is the endpoint postfix for FENCE access token
	FenceAccessTokenEndpoint = "/user/credentials/api/access_token"

	// FenceDataUploadEndpoint is the endpoint postfix for FENCE data upload
	FenceDataUploadEndpoint = FenceDataEndpoint + "/upload"

	// FenceDataDownloadEndpoint is the endpoint postfix for FENCE data download
	FenceDataDownloadEndpoint = FenceDataEndpoint + "/download"

	// FenceDataMultipartInitEndpoint is the endpoint postfix for FENCE multipart init
	FenceDataMultipartInitEndpoint = FenceDataEndpoint + "/multipart/init"

	// FenceDataMultipartUploadEndpoint is the endpoint postfix for FENCE multipart upload
	FenceDataMultipartUploadEndpoint = FenceDataEndpoint + "/multipart/upload"

	// FenceDataMultipartCompleteEndpoint is the endpoint postfix for FENCE multipart complete
	FenceDataMultipartCompleteEndpoint = FenceDataEndpoint + "/multipart/complete"

	// PathSeparator is os dependent path separator char
	PathSeparator = string(os.PathSeparator)

	// DefaultTimeout is used to set timeout value for http client
	DefaultTimeout = 120 * time.Second

	HeaderContentType   = "Content-Type"
	MIMEApplicationJSON = "application/json"

	// FileSizeLimit is the maximum single file size for non-multipart upload (5GB)
	FileSizeLimit = 5 * GB

	// MultipartFileSizeLimit is the maximum single file size for multipart upload (5TB)
	MultipartFileSizeLimit = 5 * TB
	MinMultipartChunkSize  = 10 * MB

	// MaxRetryCount is the maximum retry number per record
	MaxRetryCount = 5
	MaxWaitTime   = 300

	MaxMultipartParts    = 10000
	MaxConcurrentUploads = 10
	MaxRetries           = 5
)

Variables

var (
	// MinChunkSize is configurable via git config and initialized in init()
	MinChunkSize = 10 * MB
)

Functions

func CanDownloadFile

func CanDownloadFile(signedURL string) error

CanDownloadFile checks if a file can be downloaded from the given signed URL by issuing a ranged GET for a single byte to mimic HEAD behavior.

func GetAbsolutePath

func GetAbsolutePath(filePath string) (string, error)

GetAbsolutePath parses input file path to its absolute path and removes the "~" in the beginning

func GetOid

func GetOid(ctx context.Context) string

GetOid returns the OID from the context, or empty string if not found.

func IsHidden

func IsHidden(filePath string) (bool, error)

func LoadFailedLog

func LoadFailedLog(path string) (map[string]RetryObject, error)

func ParseFilePaths

func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error)

ParseFilePaths generates all possible file paths

func ParseRootPath

func ParseRootPath(filePath string) (string, error)

ParseRootPath parses dirname that has "~" in the beginning

func ProjectToResource

func ProjectToResource(project string) (string, error)

func ToJSONReader

func ToJSONReader(payload any) (io.Reader, error)

func WithOid

func WithOid(ctx context.Context, oid string) context.Context

WithOid returns a new context with the provided OID.

func WithProgress

func WithProgress(ctx context.Context, cb ProgressCallback) context.Context

WithProgress returns a new context with the provided ProgressCallback.

Types

type AccessTokenStruct

type AccessTokenStruct struct {
	AccessToken string `json:"access_token"`
}

type FileDownloadResponseObject

type FileDownloadResponseObject struct {
	DownloadPath string
	Filename     string
	GUID         string
	PresignedURL string
	Range        int64
	Overwrite    bool
	Skip         bool
	Response     *http.Response
	Writer       io.Writer
}

FileDownloadResponseObject defines a object for file download

type FileMetadata

type FileMetadata struct {
	Authz   []string `json:"authz"`
	Aliases []string `json:"aliases"`
	// Metadata is an encoded JSON string of any arbitrary metadata the user wishes to upload.
	Metadata map[string]any `json:"metadata"`
}

FileMetadata defines the metadata accepted by the new object management API, Shepherd

type FileUploadRequestObject

type FileUploadRequestObject struct {
	SourcePath   string
	ObjectKey    string
	FileMetadata FileMetadata
	GUID         string
	PresignedURL string
	Bucket       string `json:"bucket,omitempty"`
}

FileUploadRequestObject defines a object for file upload

type ManifestObject

type ManifestObject struct {
	GUID      string `json:"object_id"`
	SubjectID string `json:"subject_id"`
	Title     string `json:"title"`
	Size      int64  `json:"size"`
}

type ProgressCallback

type ProgressCallback func(ProgressEvent) error

ProgressCallback emits transfer progress updates.

func GetProgress

func GetProgress(ctx context.Context) ProgressCallback

GetProgress returns the ProgressCallback from the context, or nil if not found.

type ProgressEvent

type ProgressEvent struct {
	Event          string         `json:"event"`
	Oid            string         `json:"oid"`
	BytesSoFar     int64          `json:"bytesSoFar"`
	BytesSinceLast int64          `json:"bytesSinceLast"`
	Message        string         `json:"message,omitempty"`
	Level          string         `json:"level,omitempty"`
	Attrs          map[string]any `json:"attrs,omitempty"`
}

ProgressEvent matches the Git LFS custom transfer progress payload.

type RetryObject

type RetryObject struct {
	SourcePath   string
	ObjectKey    string
	FileMetadata FileMetadata
	GUID         string
	RetryCount   int
	Multipart    bool
	Bucket       string
}

RetryObject defines a object for retry upload