vfs

package
v1.15.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 17, 2020 License: Apache-2.0 Imports: 44 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Context = VFSContext{
	// contains filtered or unexported fields
}

Functions

func AWSErrorCode

func AWSErrorCode(err error) string

AWSErrorCode returns the aws error code, if it is an awserr.Error, otherwise ""

func CopyFile

func CopyFile(src, dest Path, acl ACL) error

CopyFile copies the file at src to dest. It uses a TempFile, rather than buffering in memory.

func CopyTree

func CopyTree(src Path, dest Path, aclOracle ACLOracle) error

CopyTree copies all files in src to dest. It copies the whole recursive subtree of files.

func IsClusterReadable

func IsClusterReadable(p Path) bool

func IsDirectory

func IsDirectory(p Path) bool

func NewAliOSSClient

func NewAliOSSClient() (*oss.Client, error)

func NewSwiftClient

func NewSwiftClient() (*gophercloud.ServiceClient, error)

func RelativePath

func RelativePath(base Path, child Path) (string, error)

func RetryWithBackoff

func RetryWithBackoff(backoff wait.Backoff, condition func() (bool, error)) (bool, error)

RetryWithBackoff runs until a condition function returns true, or until Steps attempts have been taken As compared to wait.ExponentialBackoff, this function returns the results from the function on the final attempt

func SyncDir

func SyncDir(src *VFSScan, destBase Path) error

func VFSPath

func VFSPath(url string) (string, error)

Types

type ACL

type ACL interface {
}

type ACLOracle

type ACLOracle func(Path) (ACL, error)

type Change

type Change struct {
	ChangeType ChangeType
	Path       Path
	Hash       *hashing.Hash
}

type ChangeType

type ChangeType string
const (
	ChangeType_Added    ChangeType = "ADDED"
	ChangeType_Removed  ChangeType = "REMOVED"
	ChangeType_Modified ChangeType = "MODIFIED"
)

type FSPath

type FSPath struct {
	// contains filtered or unexported fields
}

func NewFSPath

func NewFSPath(location string) *FSPath

func (*FSPath) Base

func (p *FSPath) Base() string

func (*FSPath) CreateFile

func (p *FSPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*FSPath) Hash

func (p *FSPath) Hash(a hashing.HashAlgorithm) (*hashing.Hash, error)

func (*FSPath) Join

func (p *FSPath) Join(relativePath ...string) Path

func (*FSPath) Path

func (p *FSPath) Path() string

func (*FSPath) PreferredHash

func (p *FSPath) PreferredHash() (*hashing.Hash, error)

func (*FSPath) ReadDir

func (p *FSPath) ReadDir() ([]Path, error)

func (*FSPath) ReadFile

func (p *FSPath) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*FSPath) ReadTree

func (p *FSPath) ReadTree() ([]Path, error)

func (*FSPath) Remove

func (p *FSPath) Remove() error

func (*FSPath) String

func (p *FSPath) String() string

func (*FSPath) WriteFile

func (p *FSPath) WriteFile(data io.ReadSeeker, acl ACL) error

func (*FSPath) WriteTo

func (p *FSPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo

type GSAcl

type GSAcl struct {
	Acl []*storage.ObjectAccessControl
}

GSAcl is an ACL implementation for objects on Google Cloud Storage

type GSPath

type GSPath struct {
	// contains filtered or unexported fields
}

GSPath is a vfs path for Google Cloud Storage

func NewGSPath

func NewGSPath(client *storage.Service, bucket string, key string) *GSPath

func (*GSPath) Base

func (p *GSPath) Base() string

func (*GSPath) Bucket

func (p *GSPath) Bucket() string

func (*GSPath) Client

func (p *GSPath) Client() *storage.Service

Client returns the storage.Service bound to this path

func (*GSPath) CreateFile

func (p *GSPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*GSPath) Hash

func (p *GSPath) Hash(a hashing.HashAlgorithm) (*hashing.Hash, error)

func (*GSPath) Join

func (p *GSPath) Join(relativePath ...string) Path

func (*GSPath) Object

func (p *GSPath) Object() string

func (*GSPath) Path

func (p *GSPath) Path() string

func (*GSPath) PreferredHash

func (p *GSPath) PreferredHash() (*hashing.Hash, error)

func (*GSPath) ReadDir

func (p *GSPath) ReadDir() ([]Path, error)

ReadDir implements Path::ReadDir

func (*GSPath) ReadFile

func (p *GSPath) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*GSPath) ReadTree

func (p *GSPath) ReadTree() ([]Path, error)

ReadTree implements Path::ReadTree

func (*GSPath) Remove

func (p *GSPath) Remove() error

func (*GSPath) String

func (p *GSPath) String() string

func (*GSPath) WriteFile

func (p *GSPath) WriteFile(data io.ReadSeeker, acl ACL) error

func (*GSPath) WriteTo

func (p *GSPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo::WriteTo

type HasClusterReadable

type HasClusterReadable interface {
	IsClusterReadable() bool
}

type HasHash

type HasHash interface {
	// Returns the hash of the file contents, with the preferred hash algorithm
	PreferredHash() (*hashing.Hash, error)

	// Gets the hash, or nil if the hash cannot be (easily) computed
	Hash(algorithm hashing.HashAlgorithm) (*hashing.Hash, error)
}

type KubernetesContext

type KubernetesContext struct {
}

KubernetesContext is the context for a Kubernetes VFS implementation

func NewKubernetesContext

func NewKubernetesContext() *KubernetesContext

NewKubernetesContext builds a KubernetesContext This will likely take a kubernetes rest client object (or similar) once the implementation is more complete

type KubernetesPath

type KubernetesPath struct {
	// contains filtered or unexported fields
}

KubernetesPath is a path for a VFS backed by the kubernetes API Currently all operations are no-ops

func (*KubernetesPath) Base

func (p *KubernetesPath) Base() string

func (*KubernetesPath) CreateFile

func (p *KubernetesPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*KubernetesPath) Hash

func (*KubernetesPath) Host

func (p *KubernetesPath) Host() string

func (*KubernetesPath) Join

func (p *KubernetesPath) Join(relativePath ...string) Path

func (*KubernetesPath) Key

func (p *KubernetesPath) Key() string

func (*KubernetesPath) Path

func (p *KubernetesPath) Path() string

func (*KubernetesPath) PreferredHash

func (p *KubernetesPath) PreferredHash() (*hashing.Hash, error)

func (*KubernetesPath) ReadDir

func (p *KubernetesPath) ReadDir() ([]Path, error)

func (*KubernetesPath) ReadFile

func (p *KubernetesPath) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*KubernetesPath) ReadTree

func (p *KubernetesPath) ReadTree() ([]Path, error)

func (*KubernetesPath) Remove

func (p *KubernetesPath) Remove() error

func (*KubernetesPath) String

func (p *KubernetesPath) String() string

func (*KubernetesPath) WriteFile

func (p *KubernetesPath) WriteFile(data io.ReadSeeker, acl ACL) error

func (*KubernetesPath) WriteTo

func (p *KubernetesPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo

type MemFSContext

type MemFSContext struct {
	// contains filtered or unexported fields
}

func NewMemFSContext

func NewMemFSContext() *MemFSContext

func (*MemFSContext) MarkClusterReadable

func (c *MemFSContext) MarkClusterReadable()

MarkClusterReadable pretends the current memfscontext is cluster readable; this is useful for tests

type MemFSPath

type MemFSPath struct {
	// contains filtered or unexported fields
}

func NewMemFSPath

func NewMemFSPath(context *MemFSContext, location string) *MemFSPath

func (*MemFSPath) Base

func (p *MemFSPath) Base() string

func (*MemFSPath) CreateFile

func (p *MemFSPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*MemFSPath) HasChildren

func (c *MemFSPath) HasChildren() bool

func (*MemFSPath) IsClusterReadable

func (c *MemFSPath) IsClusterReadable() bool

func (*MemFSPath) Join

func (p *MemFSPath) Join(relativePath ...string) Path

func (*MemFSPath) Path

func (p *MemFSPath) Path() string

func (*MemFSPath) ReadDir

func (p *MemFSPath) ReadDir() ([]Path, error)

func (*MemFSPath) ReadFile

func (p *MemFSPath) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*MemFSPath) ReadTree

func (p *MemFSPath) ReadTree() ([]Path, error)

func (*MemFSPath) Remove

func (p *MemFSPath) Remove() error

func (*MemFSPath) String

func (p *MemFSPath) String() string

func (*MemFSPath) WriteFile

func (p *MemFSPath) WriteFile(r io.ReadSeeker, acl ACL) error

func (*MemFSPath) WriteTo

func (p *MemFSPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo

type OSSPath

type OSSPath struct {
	// contains filtered or unexported fields
}

OSSPath is a vfs path for Aliyun Open Storage Service

func NewOSSPath

func NewOSSPath(client *oss.Client, bucket string, key string) (*OSSPath, error)

func (*OSSPath) Base

func (p *OSSPath) Base() string

func (*OSSPath) CreateFile

func (p *OSSPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*OSSPath) Hash

func (p *OSSPath) Hash(a hashing.HashAlgorithm) (*hashing.Hash, error)

func (*OSSPath) Join

func (p *OSSPath) Join(relativePath ...string) Path

func (*OSSPath) Path

func (p *OSSPath) Path() string

func (*OSSPath) PreferredHash

func (p *OSSPath) PreferredHash() (*hashing.Hash, error)

func (*OSSPath) ReadDir

func (p *OSSPath) ReadDir() ([]Path, error)

func (*OSSPath) ReadFile

func (p *OSSPath) ReadFile() ([]byte, error)

func (*OSSPath) ReadTree

func (p *OSSPath) ReadTree() ([]Path, error)

func (*OSSPath) Remove

func (p *OSSPath) Remove() error

func (*OSSPath) String

func (p *OSSPath) String() string

func (*OSSPath) WriteFile

func (p *OSSPath) WriteFile(data io.ReadSeeker, acl ACL) error

func (*OSSPath) WriteTo

func (p *OSSPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriteTo

type OpenstackConfig

type OpenstackConfig struct {
}

func (OpenstackConfig) GetCredential

func (oc OpenstackConfig) GetCredential() (gophercloud.AuthOptions, error)

func (OpenstackConfig) GetRegion

func (oc OpenstackConfig) GetRegion() (string, error)

func (OpenstackConfig) GetServiceConfig

func (oc OpenstackConfig) GetServiceConfig(name string) (gophercloud.EndpointOpts, error)

type Path

type Path interface {
	io.WriterTo

	Join(relativePath ...string) Path

	// ReadFile returns the contents of the file, or an error if the file could not be read.
	// If the file did not exist, err = os.ErrNotExist
	// As this reads the entire file into memory, consider using WriteTo for bigger files
	ReadFile() ([]byte, error)

	WriteFile(data io.ReadSeeker, acl ACL) error
	// CreateFile writes the file contents, but only if the file does not already exist
	CreateFile(data io.ReadSeeker, acl ACL) error

	// Remove deletes the file
	Remove() error

	// Base returns the base name (last element)
	Base() string

	// Path returns a string representing the full path
	Path() string

	// ReadDir lists the files in a particular Path
	ReadDir() ([]Path, error)

	// ReadTree lists all files (recursively) in the subtree rooted at the current Path
	/// Note: returns only files, not directories
	ReadTree() ([]Path, error)
}

Path is a path in the VFS space, which we can read, write, list etc

type S3Acl

type S3Acl struct {
	RequestACL *string
}

S3Acl is an ACL implementation for objects on S3

type S3BucketDetails

type S3BucketDetails struct {
	// contains filtered or unexported fields
}

type S3Context

type S3Context struct {
	// contains filtered or unexported fields
}

func NewS3Context

func NewS3Context() *S3Context

type S3Path

type S3Path struct {
	// contains filtered or unexported fields
}

func (*S3Path) Base

func (p *S3Path) Base() string

func (*S3Path) Bucket

func (p *S3Path) Bucket() string

func (*S3Path) CreateFile

func (p *S3Path) CreateFile(data io.ReadSeeker, acl ACL) error

func (*S3Path) Hash

func (p *S3Path) Hash(a hashing.HashAlgorithm) (*hashing.Hash, error)

func (*S3Path) Join

func (p *S3Path) Join(relativePath ...string) Path

func (*S3Path) Key

func (p *S3Path) Key() string

func (*S3Path) Path

func (p *S3Path) Path() string

func (*S3Path) PreferredHash

func (p *S3Path) PreferredHash() (*hashing.Hash, error)

func (*S3Path) ReadDir

func (p *S3Path) ReadDir() ([]Path, error)

func (*S3Path) ReadFile

func (p *S3Path) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*S3Path) ReadTree

func (p *S3Path) ReadTree() ([]Path, error)

func (*S3Path) Remove

func (p *S3Path) Remove() error

func (*S3Path) String

func (p *S3Path) String() string

func (*S3Path) WriteFile

func (p *S3Path) WriteFile(data io.ReadSeeker, aclObj ACL) error

func (*S3Path) WriteTo

func (p *S3Path) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo

type SSHAcl

type SSHAcl struct {
	Mode os.FileMode
}

type SSHPath

type SSHPath struct {
	// contains filtered or unexported fields
}

func NewSSHPath

func NewSSHPath(client *ssh.Client, server string, path string, sudo bool) *SSHPath

func (*SSHPath) Base

func (p *SSHPath) Base() string

func (*SSHPath) CreateFile

func (p *SSHPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*SSHPath) Join

func (p *SSHPath) Join(relativePath ...string) Path

func (*SSHPath) Path

func (p *SSHPath) Path() string

func (*SSHPath) ReadDir

func (p *SSHPath) ReadDir() ([]Path, error)

func (*SSHPath) ReadFile

func (p *SSHPath) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*SSHPath) ReadTree

func (p *SSHPath) ReadTree() ([]Path, error)

func (*SSHPath) Remove

func (p *SSHPath) Remove() error

func (*SSHPath) String

func (p *SSHPath) String() string

func (*SSHPath) WriteFile

func (p *SSHPath) WriteFile(data io.ReadSeeker, acl ACL) error

func (*SSHPath) WriteTo

func (p *SSHPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo

type SwiftPath

type SwiftPath struct {
	// contains filtered or unexported fields
}

SwiftPath is a vfs path for Openstack Cloud Storage.

func NewSwiftPath

func NewSwiftPath(client *gophercloud.ServiceClient, bucket string, key string) (*SwiftPath, error)

func (*SwiftPath) Base

func (p *SwiftPath) Base() string

func (*SwiftPath) Bucket

func (p *SwiftPath) Bucket() string

func (*SwiftPath) CreateFile

func (p *SwiftPath) CreateFile(data io.ReadSeeker, acl ACL) error

func (*SwiftPath) Hash

func (*SwiftPath) Join

func (p *SwiftPath) Join(relativePath ...string) Path

func (*SwiftPath) Path

func (p *SwiftPath) Path() string

func (*SwiftPath) PreferredHash

func (p *SwiftPath) PreferredHash() (*hashing.Hash, error)

func (*SwiftPath) ReadDir

func (p *SwiftPath) ReadDir() ([]Path, error)

ReadDir implements Path::ReadDir.

func (*SwiftPath) ReadFile

func (p *SwiftPath) ReadFile() ([]byte, error)

ReadFile implements Path::ReadFile

func (*SwiftPath) ReadTree

func (p *SwiftPath) ReadTree() ([]Path, error)

ReadTree implements Path::ReadTree.

func (*SwiftPath) Remove

func (p *SwiftPath) Remove() error

func (*SwiftPath) String

func (p *SwiftPath) String() string

func (*SwiftPath) WriteFile

func (p *SwiftPath) WriteFile(data io.ReadSeeker, acl ACL) error

func (*SwiftPath) WriteTo

func (p *SwiftPath) WriteTo(out io.Writer) (int64, error)

WriteTo implements io.WriterTo

type VFS

type VFS interface {
}

type VFSContext

type VFSContext struct {
	// contains filtered or unexported fields
}

VFSContext is a 'context' for VFS, that is normally a singleton but allows us to configure S3 credentials, for example

func (*VFSContext) BuildVfsPath

func (c *VFSContext) BuildVfsPath(p string) (Path, error)

func (*VFSContext) ReadFile

func (c *VFSContext) ReadFile(location string, options ...VFSOption) ([]byte, error)

ReadLocation reads a file from a vfs URL It supports additional schemes which don't (yet) have full VFS implementations:

metadata: reads from instance metadata on GCE/AWS
http / https: reads from HTTP

func (*VFSContext) ResetMemfsContext

func (c *VFSContext) ResetMemfsContext(clusterReadable bool)

type VFSOption

type VFSOption func(options *vfsOptions)

func WithBackoff

func WithBackoff(backoff wait.Backoff) VFSOption

WithBackoff specifies a custom VFS backoff policy

type VFSScan

type VFSScan struct {
	Base Path
	// contains filtered or unexported fields
}

VFSScan scans a source Path for changed files

func NewVFSScan

func NewVFSScan(base Path) *VFSScan

func (*VFSScan) Scan

func (v *VFSScan) Scan() ([]Change, error)

Scans for changes files. On the first call will return all files as ChangeType_Added. On subsequent calls will return any changed files (using their hashes)

type WriteOption

type WriteOption string
const (
	WriteOptionCreate       WriteOption = "Create"
	WriteOptionOnlyIfExists WriteOption = "IfExists"
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL