corfs

Documentation

Index

• func NewPrefixEnvCredentials(prefix string) *credentials.Credentials
• type FileInfo
• type FileSystem
  • func InferFilesystem(location string) FileSystem
  • func InitFilesystem(fsType FileSystemType) (FileSystem, error)
• type FileSystemType
  • func FilesystemType(fs FileSystem) FileSystemType
• type LocalFileSystem
  • func (l *LocalFileSystem) Delete(filePath string) error
  • func (l *LocalFileSystem) Init() error
  • func (l *LocalFileSystem) Join(elem ...string) string
  • func (l *LocalFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
  • func (l *LocalFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
  • func (l *LocalFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
  • func (l *LocalFileSystem) Split(path string) []string
  • func (l *LocalFileSystem) Stat(filePath string) (FileInfo, error)
• type MinioFileSystem
  • func (s *MinioFileSystem) Delete(filePath string) error
  • func (s *MinioFileSystem) Init() error
  • func (s *MinioFileSystem) Join(elem ...string) string
  • func (s *MinioFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
  • func (s *MinioFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
  • func (s *MinioFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
  • func (s *MinioFileSystem) Split(path string) []string
  • func (s *MinioFileSystem) Stat(filePath string) (FileInfo, error)
• type PrefixEnvProvider
  • func (e *PrefixEnvProvider) IsExpired() bool
  • func (e *PrefixEnvProvider) Retrieve() (credentials.Value, error)
• type S3FileSystem
  • func (s *S3FileSystem) Delete(filePath string) error
  • func (s *S3FileSystem) Init() error
  • func (s S3FileSystem) Join(elem ...string) string
  • func (s *S3FileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
  • func (s *S3FileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
  • func (s *S3FileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
  • func (s S3FileSystem) Split(path string) []string
  • func (s *S3FileSystem) Stat(filePath string) (FileInfo, error)

Functions

func NewPrefixEnvCredentials

func NewPrefixEnvCredentials(prefix string) *credentials.Credentials

NewEnvCredentials returns a pointer to a new Credentials object wrapping the environment variable provider.

Types

type FileInfo

type FileInfo struct {
	Name string // file path
	Size int64  // file size in bytes
}

FileInfo provides information about a file

type FileSystem

type FileSystem interface {
	ListFiles(pathGlob string) ([]FileInfo, error)
	Stat(filePath string) (FileInfo, error)
	OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
	OpenWriter(filePath string) (io.WriteCloser, error)
	Delete(filePath string) error
	Join(elem ...string) string
	Split(path string) []string
	Init() error
}

FileSystem provides the file backend for MapReduce jobs. Input data is read from a file system. Intermediate and output data is written to a file system. This is abstracted to allow remote filesystems like S3 to be supported.

func InferFilesystem

func InferFilesystem(location string) FileSystem

InferFilesystem initializes a filesystem by inferring its type from a file address. For example, locations starting with "s3://" will resolve to an S3 filesystem.

func InitFilesystem

func InitFilesystem(fsType FileSystemType) (FileSystem, error)

InitFilesystem intializes a filesystem of the given type

type FileSystemType

type FileSystemType int

FileSystemType is an identifier for supported FileSystems

const (
	Local FileSystemType = iota
	S3
	MINIO
)

Identifiers for supported FileSystemTypes

func FilesystemType

func FilesystemType(fs FileSystem) FileSystemType

type LocalFileSystem

type LocalFileSystem struct{}

LocalFileSystem wraps "os" to provide access to the local filesystem.

func (*LocalFileSystem) Delete

func (l *LocalFileSystem) Delete(filePath string) error

Delete deletes the file at filePath.

func (*LocalFileSystem) Init

func (l *LocalFileSystem) Init() error

Init initializes the filesystem.

func (*LocalFileSystem) Join

func (l *LocalFileSystem) Join(elem ...string) string

Join joins file path elements

func (*LocalFileSystem) ListFiles

func (l *LocalFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)

ListFiles lists files that match pathGlob.

func (*LocalFileSystem) OpenReader

func (l *LocalFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)

OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.

func (*LocalFileSystem) OpenWriter

func (l *LocalFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)

OpenWriter opens a writer to the file at filePath.

func (*LocalFileSystem) Split

func (l *LocalFileSystem) Split(path string) []string

Join joins file path elements

func (*LocalFileSystem) Stat

func (l *LocalFileSystem) Stat(filePath string) (FileInfo, error)

Stat returns information about the file at filePath.

type MinioFileSystem

type MinioFileSystem struct {
	// contains filtered or unexported fields
}

S3FileSystem abstracts AWS S3 as a filesystem

func (*MinioFileSystem) Delete

func (s *MinioFileSystem) Delete(filePath string) error

Delete deletes the file at filePath.

func (*MinioFileSystem) Init

func (s *MinioFileSystem) Init() error

Init initializes the filesystem.

func (*MinioFileSystem) Join

func (s *MinioFileSystem) Join(elem ...string) string

Join joins file path elements

func (*MinioFileSystem) ListFiles

func (s *MinioFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)

ListFiles lists files that match pathGlob.

func (*MinioFileSystem) OpenReader

func (s *MinioFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)

OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.

func (*MinioFileSystem) OpenWriter

func (s *MinioFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)

OpenWriter opens a writer to the file at filePath.

func (*MinioFileSystem) Split

func (s *MinioFileSystem) Split(path string) []string

Join joins file path elements

func (*MinioFileSystem) Stat

func (s *MinioFileSystem) Stat(filePath string) (FileInfo, error)

Stat returns information about the file at filePath.

type PrefixEnvProvider

type PrefixEnvProvider struct {
	// contains filtered or unexported fields
}

A EnvProvider retrieves credentials from the environment variables of the running process. Environment credentials never expire.

Environment variables used:

* Access Key ID: [PREFIX]+AWS_ACCESS_KEY_ID or [PREFIX]+AWS_ACCESS_KEY

* Secret Access Key: [PREFIX]+AWS_SECRET_ACCESS_KEY or [PREFIX]+AWS_SECRET_KEY

func (*PrefixEnvProvider) IsExpired

func (e *PrefixEnvProvider) IsExpired() bool

IsExpired returns if the credentials have been retrieved.

func (*PrefixEnvProvider) Retrieve

func (e *PrefixEnvProvider) Retrieve() (credentials.Value, error)

Retrieve retrieves the keys from the environment.

type S3FileSystem

type S3FileSystem struct {
	// contains filtered or unexported fields
}

S3FileSystem abstracts AWS S3 as a filesystem

func (*S3FileSystem) Delete

func (s *S3FileSystem) Delete(filePath string) error

Delete deletes the file at filePath.

func (*S3FileSystem) Init

func (s *S3FileSystem) Init() error

Init initializes the filesystem.

func (S3FileSystem) Join

func (s S3FileSystem) Join(elem ...string) string

Join joins file path elements

func (*S3FileSystem) ListFiles

func (s *S3FileSystem) ListFiles(pathGlob string) ([]FileInfo, error)

ListFiles lists files that match pathGlob.

func (*S3FileSystem) OpenReader

func (s *S3FileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)

OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.

func (*S3FileSystem) OpenWriter

func (s *S3FileSystem) OpenWriter(filePath string) (io.WriteCloser, error)

OpenWriter opens a writer to the file at filePath.

func (S3FileSystem) Split

func (s S3FileSystem) Split(path string) []string

Join joins file path elements

func (*S3FileSystem) Stat

func (s *S3FileSystem) Stat(filePath string) (FileInfo, error)

Stat returns information about the file at filePath.