filesystem

Documentation

Index

• func IsDir(filepath string) bool
• func SplitS3URL(url string) (prefix, bucket, key string, err error)
• type FileLocation
  • func List(filepath string) ([]*FileLocation, error)
  • func NewFileLocation(loc string, ft FileType) *FileLocation
• type FileType
  • func StringToFileType(ts string) FileType
• type HdfsFileSystem
  • func (fs *HdfsFileSystem) Accept(fl *FileLocation) bool
  • func (fs *HdfsFileSystem) IsDir(fl *FileLocation) bool
  • func (fs *HdfsFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (fs *HdfsFileSystem) Open(fl *FileLocation) (VirtualFile, error)
• type LocalFileSystem
  • func (fs *LocalFileSystem) Accept(fl *FileLocation) bool
  • func (fs *LocalFileSystem) IsDir(fl *FileLocation) bool
  • func (fs *LocalFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (fs *LocalFileSystem) Open(fl *FileLocation) (VirtualFile, error)
• type S3FileSystem
  • func (self *S3FileSystem) Accept(fl *FileLocation) bool
  • func (self *S3FileSystem) IsDir(fl *FileLocation) bool
  • func (self *S3FileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (self *S3FileSystem) Open(fl *FileLocation) (VirtualFile, error)
• type VirtualFile
  • func Open(filepath string) (VirtualFile, error)
• type VirtualFileHdfs
  • func (self *VirtualFileHdfs) Close() (err error)
  • func (self *VirtualFileHdfs) Read(p []byte) (n int, err error)
  • func (self *VirtualFileHdfs) ReadAt(b []byte, off int64) (n int, err error)
  • func (self *VirtualFileHdfs) Seek(offset int64, whence int) (int64, error)
  • func (self *VirtualFileHdfs) Size() int64
• type VirtualFileLocal
  • func (vf *VirtualFileLocal) Size() int64
• type VirtualFileS3
  • func (self *VirtualFileS3) Close() error
  • func (self *VirtualFileS3) Size() int64
• type VirtualFileSystem

Functions

func IsDir

func IsDir(filepath string) bool

func SplitS3URL

func SplitS3URL(url string) (prefix, bucket, key string, err error)

Types

type FileLocation

type FileLocation struct {
	Location string
	FileType FileType
}

func List

func List(filepath string) ([]*FileLocation, error)

func NewFileLocation

func NewFileLocation(loc string, ft FileType) *FileLocation

type FileType

type FileType int32

const (
	UNKNOWNFILETYPE FileType = iota
	CSV
	PARQUET
	ORC
)

func StringToFileType

func StringToFileType(ts string) FileType

type HdfsFileSystem

type HdfsFileSystem struct {
}

Get the namenode from 1) the hdfs://namenode/... string 2) Or from env HADOOP_NAMENODE. 3) Or based on env HADOOP_CONF_DIR or HADOOP_HOME to locate hdfs-site.xml and core-site.xml

func (*HdfsFileSystem) Accept

func (fs *HdfsFileSystem) Accept(fl *FileLocation) bool

func (*HdfsFileSystem) IsDir

func (fs *HdfsFileSystem) IsDir(fl *FileLocation) bool

func (*HdfsFileSystem) List

func (fs *HdfsFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

List generates a full list of file locations under the given location, which should have a prefix of hdfs://

func (*HdfsFileSystem) Open

func (fs *HdfsFileSystem) Open(fl *FileLocation) (VirtualFile, error)

type LocalFileSystem

type LocalFileSystem struct {
}

func (*LocalFileSystem) Accept

func (fs *LocalFileSystem) Accept(fl *FileLocation) bool

func (*LocalFileSystem) IsDir

func (fs *LocalFileSystem) IsDir(fl *FileLocation) bool

func (*LocalFileSystem) List

func (fs *LocalFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

func (*LocalFileSystem) Open

func (fs *LocalFileSystem) Open(fl *FileLocation) (VirtualFile, error)

type S3FileSystem

type S3FileSystem struct {
}

func (*S3FileSystem) Accept

func (self *S3FileSystem) Accept(fl *FileLocation) bool

func (*S3FileSystem) IsDir

func (self *S3FileSystem) IsDir(fl *FileLocation) bool

func (*S3FileSystem) List

func (self *S3FileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

func (*S3FileSystem) Open

func (self *S3FileSystem) Open(fl *FileLocation) (VirtualFile, error)

type VirtualFile

type VirtualFile interface {
	io.ReaderAt
	io.ReadCloser
	io.Seeker
	Size() int64
}

func Open

func Open(filepath string) (VirtualFile, error)

type VirtualFileHdfs

type VirtualFileHdfs struct {
	FileReader *hdfs.FileReader
	Client     *hdfs.Client
}

/////////////////////////////////

func (*VirtualFileHdfs) Close

func (self *VirtualFileHdfs) Close() (err error)

func (*VirtualFileHdfs) Read

func (self *VirtualFileHdfs) Read(p []byte) (n int, err error)

func (*VirtualFileHdfs) ReadAt

func (self *VirtualFileHdfs) ReadAt(b []byte, off int64) (n int, err error)

func (*VirtualFileHdfs) Seek

func (self *VirtualFileHdfs) Seek(offset int64, whence int) (int64, error)

func (*VirtualFileHdfs) Size

func (self *VirtualFileHdfs) Size() int64

type VirtualFileLocal

type VirtualFileLocal struct {
	*os.File
}

func (*VirtualFileLocal) Size

func (vf *VirtualFileLocal) Size() int64

type VirtualFileS3

type VirtualFileS3 struct {
	*os.File
	FileName string
	FileSize int64
}

func (*VirtualFileS3) Close

func (self *VirtualFileS3) Close() error

func (*VirtualFileS3) Size

func (self *VirtualFileS3) Size() int64

type VirtualFileSystem

type VirtualFileSystem interface {
	Accept(*FileLocation) bool
	Open(*FileLocation) (VirtualFile, error)
	List(*FileLocation) ([]*FileLocation, error)
	IsDir(*FileLocation) bool
}