ParquetFile

Documentation

Index

• func ConvertToThriftReader(file ParquetFile, offset int64, size int64) *thrift.TBufferedTransport
• func GetMemFileFs() afero.Fs
• func SetInMemFileFs(fs *afero.Fs)
• type HdfsFile
  • func (self *HdfsFile) Close()
  • func (self *HdfsFile) Create(name string) (ParquetFile, error)
  • func (self *HdfsFile) Open(name string) (ParquetFile, error)
  • func (self *HdfsFile) Read(b []byte) (n int, err error)
  • func (self *HdfsFile) Seek(offset int, pos int) (int64, error)
  • func (self *HdfsFile) Write(b []byte) (n int, err error)
• type LocalFile
  • func (self *LocalFile) Close()
  • func (self *LocalFile) Create(name string) (ParquetFile, error)
  • func (self *LocalFile) Open(name string) (ParquetFile, error)
  • func (self *LocalFile) Read(b []byte) (n int, err error)
  • func (self *LocalFile) Seek(offset int, pos int) (int64, error)
  • func (self *LocalFile) Write(b []byte) (n int, err error)
• type MemFile
  • func (fs *MemFile) Close()
  • func (fs *MemFile) Create(name string) (ParquetFile, error)
  • func (fs *MemFile) Open(name string) (ParquetFile, error)
  • func (fs *MemFile) Read(b []byte) (n int, err error)
  • func (fs *MemFile) Seek(offset int, pos int) (int64, error)
  • func (fs *MemFile) Write(b []byte) (n int, err error)
• type OnCloseFunc
• type ParquetFile
  • func NewHdfsFileReader(hosts []string, user string, name string) (ParquetFile, error)
  • func NewHdfsFileWriter(hosts []string, user string, name string) (ParquetFile, error)
  • func NewLocalFileReader(name string) (ParquetFile, error)
  • func NewLocalFileWriter(name string) (ParquetFile, error)
  • func NewMemFileWriter(name string, f OnCloseFunc) (ParquetFile, error)
  • func NewWriterFile(writer io.Writer) ParquetFile
• type WriterFile
  • func (self *WriterFile) Close()
  • func (self *WriterFile) Create(name string) (ParquetFile, error)
  • func (self *WriterFile) Open(name string) (ParquetFile, error)
  • func (self *WriterFile) Read(b []byte) (int, error)
  • func (self *WriterFile) Seek(offset int, pos int) (int64, error)
  • func (self *WriterFile) Write(b []byte) (int, error)

Functions

func ConvertToThriftReader

func ConvertToThriftReader(file ParquetFile, offset int64, size int64) *thrift.TBufferedTransport

Convert a file reater to Thrift reader

func GetMemFileFs

func GetMemFileFs() afero.Fs

GetMemFileFs - returns the current memory file-system being used by ParquetFile

func SetInMemFileFs

func SetInMemFileFs(fs *afero.Fs)

SetInMemFileFs - overrides local in-memory fileSystem NOTE: this is set by NewMemFileWriter is created and memFs is still nil

Types

type HdfsFile

type HdfsFile struct {
	Hosts []string
	User  string

	Client     *hdfs.Client
	FilePath   string
	FileReader *hdfs.FileReader
	FileWriter *hdfs.FileWriter
}

func (*HdfsFile) Close

func (self *HdfsFile) Close()

func (*HdfsFile) Create

func (self *HdfsFile) Create(name string) (ParquetFile, error)

func (*HdfsFile) Open

func (self *HdfsFile) Open(name string) (ParquetFile, error)

func (*HdfsFile) Read

func (self *HdfsFile) Read(b []byte) (n int, err error)

func (*HdfsFile) Seek

func (self *HdfsFile) Seek(offset int, pos int) (int64, error)

func (*HdfsFile) Write

func (self *HdfsFile) Write(b []byte) (n int, err error)

type LocalFile

type LocalFile struct {
	FilePath string
	File     *os.File
}

func (*LocalFile) Close

func (self *LocalFile) Close()

func (*LocalFile) Create

func (self *LocalFile) Create(name string) (ParquetFile, error)

func (*LocalFile) Open

func (self *LocalFile) Open(name string) (ParquetFile, error)

func (*LocalFile) Read

func (self *LocalFile) Read(b []byte) (n int, err error)

func (*LocalFile) Seek

func (self *LocalFile) Seek(offset int, pos int) (int64, error)

func (*LocalFile) Write

func (self *LocalFile) Write(b []byte) (n int, err error)

type MemFile

type MemFile struct {
	FilePath string
	File     afero.File
	OnClose  OnCloseFunc
}

MemFile - ParquetFile type for in-memory file operations

func (*MemFile) Close

func (fs *MemFile) Close()

Close - close file and execute OnCloseFunc

func (*MemFile) Create

func (fs *MemFile) Create(name string) (ParquetFile, error)

Create - create in-memory file

func (*MemFile) Open

func (fs *MemFile) Open(name string) (ParquetFile, error)

Open - open file in-memory

func (*MemFile) Read

func (fs *MemFile) Read(b []byte) (n int, err error)

Read - read file

func (*MemFile) Seek

func (fs *MemFile) Seek(offset int, pos int) (int64, error)

Seek - seek function

func (*MemFile) Write

func (fs *MemFile) Write(b []byte) (n int, err error)

Write - write file in-memory

type OnCloseFunc

type OnCloseFunc func(string, io.Reader)

OnCloseFunc function type, handles what to do after converted file is closed in-memory. Close() will pass the filename string and data as io.reader

type ParquetFile

type ParquetFile interface {
	Seek(offset int, pos int) (int64, error)
	Read(b []byte) (n int, err error)
	Write(b []byte) (n int, err error)
	Close()
	Open(name string) (ParquetFile, error)
	Create(name string) (ParquetFile, error)
}

func NewHdfsFileReader

func NewHdfsFileReader(hosts []string, user string, name string) (ParquetFile, error)

func NewHdfsFileWriter

func NewHdfsFileWriter(hosts []string, user string, name string) (ParquetFile, error)

func NewLocalFileReader

func NewLocalFileReader(name string) (ParquetFile, error)

func NewLocalFileWriter

func NewLocalFileWriter(name string) (ParquetFile, error)

func NewMemFileWriter

func NewMemFileWriter(name string, f OnCloseFunc) (ParquetFile, error)

NewMemFileWriter - intiates and creates an instance of MemFiles NOTE: there is no NewMemFileReader as this particular type was written to handle in-memory converstions and offloading. The results of conversion can then be stored and read via HDFS, LocalFS, etc without the need for loading the file back into memory directly

func NewWriterFile

func NewWriterFile(writer io.Writer) ParquetFile

type WriterFile

type WriterFile struct {
	Writer io.Writer
}

func (*WriterFile) Close

func (self *WriterFile) Close()

func (*WriterFile) Create

func (self *WriterFile) Create(name string) (ParquetFile, error)

func (*WriterFile) Open

func (self *WriterFile) Open(name string) (ParquetFile, error)

func (*WriterFile) Read

func (self *WriterFile) Read(b []byte) (int, error)

func (*WriterFile) Seek

func (self *WriterFile) Seek(offset int, pos int) (int64, error)

func (*WriterFile) Write

func (self *WriterFile) Write(b []byte) (int, error)