fileproc

README

fileproc

fileproc is an framework to process file by line concurrently.

Documentation

Index

• func ProcTerm(num, maxSize int, m Mapper, r Reducer, fw FileWrapper) error
• type FileProcessor
  • func NewFileProcessor(num, maxSize, splitCnt int, seq, prefixTime bool, m Mapper, r Reducer, ...) *FileProcessor
  • func (p *FileProcessor) ProcFile(path, dir, ext string) error
  • func (p *FileProcessor) ProcPath(path, dir, ext string) error
  • func (p *FileProcessor) ProcPathReverse(path, dir, ext string) error
  • func (p *FileProcessor) Stat() (int, int, int)
• type FileWrapper
  • func DummyWrapper() FileWrapper
• type FileWriter
  • func NewFileWriter(fw FileWrapper, dir, fn, ext string, splitCnt int) FileWriter
  • func WithSequence(parent FileWriter) FileWriter
• type LineInfoSlice
  • func (lis LineInfoSlice) Len() int
  • func (lis LineInfoSlice) Less(i, j int) bool
  • func (lis LineInfoSlice) Swap(i, j int)
• type Mapper
• type Reducer
• type TermWriter
  • func NewTermWriter(fw FileWrapper) *TermWriter
  • func (w *TermWriter) Close() error
  • func (w *TermWriter) Open() error
  • func (w *TermWriter) Write(li lineInfo) (int, error)

Functions

func ProcTerm

func ProcTerm(num, maxSize int, m Mapper, r Reducer, fw FileWrapper) error

Types

type FileProcessor

type FileProcessor struct {
	SplitCnt   int
	Seq        bool
	PrefixTime bool
	FileWrapper
	// contains filtered or unexported fields
}

func NewFileProcessor

func NewFileProcessor(num, maxSize, splitCnt int, seq, prefixTime bool, m Mapper, r Reducer, fw FileWrapper) *FileProcessor

func (*FileProcessor) ProcFile

func (p *FileProcessor) ProcFile(path, dir, ext string) error

func (*FileProcessor) ProcPath

func (p *FileProcessor) ProcPath(path, dir, ext string) error

func (*FileProcessor) ProcPathReverse

func (p *FileProcessor) ProcPathReverse(path, dir, ext string) error

func (*FileProcessor) Stat

func (p *FileProcessor) Stat() (int, int, int)

type FileWrapper

type FileWrapper interface {
	BeforeWrite(*os.File) error
	AfterWrite(*os.File) error
}

func DummyWrapper

func DummyWrapper() FileWrapper

type FileWriter

type FileWriter interface {
	Open() error
	Write(li lineInfo) (int, error)
	Close() error
}

func NewFileWriter

func NewFileWriter(fw FileWrapper, dir, fn, ext string, splitCnt int) FileWriter

func WithSequence

func WithSequence(parent FileWriter) FileWriter

WithSequence keep output follow input sequence

type LineInfoSlice

type LineInfoSlice []lineInfo

func (LineInfoSlice) Len

func (lis LineInfoSlice) Len() int

func (LineInfoSlice) Less

func (lis LineInfoSlice) Less(i, j int) bool

ascending order

func (LineInfoSlice) Swap

func (lis LineInfoSlice) Swap(i, j int)

type Mapper

type Mapper interface {
	Map(data []byte) []byte
}

type Reducer

type Reducer interface {
	Reduce(data []byte) []byte
}

type TermWriter

type TermWriter struct {
	FileWrapper
}

func NewTermWriter

func NewTermWriter(fw FileWrapper) *TermWriter

func (*TermWriter) Close

func (w *TermWriter) Close() error

func (*TermWriter) Open

func (w *TermWriter) Open() error

func (*TermWriter) Write

func (w *TermWriter) Write(li lineInfo) (int, error)