procs

Documentation

Index

• Variables
• func GetGroup(c *scat.Chunk) (group []*scat.Chunk, ok bool)
• func GetGroupErr(c *scat.Chunk) (err error, ok bool)
• func Process(proc Proc, chunk *scat.Chunk) error
• func SingleRes(c *scat.Chunk, err error) <-chan Res
• type Cascade
  • func (casc Cascade) Finish() error
  • func (casc Cascade) Process(c *scat.Chunk) <-chan Res
• type Chain
  • func (procs Chain) Finish() error
  • func (chain Chain) Process(c *scat.Chunk) <-chan Res
• type ChunkFunc
  • func (ChunkFunc) Finish() error
  • func (fn ChunkFunc) Process(c *scat.Chunk) <-chan Res
• type ChunkIterFunc
  • func (ChunkIterFunc) Finish() error
  • func (fn ChunkIterFunc) Process(c *scat.Chunk) <-chan Res
• type CmdFunc
  • func (CmdFunc) Finish() error
  • func (fn CmdFunc) Process(c *scat.Chunk) <-chan Res
• type CmdInFunc
  • func (CmdInFunc) Finish() error
  • func (fn CmdInFunc) Process(c *scat.Chunk) <-chan Res
• type CmdOutFunc
  • func (CmdOutFunc) Finish() error
  • func (fn CmdOutFunc) Process(c *scat.Chunk) <-chan Res
• type DiscardChunks
  • func (dc DiscardChunks) Process(c *scat.Chunk) <-chan Res
• type DynProcer
• type EndProc
• type ErrProc
• type Filter
  • func (p Filter) Process(c *scat.Chunk) <-chan Res
• type Group
  • func NewGroup(size int) Group
• type Gzip
  • func (gz Gzip) Proc() Proc
  • func (gz Gzip) Unproc() Proc
• type IndexProc
  • func NewIndexProc(w io.Writer) IndexProc
• type InplaceFunc
  • func (InplaceFunc) Finish() error
  • func (fn InplaceFunc) Process(c *scat.Chunk) <-chan Res
• type MissingDataError
  • func (err MissingDataError) Error() string
• type OnEnd
  • func (oe OnEnd) Process(c *scat.Chunk) <-chan Res
• type PathCmdInFn
• type Proc
  • func NewBacklog(nslots int, proc Proc) Proc
  • func NewConcur(max int, dynp DynProcer) Proc
  • func NewJoin(w io.Writer) Proc
  • func NewPathCmdIn(newCmd PathCmdInFn, tmp *tmpdedup.Dir) Proc
  • func NewSplitSize(min, max uint) Proc
• type ProcFunc
  • func (ProcFunc) Finish() error
  • func (fn ProcFunc) Process(c *scat.Chunk) <-chan Res
• type ProcUnprocer
  • func NewParity(ndata, nparity int) (p ProcUnprocer, err error)
• type Procer
• type Res
• type Sort
  • func (s *Sort) Finish() error
  • func (s *Sort) Process(c *scat.Chunk) <-chan Res
• type Unprocer
• type WrapperProc
• type WriterTo
  • func (wt WriterTo) Finish() error
  • func (wt WriterTo) Process(c *scat.Chunk) <-chan Res

Variables

var (
	ErrIndexUnprocessedChunk = errors.New("unprocessed chunk")
	ErrIndexProcessEnded     = errors.New("process already ended")
	ErrIndexDup              = errors.New("won't process dup chunk")
)

var (
	ErrShort           = errors.New("missing final chunks")
	ErrUnreturnedSlots = errors.New("unreturned slots left")
)

var ErrIntegrityCheckFailed = errors.New("checksum verification failed")

Functions

func GetGroup

func GetGroup(c *scat.Chunk) (group []*scat.Chunk, ok bool)

func GetGroupErr

func GetGroupErr(c *scat.Chunk) (err error, ok bool)

func Process

func Process(proc Proc, chunk *scat.Chunk) error

func SingleRes

func SingleRes(c *scat.Chunk, err error) <-chan Res

Types

type Cascade

type Cascade []Proc

func (Cascade) Finish

func (casc Cascade) Finish() error

func (Cascade) Process

func (casc Cascade) Process(c *scat.Chunk) <-chan Res

type Chain

type Chain []Proc

func (Chain) Finish

func (procs Chain) Finish() error

func (Chain) Process

func (chain Chain) Process(c *scat.Chunk) <-chan Res

type ChunkFunc

type ChunkFunc func(*scat.Chunk) (*scat.Chunk, error)

func (ChunkFunc) Finish

func (ChunkFunc) Finish() error

func (ChunkFunc) Process

func (fn ChunkFunc) Process(c *scat.Chunk) <-chan Res

type ChunkIterFunc

type ChunkIterFunc func(*scat.Chunk) scat.ChunkIter

func (ChunkIterFunc) Finish

func (ChunkIterFunc) Finish() error

func (ChunkIterFunc) Process

func (fn ChunkIterFunc) Process(c *scat.Chunk) <-chan Res

type CmdFunc

type CmdFunc func(*scat.Chunk) (*exec.Cmd, error)

func (CmdFunc) Finish

func (CmdFunc) Finish() error

func (CmdFunc) Process

func (fn CmdFunc) Process(c *scat.Chunk) <-chan Res

type CmdInFunc

type CmdInFunc CmdFunc

func (CmdInFunc) Finish

func (CmdInFunc) Finish() error

func (CmdInFunc) Process

func (fn CmdInFunc) Process(c *scat.Chunk) <-chan Res

type CmdOutFunc

type CmdOutFunc CmdFunc

func (CmdOutFunc) Finish

func (CmdOutFunc) Finish() error

func (CmdOutFunc) Process

func (fn CmdOutFunc) Process(c *scat.Chunk) <-chan Res

type DiscardChunks

type DiscardChunks struct {
	Proc
}

func (DiscardChunks) Process

func (dc DiscardChunks) Process(c *scat.Chunk) <-chan Res

type DynProcer

type DynProcer interface {
	Procs(*scat.Chunk) ([]Proc, error)
	Finish() error
}

type EndProc

type EndProc interface {
	ProcessFinal(*scat.Chunk, *scat.Chunk) error
	ProcessEnd(*scat.Chunk) error
}

type ErrProc

type ErrProc interface {
	ProcessErr(*scat.Chunk, error) <-chan Res
}

type Filter

type Filter struct {
	Proc
	Filter func(Res) Res
}

func (Filter) Process

func (p Filter) Process(c *scat.Chunk) <-chan Res

type Group

type Group interface {
	Proc
	ErrProc
}

func NewGroup

func NewGroup(size int) Group

type Gzip

type Gzip struct {
}

func (Gzip) Proc

func (gz Gzip) Proc() Proc

func (Gzip) Unproc

func (gz Gzip) Unproc() Proc

type IndexProc

type IndexProc interface {
	Proc
	EndProc
}

func NewIndexProc

func NewIndexProc(w io.Writer) IndexProc

type InplaceFunc

type InplaceFunc func(*scat.Chunk) error

func (InplaceFunc) Finish

func (InplaceFunc) Finish() error

func (InplaceFunc) Process

func (fn InplaceFunc) Process(c *scat.Chunk) <-chan Res

type MissingDataError

type MissingDataError struct {
	Err error
}

func (MissingDataError) Error

func (err MissingDataError) Error() string

type OnEnd

type OnEnd struct {
	Proc
	OnEnd func(error)
}

func (OnEnd) Process

func (oe OnEnd) Process(c *scat.Chunk) <-chan Res

type PathCmdInFn

type PathCmdInFn func(*scat.Chunk, string) (*exec.Cmd, error)

type Proc

type Proc interface {
	Process(*scat.Chunk) <-chan Res
	Finish() error
}

var (
	ChecksumProc   Proc = InplaceFunc(checksumProcess)
	ChecksumUnproc Proc = InplaceFunc(checksumUnprocess)
)

var IndexUnproc Proc = ChunkIterFunc(indexUnprocess)

var Nop Proc

var Split Proc

func NewBacklog

func NewBacklog(nslots int, proc Proc) Proc

func NewConcur

func NewConcur(max int, dynp DynProcer) Proc

func NewJoin

func NewJoin(w io.Writer) Proc

func NewPathCmdIn

func NewPathCmdIn(newCmd PathCmdInFn, tmp *tmpdedup.Dir) Proc

func NewSplitSize

func NewSplitSize(min, max uint) Proc

type ProcFunc

type ProcFunc func(*scat.Chunk) <-chan Res

func (ProcFunc) Finish

func (ProcFunc) Finish() error

func (ProcFunc) Process

func (fn ProcFunc) Process(c *scat.Chunk) <-chan Res

type ProcUnprocer

type ProcUnprocer interface {
	Procer
	Unprocer
}

func NewParity

func NewParity(ndata, nparity int) (p ProcUnprocer, err error)

type Procer

type Procer interface {
	Proc() Proc
}

type Res

type Res struct {
	Chunk *scat.Chunk
	Err   error
}

type Sort

type Sort struct {
	// contains filtered or unexported fields
}

func (*Sort) Finish

func (s *Sort) Finish() error

func (*Sort) Process

func (s *Sort) Process(c *scat.Chunk) <-chan Res

type Unprocer

type Unprocer interface {
	Unproc() Proc
}

type WrapperProc

type WrapperProc interface {
	Proc
	Underlying() Proc
}

type WriterTo

type WriterTo struct {
	W io.Writer
}

func (WriterTo) Finish

func (wt WriterTo) Finish() error

func (WriterTo) Process

func (wt WriterTo) Process(c *scat.Chunk) <-chan Res