stream

package
v0.0.0-...-eb6fe92 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 26, 2018 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var MAXIMUM_SLICE_LENGTH = 400

Functions

This section is empty.

Types

type AbstractBlock 

type AbstractBlock struct {
	Algorithm string `xml:"-" json:"-"` // the compression algorithm used: snappy, gzip, or none.
	BigEndian bool   `xml:"-" json:"-"` // If true, then encode numbers using a big-endian byte order, else encodes using littl-endian byte order.
}

AbstractBlock is an abstract struct extended by MemoryBlock and TempFileBlock.

func (AbstractBlock) GetAlgorithm 

func (ab AbstractBlock) GetAlgorithm() string

Returns the compress algorithm, which can be: snappy, gzip, or none.

func (AbstractBlock) UseBigEndian 

func (ab AbstractBlock) UseBigEndian() bool

UseBigEndian returns true is

type Block 

type Block interface {
	Init(b []byte) error               // initialize block
	Size() (int64, error)              // get size of block in bytes
	Reader() (*Reader, error)          // get reader for this block
	Iterator() (*BlockIterator, error) // get iterator for this block
	Get(position int) ([]byte, error)  // get object at the given position
	Remove() error                     // remove block
}

Block is an interface for a compressed array of objects in a binary representation

type BlockIterator 

type BlockIterator struct {
	Reader    *Reader
	BigEndian bool
}

BlockIterator is an iterator for reading the data in a block

func (*BlockIterator) Close 

func (it *BlockIterator) Close() error

Close closes the iterator's underlying Reader.

func (*BlockIterator) Next 

func (it *BlockIterator) Next() ([]byte, error)

Next returns the bytes of the next object in the block, and an error if any.

func (*BlockIterator) Skip 

func (it *BlockIterator) Skip(n int) error

Skip advances the iterator forward by "n" objects. Returns an error, if any.

type Iterator 

type Iterator interface {
	Next() ([]byte, error) // returns the current object and advances forward.
	Close() error          // close the underlying Reader.
}

Iterator is an interface implemented by StreamIterator and BlockIterator used for iterating through objects.

type MemoryBlock 

type MemoryBlock struct {
	AbstractBlock
	Bytes []byte `xml:"-" json:"-"` // The compressed block of bytes
}

MemoryBlock holds a block of objects in memory. The objects may be compressed using the Algorithm.

func NewMemoryBlock 

func NewMemoryBlock(algorithm string, bigEndian bool) *MemoryBlock

NewMemoryBlock returns a new MemoryBlock. Algorithm can be snappy, gzip, or none. If bigEndian is true, then encodes numbers using a big-endian byte order, else encodes using littl-endian byte order.

func (*MemoryBlock) Get 

func (mb *MemoryBlock) Get(position int) ([]byte, error)

Get returns the bytes for an object at an arbitrary position, and an error if any. Given the compressed nature of the data, Get starts reading from the beginning every time. If you're iterating through the data, use Iterator. Only use this function for random access.

func (*MemoryBlock) Init 

func (mb *MemoryBlock) Init(b []byte) error

Init initializes a block's data.

func (*MemoryBlock) Iterator 

func (mb *MemoryBlock) Iterator() (*BlockIterator, error)

Iterator returns a BlockIterator for iterating through the bytes, and an error if any.

func (*MemoryBlock) Reader 

func (mb *MemoryBlock) Reader() (*Reader, error)

Reader returns a *Reader for reading the compressed bytes, and an error if any.

func (*MemoryBlock) Remove 

func (mb *MemoryBlock) Remove() error

Remove clears the Bytes array.

func (*MemoryBlock) Size 

func (mb *MemoryBlock) Size() (int64, error)

Size returns the number of bytes as an int64.

type Reader 

type Reader struct {
	Reader     io.Reader
	ReadCloser io.ReadCloser
	File       *os.File
}

Reader is a struct for easily closing underlying file points once a read is complete. Reader should only be used in situations where an *os.File is only used by a single io.Rreader.

func (*Reader) Close 

func (r *Reader) Close() error

Close closes the ReadCloser, if any and the underlying os.File.

func (*Reader) Read 

func (r *Reader) Read(p []byte) (n int, err error)

Read reads the

type Stream 

type Stream struct {
	BlockType   string        `xml:"-" json:"-"`
	TempDir     string        `xml:"-" json:"-"`
	Algorithm   string        `xml:"-" json:"-"`
	BigEndian   bool          `xml:"-" json:"-"`
	BlockSize   int           `xml:"-" json:"-"`
	Blocks      []Block       `xml:"-" json:"-"`
	Buffer      *bytes.Buffer `xml:"-" json:"-"`
	Writer      Writer        `xml:"-" json:"-"`
	WriteCloser WriteCloser   `xml:"-" json:"-"`
}

func New 

func New(alg string, endianness string, blockSize int, block_type string, tempDir string) (*Stream, error)

func (*Stream) AppendBlock 

func (s *Stream) AppendBlock(b []byte) error

func (*Stream) BlockIterator 

func (s *Stream) BlockIterator(n int) (*BlockIterator, error)

NewIterator returns a new iterator for the stream

func (*Stream) Close 

func (s *Stream) Close() error

func (*Stream) Flush 

func (s *Stream) Flush() error

func (*Stream) Get 

func (s *Stream) Get(position int) ([]byte, error)

func (*Stream) Init 

func (s *Stream) Init() error

func (*Stream) Iterator 

func (s *Stream) Iterator() (*StreamIterator, error)

func (*Stream) Reader 

func (s *Stream) Reader(n int) (*Reader, error)

func (*Stream) Remove 

func (s *Stream) Remove() error

func (*Stream) Rotate 

func (s *Stream) Rotate() error

func (*Stream) Size 

func (s *Stream) Size() (int64, error)

func (*Stream) Write 

func (s *Stream) Write(b []byte) (n int, err error)

func (*Stream) WriteObject 

func (s *Stream) WriteObject(obj encoding.BinaryMarshaler) (n int, err error)

type StreamIterator 

type StreamIterator struct {
	Blocks        []Block        `xml:"-" json:"-"`
	BlockIndex    int            `xml:"-" json:"-"`
	BlockIterator *BlockIterator `xml:"-" json:"-"`
}

func NewStreamIterator 

func NewStreamIterator(blocks []Block) (*StreamIterator, error)

func (*StreamIterator) Next 

func (si *StreamIterator) Next() ([]byte, error)

type TempFileBlock 

type TempFileBlock struct {
	AbstractBlock
	TempDir         string `xml:"-" json:"-"`
	TempDirExpanded string `xml:"-" json:"-"`
	TempFile        string `xml:"-" json:"-"`
}

TempFileBlock is struct for reading & writing a compressed block of objects to a temporary file on disk.

func NewTempFileBlock 

func NewTempFileBlock(algorithm string, bigEndian bool, tempDir string) *TempFileBlock

NewTempFileBlock returns a new TempFileBlock.

func (*TempFileBlock) Get 

func (tfb *TempFileBlock) Get(position int) ([]byte, error)

func (*TempFileBlock) Init 

func (tfb *TempFileBlock) Init(b []byte) error

Init initializes a TempFileBlock by writing "b" to a temp file in the TempDir directory. The path to the temp file is saved to tfb.TempFile.

func (*TempFileBlock) Iterator 

func (tfb *TempFileBlock) Iterator() (*BlockIterator, error)

Iterator returns a BlockIterator for iterating through the blocks data, and an error if any.

func (*TempFileBlock) Reader 

func (tfb *TempFileBlock) Reader() (*Reader, error)

Reader returns a Reader for reading the data in the block, and an error if any.

func (*TempFileBlock) Remove 

func (tfb *TempFileBlock) Remove() error

Remove removes the temp file from disk.

func (*TempFileBlock) Size 

func (tfb *TempFileBlock) Size() (int64, error)

Size returns the number of bytes in the block, by using os.FileInfo.Size(), and an error if any.

type WriteCloser 

type WriteCloser interface {
	Writer
	io.Closer
}

type Writer 

type Writer interface {
	io.Writer
	Flush() error
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.