recordioutil

Documentation

Index

• func NewWriter(w io.Writer, opts WriterOpts) (deprecated.LegacyPackedWriter, error)
• func RegisterWriterFlags(flag *flag.FlagSet, opts *WriterFlags)
• func ScannerOptsFromName(path string) (opts recordio.ScannerOpts, err error)
• func SuffixFromWriterOpts(opts WriterOpts) string
• type CompressionLevelFlag
  • func (f *CompressionLevelFlag) Set(v string) error
  • func (f *CompressionLevelFlag) String() string
• type FlateTransform
  • func NewFlateTransform(level int) *FlateTransform
  • func (f *FlateTransform) CompressTransform(bufs [][]byte) ([]byte, error)
  • func (f *FlateTransform) DecompressTransform(buf []byte) ([]byte, error)
• type WriterFlags
• type WriterOpts
  • func WriterOptsFromFlags(flags *WriterFlags) WriterOpts

Functions

func NewWriter

func NewWriter(w io.Writer, opts WriterOpts) (deprecated.LegacyPackedWriter, error)

NewWriter returns a recordio.LegacyPackedWriter that can optionally compress and encrypt the items it writes.

func RegisterWriterFlags

func RegisterWriterFlags(flag *flag.FlagSet, opts *WriterFlags)

RegisterWriterFlags registers the flags in WriterFlags with the supplied FlagSet. If the values of ItemsPerRecord, MegaBytesPerRecord or CompressionFlag.Level have their go default values then appropriate defaults will be used instead.

func ScannerOptsFromName

func ScannerOptsFromName(path string) (opts recordio.ScannerOpts, err error)

ScannerV2OptsFromName returns the options to use for the supplied pathname based on its suffix.

func SuffixFromWriterOpts

func SuffixFromWriterOpts(opts WriterOpts) string

SuffixFromWriterOpts returns the file name suffix to use for the specified Writer options.

Types

type CompressionLevelFlag

type CompressionLevelFlag struct {
	// Specified will be true if the flag has been specified on the command line.
	Specified bool
	// Level corresponds to the consts in the flate package
	Level int
}

CompressionLevelFlag represents a command line flag to specify a compression level

func (*CompressionLevelFlag) Set

func (f *CompressionLevelFlag) Set(v string) error

Set implements flag.Value.

func (*CompressionLevelFlag) String

func (f *CompressionLevelFlag) String() string

String implements flag.Value.

type FlateTransform

type FlateTransform struct {
	// contains filtered or unexported fields
}

FlateTransform represents a 'transform' that can be used with recordio.PackedWriter and Scanner to compress/decompress items written to a single record.

func NewFlateTransform

func NewFlateTransform(level int) *FlateTransform

NewFlateTransform creates a new Flate instance for use with recordio.PackedWriter and PackedScanner. Level indicates the compression level to use as per the flate package's contstants. The chosen level has no effect when decompressing.

func (*FlateTransform) CompressTransform

func (f *FlateTransform) CompressTransform(bufs [][]byte) ([]byte, error)

CompressTransform is intended for use Recordio.PackedWriterOpts.Transform.

func (*FlateTransform) DecompressTransform

func (f *FlateTransform) DecompressTransform(buf []byte) ([]byte, error)

DecompressTransform is intended for use Recordio.PackedScannerOpts.Transform.

type WriterFlags

type WriterFlags struct {
	CompressionFlag    CompressionLevelFlag
	MegaBytesPerRecord uint
	ItemsPerRecord     uint
}

WriterFlags represents the flags required to configure a recordioutil.Writer.

type WriterOpts

type WriterOpts struct {
	// Marshal is called to marshal an object to a byte slice.
	Marshal recordio.MarshalFunc

	// Index is called whenever a new record is written.
	Index deprecated.RecordIndex

	// MaxItems sets recordio.LegacyPackedWriterOpts when calling NewLegacyPackedWriter
	MaxItems uint32

	// MaxBytes sets recordio.LegacyPackedWriterOpts when calling NewLegacyPackedWriter.
	MaxBytes uint32

	// Flushed is passed to recordio.PackedWriterOpts.
	Flushed func() error

	// FlateLevel indicates the compression level to use, it accepts
	// the values supported by the flate package. The default value (0)
	// is flate.NoCompression.
	// Compression is always performed before encryption.
	FlateLevel int
}

WriterOpts represents the options accepted by NewWriter.

func WriterOptsFromFlags

func WriterOptsFromFlags(flags *WriterFlags) WriterOpts

WriterOptsFromFlags determines the WriterOpts to use based on the supplied command line flags.