Version: v1.9.0 Latest Latest

This package is not in the latest version of its module.

Go to latest
Published: Jul 27, 2023 License: BSD-3-Clause Imports: 19 Imported by: 0



Package zngio provides an API for reading and writing zng values and directives in binary zng format. The Reader and Writer types implement the the zio.Reader and zio.Writer interfaces. Since these methods read and write only zed.Records, but the zng format includes additional functionality, other methods are available to read/write zng comments and include virtual channel numbers in the stream. Virtual channels provide a way to indicate which output of a flowgraph a result came from when a flowgraph computes multiple output channels. The zng values in this zng value are "machine format" as prescirbed by the ZNG spec. The vanilla zio.Reader and zio.Writer implementations ignore application-specific payloads (e.g., channel encodings).



View Source
const (
	TypesFrame   = 0
	ValuesFrame  = 1
	ControlFrame = 2
View Source
const (
	EOS                 = 0xff
	ControlFormatZNG    = 0
	ControlFormatJSON   = 1
	ControlFormatZSON   = 2
	ControlFormatString = 3
	ControlFormatBinary = 4
View Source
const (
	ReadSize  = 512 * 1024
	MaxSize   = 1024 * 1024 * 1024
	TypeLimit = 10000
View Source
const (
	Magic          = "ZNG Trailer"
	TrailerMaxSize = 4096
View Source
const (
	TypeDefRecord = 0
	TypeDefArray  = 1
	TypeDefSet    = 2
	TypeDefMap    = 3
	TypeDefUnion  = 4
	TypeDefEnum   = 5
	TypeDefError  = 6
	TypeDefName   = 7
View Source
const DefaultFrameThresh = 512 * 1024

DefaultFrameThresh is a reasonable default for WriterOpts.FrameThresh.


View Source
var ErrTrailerNotFound = errors.New("trailer not found")


func MarshalTrailer added in v1.0.0

func MarshalTrailer(typ string, version int, sections []int64, meta interface{}) (zed.Value, error)

func ReadTrailerAsBytes added in v1.0.0

func ReadTrailerAsBytes(r io.ReaderAt, fileSize int64) ([]byte, error)


type CompressionFormat added in v1.0.0

type CompressionFormat int
const CompressionFormatLZ4 CompressionFormat = 0x00

type Control added in v1.0.0

type Control struct {
	Format int
	Bytes  []byte

type Decoder added in v1.0.0

type Decoder struct {
	// contains filtered or unexported fields

func NewDecoder added in v1.0.0

func NewDecoder(zctx *zed.Context) *Decoder

type Encoder

type Encoder struct {
	// contains filtered or unexported fields

func NewEncoder

func NewEncoder() *Encoder

func (*Encoder) Encode

func (e *Encoder) Encode(external zed.Type) (zed.Type, error)

Encode takes a type from outside this context and constructs a type from inside this context and emits ZNG typedefs for any type needed to construct the new type into the buffer provided.

func (*Encoder) Flush added in v1.0.0

func (e *Encoder) Flush()

func (*Encoder) Lookup

func (e *Encoder) Lookup(external zed.Type) zed.Type

func (*Encoder) Reset

func (e *Encoder) Reset()

type Reader

type Reader struct {
	// contains filtered or unexported fields

func NewReader

func NewReader(sctx *zed.Context, reader io.Reader) *Reader

func NewReaderWithOpts

func NewReaderWithOpts(zctx *zed.Context, reader io.Reader, opts ReaderOpts) *Reader

func (*Reader) Close added in v1.0.0

func (r *Reader) Close() error

Close guarantees that the underlying io.Reader is not read after it returns.

func (*Reader) NewScanner

func (r *Reader) NewScanner(ctx context.Context, filter zbuf.Filter) (zbuf.Scanner, error)

func (*Reader) Read

func (r *Reader) Read() (*zed.Value, error)

func (*Reader) ReadPayload

func (r *Reader) ReadPayload() (*zed.Value, *Control, error)

type ReaderOpts

type ReaderOpts struct {
	Validate bool
	Size     int
	Max      int
	Threads  int

type Trailer added in v1.0.0

type Trailer struct {
	Magic    string    `zed:"magic"`
	Type     string    `zed:"type"`
	Version  int       `zed:"version"`
	Sections []int64   `zed:"sections"`
	Meta     zed.Value `zed:"meta"`

func ReadTrailer added in v1.0.0

func ReadTrailer(r io.ReaderAt, fileSize int64) (*Trailer, error)

type Writer

type Writer struct {
	// contains filtered or unexported fields

func NewWriter

func NewWriter(w io.WriteCloser) *Writer

NewWriter returns a writer to w with reasonable default options. Specifically, it enables compression and sets the frame threshold to DefaultFrameThresh.

func NewWriterWithOpts added in v1.2.0

func NewWriterWithOpts(w io.WriteCloser, opts WriterOpts) *Writer

NewWriterWithOpts returns a writer to w with opts.

func (*Writer) Close

func (w *Writer) Close() error

func (*Writer) DisableCompression added in v1.0.0

func (w *Writer) DisableCompression()

func (*Writer) EndStream

func (w *Writer) EndStream() error

func (*Writer) Position

func (w *Writer) Position() int64

Position may be called after EndStream to get a seekable offset into the output for the next stream. Calling Position at any other team returns unusable seek offsets.

func (*Writer) Write

func (w *Writer) Write(val *zed.Value) error

func (*Writer) WriteControl

func (w *Writer) WriteControl(b []byte, format uint8) error

type WriterOpts

type WriterOpts struct {
	Compress bool
	// FrameThresh is the minimum frame size in uncompressed bytes.
	FrameThresh int

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL