ocf

package
v1.6.1-alpha.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 29, 2021 License: MIT Imports: 12 Imported by: 0

Documentation

Overview

Package ocf implements encoding and decoding of Avro Object Container Files as defined by the Avro specification.

See the Avro specification for an understanding of Avro: http://avro.apache.org/docs/current/

Index

Examples

Constants

This section is empty.

Variables

View Source
var HeaderSchema = avro.MustParse(`{
	"type": "record", 
	"name": "org.apache.avro.file.Header",
	"fields": [
		{"name": "magic", "type": {"type": "fixed", "name": "Magic", "size": 4}},
		{"name": "meta", "type": {"type": "map", "values": "bytes"}},
		{"name": "sync", "type": {"type": "fixed", "name": "Sync", "size": 16}}
	]
}`)

HeaderSchema is the Avro schema of a container file header.

Functions

This section is empty.

Types

type Codec

type Codec interface {
	// Decode decodes the given bytes.
	Decode([]byte) ([]byte, error)
	// Encode encodes the given bytes.
	Encode([]byte) []byte
}

Codec represents a compression codec.

type CodecName

type CodecName string

CodecName represents a compression codec name.

const (
	Null    CodecName = "null"
	Deflate CodecName = "deflate"
	Snappy  CodecName = "snappy"
)

Supported compression codecs.

type Decoder

type Decoder struct {
	// contains filtered or unexported fields
}

Decoder reads and decodes Avro values from a container file.

func NewDecoder

func NewDecoder(r io.Reader) (*Decoder, error)

NewDecoder returns a new decoder that reads from reader r.

Example
package main

import (
	"log"
	"os"

	"github.com/xl4hub/hamba-avro/ocf"
)

func main() {
	type SimpleRecord struct {
		A int64  `avro:"a"`
		B string `avro:"b"`
	}

	f, err := os.Open("/your/avro/file.avro")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	dec, err := ocf.NewDecoder(f)
	if err != nil {
		log.Fatal(err)
	}

	for dec.HasNext() {
		var record SimpleRecord
		err = dec.Decode(&record)
		if err != nil {
			log.Fatal(err)
		}

		// Do something with the data
	}

	if dec.Error() != nil {
		log.Fatal(err)
	}
}
Output:

func (*Decoder) Decode

func (d *Decoder) Decode(v interface{}) error

Decode reads the next Avro encoded value from its input and stores it in the value pointed to by v.

func (*Decoder) Error

func (d *Decoder) Error() error

Error returns the last reader error.

func (*Decoder) HasNext

func (d *Decoder) HasNext() bool

HasNext determines if there is another value to read.

func (*Decoder) Metadata

func (d *Decoder) Metadata() map[string][]byte

Metadata returns the header metadata.

type DeflateCodec

type DeflateCodec struct{}

DeflateCodec is a flate compression codec.

func (*DeflateCodec) Decode

func (*DeflateCodec) Decode(b []byte) ([]byte, error)

Decode decodes the given bytes.

func (*DeflateCodec) Encode

func (*DeflateCodec) Encode(b []byte) []byte

Encode encodes the given bytes.

type Encoder

type Encoder struct {
	// contains filtered or unexported fields
}

Encoder writes Avro container file to an output stream.

func NewEncoder

func NewEncoder(s string, w io.Writer, opts ...EncoderFunc) (*Encoder, error)

NewEncoder returns a new encoder that writes to w using schema s.

Example
package main

import (
	"log"
	"os"

	"github.com/xl4hub/hamba-avro/ocf"
)

func main() {
	schema := `{
	    "type": "record",
	    "name": "simple",
	    "namespace": "org.hamba.avro",
	    "fields" : [
	        {"name": "a", "type": "long"},
	        {"name": "b", "type": "string"}
	    ]
	}`

	type SimpleRecord struct {
		A int64  `avro:"a"`
		B string `avro:"b"`
	}

	f, err := os.Open("/your/avro/file.avro")
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	enc, err := ocf.NewEncoder(schema, f)
	if err != nil {
		log.Fatal(err)
	}

	var record SimpleRecord
	err = enc.Encode(record)
	if err != nil {
		log.Fatal(err)
	}

	if err := enc.Flush(); err != nil {
		log.Fatal(err)
	}

	if err := f.Sync(); err != nil {
		log.Fatal(err)
	}
}
Output:

func (*Encoder) Close

func (e *Encoder) Close() error

Close closes the encoder, flushing the writer.

func (*Encoder) Encode

func (e *Encoder) Encode(v interface{}) error

Encode writes the Avro encoding of v to the stream.

func (*Encoder) Flush

func (e *Encoder) Flush() error

Flush flushes the underlying writer.

type EncoderFunc

type EncoderFunc func(cfg *encoderConfig)

EncoderFunc represents an configuration function for Encoder.

func WithBlockLength

func WithBlockLength(length int) EncoderFunc

WithBlockLength sets the block length on the encoder.

func WithCodec

func WithCodec(codec CodecName) EncoderFunc

WithCodec sets the compression codec on the encoder.

func WithMetadata

func WithMetadata(meta map[string][]byte) EncoderFunc

WithMetadata sets the metadata on the encoder header.

type Header struct {
	Magic [4]byte           `avro:"magic"`
	Meta  map[string][]byte `avro:"meta"`
	Sync  [16]byte          `avro:"sync"`
}

Header represents an Avro container file header.

type NullCodec

type NullCodec struct{}

NullCodec is a no op codec.

func (*NullCodec) Decode

func (*NullCodec) Decode(b []byte) ([]byte, error)

Decode decodes the given bytes.

func (*NullCodec) Encode

func (*NullCodec) Encode(b []byte) []byte

Encode encodes the given bytes.

type SnappyCodec

type SnappyCodec struct{}

SnappyCodec is a snappy compression codec.

func (*SnappyCodec) Decode

func (*SnappyCodec) Decode(b []byte) ([]byte, error)

Decode decodes the given bytes.

func (*SnappyCodec) Encode

func (*SnappyCodec) Encode(b []byte) []byte

Encode encodes the given bytes.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL