columnifier

Documentation

Index

• func NewParquetColumnifier(st string, sf string, rt string, output string, config Config) (*parquetColumnifier, error)
• type Columnifier
  • func NewColumnifier(st string, sf string, rt string, o string, config Config) (Columnifier, error)
• type Config
  • func NewConfig(parquetPageSize, parquetRowGroupSize int64, parquetCompressionCodec string) (*Config, error)
• type Parquet

Functions

func NewParquetColumnifier

func NewParquetColumnifier(st string, sf string, rt string, output string, config Config) (*parquetColumnifier, error)

NewParquetColumnifier creates a new parquetColumnifier.

Types

type Columnifier

type Columnifier interface {
	WriteFromReader(reader io.Reader) (int, error)
	WriteFromFiles(paths []string) (int, error)
	Close() error
}

Columnifier is the interface that converts input file to columnar format file.

func NewColumnifier

func NewColumnifier(st string, sf string, rt string, o string, config Config) (Columnifier, error)

NewColumnifier creates a new Columnifier.

type Config

type Config struct {
	Parquet Parquet
}

func NewConfig

func NewConfig(parquetPageSize, parquetRowGroupSize int64, parquetCompressionCodec string) (*Config, error)

type Parquet

type Parquet struct {
	PageSize         int64
	RowGroupSize     int64
	CompressionCodec parquet.CompressionCodec
}