config

Documentation

Index

• func ParquetFields(fields []Field) []parquet.Field
• type Archiver
• type Field
  • func SchemaToConfigFields(schema parquet.Schema) []Field
• type Global
• type Librarian
  • func NewLibrarianFromFile(fpath string) (*Librarian, error)
• type Logger
• type Preserver
• type Repository
• type Source

Functions

func ParquetFields

func ParquetFields(fields []Field) []parquet.Field

Types

type Archiver

type Archiver struct {
	Name       string     `yaml:"name"`
	Source     Source     `yaml:"source"`
	Preserver  Preserver  `yaml:"preserver"`
	Repository Repository `yaml:"repository"`
}

type Field

type Field struct {
	Name           string `yaml:"name"`
	Type           string `yaml:"type"`
	ConvertedType  string `yaml:"converted_type,omitempty"`
	RepetitionType string `yaml:"repetition_type,omitempty"`
	Scale          *int   `yaml:"scale,omitempty"`
	Precision      *int   `yaml:"precision,omitempty"`
	Length         *int   `yaml:"length,omitempty"`
}

func SchemaToConfigFields

func SchemaToConfigFields(schema parquet.Schema) []Field

type Global

type Global struct {
	Logger Logger `yaml:"logger"`
}

type Librarian

type Librarian struct {
	Global   Global   `yaml:"global"`
	Archiver Archiver `yaml:"archiver"`
}

func NewLibrarianFromFile

func NewLibrarianFromFile(fpath string) (*Librarian, error)

type Logger

type Logger struct {
	Level string `yaml:"level"`
}

type Preserver

type Preserver struct {
	Type string `yaml:"type"`

	// BatchSizeNumRecords specifies the number of records
	// that will be contained in each file part.
	BatchSizeNumRecords int `yaml:"batch_size_num_records"`
	Parquet             struct {
		Schema []Field `yaml:"schema"`
	} `yaml:"parquet"`
}

type Repository

type Repository struct {
	Type        string `yaml:"type"`
	LocalConfig struct {
		Path string `yaml:"path"`
	} `yaml:"local"`

	S3Config struct {
		Bucket         string `yaml:"bucket"`
		Region         string `yaml:"region"`
		Prefix         string `yaml:"prefix"`
		Endpoint       string `yaml:"endpoint"`
		ForcePathStyle bool   `yaml:"force_path_style"`
	} `yaml:"s3"`
}

type Source

type Source struct {
	ConnectionString string `yaml:"connection_string"`
	Schema           string `yaml:"schema"`
	Table            string `yaml:"table"`
	Query            string `yaml:"query"`
}