tabular

Documentation

Index

• func BuildRepresentations(rows []map[string]string, relPath string, cfg config.TabularConfig) ([]ingest.Representation, error)
• type CSVLoader
  • func NewCSVLoader(cfg config.TabularConfig) *CSVLoader
  • func (l *CSVLoader) Extensions() []string
  • func (l *CSVLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)
• type Column
  • func DetectSchema(rows []map[string]string) []Column
• type ColumnKind
• type ExcelLoader
  • func NewExcelLoader(cfg config.TabularConfig) *ExcelLoader
  • func (l *ExcelLoader) Extensions() []string
  • func (l *ExcelLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)
• type JSONLoader
  • func NewJSONLoader(cfg config.TabularConfig) *JSONLoader
  • func (l *JSONLoader) Extensions() []string
  • func (l *JSONLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)
• type ParquetLoader
  • func NewParquetLoader(cfg config.TabularConfig) *ParquetLoader
  • func (l *ParquetLoader) Extensions() []string
  • func (l *ParquetLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)
• type SQLiteLoader
  • func NewSQLiteLoader(cfg config.TabularConfig) *SQLiteLoader
  • func (l *SQLiteLoader) Extensions() []string
  • func (l *SQLiteLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)

Functions

func BuildRepresentations

func BuildRepresentations(rows []map[string]string, relPath string, cfg config.TabularConfig) ([]ingest.Representation, error)

BuildRepresentations converts table rows into ingest.Representation slices according to config rules (sampling, token thresholds, etc.). rows parameter MAY be truncated already (e.g. sampling pre-applied by loader).

Types

type CSVLoader

type CSVLoader struct {
	// contains filtered or unexported fields
}

func NewCSVLoader

func NewCSVLoader(cfg config.TabularConfig) *CSVLoader

func (*CSVLoader) Extensions

func (l *CSVLoader) Extensions() []string

func (*CSVLoader) Load

func (l *CSVLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)

type Column

type Column struct {
	Name string
	Kind ColumnKind
}

Column contains name & inferred kind.

func DetectSchema

func DetectSchema(rows []map[string]string) []Column

DetectSchema inspects the first N rows and returns a slice of Column with the best-guess kinds.

type ColumnKind

type ColumnKind int

const (
	KindUnknown     ColumnKind = iota
	KindText                   // long or free-form string
	KindCategorical            // finite small vocabulary of strings
	KindNumeric                // any int / float
	KindDateTime               // ISO / RFC3339 determinable timestamp
	KindBinary                 // base64 / raw bytes, ignored
)

type ExcelLoader

type ExcelLoader struct {
	// contains filtered or unexported fields
}

func NewExcelLoader

func NewExcelLoader(cfg config.TabularConfig) *ExcelLoader

func (*ExcelLoader) Extensions

func (l *ExcelLoader) Extensions() []string

func (*ExcelLoader) Load

func (l *ExcelLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)

type JSONLoader

type JSONLoader struct {
	// contains filtered or unexported fields
}

JSONLoader handles .json (array) and .jsonl files containing objects.

func NewJSONLoader

func NewJSONLoader(cfg config.TabularConfig) *JSONLoader

func (*JSONLoader) Extensions

func (l *JSONLoader) Extensions() []string

func (*JSONLoader) Load

func (l *JSONLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)

type ParquetLoader

type ParquetLoader struct {
	// contains filtered or unexported fields
}

func NewParquetLoader

func NewParquetLoader(cfg config.TabularConfig) *ParquetLoader

func (*ParquetLoader) Extensions

func (l *ParquetLoader) Extensions() []string

func (*ParquetLoader) Load

func (l *ParquetLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)

type SQLiteLoader

type SQLiteLoader struct {
	// contains filtered or unexported fields
}

SQLiteLoader treats each table as a separate dataset similar to a file. Supported extensions: .sqlite .db .sqlite3.

func NewSQLiteLoader

func NewSQLiteLoader(cfg config.TabularConfig) *SQLiteLoader

func (*SQLiteLoader) Extensions

func (l *SQLiteLoader) Extensions() []string

func (*SQLiteLoader) Load

func (l *SQLiteLoader) Load(ctx context.Context, relPath string, absPath string) ([]ingest.Representation, error)