importer

Documentation

Overview

Package importer exposes the shared data-import pipeline used by GraphQL and the CLI.

Index

• Constants
• func DetectCSVDelimiter(data []byte) (string, error)
• func ErrorKeyFromError(err error) string
• func ReadFile(path string, maxBytes int64) ([]byte, error)
• type ColumnMapping
  • func BuildDefaultMappings(sourceColumns []string, targetColumns []engine.Column, useHeaderMapping bool, ...) ([]ColumnMapping, []string, error)
• type ExecuteRequest
• type ExecuteResult
  • func Execute(plugin engine.PluginFunctions, config *engine.PluginConfig, ...) (*ExecuteResult, error)
• type FileFormat
  • func FormatFromPath(path string) FileFormat
  • func (f FileFormat) IsValid() bool
• type FileOptions
• type Mode
  • func (m Mode) IsValid() bool
• type ParsedFile
  • func ParseFile(data []byte, options *FileOptions, maxRows int, enforceRowCap bool) (*ParsedFile, error)
• type ResolvedMapping
  • func ResolveMappings(sourceColumns []string, mappings []ColumnMapping, ...) ([]ResolvedMapping, error)

Constants

const (

	// DefaultPreviewLimit is the default number of rows returned for file previews.
	DefaultPreviewLimit = 25
	// MaxFileSizeBytes is the maximum supported size for table import files.
	MaxFileSizeBytes = int64(50 * 1024 * 1024)
)

Functions

func DetectCSVDelimiter

func DetectCSVDelimiter(data []byte) (string, error)

DetectCSVDelimiter detects a CSV delimiter from a sample of input data.

func ErrorKeyFromError

func ErrorKeyFromError(err error) string

ErrorKeyFromError extracts an import localization key from an importer error.

func ReadFile

func ReadFile(path string, maxBytes int64) ([]byte, error)

ReadFile reads a local import file with a size cap.

Types

type ColumnMapping

type ColumnMapping struct {
	SourceColumn string
	TargetColumn *string
	Skip         bool
}

ColumnMapping maps a source column from the input file to a target column in the destination storage unit.

func BuildDefaultMappings

func BuildDefaultMappings(
	sourceColumns []string,
	targetColumns []engine.Column,
	useHeaderMapping bool,
	allowAutoGenerated bool,
) ([]ColumnMapping, []string, error)

BuildDefaultMappings creates a default mapping plan from parsed source columns to target database columns.

type ExecuteRequest

type ExecuteRequest struct {
	Schema             string
	StorageUnit        string
	Mode               Mode
	Parsed             *ParsedFile
	Mapping            []ColumnMapping
	AllowAutoGenerated bool
	BatchSize          int
	TargetColumns      []engine.Column
}

ExecuteRequest describes a parsed import ready to be applied to a database.

type ExecuteResult

type ExecuteResult struct {
	RowsImported int
}

ExecuteResult summarizes a completed table import.

func Execute

func Execute(plugin engine.PluginFunctions, config *engine.PluginConfig, request *ExecuteRequest) (*ExecuteResult, error)

Execute applies a parsed import file to the target storage unit using the configured import mode and mappings.

type FileFormat

type FileFormat string

FileFormat identifies the supported structured file formats for table import.

const (
	// FileFormatCSV represents a delimited text file.
	FileFormatCSV FileFormat = "CSV"
	// FileFormatExcel represents an Excel workbook.
	FileFormatExcel FileFormat = "EXCEL"
)

func FormatFromPath

func FormatFromPath(path string) FileFormat

FormatFromPath infers the file format from the file extension.

func (FileFormat) IsValid

func (f FileFormat) IsValid() bool

IsValid reports whether the file format is supported.

type FileOptions

type FileOptions struct {
	Format    FileFormat
	Delimiter *string
	Sheet     *string
	HasHeader bool
}

FileOptions configures how an input file should be parsed.

type Mode

type Mode string

Mode controls how imported rows should be applied to the target storage unit.

const (
	// ModeAppend inserts rows and skips duplicates when the database supports it.
	ModeAppend Mode = "APPEND"
	// ModeOverwrite clears existing rows before importing.
	ModeOverwrite Mode = "OVERWRITE"
	// ModeUpsert inserts new rows and updates existing rows by primary key.
	ModeUpsert Mode = "UPSERT"
)

func (Mode) IsValid

func (m Mode) IsValid() bool

IsValid reports whether the import mode is supported.

type ParsedFile

type ParsedFile struct {
	Columns   []string
	Rows      [][]string
	Truncated bool
	Sheet     string
}

ParsedFile contains the normalized contents of an imported file.

func ParseFile

func ParseFile(data []byte, options *FileOptions, maxRows int, enforceRowCap bool) (*ParsedFile, error)

ParseFile parses a structured import file into normalized columns and rows.

type ResolvedMapping

type ResolvedMapping struct {
	SourceIndex int
	TargetName  string
	TargetType  string
	IsNullable  bool
}

ResolvedMapping stores the validated destination metadata for a source column mapping.

func ResolveMappings

func ResolveMappings(
	sourceColumns []string,
	mappings []ColumnMapping,
	targetColumns []engine.Column,
	allowAutoGenerated bool,
) ([]ResolvedMapping, error)

ResolveMappings validates an explicit mapping plan against the target table columns and returns the normalized metadata needed for insertion.