filehandler

Documentation

Index

• func CleanupDecompressedFiles(files []DecompressedFileInfo)
• func DecompressFile(filePath string) (string, error)
• func DetectFormatsFromFiles(files []string) (map[string]Format, error)
• func GetUncompressedPath(filePath string) string
• func GroupFilesByFormat(files []string) (map[Format][]string, error)
• func IsCompressed(filePath string) bool
• func IsFormatSupported(format string) bool
• func IsMQURL(urlStr string) bool
• type Compression
  • func DetectCompression(filePath string) Compression
• type DecompressedFileInfo
  • func DecompressFiles(files []string) ([]DecompressedFileInfo, error)
• type FileHandler
• type Format
  • func DetectFormat(filePath string) (Format, error)
  • func DetectFormatFromFiles(files []string) (Format, error)
  • func GetInnerFormat(filePath string) (Format, error)
  • func SupportedFormats() []Format
• type HandlerFactory
  • func NewHandlerFactory(storage storage.Storage, bar *progressbar.ProgressBar, limitLines int, ...) *HandlerFactory

Functions

func CleanupDecompressedFiles

func CleanupDecompressedFiles(files []DecompressedFileInfo)

CleanupDecompressedFiles removes temporary decompressed files

func DecompressFile

func DecompressFile(filePath string) (string, error)

DecompressFile decompresses a file to a temporary file and returns the temp file path The caller is responsible for cleaning up the temp file

func DetectFormatsFromFiles

func DetectFormatsFromFiles(files []string) (map[string]Format, error)

DetectFormatsFromFiles returns a map of file path to format for each file

func GetUncompressedPath

func GetUncompressedPath(filePath string) string

GetUncompressedPath returns the path without the compression extension e.g., "data.csv.gz" -> "data.csv"

func GroupFilesByFormat

func GroupFilesByFormat(files []string) (map[Format][]string, error)

GroupFilesByFormat groups files by their format

func IsCompressed

func IsCompressed(filePath string) bool

IsCompressed checks if a file path has a compression extension

func IsFormatSupported

func IsFormatSupported(format string) bool

IsFormatSupported checks if a format is supported

func IsMQURL

func IsMQURL(urlStr string) bool

IsMQURL checks if the given URL is a message queue URL

Types

type Compression

type Compression string

Compression represents a supported compression format

const (
	CompressionNone  Compression = ""
	CompressionGzip  Compression = "gzip"
	CompressionBzip2 Compression = "bzip2"
	CompressionXZ    Compression = "xz"
	CompressionZstd  Compression = "zstd"
)

func DetectCompression

func DetectCompression(filePath string) Compression

DetectCompression detects if a file is compressed and returns the compression type

type DecompressedFileInfo

type DecompressedFileInfo struct {
	OriginalPath   string
	DecompressPath string
	WasCompressed  bool
	Format         Format
}

DecompressedFileInfo holds information about a decompressed file

func DecompressFiles

func DecompressFiles(files []string) ([]DecompressedFileInfo, error)

DecompressFiles decompresses all compressed files in the list Returns a list of DecompressedFileInfo with paths to the decompressed files

type FileHandler

type FileHandler interface {
	Import() error
	Lines() int
	Close() error
}

type Format

type Format string

Format represents a supported file format

const (
	FormatCSV      Format = "csv"
	FormatJSON     Format = "json"
	FormatJSONL    Format = "jsonl"
	FormatXML      Format = "xml"
	FormatExcel    Format = "excel"
	FormatParquet  Format = "parquet"
	FormatYAML     Format = "yaml"
	FormatAVRO     Format = "avro"
	FormatORC      Format = "orc"
	FormatPostgres Format = "postgres"
	FormatMySQL    Format = "mysql"
	FormatDuckDB   Format = "duckdb"
	FormatMongoDB  Format = "mongodb"
	FormatDynamoDB Format = "dynamodb"
	FormatSQLite   Format = "sqlite"
	FormatMQ       Format = "mq"    // Message Queue (SQS, Kafka, RabbitMQ, etc.)
	FormatMixed    Format = "mixed" // Mixed file formats (for JOINs across different formats)
)

func DetectFormat

func DetectFormat(filePath string) (Format, error)

DetectFormat detects the file format from file extension or URL scheme

func DetectFormatFromFiles

func DetectFormatFromFiles(files []string) (Format, error)

DetectFormatFromFiles detects the format from a list of files If all files have the same format, returns that format If files have mixed formats, returns FormatMixed

func GetInnerFormat

func GetInnerFormat(filePath string) (Format, error)

GetInnerFormat detects the format of a compressed file e.g., "data.csv.gz" -> FormatCSV

func SupportedFormats

func SupportedFormats() []Format

SupportedFormats returns a list of supported file formats

type HandlerFactory

type HandlerFactory struct {
	// contains filtered or unexported fields
}

HandlerFactory creates file handlers based on format

func NewHandlerFactory

func NewHandlerFactory(storage storage.Storage, bar *progressbar.ProgressBar, limitLines int, collection string, delimiter rune) *HandlerFactory

NewHandlerFactory creates a new handler factory