impl

package
v0.1.20 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 8, 2022 License: MIT Imports: 26 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ColumnBuilders map[arrow.Type]func(fixedField *FixedField, builder *array.RecordBuilder, columnsize int, fieldNr int, columnsizeCap int) *ColumnBuilder

Functions

func ConsumeLine 

func ConsumeLine(line string, fstc *FixedSizeTableChunk)

TODO fix for utf8 !!! this is only for Ascii/8851-9 one byte coded glyphs

func CreateFixedSizeTableFromFile 

func CreateFixedSizeTableFromFile(fst *FixedSizeTable, row *FixedRow, reader *io.Reader, size int64) error

func DateStringT1ToUnix 

func DateStringT1ToUnix(dateString string) (error, int64)

2020-07-09-09.59.59.99375

func DateStringT1ToUnixMicro added in v0.1.9 

func DateStringT1ToUnixMicro(dateString string) (error, time.Time)

func DateStringT1ToUnixNano 

func DateStringT1ToUnixNano(dateString string) (error, time.Time)

2000-05-13-09.00.00.000000000

func FindLastNLCR added in v0.1.14 

func FindLastNLCR(bytes []byte) int

func FindLastNL_NO_CR added in v0.1.14 

func FindLastNL_NO_CR(bytes []byte) int

func GetSplitBytePositions 

func GetSplitBytePositions(fullString string, substring []Substring)

func IsError 

func IsError(err error) bool

func ParalizeChunks 

func ParalizeChunks(fst *FixedSizeTable, reader *io.Reader, size int64) error

func PrintMemUsage added in v0.1.3 

func PrintMemUsage()

Borrowed this code from https://golangcode.com/print-the-current-memory-usage/

func PrintPerfomance 

func PrintPerfomance(elapsed time.Duration, fst *FixedSizeTable)

func ReleaseRecordBuilders added in v0.1.4 

func ReleaseRecordBuilders(fst *FixedSizeTable)

func ReleaseRecordsForSchema added in v0.1.4 

func ReleaseRecordsForSchema(fst *FixedSizeTable, i int)

func SaveFeather 

func SaveFeather(w *os.File, fst *FixedSizeTable) error

func SaveToCSV added in v0.1.20 

func SaveToCSV(schema *arrow.Schema, record []arrow.Record, writer io.Writer, i int64) error

func SaveToParquet 

func SaveToParquet(schema *arrow.Schema, record []arrow.Record, writer io.Writer, i int64) error

Types

type ColumnBuilder 

type ColumnBuilder interface {
	ParseValue(name string) bool
	FinishColumn() bool
	Nullify()
}

func CreateColumBuilder 

func CreateColumBuilder(fixedField *FixedField, builder *array.RecordBuilder, columnsize int, fieldNr int, columnsizeCap int) *ColumnBuilder

type ColumnBuilderBoolean 

type ColumnBuilderBoolean struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderBoolean) FinishColumn 

func (c *ColumnBuilderBoolean) FinishColumn() bool

func (*ColumnBuilderBoolean) Nullify 

func (c *ColumnBuilderBoolean) Nullify()

func (*ColumnBuilderBoolean) ParseValue 

func (c *ColumnBuilderBoolean) ParseValue(name string) bool

make configurable

type ColumnBuilderDate32 

type ColumnBuilderDate32 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderDate32) FinishColumn 

func (c *ColumnBuilderDate32) FinishColumn() bool

func (*ColumnBuilderDate32) Nullify 

func (c *ColumnBuilderDate32) Nullify()

func (*ColumnBuilderDate32) ParseValue 

func (c *ColumnBuilderDate32) ParseValue(name string) bool

type ColumnBuilderDate64 

type ColumnBuilderDate64 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderDate64) FinishColumn 

func (c *ColumnBuilderDate64) FinishColumn() bool

func (*ColumnBuilderDate64) Nullify 

func (c *ColumnBuilderDate64) Nullify()

func (*ColumnBuilderDate64) ParseValue 

func (c *ColumnBuilderDate64) ParseValue(name string) bool

type ColumnBuilderFloat32 

type ColumnBuilderFloat32 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderFloat32) FinishColumn 

func (c *ColumnBuilderFloat32) FinishColumn() bool

func (*ColumnBuilderFloat32) Nullify 

func (c *ColumnBuilderFloat32) Nullify()

func (*ColumnBuilderFloat32) ParseValue 

func (c *ColumnBuilderFloat32) ParseValue(name string) bool

type ColumnBuilderFloat64 

type ColumnBuilderFloat64 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderFloat64) FinishColumn 

func (c *ColumnBuilderFloat64) FinishColumn() bool

func (*ColumnBuilderFloat64) Nullify 

func (c *ColumnBuilderFloat64) Nullify()

func (*ColumnBuilderFloat64) ParseValue 

func (c *ColumnBuilderFloat64) ParseValue(name string) bool

type ColumnBuilderInt16 

type ColumnBuilderInt16 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderInt16) FinishColumn 

func (c *ColumnBuilderInt16) FinishColumn() bool

func (*ColumnBuilderInt16) Nullify 

func (c *ColumnBuilderInt16) Nullify()

func (*ColumnBuilderInt16) ParseValue 

func (c *ColumnBuilderInt16) ParseValue(name string) bool

type ColumnBuilderInt32 

type ColumnBuilderInt32 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderInt32) FinishColumn 

func (c *ColumnBuilderInt32) FinishColumn() bool

func (*ColumnBuilderInt32) Nullify 

func (c *ColumnBuilderInt32) Nullify()

func (*ColumnBuilderInt32) ParseValue 

func (c *ColumnBuilderInt32) ParseValue(name string) bool

type ColumnBuilderInt64 

type ColumnBuilderInt64 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderInt64) FinishColumn 

func (c *ColumnBuilderInt64) FinishColumn() bool

func (*ColumnBuilderInt64) Nullify 

func (c *ColumnBuilderInt64) Nullify()

func (*ColumnBuilderInt64) ParseValue 

func (c *ColumnBuilderInt64) ParseValue(name string) bool

type ColumnBuilderInt8 

type ColumnBuilderInt8 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderInt8) FinishColumn 

func (c *ColumnBuilderInt8) FinishColumn() bool

func (*ColumnBuilderInt8) Nullify 

func (c *ColumnBuilderInt8) Nullify()

func (*ColumnBuilderInt8) ParseValue 

func (c *ColumnBuilderInt8) ParseValue(name string) bool

type ColumnBuilderString 

type ColumnBuilderString struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderString) FinishColumn 

func (c *ColumnBuilderString) FinishColumn() bool

func (*ColumnBuilderString) Nullify 

func (c *ColumnBuilderString) Nullify()

func (*ColumnBuilderString) ParseValue 

func (c *ColumnBuilderString) ParseValue(name string) bool

type ColumnBuilderUint16 

type ColumnBuilderUint16 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderUint16) FinishColumn 

func (c *ColumnBuilderUint16) FinishColumn() bool

func (*ColumnBuilderUint16) Nullify 

func (c *ColumnBuilderUint16) Nullify()

func (*ColumnBuilderUint16) ParseValue 

func (c *ColumnBuilderUint16) ParseValue(name string) bool

type ColumnBuilderUint32 

type ColumnBuilderUint32 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderUint32) FinishColumn 

func (c *ColumnBuilderUint32) FinishColumn() bool

func (*ColumnBuilderUint32) Nullify 

func (c *ColumnBuilderUint32) Nullify()

func (*ColumnBuilderUint32) ParseValue 

func (c *ColumnBuilderUint32) ParseValue(name string) bool

type ColumnBuilderUint64 

type ColumnBuilderUint64 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderUint64) FinishColumn 

func (c *ColumnBuilderUint64) FinishColumn() bool

func (*ColumnBuilderUint64) Nullify 

func (c *ColumnBuilderUint64) Nullify()

func (*ColumnBuilderUint64) ParseValue 

func (c *ColumnBuilderUint64) ParseValue(name string) bool

type ColumnBuilderUint8 

type ColumnBuilderUint8 struct {
	// contains filtered or unexported fields
}

func (*ColumnBuilderUint8) FinishColumn 

func (c *ColumnBuilderUint8) FinishColumn() bool

func (*ColumnBuilderUint8) Nullify 

func (c *ColumnBuilderUint8) Nullify()

func (*ColumnBuilderUint8) ParseValue 

func (c *ColumnBuilderUint8) ParseValue(name string) bool

type FixedField 

type FixedField struct {
	Len         int
	DestinField arrow.Field
	SourceType  arrow.DataType
	TableId     int
}

type FixedRow 

type FixedRow struct {
	FixedField []FixedField
}

func (FixedRow) CalRowLength 

func (f FixedRow) CalRowLength() int

type FixedSizeTable 

type FixedSizeTable struct {
	// pointer to bytebuffer
	Bytes       []byte
	TableChunks []FixedSizeTableChunk
	Row         *FixedRow

	Schema []arrow.Schema

	Records              [][]arrow.Record
	TableColAmount       []int
	Header               string
	Footer               string
	HasHeader            bool
	HasFooter            bool
	CalcHash             bool
	SourceEncoding       string
	ConsumeLineFunc      func(line string, fstc *FixedSizeTableChunk)
	FindLastNL           func(bytes []byte) int
	CustomParams         interface{}
	CustomColumnBuilders map[arrow.Type]func(fixedField *FixedField, builder *array.RecordBuilder, columnsize int, fieldNr int, columnsizeCap int) *ColumnBuilder

	Cores              int
	LinesParsed        int
	Hash               []byte
	DurationReadChunk  time.Duration
	DurationToArrow    time.Duration
	DurationToExport   time.Duration
	DurationDoneExport time.Duration
	ColumnsizeCap      int
	// contains filtered or unexported fields
}

type FixedSizeTableChunk 

type FixedSizeTableChunk struct {
	Chunkr         int
	FixedSizeTable *FixedSizeTable
	ColumnBuilders []ColumnBuilder
	RecordBuilder  []*array.RecordBuilder
	Record         []arrow.Record
	Bytes          []byte

	LinesParsed       int
	DurationReadChunk time.Duration
	DurationToArrow   time.Duration
	DurationToExport  time.Duration
}

type Substring 

type Substring struct {
	RuneLen int
	Sub     string
	Null    bool
}

func CreateSubstring 

func CreateSubstring(fst *FixedSizeTable) []Substring

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.