loader

Documentation

Index

• Constants
• func CSVtoNumpyMulti(csvReader *csv.Reader, tbk io.TimeBucketKey, cvm *CSVMetadata, chunkSize int, ...) (npm *io.NumpyMultiDataset, endReached bool, err error)
• type CSVConfig
• type CSVMetadata
  • func ReadMetadata(dataFD, controlFD *os.File, dbDataShapes []io.DataShape) (csvReader *csv.Reader, cvm *CSVMetadata, err error)

Constants

const String16RuneSize = 16

Functions

func CSVtoNumpyMulti

func CSVtoNumpyMulti(csvReader *csv.Reader, tbk io.TimeBucketKey, cvm *CSVMetadata, chunkSize int,
	isVariable bool,
) (npm *io.NumpyMultiDataset, endReached bool, err error)

Types

type CSVConfig

type CSVConfig struct {
	FirstRowHasColumnNames bool     `yaml:"firstRowHasColumnNames"`
	TimeFormat             string   `yaml:"timeFormat"`
	Timezone               string   `yaml:"timeZone"`
	ColumnNameMap          []string `yaml:"columnNameMap"`
}

CSVConfig is constructed from the control file that specifies the formatting of the csv data.

type CSVMetadata

type CSVMetadata struct {
	Config *CSVConfig // Configuration of the CSV file, including the names of the columns
	// DSV is data shapes inside this CSV file. The first 2 columns are "Epoch-date" and "Epoch-time".
	// If the schema of existent bucket is "Epoch,Ask,Bid", DSV is ["Epoch-date", "Epoch-time", "Epoch", "Ask", "Bid"].
	DSV []io.DataShape
	// ColumnIndex maps the index of the columns in the CSV file to each time bucket in the DB.
	// ColumnIndex[i+2]=-1 when the column of DSV[i] doesn't exist in the provided CSV file.
	// e.g. when the bucket is "Epoch,Ask,Bid" and Column[3] = -1, it means the provided CSV doesn't have "Ask" column.
	ColumnIndex []int
}

func ReadMetadata

func ReadMetadata(dataFD, controlFD *os.File, dbDataShapes []io.DataShape) (csvReader *csv.Reader, cvm *CSVMetadata, err error)

ReadMetadata returns formatting info about the csv file containing the data to be loaded into the database.