Documentation
¶
Overview ¶
Package data provides Parquet read/write, CSV helpers, download utilities, and shared market-data types used across ETL and DuckDB queries.
Index ¶
- Constants
- Variables
- func AnyTimestampToTime(ts int64) *time.Time
- func DrainParquet[T any](ctx context.Context, recordCh <-chan *T, errCh <-chan error) ([]*T, error)
- func EncodeGetQuery(url string, req any) (string, error)
- func ExtractZip(zipFilePath, destinationFolder string) error
- func FillStruct(t any, record []string, tagNames ...string) (err error)
- func IsToday(t time.Time) bool
- func JsonDecode[T any](w io.Reader) (*T, error)
- func LastMomentOfYesterday() time.Time
- func Params(req any) (string, error)
- func ParquetOpenTimeMSExpr(year, month, day int) string
- func ParseFloat(s string) float64
- func PopulateStructFromSlice(data *[]any, v any) error
- func QueryParquets(db *sql.DB, query string, dq any) (resCh chan map[string]any, errCh chan error)
- func ReadHeaderlessCSV[T any](reader io.Reader) (ch ...)
- func ReadHeaderlessCSVChan[T any](reader io.Reader) (ch chan *T, errCh chan error)
- func ReadParquet[T any](path string) (rCh chan *T, errCh chan error)
- func SetStructField(field reflect.Value, value string) (err error)
- func SortBy[S ~[]E, E any, K cmp.Ordered](s S, key func(E) K)
- func ToMicroseconds(ts int64) int64
- func WriteParquet[T any](path string) (rCh chan *T, errCh chan error)
- type Buffer
- func (b *Buffer) Bytes() []byte
- func (b *Buffer) Close() error
- func (b *Buffer) Decompress() (out *Buffer, err error)
- func (b *Buffer) Persist(path string) (err error)
- func (b *Buffer) Read(p []byte) (n int, err error)
- func (b *Buffer) Seek(offset int64, whence int) (int64, error)
- func (b *Buffer) String() string
- func (b *Buffer) Write(p []byte) (n int, err error)
- func (b *Buffer) WriteAt(p []byte, off int64) (n int, err error)
- type Frame
- type ResponseDate
- type TimestampType
Constants ¶
const ( NoFrame Frame = 0 Nanosecond Frame = 1 Microsecond = 1000 * Nanosecond Millisecond = 1000 * Microsecond Second = 1000 * Millisecond Minute = 60 * Second ThreeMinute = 3 * Minute FiveMinute = 5 * Minute FifteenMin = 15 * Minute Hour = 60 * Minute TwoHour = 2 * Hour OneDay = 24 * Hour OneWeek = 7 * OneDay )
Variables ¶
var ( ErrFileExists = errors.New("file exists") ErrWriteFile = errors.New("failed to write file") )
var Frames = map[string]Frame{ "": NoFrame, "1ns": Nanosecond, "1us": Microsecond, "1s": Second, "1m": Minute, "3m": ThreeMinute, "5m": FiveMinute, "15m": FifteenMin, "1h": Hour, "2h": TwoHour, "4h": 4 * Hour, "1d": OneDay, "1w": OneWeek, }
Frames maps Binance interval strings to Frame values.
Functions ¶
func AnyTimestampToTime ¶
AnyTimestampToTime converts variants of timestamp to time.Time:
- 1: microseconds
- 2: seconds
- 3: milliseconds
func DrainParquet ¶
DrainParquet reads all records from parquet channels until both close.
func EncodeGetQuery ¶
EncodeGetQuery encodes the request parameters into a query string.
func ExtractZip ¶
ExtractZip extracts a zip file to a destination folder.
func FillStruct ¶
FillStruct fills struct fields based on csv tag numbers
func LastMomentOfYesterday ¶
LastMomentOfYesterday return's midnight.
func ParquetOpenTimeMSExpr ¶
ParquetOpenTimeMSExpr returns a DuckDB expression for open_time column → epoch milliseconds. midnight is UTC midnight of the parquet file's calendar day.
func ParseFloat ¶
func PopulateStructFromSlice ¶
PopulateStructFromSlice maps []any to struct using `csv:"index"` tags
func QueryParquets ¶
func ReadHeaderlessCSV ¶
ReadHeaderlessCSV reads headerless CSV (like Binance data) and sends records to channel.
func ReadHeaderlessCSVChan ¶
ReadHeaderlessCSVChan reads headerless CSV (like Binance data) and sends records to channel Maps CSV columns positionally to struct fields based on csv tag numbers
func ReadParquet ¶
ReadParquet reads records from a parquet file
func SetStructField ¶
SetStructField sets field value based on type
func ToMicroseconds ¶
func WriteParquet ¶
WriteParquet writes records to a parquet file
Types ¶
type Buffer ¶
type Buffer struct {
// contains filtered or unexported fields
}
Buffer is a custom type that implements io.WriterAt
func Decompress ¶
Decompress ZIP bytes and returns the original content
func DownloadFile ¶
DownloadFile downloads a file from the given URL to the specified destination path.
func ReadIntoBuffer ¶
ReadIntoBuffer reads the content of a file into a Buffer
func (*Buffer) Decompress ¶
Decompress buffers content if it's compressed'
type Frame ¶
Frame is a candle aggregation interval (duration-based for arithmetic).
func StringToFrame ¶
type ResponseDate ¶
func (ResponseDate) GoString ¶
func (r ResponseDate) GoString() string
func (ResponseDate) MarshalJSON ¶
func (r ResponseDate) MarshalJSON() ([]byte, error)
MarshalJSON implements the json.Marshaler interface
func (*ResponseDate) UnmarshalJSON ¶
func (r *ResponseDate) UnmarshalJSON(b []byte) error
UnmarshalJSON implements the json.Unmarshaler interface
type TimestampType ¶
type TimestampType int
const ( TimestampInSeconds TimestampType = iota + 1 TimestampInMillis TimestampInMicros )
func TypeOfTimestamp ¶
func TypeOfTimestamp(ts int64) TimestampType
TypeOfTimestamp returns the type of timestamp: seconds, milliseconds or microseconds
Source Files