implementations

Documentation

Index

• Variables
• type AbstractFileAdapter
  • func (a *AbstractFileAdapter) AddFileExtension(fileName string) string
  • func (a *AbstractFileAdapter) Compression() types.FileCompression
  • func (a *AbstractFileAdapter) Format() types.FileFormat
  • func (a *AbstractFileAdapter) Path(fileName string) string
• type DummyFlattener
  • func NewDummyFlattener() *DummyFlattener
  • func (df *DummyFlattener) FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)
• type FileAdapter
• type FileConfig
• type Flattener
  • func NewFlattener() Flattener
• type FlattenerImpl
  • func (f *FlattenerImpl) FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)
• type GoogleCloudStorage
  • func NewGoogleCloudStorage(config *GoogleConfig) (*GoogleCloudStorage, error)
  • func (gcs *GoogleCloudStorage) Close() error
  • func (gcs *GoogleCloudStorage) DeleteObject(key string) (err error)
  • func (gcs *GoogleCloudStorage) Download(key string) (fileBytes []byte, err error)
  • func (gcs *GoogleCloudStorage) Upload(fileName string, fileReader io.ReadSeeker) (err error)
  • func (gcs *GoogleCloudStorage) UploadBytes(fileName string, fileBytes []byte) error
  • func (gcs *GoogleCloudStorage) ValidateWritePermission() error
• type GoogleConfig
  • func (gc *GoogleConfig) Validate() error
• type S3
  • func NewS3(s3Config *S3Config) (*S3, error)
  • func (a *S3) Close() error
  • func (a *S3) DeleteObject(key string) error
  • func (a *S3) Download(fileName string) ([]byte, error)
  • func (a *S3) Upload(fileName string, fileReader io.ReadSeeker) error
  • func (a *S3) UploadBytes(fileName string, fileBytes []byte) error
  • func (a *S3) ValidateWritePermission() error
• type S3Config
  • func (s3c *S3Config) Validate() error

Variables

var DefaultFlattener = NewFlattener()

var ErrMalformedBQDataset = errors.New("bq_dataset must be alphanumeric (plus underscores) and must be at most 1024 characters long")

Types

type AbstractFileAdapter

type AbstractFileAdapter struct {
	// contains filtered or unexported fields
}

func (*AbstractFileAdapter) AddFileExtension

func (a *AbstractFileAdapter) AddFileExtension(fileName string) string

func (*AbstractFileAdapter) Compression

func (a *AbstractFileAdapter) Compression() types.FileCompression

func (*AbstractFileAdapter) Format

func (a *AbstractFileAdapter) Format() types.FileFormat

func (*AbstractFileAdapter) Path

func (a *AbstractFileAdapter) Path(fileName string) string

type DummyFlattener

type DummyFlattener struct {
}

func NewDummyFlattener

func NewDummyFlattener() *DummyFlattener

func (*DummyFlattener) FlattenObject

func (df *DummyFlattener) FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)

FlattenObject return the same json object

type FileAdapter

type FileAdapter interface {
	io.Closer
	UploadBytes(fileName string, fileBytes []byte) error
	Upload(fileName string, fileReader io.ReadSeeker) error
	Download(fileName string) ([]byte, error)
	DeleteObject(key string) error
	Path(fileName string) string
	AddFileExtension(fileName string) string
	Format() types.FileFormat
	Compression() types.FileCompression
}

type FileConfig

type FileConfig struct {
	Folder      string                `mapstructure:"folder" json:"folder,omitempty" yaml:"folder,omitempty"`
	Format      types.FileFormat      `mapstructure:"format,omitempty" json:"format,omitempty" yaml:"format,omitempty"`
	Compression types.FileCompression `mapstructure:"compression,omitempty" json:"compression,omitempty" yaml:"compression,omitempty"`
}

type Flattener

type Flattener interface {
	FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)
}

func NewFlattener

func NewFlattener() Flattener

type FlattenerImpl

type FlattenerImpl struct {
	// contains filtered or unexported fields
}

func (*FlattenerImpl) FlattenObject

func (f *FlattenerImpl) FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)

FlattenObject flatten object e.g. from {"key1":{"key2":123}} to {"key1_key2":123} from {"$key1":1} to {"_key1":1} from {"(key1)":1} to {"_key1_":1}

type GoogleCloudStorage

type GoogleCloudStorage struct {
	AbstractFileAdapter
	// contains filtered or unexported fields
}

func NewGoogleCloudStorage

func NewGoogleCloudStorage(config *GoogleConfig) (*GoogleCloudStorage, error)

func (*GoogleCloudStorage) Close

func (gcs *GoogleCloudStorage) Close() error

Close closes gcp client and returns err if occurred

func (*GoogleCloudStorage) DeleteObject

func (gcs *GoogleCloudStorage) DeleteObject(key string) (err error)

DeleteObject deletes object from google cloud storage bucket

func (*GoogleCloudStorage) Download

func (gcs *GoogleCloudStorage) Download(key string) (fileBytes []byte, err error)

Download downloads file from google cloud storage bucket

func (*GoogleCloudStorage) Upload

func (gcs *GoogleCloudStorage) Upload(fileName string, fileReader io.ReadSeeker) (err error)

UploadBytes creates named file on google cloud storage with payload

func (*GoogleCloudStorage) UploadBytes

func (gcs *GoogleCloudStorage) UploadBytes(fileName string, fileBytes []byte) error

func (*GoogleCloudStorage) ValidateWritePermission

func (gcs *GoogleCloudStorage) ValidateWritePermission() error

ValidateWritePermission tries to create temporary file and remove it. returns nil if file creation was successful.

type GoogleConfig

type GoogleConfig struct {
	FileConfig `mapstructure:",squash" json:",inline" yaml:",inline"`
	Bucket     string `mapstructure:"gcsBucket,omitempty" json:"gcsBucket,omitempty" yaml:"gcsBucket,omitempty"`
	Project    string `mapstructure:"project,omitempty" json:"project,omitempty" yaml:"project,omitempty"`
	Dataset    string `mapstructure:"bqDataset,omitempty" json:"bqDataset,omitempty" yaml:"bqDataset,omitempty"`
	KeyFile    any    `mapstructure:"keyFile,omitempty" json:"keyFile,omitempty" yaml:"keyFile,omitempty"`

	//will be set on validation
	Credentials option.ClientOption
}

func (*GoogleConfig) Validate

func (gc *GoogleConfig) Validate() error

type S3

type S3 struct {
	AbstractFileAdapter
	// contains filtered or unexported fields
}

S3 is a S3 adapter for uploading/deleting files

func NewS3

func NewS3(s3Config *S3Config) (*S3, error)

NewS3 returns configured S3 adapter

func (*S3) Close

func (a *S3) Close() error

Close returns nil

func (*S3) DeleteObject

func (a *S3) DeleteObject(key string) error

DeleteObject deletes object from s3 bucket by key

func (*S3) Download

func (a *S3) Download(fileName string) ([]byte, error)

Download downloads file from s3 bucket

func (*S3) Upload

func (a *S3) Upload(fileName string, fileReader io.ReadSeeker) error

Upload creates named file on s3 with payload

func (*S3) UploadBytes

func (a *S3) UploadBytes(fileName string, fileBytes []byte) error

func (*S3) ValidateWritePermission

func (a *S3) ValidateWritePermission() error

ValidateWritePermission tries to create temporary file and remove it. returns nil if file creation was successful.

type S3Config

type S3Config struct {
	FileConfig `mapstructure:",squash" json:",inline" yaml:",inline"`
	AccessKey  string `mapstructure:"accessKeyId,omitempty" json:"accessKeyId,omitempty" yaml:"accessKeyId,omitempty"`
	SecretKey  string `mapstructure:"secretAccessKey,omitempty" json:"secretAccessKey,omitempty" yaml:"secretAccessKey,omitempty"`
	Bucket     string `mapstructure:"bucket,omitempty" json:"bucket,omitempty" yaml:"bucket,omitempty"`
	Region     string `mapstructure:"region,omitempty" json:"region,omitempty" yaml:"region,omitempty"`
	Endpoint   string `mapstructure:"endpoint,omitempty" json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
}

S3Config is a dto for config deserialization

func (*S3Config) Validate

func (s3c *S3Config) Validate() error

Validate returns err if invalid