implementations

package
v0.0.0-...-d5f2c4f Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 17, 2025 License: MIT Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const GCSBulkerTypeId = "gcs"
View Source 
const (
	S3BulkerTypeId = "s3"
)

Variables

View Source 
var ErrMalformedBQDataset = errors.New("bq_dataset must be alphanumeric (plus underscores) and must be at most 1024 characters long")

Functions

This section is empty.

Types

type AbstractFileAdapter 

type AbstractFileAdapter struct {
	// contains filtered or unexported fields
}

func (*AbstractFileAdapter) AddFileExtension 

func (a *AbstractFileAdapter) AddFileExtension(fileName string) string

func (*AbstractFileAdapter) Compression 

func (a *AbstractFileAdapter) Compression() types.FileCompression

func (*AbstractFileAdapter) Format 

func (a *AbstractFileAdapter) Format() types.FileFormat

func (*AbstractFileAdapter) Path 

func (a *AbstractFileAdapter) Path(fileName string) string

type DummyFlattener 

type DummyFlattener struct {
}

func NewDummyFlattener 

func NewDummyFlattener() *DummyFlattener

func (*DummyFlattener) FlattenObject 

func (df *DummyFlattener) FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)

FlattenObject return the same json object

type FileAdapter 

type FileAdapter interface {
	io.Closer
	Type() string
	Ping() error
	UploadBytes(fileName string, fileBytes []byte) error
	Upload(fileName string, fileReader io.ReadSeeker) error
	// GetObjectURL returns http URL to uploaded object.
	// It is short-lived signed URL if the corresponding setting (e.g. S3Config.UsePresignedURL ) is enabled on adapter config
	// Otherwise, it returns URL build with public URL template which won't work if the object is private
	GetObjectURL(fileName string) (url string, err error)
	Download(fileName string) ([]byte, error)
	DeleteObject(key string) error
	Path(fileName string) string
	AddFileExtension(fileName string) string
	Format() types.FileFormat
	Compression() types.FileCompression
}

type FileConfig 

type FileConfig struct {
	Folder      string                `mapstructure:"folder" json:"folder,omitempty" yaml:"folder,omitempty"`
	Format      types.FileFormat      `mapstructure:"format,omitempty" json:"format,omitempty" yaml:"format,omitempty"`
	Compression types.FileCompression `mapstructure:"compression,omitempty" json:"compression,omitempty" yaml:"compression,omitempty"`
}

type Flattener 

type Flattener interface {
	FlattenObject(object types.Object, notFlatteningKeys types2.Set[string]) (types.Object, error)
}

func NewFlattener 

func NewFlattener(nameTransformer func(string) string, omitNilValues, stringifyObjects bool) Flattener

type FlattenerImpl 

type FlattenerImpl struct {
	// contains filtered or unexported fields
}

func (*FlattenerImpl) FlattenObject 

func (f *FlattenerImpl) FlattenObject(object types.Object, notFlatteningKeys types2.Set[string]) (types.Object, error)

FlattenObject flatten object e.g. from {"key1":{"key2":123}} to {"key1_key2":123} from {"$key1":1} to {"_key1":1} from {"(key1)":1} to {"_key1_":1}

type GoogleCloudStorage 

type GoogleCloudStorage struct {
	AbstractFileAdapter
	// contains filtered or unexported fields
}

func NewGoogleCloudStorage 

func NewGoogleCloudStorage(config *GoogleConfig) (*GoogleCloudStorage, error)

func (*GoogleCloudStorage) Close 

func (gcs *GoogleCloudStorage) Close() error

Close closes gcp client and returns err if occurred

func (*GoogleCloudStorage) DeleteObject 

func (gcs *GoogleCloudStorage) DeleteObject(key string) (err error)

DeleteObject deletes object from google cloud storage bucket

func (*GoogleCloudStorage) Download 

func (gcs *GoogleCloudStorage) Download(key string) (fileBytes []byte, err error)

Download downloads file from google cloud storage bucket

func (*GoogleCloudStorage) GetObjectURL 

func (gcs *GoogleCloudStorage) GetObjectURL(fileName string) (string, error)

func (*GoogleCloudStorage) Ping 

func (gcs *GoogleCloudStorage) Ping() error

func (*GoogleCloudStorage) Type 

func (gcs *GoogleCloudStorage) Type() string

func (*GoogleCloudStorage) Upload 

func (gcs *GoogleCloudStorage) Upload(fileName string, fileReader io.ReadSeeker) (err error)

UploadBytes creates named file on google cloud storage with payload

func (*GoogleCloudStorage) UploadBytes 

func (gcs *GoogleCloudStorage) UploadBytes(fileName string, fileBytes []byte) error

func (*GoogleCloudStorage) ValidateWritePermission 

func (gcs *GoogleCloudStorage) ValidateWritePermission() error

ValidateWritePermission tries to create temporary file and remove it. returns nil if file creation was successful.

type GoogleConfig 

type GoogleConfig struct {
	FileConfig `mapstructure:",squash" json:",inline" yaml:",inline"`
	Bucket     string `mapstructure:"gcsBucket,omitempty" json:"gcsBucket,omitempty" yaml:"gcsBucket,omitempty"`
	Project    string `mapstructure:"project,omitempty" json:"project,omitempty" yaml:"project,omitempty"`
	Dataset    string `mapstructure:"bqDataset,omitempty" json:"bqDataset,omitempty" yaml:"bqDataset,omitempty"`
	KeyFile    any    `mapstructure:"keyFile,omitempty" json:"keyFile,omitempty" yaml:"keyFile,omitempty"`

	//will be set on validation
	Credentials option.ClientOption
}

func (*GoogleConfig) Validate 

func (gc *GoogleConfig) Validate() error

type S3 

type S3 struct {
	AbstractFileAdapter
	// contains filtered or unexported fields
}

S3 is a S3 adapter for uploading/deleting files

func NewS3 

func NewS3(s3Config *S3Config) (*S3, error)

NewS3 returns configured S3 adapter

func (*S3) Close 

func (a *S3) Close() error

Close returns nil

func (*S3) DeleteObject 

func (a *S3) DeleteObject(key string) error

DeleteObject deletes object from s3 bucket by key

func (*S3) Download 

func (a *S3) Download(fileName string) ([]byte, error)

Download downloads file from s3 bucket

func (*S3) GetObjectURL 

func (a *S3) GetObjectURL(fileName string) (string, error)

func (*S3) Ping 

func (a *S3) Ping() (err error)

func (*S3) Type 

func (a *S3) Type() string

func (*S3) Upload 

func (a *S3) Upload(fileName string, fileReader io.ReadSeeker) error

Upload creates named file on s3 with payload

func (*S3) UploadBytes 

func (a *S3) UploadBytes(fileName string, fileBytes []byte) error

func (*S3) ValidateWritePermission 

func (a *S3) ValidateWritePermission() error

ValidateWritePermission tries to create temporary file and remove it. returns nil if file creation was successful.

type S3Config 

type S3Config struct {
	FileConfig           `mapstructure:",squash" json:",inline" yaml:",inline"`
	AuthenticationMethod string `mapstructure:"authenticationMethod,omitempty" json:"authenticationMethod,omitempty" yaml:"authenticationMethod,omitempty"`
	AccessKeyID          string `mapstructure:"accessKeyId,omitempty" json:"accessKeyId,omitempty" yaml:"accessKeyId,omitempty"`
	SecretAccessKey      string `mapstructure:"secretAccessKey,omitempty" json:"secretAccessKey,omitempty" yaml:"secretAccessKey,omitempty"`
	Bucket               string `mapstructure:"bucket,omitempty" json:"bucket,omitempty" yaml:"bucket,omitempty"`
	Region               string `mapstructure:"region,omitempty" json:"region,omitempty" yaml:"region,omitempty"`
	Endpoint             string `mapstructure:"endpoint,omitempty" json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
	UsePresignedURL      bool   `mapstructure:"usePresignedURL,omitempty" json:"usePresignedURL,omitempty" yaml:"usePresignedURL,omitempty"`

	RoleARN       string        `mapstructure:"roleARN" json:"roleARN" yaml:"roleARN"`
	RoleARNExpiry time.Duration `json:"roleARNExpiry"` // default: 15m
	ExternalID    string        `mapstructure:"externalID" json:"externalID" yaml:"externalID"`
}

S3Config is a dto for config deserialization

func (*S3Config) Sanitize 

func (s3c *S3Config) Sanitize()

func (*S3Config) Validate 

func (s3c *S3Config) Validate() error

Validate returns err if invalid

Source Files

View all Source files

Directories

Path Synopsis
testcontainers
sql
redshift_driver
testcontainers
testcontainers/clickhouse
testcontainers/clickhouse_noshards

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.