utils

Documentation

Index

• func BuildConnString(config *Config, secret string) string
• func ConvertValue(value bigquery.Value, dataType string) interface{}
• func FetchColumns(ctx context.Context, pool *pgxpool.Pool, tableName string) (map[string]string, []string, error)
• func FetchSecret(ctx context.Context, secretName string) (string, error)
• func GetIndex(schema bigquery.Schema, colName string) int
• func InitializeBigQueryClient(ctx context.Context, projectID string) (*bigquery.Client, error)
• func InitializePostgresPool(ctx context.Context, connString string) (*pgxpool.Pool, error)
• type Config
  • func LoadConfig(filename string) (*Config, error)
• type File
• type Table

Functions

func BuildConnString

func BuildConnString(config *Config, secret string) string

BuildConnString builds the PostgreSQL connection string from the config and secret

func ConvertValue

func ConvertValue(value bigquery.Value, dataType string) interface{}

ConvertValue converts a BigQuery value to a PostgreSQL compatible value

func FetchColumns

func FetchColumns(ctx context.Context, pool *pgxpool.Pool, tableName string) (map[string]string, []string, error)

FetchColumns fetches column names for the given table from the source database.

func FetchSecret

func FetchSecret(ctx context.Context, secretName string) (string, error)

FetchSecret retrieves the secret value from Google Secret Manager

func GetIndex

func GetIndex(schema bigquery.Schema, colName string) int

GetIndex returns the index of a column name in the BigQuery schema

func InitializeBigQueryClient

func InitializeBigQueryClient(ctx context.Context, projectID string) (*bigquery.Client, error)

InitializeBigQueryClient initializes a BigQuery client.

func InitializePostgresPool

func InitializePostgresPool(ctx context.Context, connString string) (*pgxpool.Pool, error)

InitializePostgresPool initializes a PostgreSQL connection pool.

Types

type Config

type Config struct {
	Postgres struct {
		Host       string `yaml:"host"`
		Port       int    `yaml:"port"`
		User       string `yaml:"user"`
		DBName     string `yaml:"dbname"`
		SSLMode    string `yaml:"sslmode"`
		SecretName string `yaml:"secret_name"`
	} `yaml:"postgres"`
	GCS struct {
		BucketName     string `yaml:"bucket_name"`
		ProjectID      string `yaml:"project_id"`
		Dataset        string `yaml:"dataset"`
		Files          []File `yaml:"files"`
		ConcurrentJobs int    `yaml:"concurrent_jobs"`
	} `yaml:"gcs"`
	BQ struct {
		ProjectID string  `yaml:"project_id"`
		Dataset   string  `yaml:"dataset"`
		Tables    []Table `yaml:"tables"`
	} `yaml:"bq"`
}

Config represents the YAML configuration structure

func LoadConfig

func LoadConfig(filename string) (*Config, error)

LoadConfig loads configuration from a YAML file

type File

type File struct {
	Name  string `yaml:"name"`
	Table string `yaml:"table"`
}

type Table

type Table struct {
	Name  string `yaml:"name"`
	Table string `yaml:"table"`
}