dbqcore

Aug 25, 2025

Changes in this version

+ const CheckTypeSchemaCheck

+ type BetweenRange struct

+ Max interface{}

+ Min interface{}

+ type CheckExpression struct

+ FunctionName string

+ FunctionParameters []string

+ Operator string

+ Scope CheckScope

+ ThresholdValue interface{}

+ func ParseCheckExpression(expression string) (*CheckExpression, error)

+ type CheckScope string

+ const ScopeColumn

+ const ScopeSchema

+ const ScopeTable

type ChecksFileConfig

+ Rules []ValidationRule

+ type ColumnsNotPresentConfig struct

+ Columns []string

+ Pattern string

type DataQualityCheck

+ Expression string

+ ParsedCheck *CheckExpression

+ SchemaCheck *SchemaCheckConfig

+ func (c *DataQualityCheck) UnmarshalYAML(node *yaml.Node) error

+ type DbqDataSourceAdapter interface

+ ExecuteQuery func(ctx context.Context, query string) (string, error)

+ InterpretDataQualityCheck func(check *DataQualityCheck, dataset string, defaultWhere string) (string, error)

type DbqDataValidator

+ func NewDbqDataValidator(logger *slog.Logger) DbqDataValidator

+ type DbqDataValidatorImpl struct

+ func (d DbqDataValidatorImpl) RunCheck(ctx context.Context, adapter DbqDataSourceAdapter, check *DataQualityCheck, ...) *ValidationResult

+ type ExpectColumnsConfig struct

+ Columns []string

+ type ExpectColumnsOrderedConfig struct

+ ColumnsOrder []string

type OnFailAction

+ const OnFailActionWarn

+ type SchemaCheckConfig struct

+ ColumnsNotPresent *ColumnsNotPresentConfig

+ ExpectColumns *ExpectColumnsConfig

+ ExpectColumnsOrdered *ExpectColumnsOrderedConfig

type ValidationResult

+ QueryResultValue string

Aug 11, 2025

Changes in this version

type DataQualityCheck

+ OnError OnErrorAction

+ type OnErrorAction string

+ const OnErrorActionAlert

+ const OnErrorActionIgnore

Jul 28, 2025

Changes in this version

type ValidationResult

+ Error string

Jul 28, 2025

Changes in this version

type DataSourceType

+ const DataSourceTypeMysql

Jul 28, 2025

Changes in this version

type DbqDataProfiler

+ GetBlankCount func(ctx context.Context, dataset string, column *ColumnInfo) (int64, error)

+ GetColumns func(ctx context.Context, databaseName string, tableName string) ([]*ColumnInfo, error)

+ GetMostFrequentValue func(ctx context.Context, dataset string, column *ColumnInfo) (*string, error)

+ GetNullCount func(ctx context.Context, dataset string, column *ColumnInfo) (uint64, error)

+ GetNumericStats func(ctx context.Context, dataset string, column *ColumnInfo) (*NumericStats, error)

+ GetSampleData func(ctx context.Context, dataset string) ([]map[string]interface{}, error)

+ GetTotalRows func(ctx context.Context, dataset string) (uint64, error)

+ IsNumericType func(dataType string) bool

+ IsStringType func(dataType string) bool

+ type NumericStats struct

+ AvgValue *float64

+ MaxValue *float64

+ MinValue *float64

+ StddevValue *float64

Jul 26, 2025

Changes in this version

+ type ChecksFileConfig struct

+ Validations []ValidationRule

+ Version string

+ func LoadChecksFileConfig(fileName string) (*ChecksFileConfig, error)

+ type ConnectionConfig struct

+ Database string

+ Host string

+ Password string

+ Port int

+ Username string

+ type DataQualityCheck struct

+ Description string

+ ID string

+ OnFail OnFailAction

+ Query string

+ type DataQualityCheckType string

+ type DbqDataProfiler interface

+ ProfileDataset func(ctx context.Context, dataset string, sample bool, maxConcurrent int) (*TableMetrics, error)

+ type DbqDataValidator interface

+ RunCheck func(ctx context.Context, check *DataQualityCheck, dataset string, ...) (bool, string, error)

+ type ValidationRule struct

+ Checks []DataQualityCheck

+ Dataset string

+ Where string

Jul 25, 2025

Changes in this version

+ type DataSourceType string

+ const DataSourceTypeClickhouse

+ const DataSourceTypePostgresql

Jul 25, 2025

Changes in this version

+ type ValidationResult struct

+ ActualResult string

+ CheckID string

+ Message string

+ Pass bool

Jul 21, 2025

Changes in this version

type TableMetrics

+ DbqErrors []error

type TaskPool

+ func (tp *TaskPool) Errors() []error

Jul 14, 2025

Changes in this version

+ type TaskPool struct

+ func NewTaskPool(poolSize int, logger *slog.Logger) *TaskPool

+ func (tp *TaskPool) Enqueue(id string, task func() error)

+ func (tp *TaskPool) Join()

May 21, 2025

Changes in this version

+ const Version

+ func GetDbqCoreLibVersion() string

May 15, 2025

Changes in this version

+ const CheckTypeRawQuery

+ const DbqCoreLibVersion

+ const OnFailActionError

+ const OnFailActionWarning

+ func GetDbqVersion() string

+ type Check struct

+ Description string

+ ID string

+ OnFail OnFailAction

+ Query string

+ type ChecksConfig struct

+ Validations []Validation

+ Version string

+ func LoadChecksConfig(fileName string) (*ChecksConfig, error)

+ type ClickhouseDbqConnector struct

+ func (c *ClickhouseDbqConnector) ImportDatasets(filter string) ([]string, error)

+ func (c *ClickhouseDbqConnector) Ping() (string, error)

+ func (c *ClickhouseDbqConnector) ProfileDataset(dataset string, sample bool) (*TableMetrics, error)

+ func (c *ClickhouseDbqConnector) RunCheck(check *Check, dataset string, defaultWhere string) (bool, string, error)

+ type ColumnInfo struct

+ Comment string

+ Name string

+ Position uint

+ Type string

+ type ColumnMetrics struct

+ AvgValue *float64

+ BlankCount *int64

+ ColumnComment string

+ ColumnName string

+ ColumnPosition uint

+ DataType string

+ MaxValue *float64

+ MinValue *float64

+ MostFrequentValue *string

+ NullCount uint64

+ ProfilingDurationMs int64

+ StddevValue *float64

+ type ConfigDetails struct

+ Database string

+ Host string

+ Password string

+ Port int

+ Username string

+ type DataSource struct

+ Configuration ConfigDetails

+ Datasets []string

+ ID string

+ Type string

+ type DbqConfig struct

+ DataSources []DataSource

+ Version string

+ type DbqConnector interface

+ ImportDatasets func(filter string) ([]string, error)

+ Ping func() (string, error)

+ ProfileDataset func(dataset string, sample bool) (*TableMetrics, error)

+ RunCheck func(check *Check, dataset string, defaultWhere string) (bool, string, error)

+ func NewClickhouseDbqConnector(dataSource DataSource) (DbqConnector, error)

+ type OnFailAction string

+ type ProfileResultOutput struct

+ Profiles map[string]*TableMetrics

+ type TableMetrics struct

+ ColumnsMetrics map[string]*ColumnMetrics

+ DatabaseName string

+ ProfiledAt int64

+ ProfilingDurationMs int64

+ RowsSample []map[string]interface{}

+ TableName string

+ TotalRows uint64

+ type Validation struct

+ Checks []Check

+ Dataset string

+ Where string