abstract

package
Version: v0.0.0-...-aa414c9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 17, 2021 License: Apache-2.0 Imports: 7 Imported by: 4

Documentation

Index

Constants

View Source
const DefaultManifestFilename string = "MANIFEST.yaml"
View Source
const (
	L_SCHEMA = "schema"
)

Variables

This section is empty.

Functions

func NewDatabaseBuilder

func NewDatabaseBuilder() *databaseBuilder

NewDatabaseBuilder ... builder for a database asset type

func NewTableBuilder

func NewTableBuilder() *tableBuilder

NewTableBuilder ... table builder

Types

type Asset

type Asset struct {
	// asset last found by crawler at - only added by service (not crawler/manifest itself, i.e. no yaml)
	LastDiscoveredAt time.Time `json:"last-discovered-at" yaml:"last-discovered-at,omitempty"`
	// asset publication datetime
	PublishedOn time.Time `yaml:"published-on" json:"published-on"`
	// name of the asset
	Name string `yaml:"name" json:"name"`
	// description of the asset
	Description string `yaml:"description" json:"description"`
	// the list of assets this depends on
	DependsOn []string `yaml:"depends-on" json:"depends-on"`
	// asset type
	Type AssetType `yaml:"type" json:"type"`
	// labels for the specific asset
	Labels map[string]interface{} `yaml:"labels" json:"labels"`
	// tags are flags used to simplify asset search
	Tags []string `yaml:"tags" json:"tags"`
	// versions specify available variants of the same asset
	Versions map[string]interface{} `yaml:"versions" json:"versions"`
}

Asset ... managed resource

func NewDatasetAsset

func NewDatasetAsset() *Asset

func ParseAsset

func ParseAsset(data []byte) (*Asset, error)

ParseAsset ... Parse an asset specification file

func (*Asset) Validate

func (asset *Asset) Validate() error

Validate ... Validate asset specification file

type AssetDAOProvider

type AssetDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Upsert(asset *Asset) error
	GetById(id string) (*Asset, error)
	GetByName(id string) (*Asset, error)
	SearchAssetsByTags(tags []string) (*[]Asset, error)
	ListAllAssets() (*[]Asset, error)
	CloseConnection()
}

AssetDAOProvider ... The interface each dao must implement

type AssetType

type AssetType string

AssetType ... Asset type information

func (*AssetType) Validate

func (assetType *AssetType) Validate() error

type ColumnInfo

type ColumnInfo struct {
	Type    string
	Comment string
}

ColumnInfo ... Type and description for a table column

type ConnectorProvider

type ConnectorProvider interface {
	ValidateDataSourceDefinition(*conf.DataSourceDefinition) error
	InitConnection(*conf.DataSourceDefinition)
	CloseConnection()
}

ConnectorProvider ... The interface each connector must implement

type Crawler

type Crawler interface {
	InitConnection(cfg *conf.Config) (Crawler, error)
	WalkWithFilter(root string, filenameFilter string) ([]Asset, error)
}

type DBInfo

type DBInfo struct {
	Name    string
	Comment string
}

DBInfo ... Name and description for a database

func GetDBInfoByName

func GetDBInfoByName(dbName string) (DBInfo, error)

Partial constructor for DBInfo

func (*DBInfo) BuildAsset

func (db *DBInfo) BuildAsset() (*Asset, error)

type DeequAnalyzer

type DeequAnalyzer struct {
	AnalyzerName string `json:"analyzerName"`
	Column       string `json:"column"`
}

type DeequAnalyzerContext

type DeequAnalyzerContext struct {
	MetricMap []DeequMetricInstance `json:"metricMap"`
}

type DeequMetric

type DeequMetric struct {
	ResultKey       *DeequResultKey       `json:"resultKey,omitempty"`
	AnalyzerContext *DeequAnalyzerContext `json:"analyzerContext,omitempty"`
}

DeequMetric ...

type DeequMetricInstance

type DeequMetricInstance struct {
	Analyzer DeequAnalyzer    `json:"analyzer"`
	Metric   DeequMetricValue `json:"metric"`
}

type DeequMetricValue

type DeequMetricValue struct {
	MetricName string  `json:"metricName"`
	Entity     string  `json:"entity"`
	Instance   string  `json:"instance"`
	Name       string  `json:"name"`
	Value      float64 `json:"value"`
}

type DeequResultKey

type DeequResultKey struct {
	DataSetDate int64             `json:"dataSetDate"`
	Tags        map[string]string `json:"tags"`
}

type Feature

type Feature struct {
	Name     string      `json:"name,omitempty"`
	Value    interface{} `json:"value,omitempty"`
	DataType string      `json:"data-type,omitempty"`
}

Feature ... a named variable with a data type

func (*Feature) Validate

func (f *Feature) Validate() error

Validate ... validate a feature

type FeatureSet

type FeatureSet struct {
	Name        string            `json:"name,omitempty"`
	InsertedAt  time.Time         `json:"inserted_at,omitempty"`
	Version     string            `json:"version,omitempty"`
	Features    []Feature         `json:"features,omitempty"`
	Description string            `json:"description,omitempty"`
	Labels      map[string]string `json:"labels,omitempty"`
}

FeatureSet ... a versioned set of features

func (*FeatureSet) Validate

func (fs *FeatureSet) Validate() error

Validate ... validate a featureSet

type FeatureSetDAOProvider

type FeatureSetDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Create(fs *FeatureSet) error
	GetById(id string) (*FeatureSet, error)
	GetByName(name string) (*[]FeatureSet, error)
	ListAllFeatureSets() (*[]FeatureSet, error)
	CloseConnection()
}

FeatureSetDAOProvider ... The interface each dao must implement

type Metric

type Metric struct {
	*DeequMetric
}

func (*Metric) Validate

func (m *Metric) Validate() error

Validate ... validate a metric

type MetricSet

type MetricSet struct {
	// unique metric set name
	Name string `json:"name,omitempty"`
	// insertion time
	InsertedAt time.Time `json:"inserted_at,omitempty"`
	// version relates to the environment and the pipeline version
	Version string `json:"version,omitempty"`
	// description is related to the metrics and their extraction process and not the datasource they were calculated on
	Description string `json:"description,omitempty"`
	// labels used for query purposes
	Labels map[string]string `json:"labels,omitempty"`

	// actual metrics
	Metrics []Metric `json:"metrics,omitempty"`
}

MetricSet ... a timestamped set of Metrics

func (*MetricSet) Validate

func (ms *MetricSet) Validate() error

Validate ... validate a metricSet

type MetricSetDAOProvider

type MetricSetDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Create(m *MetricSet) error
	GetById(id string) (*MetricSet, error)
	GetByName(name string) (*[]MetricSet, error)
	SearchMetricSetsByLabels(labels map[string]string) (*[]MetricSet, error)
	ListAllMetricSets() (*[]MetricSet, error)
	CloseConnection()
}

MetricSetDAOProvider ... The interface each dao must implement

type TableInfo

type TableInfo struct {
	Name    string
	Schema  map[string]ColumnInfo
	Comment string
}

TableInfo ... Name, schema and description for a table

func GetTableInfoByName

func GetTableInfoByName(tableName string) (TableInfo, error)

Partial constructor for TableInfo

func (*TableInfo) BuildAsset

func (tb *TableInfo) BuildAsset() (*Asset, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
t or T : Toggle theme light dark auto
y or Y : Canonical URL