transformers

package
v0.1.10 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 17, 2024 License: Apache-2.0 Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source
const (
	JsonDeleteOpName = "delete"
	JsonSetOpName    = "set"
)
View Source
const (
	MPassword   string = "password"
	MName       string = "name"
	MAddress    string = "addr"
	MEmail      string = "email"
	MMobile     string = "mobile"
	MTelephone  string = "tel"
	MID         string = "id"
	MCreditCard string = "credit_card"
	MURL        string = "url"
	MDefault    string = "default"
)

Variables

View Source
var CmdTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		cmdTransformerName,
		"Transform data via external program using stdin and stdout interaction",
	),

	NewCmd,

	toolkit.MustNewParameter(
		"columns",
		"affected column names. If empty use the whole tuple."+
			"The structure:"+
			`{`+
			`"name": "type:string, required:true, description: column Name",`+
			`"not_affected": "type:bool, required:false, description: is column affected in transformation", `+
			`"skip_original_data":  "type:bool, required:false, description: is original data required for transformer",`+
			`"skip_on_null_input":  "type:bool, required:false, description: skip transformation on null input"`+
			`}`,
	).SetDefaultValue([]byte("[]")).
		SetIsColumnContainer(true),

	toolkit.MustNewParameter(
		"executable",
		"path to executable file",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"args",
		"list of parameters for executable file",
	).SetDefaultValue([]byte("[]")),

	toolkit.MustNewParameter(
		"driver",
		"row driver with parameters that is used for interacting with cmd. The default is csv. "+
			`The structure is:`+
			`{"name": "text|csv|json", "params": { "format": "[text|bytes]"} }`,
	).SetDefaultValue([]byte(`{"name": "csv"}`)),

	toolkit.MustNewParameter(
		"validate",
		"try to encode-decode data received from cmd",
	).SetDefaultValue([]byte("false")),

	toolkit.MustNewParameter(
		"timeout",
		"timeout for sending and receiving data from cmd",
	).SetDefaultValue([]byte("2s")),

	toolkit.MustNewParameter(
		"expected_exit_code",
		"expected exit code",
	).SetDefaultValue([]byte("0")),

	toolkit.MustNewParameter(
		"skip_on_behaviour",
		"skip transformation call if one of the provided columns has null value (any) or each of the provided"+
			" column has null values (all). This option works together with skip_on_null_input parameter on columns."+
			`Possible values: "all", "any"`,
	).SetDefaultValue([]byte("all")).
		SetRawValueValidator(cmdValidateSkipBehaviour),
)
View Source
var DictTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Dict",
		"Replace values matched by dictionary keys",
	),

	NewDictTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"values",
		`map of value to replace in format: {"string": "string"}". The string with value "\N" supposed to be NULL value`,
	).SetRequired(true),

	toolkit.MustNewParameter(
		"default",
		`default value if not any value has been matched with dict. The string with value "\N" supposed to be NULL value. Default is empty`,
	).SetRequired(false),

	toolkit.MustNewParameter(
		"fail_not_matched",
		`fail if value is not matched with dict otherwise keep value`,
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("true")),
	toolkit.MustNewParameter(
		"validate",
		`perform encode-decode procedure using column type, ensuring that value has correct type`,
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var FakerTransformersDes = map[string]*FakerTransformerDef{

	"RandomLatitude": {
		Generator: func(opts ...options.OptionFunc) string {
			return fmt.Sprintf("%f", faker.Latitude())
		},
		SupportedTypes: []string{"float4", "float8", "numeric"},
		Description:    "Generates a random latitude value.",
	},
	"RandomLongitude": {
		Generator: func(opts ...options.OptionFunc) string {
			return fmt.Sprintf("%f", faker.Longitude())
		},
		SupportedTypes: []string{"float4", "float8", "numeric"},
		Description:    "Generates a random longitude value.",
	},

	"RandomUnixTime": {
		Generator: func(opts ...options.OptionFunc) string {
			return fmt.Sprintf("%d", faker.UnixTime())
		},
		SupportedTypes: []string{"int4", "int8", "numeric"},
		Description:    "Generates a random Unix timestamp.",
	},
	"RandomMonthName": {
		Generator:      faker.MonthName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates the name of a random month.",
	},
	"RandomYearString": {
		Generator:      faker.YearString,
		SupportedTypes: []string{"text", "varchar", "int2", "int4", "int8", "numeric"},
		Description:    "Generates a random year as a string.",
	},
	"RandomDayOfWeek": {
		Generator:      faker.DayOfWeek,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random day of the week.",
	},
	"RandomDayOfMonth": {
		Generator:      faker.DayOfMonth,
		SupportedTypes: []string{"text", "varchar", "int2", "int4", "int8", "numeric"},
		Description:    "Generates a random day of the month.",
	},
	"RandomCentury": {
		Generator:      faker.Century,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random century.",
	},
	"RandomTimezone": {
		Generator:      faker.Timezone,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random timezone.",
	},

	"RandomEmail": {
		Generator:      faker.Email,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random email address.",
	},
	"RandomMacAddress": {
		Generator:      faker.MacAddress,
		SupportedTypes: []string{"text", "varchar", "macaddr", "macaddr8"},
		Description:    "Generates a random MAC address.",
	},
	"RandomDomainName": {
		Generator:      faker.DomainName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random domain name.",
	},
	"RandomURL": {
		Generator:      faker.URL,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random URL.",
	},
	"RandomUsername": {
		Generator:      faker.Username,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random username.",
	},
	"RandomIPv4": {
		Generator:      faker.IPv4,
		SupportedTypes: []string{"text", "varchar", "inet"},
		Description:    "Generates a random IPv4 address.",
	},
	"RandomIPv6": {
		Generator:      faker.IPv6,
		SupportedTypes: []string{"text", "varchar", "inet"},
		Description:    "Generates a random IPv6 address.",
	},
	"RandomPassword": {
		Generator:      faker.Password,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random password.",
	},

	"RandomWord": {
		Generator:      faker.Word,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random word.",
	},
	"RandomSentence": {
		Generator:      faker.Sentence,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random sentence.",
	},
	"RandomParagraph": {
		Generator:      faker.Paragraph,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random paragraph.",
	},

	"RandomCCType": {
		Generator:      faker.CCType,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random credit card type.",
	},
	"RandomCCNumber": {
		Generator:      faker.CCNumber,
		SupportedTypes: []string{"text", "varchar", "int4", "int8", "numeric"},
		Description:    "Generates a random credit card number.",
	},
	"RandomCurrency": {
		Generator:      faker.Currency,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random currency code.",
	},
	"RandomAmountWithCurrency": {
		Generator:      faker.AmountWithCurrency,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random monetary amount with currency.",
	},

	"RandomTitleMale": {
		Generator:      faker.TitleMale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random title for males.",
	},
	"RandomTitleFemale": {
		Generator:      faker.TitleFemale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random title for females.",
	},
	"RandomFirstName": {
		Generator:      faker.FirstName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random first name.",
	},
	"RandomFirstNameMale": {
		Generator:      faker.FirstNameMale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random male first name.",
	},
	"RandomFirstNameFemale": {
		Generator:      faker.FirstNameFemale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random female first name.",
	},
	"RandomLastName": {
		Generator:      faker.LastName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random last name.",
	},
	"RandomName": {
		Generator:      faker.Name,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a full random name.",
	},

	"RandomPhoneNumber": {
		Generator:      faker.Phonenumber,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random phone number.",
	},
	"RandomTollFreePhoneNumber": {
		Generator:      faker.TollFreePhoneNumber,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random toll-free phone number.",
	},
	"RandomE164PhoneNumber": {
		Generator:      faker.E164PhoneNumber,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random phone number in E.164 format.",
	},
}
View Source
var HashTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Hash",
		"Generate hash of the text value using Scrypt hash function under the hood",
	),

	NewHashTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"salt",
		"hex encoded salt string. This value may be provided via environment variable GREENMASK_GLOBAL_SALT",
	).SetGetFromGlobalEnvVariable("GREENMASK_GLOBAL_SALT"),

	toolkit.MustNewParameter(
		"function",
		fmt.Sprintf("hash function name. Possible values: %s",
			strings.Join(
				[]string{sha1Name, sha256Name, sha512Name, sha3224Name, sha3256Name, sha384Name, sha5124Name, md5Name},
				", ",
			),
		),
	).SetDefaultValue([]byte(sha3256Name)).
		SetRawValueValidator(validateHashFunctionsParameter),

	toolkit.MustNewParameter(
		"max_length",
		"limit length of hash function result",
	).SetDefaultValue([]byte("0")).
		SetRawValueValidator(validateMaxLengthParameter),
)
View Source
var JsonTransformerDefinition = utils.NewDefinition(

	utils.NewTransformerProperties(
		"Json",
		"Update json document",
	),

	NewJsonTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("json", "jsonb"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"operations",
		`list of operations that contains editing operation [{"operation": "set|delete", "path": "path to the part of the document", "value": "value in any type - string, int, float, list, object, null", "value_template": "go template", "error_not_exist", "raise error if not exists - boolean"}]`,
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"apply changes in value is null",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var MaskingTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Masking",
		"Mask a value using one of masking type",
	),

	NewMaskingTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(
		toolkit.NewColumnProperties().
			SetAffected(true).
			SetAllowedColumnTypes("text", "varchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"type",
		"logical type of attribute (default, password, name, addr, email, mobile, tel, id, credit_card, url)",
	).SetRawValueValidator(maskerTypeValidator).
		SetDefaultValue(toolkit.ParamsValue(MDefault)),
)
View Source
var NoiseDateTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"NoiseDate",
		"Add some random value (shift value) in the provided interval",
	),

	NewNoiseDateTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("date", "timestamp", "timestamptz").
		SetSkipOnNull(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"ratio",
		"max random duration for noise",
	).SetRequired(true).
		SetCastDbType("interval"),

	toolkit.MustNewParameter(
		"truncate",
		fmt.Sprintf("truncate date till the part (%s)", strings.Join(truncateParts, ", ")),
	).SetRawValueValidator(ValidateDateTruncationParameterValue),
)
View Source
var NoiseFloatTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"NoiseFloat",
		"Make noise float for int",
	),
	NewNoiseFloatTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("float4", "float8", "numeric").
		SetSkipOnNull(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"ratio",
		"max random percentage for noise",
	).SetDefaultValue(toolkit.ParamsValue("0.1")),

	toolkit.MustNewParameter(
		"precision",
		"precision of noised float value (number of digits after coma)",
	).SetDefaultValue(toolkit.ParamsValue("4")),
)
View Source
var NoiseIntTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"NoiseInt",
		"Make noise value for int",
	),

	NewNoiseIntTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("int2", "int4", "int8").
		SetSkipOnNull(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"ratio",
		"max random percentage for noise",
	).SetRequired(true).
		SetDefaultValue(toolkit.ParamsValue("0.1")),
)
View Source
var RandomBoolTransformerDefinition = utils.NewDefinition(

	utils.NewTransformerProperties(
		"RandomBool",
		"Generate random bool",
	),

	NewRandomBoolTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("bool"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RandomChoiceTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomChoice",
		"Replace values chosen randomly from list",
	),

	NewRandomChoiceTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"values",
		`list of values in any format. The string with value "\N" supposed to be NULL value`,
	).SetRequired(true).
		SetUnmarshaller(randomChoiceValuesUnmarshaller),

	toolkit.MustNewParameter(
		"validate",
		`perform decode procedure via PostgreSQL driver using column type, ensuring that value has correct type`,
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("true")),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RandomDateTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomDate",
		"Generate random date in the provided interval",
	),

	NewRandomDateTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("date", "timestamp", "timestamptz"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min",
		"min threshold date (and/or time) of random value",
	).SetRequired(true).
		SetLinkParameter("column"),

	toolkit.MustNewParameter(
		"max",
		"max threshold date  (and/or time) of random value",
	).SetRequired(true).
		SetLinkParameter("column"),

	toolkit.MustNewParameter(
		"truncate",
		fmt.Sprintf("truncate date till the part (%s)", strings.Join(truncateParts, ", ")),
	).SetRawValueValidator(ValidateDateTruncationParameterValue),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RandomFloatTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomFloat",
		"Generate random float",
	),

	NewRandomFloatTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("float4", "float8", "numeric"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min",
		"min float threshold of random value. The value range depends on column type",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"max",
		"min float threshold of random value. The value range depends on column type",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"precision",
		"precision of random float value (number of digits after coma)",
	).SetDefaultValue(toolkit.ParamsValue("4")),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RandomIntTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomInt",
		"Generate random int value from min to max",
	),

	NewRandomIntTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("int2", "int4", "int8", "numeric"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min",
		"min int value threshold",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"max",
		"max int value threshold",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RandomStringTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomString",
		"Generate random string",
	),

	NewRandomStringTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min_length",
		"min length of string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"max_length",
		"max length of string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"symbols",
		"the characters range for random string",
	).SetDefaultValue(toolkit.ParamsValue("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RandomUuidTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomUuid",
		"Generate random uuid",
	),

	NewRandomUuidTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar", "uuid"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var RealAddressTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RealAddress",
		"Generate a real address",
	),

	NewRealAddressTransformer,

	toolkit.MustNewParameter(
		"columns",
		"affected column names."+
			"The structure:"+
			`{`+
			`"name": "type:string, required:true, description: column Name",`+
			`"template": "type:string, required:true, description: gotemplate with real address attributes injections",`+
			`"keep_null": "type:bool, required:false, description: keep null values",`+
			`}`,
	).SetRequired(true).
		SetIsColumnContainer(true),
)
View Source
var RegexpReplaceTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RegexpReplace",
		"Replace string using regular expression",
	),

	NewRegexpReplaceTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("varchar", "text", "bpchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"regexp",
		"regular expression",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"replace",
		"replacement value",
	).SetRequired(true),
)
View Source
var ReplaceTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Replace",
		"Replace column value to the provided",
	),

	NewReplaceTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"value",
		"value to replace",
	).SetRequired(true).
		SetLinkParameter("column"),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),

	toolkit.MustNewParameter(
		"validate",
		"validate the value via driver decoding procedure",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source
var SetNullTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"SetNull",
		"Set NULL value",
	),
	NewSetNullTransformer,
	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetNullable(true),
	).SetRequired(true),
)
View Source
var TemplateRecordTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"TemplateRecord",
		"Modify the record using gotemplate",
	),
	NewTemplateRecordTransformer,

	toolkit.MustNewParameter(
		"template",
		"gotemplate string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"validate",
		"validate template result via PostgreSQL driver decoding",
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("false")),

	toolkit.MustNewParameter(
		"columns",
		"columns that supposed to be affected by the template. The list of columns will be checked for constraint violation",
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("[]")),
)
View Source
var TemplateTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Template",
		"Modify the value using gotemplate",
	),
	NewTemplateTransformer,
	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"template",
		"gotemplate string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"validate",
		"validate template result via PostgreSQL driver decoding",
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("false")),
)

Functions

func MakeNewFakeTransformerFunction added in v0.1.1

func MakeNewFakeTransformerFunction(generator FakerFunc) utils.NewTransformerFunc

func NewCmd

func NewDictTransformer

func NewDictTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewFakeTransformer added in v0.1.1

func NewFakeTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter, generator FakerFunc,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewHashTransformer

func NewHashTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewJsonTransformer

func NewJsonTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewMaskingTransformer

func NewMaskingTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewNoiseDateTransformer

func NewNoiseDateTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewNoiseFloatTransformer

func NewNoiseFloatTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewNoiseIntTransformer

func NewNoiseIntTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomBoolTransformer

func NewRandomBoolTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomChoiceTransformer

func NewRandomChoiceTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomDateTransformer

func NewRandomDateTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomFloatTransformer

func NewRandomFloatTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomIntTransformer

func NewRandomIntTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomStringTransformer

func NewRandomStringTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomUuidTransformer

func NewRandomUuidTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRealAddressTransformer added in v0.1.1

func NewRealAddressTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRegexpReplaceTransformer

func NewRegexpReplaceTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewReplaceTransformer

func NewReplaceTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewSetNullTransformer

func NewSetNullTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewTemplateRecordTransformer

func NewTemplateRecordTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewTemplateTransformer

func NewTemplateTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

Types

type Cmd

type Cmd struct {
	*utils.CmdTransformerBase

	Columns []*Column
	// contains filtered or unexported fields
}

func (*Cmd) Done

func (c *Cmd) Done(ctx context.Context) error

func (*Cmd) GetAffectedColumns

func (c *Cmd) GetAffectedColumns() map[int]string

func (*Cmd) Init

func (c *Cmd) Init(ctx context.Context) error

func (*Cmd) Transform

func (c *Cmd) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type Column

type Column struct {
	Name             string `json:"name"`
	NotAffected      bool   `json:"not_affected"`
	SkipOnNullInput  bool   `json:"skip_on_null_input"`
	SkipOriginalData bool   `json:"skip_original_data"`
}

type ColumnContext

type ColumnContext struct {
	// contains filtered or unexported fields
}

func NewColumnContext

func NewColumnContext(columnType string, columnName string) *ColumnContext

func (*ColumnContext) DecodeValue

func (cc *ColumnContext) DecodeValue(v any) (any, error)

func (*ColumnContext) DecodeValueByColumn

func (cc *ColumnContext) DecodeValueByColumn(name string, v any) (any, error)

func (*ColumnContext) DecodeValueByType

func (cc *ColumnContext) DecodeValueByType(name string, v any) (any, error)

func (*ColumnContext) EncodeValue

func (cc *ColumnContext) EncodeValue(v any) (any, error)

func (*ColumnContext) EncodeValueByColumn

func (cc *ColumnContext) EncodeValueByColumn(name string, v any) (any, error)

func (*ColumnContext) EncodeValueByType

func (cc *ColumnContext) EncodeValueByType(name string, v any) (any, error)

func (*ColumnContext) GetColumnRawValue

func (cc *ColumnContext) GetColumnRawValue(name string) (any, error)

func (*ColumnContext) GetColumnType

func (cc *ColumnContext) GetColumnType() string

func (*ColumnContext) GetColumnValue

func (cc *ColumnContext) GetColumnValue(name string) (any, error)

func (*ColumnContext) GetRawValue

func (cc *ColumnContext) GetRawValue() (any, error)

func (*ColumnContext) GetValue

func (cc *ColumnContext) GetValue() (any, error)

type DictTransformer

type DictTransformer struct {
	// contains filtered or unexported fields
}

func (*DictTransformer) Done

func (ht *DictTransformer) Done(ctx context.Context) error

func (*DictTransformer) GetAffectedColumns

func (ht *DictTransformer) GetAffectedColumns() map[int]string

func (*DictTransformer) Init

func (ht *DictTransformer) Init(ctx context.Context) error

func (*DictTransformer) Transform

func (ht *DictTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type FakeTransformer added in v0.1.1

type FakeTransformer struct {
	// contains filtered or unexported fields
}

func (*FakeTransformer) Done added in v0.1.1

func (fts *FakeTransformer) Done(ctx context.Context) error

func (*FakeTransformer) GetAffectedColumns added in v0.1.1

func (fts *FakeTransformer) GetAffectedColumns() map[int]string

func (*FakeTransformer) Init added in v0.1.1

func (fts *FakeTransformer) Init(ctx context.Context) error

func (*FakeTransformer) Transform added in v0.1.1

func (fts *FakeTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type FakerFunc added in v0.1.1

type FakerFunc func(opts ...options.OptionFunc) string

type FakerTransformerDef added in v0.1.1

type FakerTransformerDef struct {
	SupportedTypes []string
	Description    string
	Generator      FakerFunc
}

type HashTransformer

type HashTransformer struct {
	// contains filtered or unexported fields
}

func (*HashTransformer) Done

func (ht *HashTransformer) Done(ctx context.Context) error

func (*HashTransformer) GetAffectedColumns

func (ht *HashTransformer) GetAffectedColumns() map[int]string

func (*HashTransformer) Init

func (ht *HashTransformer) Init(ctx context.Context) error

func (*HashTransformer) Transform

func (ht *HashTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type JsonContext

type JsonContext struct {
	// contains filtered or unexported fields
}

func NewJsonContext

func NewJsonContext() *JsonContext

func (*JsonContext) DecodeValueByColumn

func (jc *JsonContext) DecodeValueByColumn(name string, v any) (any, error)

func (*JsonContext) DecodeValueByType

func (jc *JsonContext) DecodeValueByType(name string, v any) (any, error)

func (*JsonContext) EncodeValueByColumn

func (jc *JsonContext) EncodeValueByColumn(name string, v any) (any, error)

func (*JsonContext) EncodeValueByType

func (jc *JsonContext) EncodeValueByType(name string, v any) (any, error)

func (*JsonContext) GetColumnRawValue

func (jc *JsonContext) GetColumnRawValue(name string) (any, error)

func (*JsonContext) GetColumnValue

func (jc *JsonContext) GetColumnValue(name string) (any, error)

func (*JsonContext) GetOriginalValue

func (jc *JsonContext) GetOriginalValue() any

func (*JsonContext) GetPath

func (jc *JsonContext) GetPath() string

func (*JsonContext) OriginalValueExists

func (jc *JsonContext) OriginalValueExists() bool

type JsonTransformer

type JsonTransformer struct {
	// contains filtered or unexported fields
}

func (*JsonTransformer) Done

func (jt *JsonTransformer) Done(ctx context.Context) error

func (*JsonTransformer) GetAffectedColumns

func (jt *JsonTransformer) GetAffectedColumns() map[int]string

func (*JsonTransformer) Init

func (jt *JsonTransformer) Init(ctx context.Context) error

func (*JsonTransformer) Transform

func (jt *JsonTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type MaskingTransformer

type MaskingTransformer struct {
	// contains filtered or unexported fields
}

func (*MaskingTransformer) Done

func (mt *MaskingTransformer) Done(ctx context.Context) error

func (*MaskingTransformer) GetAffectedColumns

func (mt *MaskingTransformer) GetAffectedColumns() map[int]string

func (*MaskingTransformer) Init

func (mt *MaskingTransformer) Init(ctx context.Context) error

func (*MaskingTransformer) Transform

func (mt *MaskingTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type NoiseDateTransformer

type NoiseDateTransformer struct {
	// contains filtered or unexported fields
}

func (*NoiseDateTransformer) Done

func (ndt *NoiseDateTransformer) Done(ctx context.Context) error

func (*NoiseDateTransformer) GetAffectedColumns

func (ndt *NoiseDateTransformer) GetAffectedColumns() map[int]string

func (*NoiseDateTransformer) Init

func (ndt *NoiseDateTransformer) Init(ctx context.Context) error

func (*NoiseDateTransformer) Transform

func (ndt *NoiseDateTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type NoiseFloatTransformer

type NoiseFloatTransformer struct {
	// contains filtered or unexported fields
}

func (*NoiseFloatTransformer) Done

func (nft *NoiseFloatTransformer) Done(ctx context.Context) error

func (*NoiseFloatTransformer) GetAffectedColumns

func (nft *NoiseFloatTransformer) GetAffectedColumns() map[int]string

func (*NoiseFloatTransformer) Init

func (nft *NoiseFloatTransformer) Init(ctx context.Context) error

func (*NoiseFloatTransformer) Transform

func (nft *NoiseFloatTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type NoiseFloatTransformerParams

type NoiseFloatTransformerParams struct {
	Ratio     float64 `mapstructure:"ratio" validate:"required,min=0,max=1"`
	Precision int16   `mapstructure:"precision"`
	Nullable  bool    `mapstructure:"nullable"`
	Fraction  float32 `mapstructure:"fraction"`
}

type NoiseIntTransformer

type NoiseIntTransformer struct {
	// contains filtered or unexported fields
}

func (*NoiseIntTransformer) Done

func (nit *NoiseIntTransformer) Done(ctx context.Context) error

func (*NoiseIntTransformer) GetAffectedColumns

func (nit *NoiseIntTransformer) GetAffectedColumns() map[int]string

func (*NoiseIntTransformer) Init

func (nit *NoiseIntTransformer) Init(ctx context.Context) error

func (*NoiseIntTransformer) Transform

func (nit *NoiseIntTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type Operation

type Operation struct {
	Operation     string      `mapstructure:"operation" json:"operation"`
	Value         interface{} `mapstructure:"value,omitempty" json:"value"`
	ValueTemplate string      `mapstructure:"value_template,omitempty" json:"value_template"`
	Path          string      `mapstructure:"path" json:"path"`
	ErrorNotExist bool        `mapstructure:"error_not_exist" json:"error_not_exist"`
	// contains filtered or unexported fields
}

func (*Operation) Apply

func (o *Operation) Apply(inp []byte, tctx *JsonContext, buf *bytes.Buffer) ([]byte, error)

type RandomBoolTransformer

type RandomBoolTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomBoolTransformer) Done

func (rbt *RandomBoolTransformer) Done(ctx context.Context) error

func (*RandomBoolTransformer) GetAffectedColumns

func (rbt *RandomBoolTransformer) GetAffectedColumns() map[int]string

func (*RandomBoolTransformer) Init

func (rbt *RandomBoolTransformer) Init(ctx context.Context) error

func (*RandomBoolTransformer) Transform

func (rbt *RandomBoolTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomChoiceTransformer

type RandomChoiceTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomChoiceTransformer) Done

func (*RandomChoiceTransformer) GetAffectedColumns

func (rct *RandomChoiceTransformer) GetAffectedColumns() map[int]string

func (*RandomChoiceTransformer) Init

func (*RandomChoiceTransformer) Transform

type RandomDateTransformer

type RandomDateTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomDateTransformer) Done

func (rdt *RandomDateTransformer) Done(ctx context.Context) error

func (*RandomDateTransformer) GetAffectedColumns

func (rdt *RandomDateTransformer) GetAffectedColumns() map[int]string

func (*RandomDateTransformer) Init

func (rdt *RandomDateTransformer) Init(ctx context.Context) error

func (*RandomDateTransformer) Transform

func (rdt *RandomDateTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomDateTransformerParams

type RandomDateTransformerParams struct {
	Min      string  `mapstructure:"min" validate:"required"`
	Max      string  `mapstructure:"max" validate:"required"`
	Truncate string  `mapstructure:"truncate" validate:"omitempty,oneof=year month day hour second nano"`
	Nullable bool    `mapstructure:"nullable"`
	Fraction float32 `mapstructure:"fraction"`
}

type RandomFloatTransformer

type RandomFloatTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomFloatTransformer) Done

func (*RandomFloatTransformer) GetAffectedColumns

func (rft *RandomFloatTransformer) GetAffectedColumns() map[int]string

func (*RandomFloatTransformer) Init

func (*RandomFloatTransformer) Transform

type RandomIntTransformer

type RandomIntTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomIntTransformer) Done

func (rit *RandomIntTransformer) Done(ctx context.Context) error

func (*RandomIntTransformer) GetAffectedColumns

func (rit *RandomIntTransformer) GetAffectedColumns() map[int]string

func (*RandomIntTransformer) Init

func (rit *RandomIntTransformer) Init(ctx context.Context) error

func (*RandomIntTransformer) Transform

func (rit *RandomIntTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomStringTransformer

type RandomStringTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomStringTransformer) Done

func (*RandomStringTransformer) GetAffectedColumns

func (rst *RandomStringTransformer) GetAffectedColumns() map[int]string

func (*RandomStringTransformer) Init

func (*RandomStringTransformer) Transform

type RandomUuidTransformer

type RandomUuidTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomUuidTransformer) Done

func (rut *RandomUuidTransformer) Done(ctx context.Context) error

func (*RandomUuidTransformer) GetAffectedColumns

func (rut *RandomUuidTransformer) GetAffectedColumns() map[int]string

func (*RandomUuidTransformer) Init

func (rut *RandomUuidTransformer) Init(ctx context.Context) error

func (*RandomUuidTransformer) Transform

func (rut *RandomUuidTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RealAddressColumn added in v0.1.1

type RealAddressColumn struct {
	Name     string `json:"name"`
	KeepNull bool   `json:"keep_null"`
	Template string `json:"template"`
	// contains filtered or unexported fields
}

type RealAddressTransformer added in v0.1.1

type RealAddressTransformer struct {
	// contains filtered or unexported fields
}

func (*RealAddressTransformer) Done added in v0.1.1

func (*RealAddressTransformer) GetAffectedColumns added in v0.1.1

func (rat *RealAddressTransformer) GetAffectedColumns() map[int]string

func (*RealAddressTransformer) Init added in v0.1.1

func (*RealAddressTransformer) Transform added in v0.1.1

type RealAddressValue added in v0.1.1

type RealAddressValue struct {
	Address    string  `json:"address1"`
	City       string  `json:"city"`
	State      string  `json:"state"`
	PostalCode string  `json:"postalCode"`
	Latitude   float64 `json:"lat"`
	Longitude  float64 `json:"lng"`
}

type RegexpReplaceTransformer

type RegexpReplaceTransformer struct {
	// contains filtered or unexported fields
}

func (*RegexpReplaceTransformer) Done

func (*RegexpReplaceTransformer) GetAffectedColumns

func (rrt *RegexpReplaceTransformer) GetAffectedColumns() map[int]string

func (*RegexpReplaceTransformer) Init

func (*RegexpReplaceTransformer) Transform

type ReplaceTransformer

type ReplaceTransformer struct {
	// contains filtered or unexported fields
}

func (*ReplaceTransformer) Done

func (rt *ReplaceTransformer) Done(ctx context.Context) error

func (*ReplaceTransformer) GetAffectedColumns

func (rt *ReplaceTransformer) GetAffectedColumns() map[int]string

func (*ReplaceTransformer) Init

func (rt *ReplaceTransformer) Init(ctx context.Context) error

func (*ReplaceTransformer) Transform

func (rt *ReplaceTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type SetNullTransformer

type SetNullTransformer struct {
	// contains filtered or unexported fields
}

func (*SetNullTransformer) Done

func (sut *SetNullTransformer) Done(ctx context.Context) error

func (*SetNullTransformer) GetAffectedColumns

func (sut *SetNullTransformer) GetAffectedColumns() map[int]string

func (*SetNullTransformer) Init

func (sut *SetNullTransformer) Init(ctx context.Context) error

func (*SetNullTransformer) Transform

func (sut *SetNullTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type TemplateRecordTransformer

type TemplateRecordTransformer struct {
	// contains filtered or unexported fields
}

func (*TemplateRecordTransformer) Done

func (*TemplateRecordTransformer) GetAffectedColumns

func (sut *TemplateRecordTransformer) GetAffectedColumns() map[int]string

func (*TemplateRecordTransformer) Init

func (*TemplateRecordTransformer) Transform

type TemplateTransformer

type TemplateTransformer struct {
	// contains filtered or unexported fields
}

func (*TemplateTransformer) Done

func (sut *TemplateTransformer) Done(ctx context.Context) error

func (*TemplateTransformer) GetAffectedColumns

func (sut *TemplateTransformer) GetAffectedColumns() map[int]string

func (*TemplateTransformer) Init

func (sut *TemplateTransformer) Init(ctx context.Context) error

func (*TemplateTransformer) Transform

func (sut *TemplateTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL