Version: v0.0.0-...-9d9e229 Latest Latest

This package is not in the latest version of its module.

Go to latest
Published: Aug 19, 2021 License: Apache-2.0 Imports: 11 Imported by: 0




View Source
const (
	// DataProcessorTask predefined sections
	ConfigDirs   = "config_dirs"
	ConfigFiles  = "config_files"
	InputDirs    = "input_dirs"
	InputFiles   = "input_files"
	InputTables  = "input_tables"
	OutputDirs   = "output_dirs"
	OutputFiles  = "output_files"
	OutputTables = "output_tables"
	ReportLevel  = "report_level"
	SummaryLevel = "summary_level"
	TraceLevel   = "trace_level"
View Source
const (
	// Names of the directories
	ConfigDirName = "config"
	InputDirName  = "input"
	OutputDirName = "output"


View Source
var DataProcessorTaskFile string

DataProcessorTaskFile defines what DataProcessorTask file should be used. It has to be exported var in order to be used in cases such as: rootCmd.PersistentFlags().StringVar(&data_processor_task.DataProcessorTaskFile, "task", "", "DataProcessorTask file")


func ReadIn

func ReadIn()

ReadIn reads/un-serializes the DataProcessorTask from DataProcessorTaskFile


type DataProcessorTask

type DataProcessorTask struct {
	// Items contains named sections of various parameters. Each section is represented as a slice.
	Items map[string][]string `json:"items,omitempty" yaml:"items,omitempty"`
	// Status represents status of the data processor task
	Status Status `json:"status,omitempty" yaml:"status,omitempty"`

	// RootDir specifies name of the root directory in case dirs have nested structure
	RootDir string `json:"root,omitempty" yaml:"root,omitempty"`
	// TaskFile specifies path/name where to/from serialize/un-serialize a task
	TaskFile string `json:"task,omitempty" yaml:"task,omitempty"`
	// Format specifies DataProcessorTask serialization formats
	Format Format `json:"-" yaml:"-"`

DataProcessorTask specifies task to be launched as external process for data processing

var Task *DataProcessorTask

Task is the DataProcessorTask un-serialized from DataProcessorTaskFile It has to be exported var in order to be used in external modules to access the Task specification

func New

func New() *DataProcessorTask

New creates new task

func (*DataProcessorTask) Add

func (t *DataProcessorTask) Add(section string, items ...string) *DataProcessorTask

Add adds item(s) to a section

func (*DataProcessorTask) AddConfigDir

func (t *DataProcessorTask) AddConfigDir(dir ...string) *DataProcessorTask

AddConfigDir adds config dir(s)

func (*DataProcessorTask) AddConfigFile

func (t *DataProcessorTask) AddConfigFile(file ...string) *DataProcessorTask

AddConfigFile adds config file(s)

func (*DataProcessorTask) AddInputDir

func (t *DataProcessorTask) AddInputDir(dir ...string) *DataProcessorTask

AddInputDir adds input dir(s)

func (*DataProcessorTask) AddInputFile

func (t *DataProcessorTask) AddInputFile(file ...string) *DataProcessorTask

AddInputFile adds input file(s)

func (*DataProcessorTask) AddInputTable

func (t *DataProcessorTask) AddInputTable(table ...string) *DataProcessorTask

AddInputTable adds input table(s)

func (*DataProcessorTask) AddOutputDir

func (t *DataProcessorTask) AddOutputDir(dir ...string) *DataProcessorTask

AddOutputDir adds output dir(s)

func (*DataProcessorTask) AddOutputFile

func (t *DataProcessorTask) AddOutputFile(file ...string) *DataProcessorTask

AddOutputFile adds output file(s)

func (*DataProcessorTask) AddOutputTable

func (t *DataProcessorTask) AddOutputTable(table ...string) *DataProcessorTask

AddOutputTable adds output table(s)

func (*DataProcessorTask) CreateTempDir

func (t *DataProcessorTask) CreateTempDir(dir, pattern string) *DataProcessorTask


func (*DataProcessorTask) Delete

func (t *DataProcessorTask) Delete(section string) *DataProcessorTask

Delete deletes a section

func (*DataProcessorTask) Exists

func (t *DataProcessorTask) Exists(section string) bool

Exists checks whether specified section exists within DataProcessorTask. Returns tru if section exists. Section may have 0 items in it and return true

func (*DataProcessorTask) Get

func (t *DataProcessorTask) Get(section string, defaultValue ...string) string

Get first item from a section or default value. Default value can be provided explicitly or "" used otherwise

func (*DataProcessorTask) GetAll

func (t *DataProcessorTask) GetAll(section string) []string

GetAll gets all entities of a section

func (*DataProcessorTask) GetConfigDir

func (t *DataProcessorTask) GetConfigDir(defaultValue ...string) string

GetConfigDir gets the first config dir

func (*DataProcessorTask) GetConfigDirs

func (t *DataProcessorTask) GetConfigDirs() []string

GetConfigDirs gets all config dirs

func (*DataProcessorTask) GetConfigFile

func (t *DataProcessorTask) GetConfigFile(defaultValue ...string) string

GetConfigFile gets the first config file

func (*DataProcessorTask) GetConfigFiles

func (t *DataProcessorTask) GetConfigFiles() []string

GetConfigFiles gets all config files

func (*DataProcessorTask) GetErrors

func (t *DataProcessorTask) GetErrors() []string

GetErrors gets slice of errors reported by the task

func (*DataProcessorTask) GetFormat

func (t *DataProcessorTask) GetFormat() Format

GetFormat gets format to serialize DataProcessorTask to

func (*DataProcessorTask) GetInputDir

func (t *DataProcessorTask) GetInputDir(defaultValue ...string) string

GetInputDir gets the first input dir

func (*DataProcessorTask) GetInputDirs

func (t *DataProcessorTask) GetInputDirs() []string

GetInputDirs gets all input dirs

func (*DataProcessorTask) GetInputFile

func (t *DataProcessorTask) GetInputFile(defaultValue ...string) string

GetInputFile gets the first input file

func (*DataProcessorTask) GetInputFiles

func (t *DataProcessorTask) GetInputFiles() []string

GetInputFiles gets all input files

func (*DataProcessorTask) GetInputTable

func (t *DataProcessorTask) GetInputTable(defaultValue ...string) string

GetInputTable gets the first input table

func (*DataProcessorTask) GetInputTables

func (t *DataProcessorTask) GetInputTables() []string

GetInputTables gets all input tables

func (*DataProcessorTask) GetOutputDir

func (t *DataProcessorTask) GetOutputDir(defaultValue ...string) string

GetOutputDir gets the first output dir

func (*DataProcessorTask) GetOutputDirs

func (t *DataProcessorTask) GetOutputDirs() []string

GetOutputDirs gets all output dirs

func (*DataProcessorTask) GetOutputFile

func (t *DataProcessorTask) GetOutputFile(defaultValue ...string) string

GetOutputFile gets the first output file

func (*DataProcessorTask) GetOutputFiles

func (t *DataProcessorTask) GetOutputFiles() []string

GetOutputFiles gets all output files

func (*DataProcessorTask) GetOutputTable

func (t *DataProcessorTask) GetOutputTable(defaultValue ...string) string

GetOutputTable gets the first output table

func (*DataProcessorTask) GetOutputTables

func (t *DataProcessorTask) GetOutputTables() []string

GetOutputTables gets all output tables

func (*DataProcessorTask) GetRootDir

func (t *DataProcessorTask) GetRootDir() string


func (*DataProcessorTask) GetStatus

func (t *DataProcessorTask) GetStatus() int32

GetStatus gets status of the task

func (*DataProcessorTask) GetTaskFile

func (t *DataProcessorTask) GetTaskFile() string


func (*DataProcessorTask) Has

func (t *DataProcessorTask) Has(section string) bool

Has checks whether DataProcessorTask has something in specified section. Returns true only in case section has > 0 items in it.

func (*DataProcessorTask) HasConfigDirs

func (t *DataProcessorTask) HasConfigDirs() bool

HasConfigDirs checks whether there are config dirs(s)

func (*DataProcessorTask) HasConfigFiles

func (t *DataProcessorTask) HasConfigFiles() bool

HasConfigFiles checks whether there are config file(s)

func (*DataProcessorTask) HasInputDirs

func (t *DataProcessorTask) HasInputDirs() bool

HasInputDirs checks whether there are input dires(s)

func (*DataProcessorTask) HasInputFiles

func (t *DataProcessorTask) HasInputFiles() bool

HasInputFiles checks whether there are input file(s)

func (*DataProcessorTask) HasInputTables

func (t *DataProcessorTask) HasInputTables() bool

HasInputTables checks whether there are input table(s)

func (*DataProcessorTask) HasOutputDirs

func (t *DataProcessorTask) HasOutputDirs() bool

HasOutputDirs checks whether there are output dir(s)

func (*DataProcessorTask) HasOutputFiles

func (t *DataProcessorTask) HasOutputFiles() bool

HasOutputFiles checks whether there are output file(s)

func (*DataProcessorTask) HasOutputTables

func (t *DataProcessorTask) HasOutputTables() bool

HasOutputTables checks whether there are output table(s)

func (*DataProcessorTask) HasReportLevel

func (t *DataProcessorTask) HasReportLevel() bool


func (*DataProcessorTask) HasSummaryLevel

func (t *DataProcessorTask) HasSummaryLevel() bool


func (*DataProcessorTask) HasTraceLevel

func (t *DataProcessorTask) HasTraceLevel() bool


func (*DataProcessorTask) IsFormatKnown

func (t *DataProcessorTask) IsFormatKnown() bool

IsFormatKnown checks whether specified format is known to parser

func (*DataProcessorTask) Len

func (t *DataProcessorTask) Len(section string) int

Len gets number of items within a section

func (*DataProcessorTask) Marshal

func (t *DataProcessorTask) Marshal() (out []byte, err error)

Marshal marshals DataProcessorTask according with specified format

func (*DataProcessorTask) ReadFrom

func (t *DataProcessorTask) ReadFrom(file string) error

ReadFrom reads DataProcessorTask from specified file and tries to understand what is the format of the specified file

func (*DataProcessorTask) SaveAs

func (t *DataProcessorTask) SaveAs(file string) error

SaveAs saves DataProcessorTask into specified file

func (*DataProcessorTask) SaveTempFile

func (t *DataProcessorTask) SaveTempFile(dir, pattern string) (string, error)

SaveTempFile saves DataProcessorTask as temp file with pattern-randomized name into specified dir

func (*DataProcessorTask) SaveTempTaskFile

func (t *DataProcessorTask) SaveTempTaskFile(dir, pattern string) error

SaveTempTaskFile saves task as temp file and sets TaskFile to produced temp file name

func (*DataProcessorTask) Sections

func (t *DataProcessorTask) Sections() []string

Sections lists all sections

func (*DataProcessorTask) Set

func (t *DataProcessorTask) Set(section string, items ...string) *DataProcessorTask

Set replaces section with specified items

func (*DataProcessorTask) SetConfigDir

func (t *DataProcessorTask) SetConfigDir(dir ...string) *DataProcessorTask

SetConfigDir sets config dir(s)

func (*DataProcessorTask) SetConfigFile

func (t *DataProcessorTask) SetConfigFile(file ...string) *DataProcessorTask

SetConfigFile adds config file(s)

func (*DataProcessorTask) SetFormat

func (t *DataProcessorTask) SetFormat(format Format) *DataProcessorTask

SetFormat sets format to serialize DataProcessorTask to

func (*DataProcessorTask) SetInputDir

func (t *DataProcessorTask) SetInputDir(dir ...string) *DataProcessorTask

SetInputDir sets input dir(s)

func (*DataProcessorTask) SetInputFile

func (t *DataProcessorTask) SetInputFile(file ...string) *DataProcessorTask

SetInputFile sets input file(s)

func (*DataProcessorTask) SetInputTable

func (t *DataProcessorTask) SetInputTable(table ...string) *DataProcessorTask

SetInputTable sets input table(s)

func (*DataProcessorTask) SetOutputDir

func (t *DataProcessorTask) SetOutputDir(dir ...string) *DataProcessorTask

SetOutputDir sets output dir(s)

func (*DataProcessorTask) SetOutputFile

func (t *DataProcessorTask) SetOutputFile(file ...string) *DataProcessorTask

SetOutputFile sets output file(s)

func (*DataProcessorTask) SetOutputTable

func (t *DataProcessorTask) SetOutputTable(table ...string) *DataProcessorTask

SetOutputTable sets output table(s)

func (*DataProcessorTask) SetReportLevel

func (t *DataProcessorTask) SetReportLevel(level int) *DataProcessorTask


func (*DataProcessorTask) SetRootDir

func (t *DataProcessorTask) SetRootDir(dir string) *DataProcessorTask


func (*DataProcessorTask) SetSummaryLevel

func (t *DataProcessorTask) SetSummaryLevel(level int) *DataProcessorTask


func (*DataProcessorTask) SetTaskFile

func (t *DataProcessorTask) SetTaskFile(file string) *DataProcessorTask


func (*DataProcessorTask) SetTraceLevel

func (t *DataProcessorTask) SetTraceLevel(level int) *DataProcessorTask


func (*DataProcessorTask) String

func (t *DataProcessorTask) String() string


func (*DataProcessorTask) Unmarshal

func (t *DataProcessorTask) Unmarshal(in []byte) (err error)

Unmarshal unmarshalls DataProcessorTask according with specified format

func (*DataProcessorTask) Walk

func (t *DataProcessorTask) Walk(f func(section string, items []string) bool) *DataProcessorTask

Walk walk over sections with a function

func (*DataProcessorTask) WalkInputFiles

func (t *DataProcessorTask) WalkInputFiles(full bool, f func(string) bool) *DataProcessorTask


func (*DataProcessorTask) WalkOutputFiles

func (t *DataProcessorTask) WalkOutputFiles(full bool, f func(string) bool) *DataProcessorTask


func (*DataProcessorTask) WalkSection

func (t *DataProcessorTask) WalkSection(section string, f func(item string) bool) *DataProcessorTask

WalkSection walk over all items in a section with a function

type Format

type Format string

Format specifies DataProcessorTask serialization formats

const (
	// DataProcessorTask serialization formats
	Empty   Format = ""
	Unknown Format = "unknown"
	YAML    Format = "yaml"
	JSON    Format = "json"

type Status

type Status struct {
	// Status represents status code
	Status int32
	// Errors is a list of errors, if any
	Errors []string

Status specifies status of the DataProcessorTask

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
t or T : Toggle theme light dark auto
y or Y : Canonical URL