creation

Documentation

Overview

Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• func GetDefaultPipelineConfigs() []common.Config
• func GetDefaultStatsAggregatorStageInstance() *common.StageConfiguration
• func GetTrashErrorStageInstance() *common.StageConfiguration
• type PipelineBean
  • func NewPipelineBean(pipelineConfig common.PipelineConfiguration, ...) (PipelineBean, []validation.Issue)
• type PipelineConfigBean
  • func NewPipelineConfigBean(pipelineConfig common.PipelineConfiguration) PipelineConfigBean
• type ServiceBean
• type StageBean
  • func NewStageBean(stageConfig *common.StageConfiguration, ...) (StageBean, error)
  • func (s *StageBean) IsProcessor() bool
  • func (s *StageBean) IsSource() bool
  • func (s *StageBean) IsTarget() bool
• type StageConfigBean
  • func NewStageConfigBean(pipelineConfig *common.StageConfiguration) StageConfigBean

Constants

const (
	ExecutionMode        = "executionMode"
	DeliveryGuarantee    = "deliveryGuarantee"
	ShouldRetry          = "shouldRetry"
	RetryAttempts        = "retryAttempts"
	MemoryLimit          = "memoryLimit"
	MemoryLmtExceeded    = "memoryLimitExceeded"
	NotifyOnStates       = "notifyOnStates"
	EmailIds             = "emailIDs"
	Constants            = "constants"
	BadRecordsHandling   = "badRecordsHandling"
	TestOriginStage      = "testOriginStage"
	RateLimit            = "rateLimit"
	MaxRunners           = "maxRunners"
	StatsAggregatorStage = "statsAggregatorStage"
	ErrorRecordPolicy    = "errorRecordPolicy"

	ClusterSlaveMemory   = "clusterSlaveMemory"
	ClusterSlaveJavaOpts = "clusterSlaveJavaOpts"
	ClusterLauncherEnv   = "clusterLauncherEnv"
	MesosDispatchUrl     = "mesosDispatcherURL"
	HdfsS3ConfigDir      = "hdfsS3ConfDir"
	WebHookConfigs       = "webhookConfigs"
)

const (
	STAGE_TYPE = "stageType"
	SOURCE     = "SOURCE"
	PROCESSOR  = "PROCESSOR"
	TARGET     = "TARGET"
)

Functions

func GetDefaultPipelineConfigs

func GetDefaultPipelineConfigs() []common.Config

func GetDefaultStatsAggregatorStageInstance

func GetDefaultStatsAggregatorStageInstance() *common.StageConfiguration

func GetTrashErrorStageInstance

func GetTrashErrorStageInstance() *common.StageConfiguration

Types

type PipelineBean

type PipelineBean struct {
	Config               PipelineConfigBean
	Stages               []StageBean
	ErrorStage           StageBean
	StatsAggregatorStage StageBean
	ElContext            context.Context
}

func NewPipelineBean

func NewPipelineBean(
	pipelineConfig common.PipelineConfiguration,
	runtimeParameters map[string]interface{},
) (PipelineBean, []validation.Issue)

type PipelineConfigBean

type PipelineConfigBean struct {
	Version              string
	ExecutionMode        string
	DeliveryGuarantee    string
	ShouldRetry          bool
	RetryAttempts        float64
	MemoryLimit          string
	MemoryLimitExceeded  string
	NotifyOnStates       []interface{}
	EmailIDs             []interface{}
	Constants            map[string]interface{}
	BadRecordsHandling   string
	TestOriginStage      string
	ErrorRecordPolicy    string
	StatsAggregatorStage string
	RateLimit            float64
	MaxRunners           float64
}

func NewPipelineConfigBean

func NewPipelineConfigBean(pipelineConfig common.PipelineConfiguration) PipelineConfigBean

type ServiceBean

type ServiceBean struct {
	Config  *common.ServiceConfiguration
	Service api.Service
}

type StageBean

type StageBean struct {
	Config        *common.StageConfiguration
	Stage         api.Stage
	SystemConfigs StageConfigBean
	Services      []ServiceBean
}

func NewStageBean

func NewStageBean(
	stageConfig *common.StageConfiguration,
	runtimeParameters map[string]interface{},
	elContext context.Context,
) (StageBean, error)

func (*StageBean) IsProcessor

func (s *StageBean) IsProcessor() bool

func (*StageBean) IsSource

func (s *StageBean) IsSource() bool

func (*StageBean) IsTarget

func (s *StageBean) IsTarget() bool

type StageConfigBean

type StageConfigBean struct {
	StageOnRecordError       string
	StageRequiredFields      []interface{}
	StageRecordPreconditions []interface{}
}

func NewStageConfigBean

func NewStageConfigBean(pipelineConfig *common.StageConfiguration) StageConfigBean