jobopts

Documentation

Overview

Package jobopts contains shared options for job submission. These options are exposed to allow user code to inspect and modify them.

Index

• Variables
• func GetEndpoint() (string, error)
• func GetEnvironmentConfig(ctx context.Context) string
• func GetEnvironmentUrn(ctx context.Context) string
• func GetExperiments() []string
• func GetJobName() string
• func GetPipelineResourceHints() resource.Hints
• func GetSdkImageOverrides() map[string]string
• func IsLoopback() bool

Variables

var (
	// Endpoint is the job service endpoint.
	Endpoint = flag.String("endpoint", "", "Job service endpoint (required).")

	// JobName is the name of the job.
	JobName = flag.String("job_name", "", "Job name (optional).")

	// EnvironmentType is the environment type to run the user code.
	EnvironmentType = flag.String("environment_type", "DOCKER",
		"Environment Type. Possible options are DOCKER, and LOOPBACK.")

	// EnvironmentConfig is the environment configuration for running the user code.
	EnvironmentConfig = flag.String("environment_config",
		"",
		"Set environment configuration for running the user code.\n"+
			"For DOCKER: Url for the docker image.\n"+
			"For PROCESS: json of the form {\"os\": \"<OS>\", "+
			"\"arch\": \"<ARCHITECTURE>\", \"command\": \"<process to execute>\", "+
			"\"env\":{\"<Environment variables 1>\": \"<ENV_VAL>\"} }. "+
			"All fields in the json are optional except command.")

	// SdkHarnessContainerImageOverrides contains patterns for overriding
	// container image names in a pipeline.
	SdkHarnessContainerImageOverrides stringSlice

	// WorkerBinary is the location of the compiled worker binary. If not
	// specified, the binary is produced via go build.
	WorkerBinary = flag.String("worker_binary", "", "Worker binary (optional)")

	// Experiments toggle experimental features in the runner.
	Experiments = flag.String("experiments", "", "Comma-separated list of experiments (optional).")

	// Async determines whether to wait for job completion.
	Async = flag.Bool("async", false, "Do not wait for job completion.")

	// Strict mode applies additional validation to user pipelines before
	// executing them and fails early if the pipelines don't pass.
	Strict = flag.Bool("beam_strict", false, "Apply additional validation to pipelines.")

	// Flag to retain docker containers created by the runner. If false, then
	// containers are deleted once the job ends, even if it failed.
	RetainDockerContainers = flag.Bool("retain_docker_containers", false, "Retain Docker containers created by the runner.")

	// Flag to set the degree of parallelism. If not set, the configured Flink default is used, or 1 if none can be found.
	Parallelism = flag.Int("parallelism", -1, "The degree of parallelism to be used when distributing operations onto Flink workers.")

	// ResourceHints flag takes whole pipeline hints for resources.
	ResourceHints stringSlice
)

Functions

func GetEndpoint

func GetEndpoint() (string, error)

GetEndpoint returns the endpoint, if non empty and exits otherwise. Runners such as Dataflow set a reasonable default. Convenience function.

func GetEnvironmentConfig

func GetEnvironmentConfig(ctx context.Context) string

GetEnvironmentConfig returns the specified configuration for specified SDK Harness, if not present, the default development container for the current user. Convenience function.

func GetEnvironmentUrn

func GetEnvironmentUrn(ctx context.Context) string

GetEnvironmentUrn returns the specified EnvironmentUrn used to run the SDK Harness, if not present, returns the docker environment urn "beam:env:docker:v1". Convenience function.

func GetExperiments

func GetExperiments() []string

GetExperiments returns the experiments.

func GetJobName

func GetJobName() string

GetJobName returns the specified job name or, if not present, a fresh autogenerated name. Convenience function.

func GetPipelineResourceHints

func GetPipelineResourceHints() resource.Hints

GetPipelineResourceHints parses known standard hints and returns the flag set hints for the pipeline. In case of duplicate hint URNs, the last value specified will be used.

func GetSdkImageOverrides

func GetSdkImageOverrides() map[string]string

GetSdkImageOverrides gets the specified overrides as a map where each key is a regular expression pattern to match, and each value is the string to replace matching containers with.

func IsLoopback

func IsLoopback() bool

IsLoopback returns whether the EnvironmentType is loopback.