spark

Documentation

Index

• Variables
• func AddGlobalLabels(labels map[string]string, globalLabels map[string]string) map[string]string
• func DriverServiceName(name string) string
• func FrameworkConfigMapName(instance string, comp Component) string
• func HeadlessServiceName(name string) string
• func HorizontalPodAutoscalerObjectMeta(sc *dcv1alpha1.SparkCluster) metav1.ObjectMeta
• func InstanceObjectName(instance string, comp Component) string
• func KeyTabConfigMapName(instance string, comp Component) string
• func MasterServiceName(name string) string
• func MetadataLabels(sc *dcv1alpha1.SparkCluster) map[string]string
• func MetadataLabelsWithComponent(sc *dcv1alpha1.SparkCluster, comp Component) map[string]string
• func NewClusterDriverNetworkPolicy(sc *dcv1alpha1.SparkCluster) *networkingv1.NetworkPolicy
• func NewClusterMasterNetworkPolicy(sc *dcv1alpha1.SparkCluster) *networkingv1.NetworkPolicy
• func NewClusterWorkerNetworkPolicy(sc *dcv1alpha1.SparkCluster) *networkingv1.NetworkPolicy
• func NewEnvoyFilter(sc *dcv1alpha1.SparkCluster) *apinetworkingv1alpha3.EnvoyFilter
• func NewFrameworkConfigMap(sc *dcv1alpha1.SparkCluster) *corev1.ConfigMap
• func NewHeadlessService(sc *dcv1alpha1.SparkCluster) *corev1.Service
• func NewHorizontalPodAutoscaler(sc *dcv1alpha1.SparkCluster) (*autoscalingv2.HorizontalPodAutoscaler, error)
• func NewKeyTabConfigMap(sc *dcv1alpha1.SparkCluster) *corev1.ConfigMap
• func NewMasterService(sc *dcv1alpha1.SparkCluster) *corev1.Service
• func NewPodSecurityPolicyRBAC(sc *dcv1alpha1.SparkCluster) (*rbacv1.Role, *rbacv1.RoleBinding)
• func NewServiceAccount(sc *dcv1alpha1.SparkCluster) *corev1.ServiceAccount
• func NewSparkDriverService(sc *dcv1alpha1.SparkCluster) *corev1.Service
• func NewStatefulSet(sc *dcv1alpha1.SparkCluster, comp Component) (*appsv1.StatefulSet, error)
• func SelectorLabels(sc *dcv1alpha1.SparkCluster) map[string]string
• func SelectorLabelsWithComponent(sc *dcv1alpha1.SparkCluster, comp Component) map[string]string
• type Component

Variables

var Meta = metadata.NewProvider(
	ApplicationName,
	func(obj client.Object) string { return objToSparkCluster(obj).Spec.Image.Tag },
	func(obj client.Object) map[string]string { return objToSparkCluster(obj).Spec.GlobalLabels },
)

Functions

func AddGlobalLabels

func AddGlobalLabels(labels map[string]string, globalLabels map[string]string) map[string]string

func DriverServiceName

func DriverServiceName(name string) string

func FrameworkConfigMapName

func FrameworkConfigMapName(instance string, comp Component) string

func HeadlessServiceName

func HeadlessServiceName(name string) string

func HorizontalPodAutoscalerObjectMeta

func HorizontalPodAutoscalerObjectMeta(sc *dcv1alpha1.SparkCluster) metav1.ObjectMeta

HorizontalPodAutoscalerObjectMeta returns the ObjectMeta object used to identify new HPA objects.

func InstanceObjectName

func InstanceObjectName(instance string, comp Component) string

InstanceObjectName returns the name that will be used to create most owned cluster resources.

func KeyTabConfigMapName

func KeyTabConfigMapName(instance string, comp Component) string

func MasterServiceName

func MasterServiceName(name string) string

MasterServiceName returns the name of the service that points to the spark master pod.

func MetadataLabels

func MetadataLabels(sc *dcv1alpha1.SparkCluster) map[string]string

MetadataLabels returns standard metadata for spark resources.

func MetadataLabelsWithComponent

func MetadataLabelsWithComponent(sc *dcv1alpha1.SparkCluster, comp Component) map[string]string

MetadataLabelsWithComponent returns standard component metadata for spark resources.

func NewClusterDriverNetworkPolicy

func NewClusterDriverNetworkPolicy(sc *dcv1alpha1.SparkCluster) *networkingv1.NetworkPolicy

func NewClusterMasterNetworkPolicy

func NewClusterMasterNetworkPolicy(sc *dcv1alpha1.SparkCluster) *networkingv1.NetworkPolicy

func NewClusterWorkerNetworkPolicy

func NewClusterWorkerNetworkPolicy(sc *dcv1alpha1.SparkCluster) *networkingv1.NetworkPolicy

func NewEnvoyFilter

func NewEnvoyFilter(sc *dcv1alpha1.SparkCluster) *apinetworkingv1alpha3.EnvoyFilter

NewEnvoyFilter creates a new EnvoyFilter resource to set idle_timeout for Istio-enabled deployments

func NewFrameworkConfigMap

func NewFrameworkConfigMap(sc *dcv1alpha1.SparkCluster) *corev1.ConfigMap

NewFrameworkConfigMap generates a configmap which represents a spark-defaults.conf file out of provided config

func NewHeadlessService

func NewHeadlessService(sc *dcv1alpha1.SparkCluster) *corev1.Service

NewHeadlessService creates a headless service that points to worker nodes

func NewHorizontalPodAutoscaler

func NewHorizontalPodAutoscaler(sc *dcv1alpha1.SparkCluster) (*autoscalingv2.HorizontalPodAutoscaler, error)

NewHorizontalPodAutoscaler generates an HPA that targets a SparkCluster resource.

The metrics-server needs to be launched separately and the worker deployment requires cpu resource requests in order for this object to have any effect.

func NewKeyTabConfigMap

func NewKeyTabConfigMap(sc *dcv1alpha1.SparkCluster) *corev1.ConfigMap

NewKeyTabConfigMap generates a configmap which represents the Kerberos KeyTab configuration out of provided config

func NewMasterService

func NewMasterService(sc *dcv1alpha1.SparkCluster) *corev1.Service

NewMasterService creates a ClusterIP service that points to the head node. Dashboard port is exposed when enabled.

func NewPodSecurityPolicyRBAC

func NewPodSecurityPolicyRBAC(sc *dcv1alpha1.SparkCluster) (*rbacv1.Role, *rbacv1.RoleBinding)

NewPodSecurityPolicyRBAC generates the role and role binding required to use a pod security policy. The role is bound to the service account used by the spark cluster pods.

func NewServiceAccount

func NewServiceAccount(sc *dcv1alpha1.SparkCluster) *corev1.ServiceAccount

NewServiceAccount generates a service account resource without API access.

func NewSparkDriverService

func NewSparkDriverService(sc *dcv1alpha1.SparkCluster) *corev1.Service

NewSparkDriverService creates a ClusterIP service that exposes the driver UI port.

func NewStatefulSet

func NewStatefulSet(sc *dcv1alpha1.SparkCluster, comp Component) (*appsv1.StatefulSet, error)

NewStatefulSet generates a Deployment configured to manage Spark cluster nodes. The configuration is based the provided spec and the desired Component workload.

func SelectorLabels

func SelectorLabels(sc *dcv1alpha1.SparkCluster) map[string]string

SelectorLabels returns a resource selector clause for spark resources.

func SelectorLabelsWithComponent

func SelectorLabelsWithComponent(sc *dcv1alpha1.SparkCluster, comp Component) map[string]string

SelectorLabelsWithComponent returns a resource component selector clause for spark resources.

Types

type Component

type Component string

Component is used to drive Kubernetes object generation for different spark types.

const (
	// ComponentNone indicates a generic spark resource.
	ComponentNone Component = "none"
	// ComponentMaster indicates a spark master resource.
	ComponentMaster Component = "master"
	// ComponentWorker indicates a spark worker resource.
	ComponentWorker Component = "worker"
	// ApplicationName defines the static name used to generate spark object metadata.
	ApplicationName = "spark"
)