extension

package
v1.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 26, 2022 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DomainPrefix = "koordinator.sh/"
	// use prefix "kubernetes.io/" for extend resource
	ResourceDomainPrefix = corev1.ResourceDefaultNamespacePrefix
	// SchedulingDomainPrefix represents the scheduling domain prefix
	SchedulingDomainPrefix = "scheduling.koordinator.sh"
	// NodeDomainPrefix represents the node domain prefix
	NodeDomainPrefix = "node.koordinator.sh"

	LabelPodQoS      = DomainPrefix + "qosClass"
	LabelPodPriority = DomainPrefix + "priority"

	LabelManagedBy = "app.kubernetes.io/managed-by"
)
View Source 
const (
	SystemQuotaName        = "system"
	RootQuotaName          = "root"
	DefaultQuotaName       = "default"
	QuotaKoordinatorPrefix = "quota.scheduling.koordinator.sh"
	LabelQuotaIsParent     = QuotaKoordinatorPrefix + "/is-parent"
	LabelQuotaParent       = QuotaKoordinatorPrefix + "/parent"
	LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource"
	LabelQuotaName         = QuotaKoordinatorPrefix + "/name"
	AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight"
	AnnotationRuntime      = QuotaKoordinatorPrefix + "/runtime"
	AnnotationRequest      = QuotaKoordinatorPrefix + "/request"
)

RootQuotaName means quotaTree's root\head.

View Source 
const (
	// AnnotationNodeCPUTopology describes the detailed CPU topology.
	AnnotationNodeCPUTopology = NodeDomainPrefix + "/cpu-topology"
	// AnnotationNodeCPUAllocs describes K8s Guaranteed Pods.
	AnnotationNodeCPUAllocs = NodeDomainPrefix + "/pod-cpu-allocs"
	// AnnotationNodeCPUSharedPools describes the CPU Shared Pool defined by Koordinator.
	// The shared pool is mainly used by Koordinator LS Pods or K8s Burstable Pods.
	AnnotationNodeCPUSharedPools = NodeDomainPrefix + "/cpu-shared-pools"

	// LabelNodeCPUBindPolicy constrains how to bind CPU logical CPUs when scheduling.
	LabelNodeCPUBindPolicy = NodeDomainPrefix + "/cpu-bind-policy"
	// LabelNodeNUMAAllocateStrategy indicates how to choose satisfied NUMA Nodes when scheduling.
	LabelNodeNUMAAllocateStrategy = NodeDomainPrefix + "/numa-allocate-strategy"
)
View Source 
const (
	// NodeCPUBindPolicyNone does not perform any bind policy
	NodeCPUBindPolicyNone = "None"
	// NodeCPUBindPolicyFullPCPUsOnly requires that the scheduler must allocate full physical cores.
	// Equivalent to kubelet CPU manager policy option full-pcpus-only=true.
	NodeCPUBindPolicyFullPCPUsOnly = "FullPCPUsOnly"
	// NodeCPUBindPolicySpreadByPCPUs requires that the scheduler must evenly allocate logical cpus across physical cores
	NodeCPUBindPolicySpreadByPCPUs = "SpreadByPCPUs"
)
View Source 
const (
	NodeNUMAAllocateStrategyLeastAllocated = string(NUMALeastAllocated)
	NodeNUMAAllocateStrategyMostAllocated  = string(NUMAMostAllocated)
)
View Source 
const (
	// AnnotationKubeletCPUManagerPolicy describes the cpu manager policy options of kubelet
	AnnotationKubeletCPUManagerPolicy = "kubelet.koordinator.sh/cpu-manager-policy"

	KubeletCPUManagerPolicyStatic                         = "static"
	KubeletCPUManagerPolicyNone                           = "none"
	KubeletCPUManagerPolicyFullPCPUsOnlyOption            = "full-pcpus-only"
	KubeletCPUManagerPolicyDistributeCPUsAcrossNUMAOption = "distribute-cpus-across-numa"
)
View Source 
const (
	AnnotationPodCPUBurst = DomainPrefix + "cpuBurst"

	AnnotationPodMemoryQoS = DomainPrefix + "memoryQOS"
)
View Source 
const (
	// Deprecated: because of the limitation of extended resource naming
	KoordBatchCPU corev1.ResourceName = DomainPrefix + "batch-cpu"
	// Deprecated: because of the limitation of extended resource naming
	KoordBatchMemory corev1.ResourceName = DomainPrefix + "batch-memory"

	BatchCPU    corev1.ResourceName = ResourceDomainPrefix + "batch-cpu"
	BatchMemory corev1.ResourceName = ResourceDomainPrefix + "batch-memory"

	KoordRDMA corev1.ResourceName = ResourceDomainPrefix + "rdma"
	KoordFPGA corev1.ResourceName = ResourceDomainPrefix + "fpga"

	KoordGPU  corev1.ResourceName = ResourceDomainPrefix + "gpu"
	NvidiaGPU corev1.ResourceName = "nvidia.com/gpu"

	GPUCore        corev1.ResourceName = ResourceDomainPrefix + "gpu-core"
	GPUMemory      corev1.ResourceName = ResourceDomainPrefix + "gpu-memory"
	GPUMemoryRatio corev1.ResourceName = ResourceDomainPrefix + "gpu-memory-ratio"

	GPUDriver string = ResourceDomainPrefix + "gpu-driver"
	GPUModel  string = ResourceDomainPrefix + "gpu-model"
)
View Source 
const (
	// AnnotationResourceSpec represents resource allocation API defined by Koordinator.
	// The user specifies the desired CPU orchestration policy by setting the annotation.
	AnnotationResourceSpec = SchedulingDomainPrefix + "/resource-spec"
	// AnnotationResourceStatus represents resource allocation result.
	// koord-scheduler patch Pod with the annotation before binding to node.
	AnnotationResourceStatus = SchedulingDomainPrefix + "/resource-status"

	// AnnotationExtendedResourceSpec specifies the resource requirements of extended resources for internal usage.
	// It annotates the requests/limits of extended resources and can be used by runtime proxy and koordlet that
	// cannot get the original pod spec in CRI requests.
	AnnotationExtendedResourceSpec = NodeDomainPrefix + "/extended-resource-spec"
)
View Source 
const (
	// LabelReservationOrder controls the preference logic for Reservation.
	// Reservation with lower order is preferred to be selected before Reservation with higher order.
	// But if it is 0, Reservation will be selected according to the capacity score.
	LabelReservationOrder = SchedulingDomainPrefix + "/reservation-order"

	// AnnotationReservationAllocated represents the reservation allocated by the pod.
	AnnotationReservationAllocated = SchedulingDomainPrefix + "/reservation-allocated"
)
View Source 
const (
	// AnnotationCustomUsageThresholds represents the user-defined resource utilization threshold.
	// For specific value definitions, see CustomUsageThresholds
	AnnotationCustomUsageThresholds = SchedulingDomainPrefix + "/usage-thresholds"

	// AnnotationDeviceAllocated represents the device allocated by the pod
	AnnotationDeviceAllocated = SchedulingDomainPrefix + "/device-allocated"
)
View Source 
const (
	AnnotationGangPrefix = "gang.scheduling.koordinator.sh"
	// AnnotationGangName specifies the name of the gang
	AnnotationGangName = AnnotationGangPrefix + "/name"

	// AnnotationGangMinNum specifies the minimum number of the gang that can be executed
	AnnotationGangMinNum = AnnotationGangPrefix + "/min-available"

	// AnnotationGangWaitTime specifies gang's max wait time in Permit Stage
	AnnotationGangWaitTime = AnnotationGangPrefix + "/waiting-time"

	// AnnotationGangTotalNum specifies the total children number of the gang
	// If not specified,it will be set with the AnnotationGangMinNum
	AnnotationGangTotalNum = AnnotationGangPrefix + "/total-number"

	// AnnotationGangMode defines the Gang Scheduling operation when failed scheduling
	// Support GangModeStrict and GangModeNonStrict, default is GangModeStrict
	AnnotationGangMode = AnnotationGangPrefix + "/mode"

	// AnnotationGangGroups defines which gangs are bundled as a group
	// The gang will go to bind only all gangs in one group meet the conditions
	AnnotationGangGroups = AnnotationGangPrefix + "/groups"

	// AnnotationGangTimeout means that the entire gang cannot be scheduled due to timeout
	// The annotation is added by the scheduler when the gang times out
	AnnotationGangTimeout = AnnotationGangPrefix + "/timeout"

	GangModeStrict    = "Strict"
	GangModeNonStrict = "NonStrict"
)

Variables

View Source 
var (
	PriorityProdValueMax int32 = 9999
	PriorityProdValueMin int32 = 9000

	PriorityMidValueMax int32 = 7999
	PriorityMidValueMin int32 = 7000

	PriorityBatchValueMax int32 = 5999
	PriorityBatchValueMin int32 = 5000

	PriorityFreeValueMax int32 = 3999
	PriorityFreeValueMin int32 = 3000
)

Define Koordinator priority as a variable value to support customizing different priority ranges

View Source 
var GetGangName = func(pod *corev1.Pod) string {
	return pod.Annotations[AnnotationGangName]
}
View Source 
var GetMinNum = func(pod *corev1.Pod) (int, error) {
	minRequiredNum, err := strconv.ParseInt(pod.Annotations[AnnotationGangMinNum], 10, 32)
	if err != nil {
		return 0, err
	}
	return int(minRequiredNum), nil
}
View Source 
var GetQuotaName = func(pod *corev1.Pod) string {
	return pod.Labels[LabelQuotaName]
}
View Source 
var (
	ResourceNameMap = map[PriorityClass]map[corev1.ResourceName]corev1.ResourceName{
		PriorityBatch: {
			corev1.ResourceCPU:    BatchCPU,
			corev1.ResourceMemory: BatchMemory,
		},
	}
)

Functions

func GetNodeCPUBindPolicy added in v1.1.0 

func GetNodeCPUBindPolicy(nodeLabels map[string]string, kubeletCPUPolicy *KubeletCPUManagerPolicy) string

func GetParentQuotaName 

func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string

func GetPodCPUBurstConfig 

func GetPodCPUBurstConfig(pod *corev1.Pod) (*slov1alpha1.CPUBurstConfig, error)

func GetPodMemoryQoSConfig 

func GetPodMemoryQoSConfig(pod *corev1.Pod) (*slov1alpha1.PodMemoryQOSConfig, error)

func GetPodSubPriority 

func GetPodSubPriority(labels map[string]string) (int32, error)

GetPodSubPriority get pod's sub-priority in Koordinator from label

func GetSharedWeight 

func GetSharedWeight(quota *v1alpha1.ElasticQuota) corev1.ResourceList

func IsAllowLentResource 

func IsAllowLentResource(quota *v1alpha1.ElasticQuota) bool

func IsForbiddenModify 

func IsForbiddenModify(quota *v1alpha1.ElasticQuota) (bool, error)

func IsParentQuota 

func IsParentQuota(quota *v1alpha1.ElasticQuota) bool

func RemoveReservationAllocated 

func RemoveReservationAllocated(pod *corev1.Pod, r *schedulingv1alpha1.Reservation) (bool, error)

func SetDeviceAllocations 

func SetDeviceAllocations(pod *corev1.Pod, allocations DeviceAllocations) error

func SetExtendedResourceSpec added in v1.1.0 

func SetExtendedResourceSpec(pod *corev1.Pod, spec *ExtendedResourceSpec) error

func SetReservationAllocated 

func SetReservationAllocated(pod *corev1.Pod, r *schedulingv1alpha1.Reservation)

func SetResourceStatus 

func SetResourceStatus(pod *corev1.Pod, status *ResourceStatus) error

func TranslateResourceNameByPriorityClass 

func TranslateResourceNameByPriorityClass(priorityClass PriorityClass, defaultResourceName corev1.ResourceName) corev1.ResourceName

TranslateResourceNameByPriorityClass translates defaultResourceName to extend resourceName by PriorityClass

Types

type CPUBindPolicy 

type CPUBindPolicy string

CPUBindPolicy defines the CPU binding policy 

const (
	// CPUBindPolicyDefault performs the default bind policy that specified in koord-scheduler configuration
	CPUBindPolicyDefault CPUBindPolicy = "Default"
	// CPUBindPolicyFullPCPUs favor cpuset allocation that pack in few physical cores
	CPUBindPolicyFullPCPUs CPUBindPolicy = "FullPCPUs"
	// CPUBindPolicySpreadByPCPUs favor cpuset allocation that evenly allocate logical cpus across physical cores
	CPUBindPolicySpreadByPCPUs CPUBindPolicy = "SpreadByPCPUs"
	// CPUBindPolicyConstrainedBurst constrains the CPU Shared Pool range of the Burstable Pod
	CPUBindPolicyConstrainedBurst CPUBindPolicy = "ConstrainedBurst"
)

type CPUExclusivePolicy 

type CPUExclusivePolicy string
const (
	// CPUExclusivePolicyNone does not perform any exclusive policy
	CPUExclusivePolicyNone CPUExclusivePolicy = "None"
	// CPUExclusivePolicyPCPULevel represents mutual exclusion in the physical core dimension
	CPUExclusivePolicyPCPULevel CPUExclusivePolicy = "PCPULevel"
	// CPUExclusivePolicyNUMANodeLevel indicates mutual exclusion in the NUMA topology dimension
	CPUExclusivePolicyNUMANodeLevel CPUExclusivePolicy = "NUMANodeLevel"
)

type CPUInfo 

type CPUInfo struct {
	ID     int32 `json:"id"`
	Core   int32 `json:"core"`
	Socket int32 `json:"socket"`
	Node   int32 `json:"node"`
}

type CPUSharedPool 

type CPUSharedPool struct {
	Socket int32  `json:"socket"`
	Node   int32  `json:"node"`
	CPUSet string `json:"cpuset,omitempty"`
}

func GetNodeCPUSharePools 

func GetNodeCPUSharePools(nodeTopoAnnotations map[string]string) ([]CPUSharedPool, error)

type CPUTopology 

type CPUTopology struct {
	Detail []CPUInfo `json:"detail,omitempty"`
}

func GetCPUTopology 

func GetCPUTopology(annotations map[string]string) (*CPUTopology, error)

type CustomAggregatedUsage added in v1.1.0 

type CustomAggregatedUsage struct {
	// UsageThresholds indicates the resource utilization threshold of the machine based on percentile statistics
	UsageThresholds map[corev1.ResourceName]int64 `json:"usageThresholds,omitempty"`
	// UsageAggregationType indicates the percentile type of the machine's utilization when filtering
	UsageAggregationType slov1alpha1.AggregationType `json:"usageAggregationType,omitempty"`
	// UsageAggregatedDuration indicates the statistical period of the percentile of the machine's utilization when filtering
	UsageAggregatedDuration *metav1.Duration `json:"usageAggregatedDuration,omitempty"`
}

type CustomUsageThresholds 

type CustomUsageThresholds struct {
	// UsageThresholds indicates the resource utilization threshold of the whole machine.
	UsageThresholds map[corev1.ResourceName]int64 `json:"usageThresholds,omitempty"`
	// ProdUsageThresholds indicates the resource utilization threshold of Prod Pods compared to the whole machine
	ProdUsageThresholds map[corev1.ResourceName]int64 `json:"prodUsageThresholds,omitempty"`
	// AggregatedUsage supports resource utilization filtering and scoring based on percentile statistics
	AggregatedUsage *CustomAggregatedUsage `json:"aggregatedUsage,omitempty"`
}

CustomUsageThresholds supports user-defined node resource utilization thresholds.

func GetCustomUsageThresholds 

func GetCustomUsageThresholds(node *corev1.Node) (*CustomUsageThresholds, error)

type DeviceAllocation 

type DeviceAllocation struct {
	Minor     int32               `json:"minor"`
	Resources corev1.ResourceList `json:"resources"`
}

type DeviceAllocations 

type DeviceAllocations map[schedulingv1alpha1.DeviceType][]*DeviceAllocation

DeviceAllocations would be injected into Pod as form of annotation during Pre-bind stage. 

{
  "gpu": [
    {
      "minor": 0,
      "resources": {
        "koordinator.sh/gpu-core": 100,
        "koordinator.sh/gpu-mem-ratio": 100,
        "koordinator.sh/gpu-mem": "16Gi"
      }
    },
    {
      "minor": 1,
      "resources": {
        "koordinator.sh/gpu-core": 100,
        "koordinator.sh/gpu-mem-ratio": 100,
        "koordinator.sh/gpu-mem": "16Gi"
      }
    }
  ]
}

func GetDeviceAllocations 

func GetDeviceAllocations(podAnnotations map[string]string) (DeviceAllocations, error)

type ExtendedResourceContainerSpec added in v1.1.0 

type ExtendedResourceContainerSpec struct {
	Limits   corev1.ResourceList `json:"limits,omitempty"`
	Requests corev1.ResourceList `json:"requests,omitempty"`
}

type ExtendedResourceSpec added in v1.1.0 

type ExtendedResourceSpec struct {
	Containers map[string]ExtendedResourceContainerSpec `json:"containers,omitempty"`
}

func GetExtendedResourceSpec added in v1.1.0 

func GetExtendedResourceSpec(annotations map[string]string) (*ExtendedResourceSpec, error)

GetExtendedResourceSpec parses ExtendedResourceSpec from annotations

type KubeletCPUManagerPolicy 

type KubeletCPUManagerPolicy struct {
	Policy       string            `json:"policy,omitempty"`
	Options      map[string]string `json:"options,omitempty"`
	ReservedCPUs string            `json:"reservedCPUs,omitempty"`
}

func GetKubeletCPUManagerPolicy 

func GetKubeletCPUManagerPolicy(annotations map[string]string) (*KubeletCPUManagerPolicy, error)

type NUMAAllocateStrategy 

type NUMAAllocateStrategy string

NUMAAllocateStrategy indicates how to choose satisfied NUMA Nodes 

const (
	// NUMAMostAllocated indicates that allocates from the NUMA Node with the least amount of available resource.
	NUMAMostAllocated NUMAAllocateStrategy = "MostAllocated"
	// NUMALeastAllocated indicates that allocates from the NUMA Node with the most amount of available resource.
	NUMALeastAllocated NUMAAllocateStrategy = "LeastAllocated"
	// NUMADistributeEvenly indicates that evenly distribute CPUs across NUMA Nodes.
	NUMADistributeEvenly NUMAAllocateStrategy = "DistributeEvenly"
)

type NUMACPUSharedPools 

type NUMACPUSharedPools []CPUSharedPool

type PodCPUAlloc 

type PodCPUAlloc struct {
	Namespace        string    `json:"namespace,omitempty"`
	Name             string    `json:"name,omitempty"`
	UID              types.UID `json:"uid,omitempty"`
	CPUSet           string    `json:"cpuset,omitempty"`
	ManagedByKubelet bool      `json:"managedByKubelet,omitempty"`
}

type PodCPUAllocs 

type PodCPUAllocs []PodCPUAlloc

func GetPodCPUAllocs 

func GetPodCPUAllocs(annotations map[string]string) (PodCPUAllocs, error)

type PriorityClass 

type PriorityClass string
const (
	PriorityProd  PriorityClass = "koord-prod"
	PriorityMid   PriorityClass = "koord-mid"
	PriorityBatch PriorityClass = "koord-batch"
	PriorityFree  PriorityClass = "koord-free"
	PriorityNone  PriorityClass = ""
)

func GetPriorityClass 

func GetPriorityClass(pod *corev1.Pod) PriorityClass

type QoSClass 

type QoSClass string
const (
	QoSLSE    QoSClass = "LSE"
	QoSLSR    QoSClass = "LSR"
	QoSLS     QoSClass = "LS"
	QoSBE     QoSClass = "BE"
	QoSSystem QoSClass = "SYSTEM"
	QoSNone   QoSClass = ""
)

func GetPodQoSClass 

func GetPodQoSClass(pod *corev1.Pod) QoSClass

func GetPodQoSClassByName 

func GetPodQoSClassByName(qos string) QoSClass

func GetQoSClassByAttrs 

func GetQoSClassByAttrs(labels, annotations map[string]string) QoSClass

type ReservationAllocated 

type ReservationAllocated struct {
	Name string    `json:"name,omitempty"`
	UID  types.UID `json:"uid,omitempty"`
}

func GetReservationAllocated 

func GetReservationAllocated(pod *corev1.Pod) (*ReservationAllocated, error)

type ResourceSpec 

type ResourceSpec struct {
	// PreferredCPUBindPolicy represents best-effort CPU bind policy.
	PreferredCPUBindPolicy CPUBindPolicy `json:"preferredCPUBindPolicy,omitempty"`
	// PreferredCPUExclusivePolicy represents best-effort CPU exclusive policy.
	PreferredCPUExclusivePolicy CPUExclusivePolicy `json:"preferredCPUExclusivePolicy,omitempty"`
}

ResourceSpec describes extra attributes of the resource requirements.

func GetResourceSpec 

func GetResourceSpec(annotations map[string]string) (*ResourceSpec, error)

GetResourceSpec parses ResourceSpec from annotations

type ResourceStatus 

type ResourceStatus struct {
	// CPUSet represents the allocated CPUs. It is Linux CPU list formatted string.
	// When LSE/LSR Pod requested, koord-scheduler will update the field.
	CPUSet string `json:"cpuset,omitempty"`
	// CPUSharedPools represents the desired CPU Shared Pools used by LS Pods.
	CPUSharedPools []CPUSharedPool `json:"cpuSharedPools,omitempty"`
}

ResourceStatus describes resource allocation result, such as how to bind CPU.

func GetResourceStatus 

func GetResourceStatus(annotations map[string]string) (*ResourceStatus, error)

GetResourceStatus parses ResourceStatus from annotations

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.