Enable preemption and borrowing based on flavor (#849)

* add preempt and borrow based on flavor * Add some details * Skip the flavors which has been considered in last schedule * update crd * fix test * add some tests * fix last scheduling context * add test for last scheduling context * add test cases * replace usage state data in lastAssignmentState with generation * add featuregate * update helm * enable feature gate in test * update comments and tests * update helm * rebase and update test * add unit test for generation * update test * fix test * modify according to commments * update AllocatableResourceGeneration logic * set last assignment to nil when assignment was skipped * add comments * add some tests * add in feature table * modify test * fix test * move FlavorResourceQuantities to cache remove Preemption setting in MakeClusterQueue
kubernetes-sigs · Oct 9, 2023 · fe55306 · fe55306
1 parent 1d74582
commit fe55306
Show file tree

Hide file tree

Showing 26 changed files with 1,529 additions and 172 deletions.
diff --git a/apis/kueue/v1beta1/clusterqueue_types.go b/apis/kueue/v1beta1/clusterqueue_types.go
@@ -72,6 +72,10 @@ type ClusterQueueSpec struct {
 	// If set to an empty selector `{}`, then all namespaces are eligible.
 	NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"`
 
+	// flavorFungibility defines whether a workload should try the next flavor
+	// before borrowing or preempting in the flavor being evaluated.
+	FlavorFungibility *FlavorFungibility `json:"flavorFungibility,omitempty"`
+
 	// preemption describes policies to preempt Workloads from this ClusterQueue
 	// or the ClusterQueue's cohort.
 	//
@@ -274,6 +278,40 @@ const (
 	PreemptionPolicyLowerOrNewerEqualPriority PreemptionPolicy = "LowerOrNewerEqualPriority"
 )
 
+type FlavorFungibilityPolicy string
+
+const (
+	Borrow        FlavorFungibilityPolicy = "Borrow"
+	Preempt       FlavorFungibilityPolicy = "Preempt"
+	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
+)
+
+// FlavorFungibility determines whether a workload should try the next flavor
+// before borrowing or preempting in current flavor.
+type FlavorFungibility struct {
+	// whenCanBorrow determines whether a workload should try the next flavor
+	// before borrowing in current flavor. The possible values are:
+	//
+	// - `Borrow` (default): allocate in current flavor if borrowing
+	//   is possible.
+	// - `TryNextFlavor`: try next flavor even if the current
+	//   flavor has enough resources to borrow.
+	//
+	// +kubebuilder:validation:Enum={Borrow,TryNextFlavor}
+	// +kubebuilder:default="Borrow"
+	WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"`
+	// whenCanPreempt determines whether a workload should try the next flavor
+	// before borrowing in current flavor. The possible values are:
+	//
+	// - `Preempt`: allocate in current flavor if it's possible to preempt some workloads.
+	// - `TryNextFlavor` (default): try next flavor even if there are enough
+	//   candidates for preemption in the current flavor.
+	//
+	// +kubebuilder:validation:Enum={Preempt,TryNextFlavor}
+	// +kubebuilder:default="TryNextFlavor"
+	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+}
+
 // ClusterQueuePreemption contains policies to preempt Workloads from this
 // ClusterQueue or the ClusterQueue's cohort.
 type ClusterQueuePreemption struct {

diff --git a/apis/kueue/v1beta1/zz_generated.deepcopy.go b/apis/kueue/v1beta1/zz_generated.deepcopy.go
diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
@@ -84,6 +84,35 @@ spec:
                   Validation of a cohort name is equivalent to that of object names:
                   subdomain in DNS (RFC 1123)."
                 type: string
+              flavorFungibility:
+                description: flavorFungibility defines whether a workload should try
+                  the next flavor before borrowing or preempting in the flavor being
+                  evaluated.
+                properties:
+                  whenCanBorrow:
+                    default: Borrow
+                    description: "whenCanBorrow determines whether a workload should
+                      try the next flavor before borrowing in current flavor. The
+                      possible values are: \n - `Borrow` (default): allocate in current
+                      flavor if borrowing is possible. - `TryNextFlavor`: try next
+                      flavor even if the current flavor has enough resources to borrow."
+                    enum:
+                    - Borrow
+                    - TryNextFlavor
+                    type: string
+                  whenCanPreempt:
+                    default: TryNextFlavor
+                    description: "whenCanPreempt determines whether a workload should
+                      try the next flavor before borrowing in current flavor. The
+                      possible values are: \n - `Preempt`: allocate in current flavor
+                      if it's possible to preempt some workloads. - `TryNextFlavor`
+                      (default): try next flavor even if there are enough candidates
+                      for preemption in the current flavor."
+                    enum:
+                    - Preempt
+                    - TryNextFlavor
+                    type: string
+                type: object
               namespaceSelector:
                 description: namespaceSelector defines which namespaces are allowed
                   to submit workloads to this clusterQueue. Beyond this basic support

diff --git a/client-go/applyconfiguration/kueue/v1beta1/clusterqueuespec.go b/client-go/applyconfiguration/kueue/v1beta1/clusterqueuespec.go
diff --git a/client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go b/client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go
diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go
diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
@@ -71,6 +71,35 @@ spec:
                   Validation of a cohort name is equivalent to that of object names:
                   subdomain in DNS (RFC 1123)."
                 type: string
+              flavorFungibility:
+                description: flavorFungibility defines whether a workload should try
+                  the next flavor before borrowing or preempting in the flavor being
+                  evaluated.
+                properties:
+                  whenCanBorrow:
+                    default: Borrow
+                    description: "whenCanBorrow determines whether a workload should
+                      try the next flavor before borrowing in current flavor. The
+                      possible values are: \n - `Borrow` (default): allocate in current
+                      flavor if borrowing is possible. - `TryNextFlavor`: try next
+                      flavor even if the current flavor has enough resources to borrow."
+                    enum:
+                    - Borrow
+                    - TryNextFlavor
+                    type: string
+                  whenCanPreempt:
+                    default: TryNextFlavor
+                    description: "whenCanPreempt determines whether a workload should
+                      try the next flavor before borrowing in current flavor. The
+                      possible values are: \n - `Preempt`: allocate in current flavor
+                      if it's possible to preempt some workloads. - `TryNextFlavor`
+                      (default): try next flavor even if there are enough candidates
+                      for preemption in the current flavor."
+                    enum:
+                    - Preempt
+                    - TryNextFlavor
+                    type: string
+                type: object
               namespaceSelector:
                 description: namespaceSelector defines which namespaces are allowed
                   to submit workloads to this clusterQueue. Beyond this basic support

diff --git a/keps/582-preempt-based-on-flavor-order/README.md b/keps/582-preempt-based-on-flavor-order/README.md
@@ -246,21 +246,21 @@ func assignFlavors(log logr.Logger, requests []workload.PodSetResources, podSets
 	var assignment Assignment
 	if lastAssignment != nil {
 		assignment = Assignment{
-			TotalBorrow: make(workload.FlavorResourceQuantities),
+			TotalBorrow: make(cache.FlavorResourceQuantities),
 			PodSets:     make([]PodSetAssignment, 0, len(requests)),
 			LastState:   *lastAssignment,
-			Usage:       make(workload.FlavorResourceQuantities),
+			Usage:       make(cache.FlavorResourceQuantities),
 		}
 	} else {
 		assignment = Assignment{
-			TotalBorrow: make(workload.FlavorResourceQuantities),
+			TotalBorrow: make(cache.FlavorResourceQuantities),
 			PodSets:     make([]PodSetAssignment, 0, len(requests)),
 			LastState: workload.AssigmentClusterQueueState{
-				LastAssignedFlavorIdx:  make([]map[corev1.ResourceName]int, 0),
+				LastAssignedFlavorIdx:  make([]map[corev1.ResourceName]int, 0, len(podSets)),
 				CohortGeneration:       0,
 				ClusterQueueGeneration: cq.Generation,
 			},
-			Usage: make(workload.FlavorResourceQuantities),
+			Usage: make(cache.FlavorResourceQuantities),
 		}
 		if cq.Cohort != nil {
 			assignment.LastState.CohortGeneration = cq.Cohort.Generation