From fd0ed8bd04f66595b1bd6d114680ac5ec5537f3e Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 26 Sep 2024 14:48:33 +0300 Subject: [PATCH] Expose Flavors in LocalQueue Status. --- apis/kueue/v1beta1/localqueue_types.go | 12 ++ apis/kueue/v1beta1/zz_generated.deepcopy.go | 20 +++ .../crd/kueue.x-k8s.io_localqueues.yaml | 18 +++ .../kueue/v1beta1/availableflavor.go | 42 ++++++ .../kueue/v1beta1/localqueuestatus.go | 14 ++ client-go/applyconfiguration/utils.go | 2 + .../crd/bases/kueue.x-k8s.io_localqueues.yaml | 18 +++ pkg/cache/cache.go | 12 ++ pkg/controller/core/localqueue_controller.go | 1 + .../en/docs/reference/kueue.v1beta1.md | 34 +++++ .../core/localqueue_controller_test.go | 121 +++++++++++++----- 11 files changed, 262 insertions(+), 32 deletions(-) create mode 100644 client-go/applyconfiguration/kueue/v1beta1/availableflavor.go diff --git a/apis/kueue/v1beta1/localqueue_types.go b/apis/kueue/v1beta1/localqueue_types.go index 622014953a..9c59605a82 100644 --- a/apis/kueue/v1beta1/localqueue_types.go +++ b/apis/kueue/v1beta1/localqueue_types.go @@ -48,6 +48,11 @@ type LocalQueueSpec struct { // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" type ClusterQueueReference string +type AvailableFlavor struct { + // name of the flavor. + Name ResourceFlavorReference `json:"name"` +} + // LocalQueueStatus defines the observed state of LocalQueue type LocalQueueStatus struct { // PendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue @@ -88,6 +93,13 @@ type LocalQueueStatus struct { // +kubebuilder:validation:MaxItems=16 // +optional FlavorUsage []LocalQueueFlavorUsage `json:"flavorUsage"` + + // availableFlavors lists all currently available ResourceFlavors + // in specified ClusterQueue. + // + // +listType=map + // +listMapKey=name + AvailableFlavors []AvailableFlavor `json:"availableFlavors,omitempty"` } const ( diff --git a/apis/kueue/v1beta1/zz_generated.deepcopy.go b/apis/kueue/v1beta1/zz_generated.deepcopy.go index cf773364d5..7aebcb7378 100644 --- a/apis/kueue/v1beta1/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta1/zz_generated.deepcopy.go @@ -234,6 +234,21 @@ func (in *AdmissionChecksStrategy) DeepCopy() *AdmissionChecksStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AvailableFlavor) DeepCopyInto(out *AvailableFlavor) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AvailableFlavor. +func (in *AvailableFlavor) DeepCopy() *AvailableFlavor { + if in == nil { + return nil + } + out := new(AvailableFlavor) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BorrowWithinCohort) DeepCopyInto(out *BorrowWithinCohort) { *out = *in @@ -707,6 +722,11 @@ func (in *LocalQueueStatus) DeepCopyInto(out *LocalQueueStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.AvailableFlavors != nil { + in, out := &in.AvailableFlavors, &out.AvailableFlavors + *out = make([]AvailableFlavor, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalQueueStatus. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index d57d1941c6..c38410ad9e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -106,6 +106,24 @@ spec: admitted to a ClusterQueue and that haven't finished yet. format: int32 type: integer + availableFlavors: + description: |- + availableFlavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map conditions: description: |- Conditions hold the latest available observations of the LocalQueue diff --git a/client-go/applyconfiguration/kueue/v1beta1/availableflavor.go b/client-go/applyconfiguration/kueue/v1beta1/availableflavor.go new file mode 100644 index 0000000000..1180191117 --- /dev/null +++ b/client-go/applyconfiguration/kueue/v1beta1/availableflavor.go @@ -0,0 +1,42 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" +) + +// AvailableFlavorApplyConfiguration represents a declarative configuration of the AvailableFlavor type for use +// with apply. +type AvailableFlavorApplyConfiguration struct { + Name *v1beta1.ResourceFlavorReference `json:"name,omitempty"` +} + +// AvailableFlavorApplyConfiguration constructs a declarative configuration of the AvailableFlavor type for use with +// apply. +func AvailableFlavor() *AvailableFlavorApplyConfiguration { + return &AvailableFlavorApplyConfiguration{} +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *AvailableFlavorApplyConfiguration) WithName(value v1beta1.ResourceFlavorReference) *AvailableFlavorApplyConfiguration { + b.Name = &value + return b +} diff --git a/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go b/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go index 38ca3bf58a..d2c1df8ab2 100644 --- a/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go +++ b/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go @@ -30,6 +30,7 @@ type LocalQueueStatusApplyConfiguration struct { Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` FlavorsReservation []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` FlavorUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorUsage,omitempty"` + AvailableFlavors []AvailableFlavorApplyConfiguration `json:"availableFlavors,omitempty"` } // LocalQueueStatusApplyConfiguration constructs a declarative configuration of the LocalQueueStatus type for use with @@ -100,3 +101,16 @@ func (b *LocalQueueStatusApplyConfiguration) WithFlavorUsage(values ...*LocalQue } return b } + +// WithAvailableFlavors adds the given value to the AvailableFlavors field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the AvailableFlavors field. +func (b *LocalQueueStatusApplyConfiguration) WithAvailableFlavors(values ...*AvailableFlavorApplyConfiguration) *LocalQueueStatusApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithAvailableFlavors") + } + b.AvailableFlavors = append(b.AvailableFlavors, *values[i]) + } + return b +} diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go index 89e2bd5ca5..4d3c4f1490 100644 --- a/client-go/applyconfiguration/utils.go +++ b/client-go/applyconfiguration/utils.go @@ -67,6 +67,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &kueuev1beta1.AdmissionCheckStatusApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("AdmissionCheckStrategyRule"): return &kueuev1beta1.AdmissionCheckStrategyRuleApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("AvailableFlavor"): + return &kueuev1beta1.AvailableFlavorApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("BorrowWithinCohort"): return &kueuev1beta1.BorrowWithinCohortApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("ClusterQueue"): diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index 768cf094a3..54f153d56d 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -91,6 +91,24 @@ spec: admitted to a ClusterQueue and that haven't finished yet. format: int32 type: integer + availableFlavors: + description: |- + availableFlavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map conditions: description: |- Conditions hold the latest available observations of the LocalQueue diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index b28e81b1cd..f8b9fdbf77 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -38,6 +38,7 @@ import ( "sigs.k8s.io/kueue/pkg/hierarchy" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/resources" + "sigs.k8s.io/kueue/pkg/util/maps" "sigs.k8s.io/kueue/pkg/workload" ) @@ -668,6 +669,7 @@ type LocalQueueUsageStats struct { ReservingWorkloads int AdmittedResources []kueue.LocalQueueFlavorUsage AdmittedWorkloads int + AvailableFlavors []kueue.AvailableFlavor } func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, error) { @@ -683,11 +685,21 @@ func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, return nil, errQNotFound } + availableFlavors := make(map[kueue.ResourceFlavorReference]kueue.AvailableFlavor) + for _, rg := range cqImpl.ResourceGroups { + for _, fl := range rg.Flavors { + availableFlavors[fl] = kueue.AvailableFlavor{ + Name: fl, + } + } + } + return &LocalQueueUsageStats{ ReservedResources: filterLocalQueueUsage(qImpl.usage, cqImpl.ResourceGroups), ReservingWorkloads: qImpl.reservingWorkloads, AdmittedResources: filterLocalQueueUsage(qImpl.admittedUsage, cqImpl.ResourceGroups), AdmittedWorkloads: qImpl.admittedWorkloads, + AvailableFlavors: maps.Values(availableFlavors), }, nil } diff --git a/pkg/controller/core/localqueue_controller.go b/pkg/controller/core/localqueue_controller.go index fada128c34..a0820765c9 100644 --- a/pkg/controller/core/localqueue_controller.go +++ b/pkg/controller/core/localqueue_controller.go @@ -328,6 +328,7 @@ func (r *LocalQueueReconciler) UpdateStatusIfChanged( queue.Status.AdmittedWorkloads = int32(stats.AdmittedWorkloads) queue.Status.FlavorsReservation = stats.ReservedResources queue.Status.FlavorUsage = stats.AdmittedResources + queue.Status.AvailableFlavors = stats.AvailableFlavors if len(conditionStatus) != 0 && len(reason) != 0 && len(msg) != 0 { meta.SetStatusCondition(&queue.Status.Conditions, metav1.Condition{ Type: kueue.LocalQueueActive, diff --git a/site/content/en/docs/reference/kueue.v1beta1.md b/site/content/en/docs/reference/kueue.v1beta1.md index 532cc77737..a08670d208 100644 --- a/site/content/en/docs/reference/kueue.v1beta1.md +++ b/site/content/en/docs/reference/kueue.v1beta1.md @@ -497,6 +497,30 @@ If empty, the AdmissionCheck will run for all workloads submitted to the Cluster +## `AvailableFlavor` {#kueue-x-k8s-io-v1beta1-AvailableFlavor} + + +**Appears in:** + +- [LocalQueueStatus](#kueue-x-k8s-io-v1beta1-LocalQueueStatus) + + + + + + + + + + + + +
FieldDescription
name [Required]
+ResourceFlavorReference +		 +

name of the flavor.

+
+ ## `BorrowWithinCohort` {#kueue-x-k8s-io-v1beta1-BorrowWithinCohort} @@ -1289,6 +1313,14 @@ workloads assigned to this LocalQueue.

workloads assigned to this LocalQueue.

+availableFlavors [Required]
+[]AvailableFlavor + + +

availableFlavors lists all currently available ResourceFlavors +in specified ClusterQueue.

+ + @@ -1613,6 +1645,8 @@ this time would be reset to null.

- [AdmissionCheckStrategyRule](#kueue-x-k8s-io-v1beta1-AdmissionCheckStrategyRule) +- [AvailableFlavor](#kueue-x-k8s-io-v1beta1-AvailableFlavor) + - [FlavorQuotas](#kueue-x-k8s-io-v1beta1-FlavorQuotas) - [FlavorUsage](#kueue-x-k8s-io-v1beta1-FlavorUsage) diff --git a/test/integration/controller/core/localqueue_controller_test.go b/test/integration/controller/core/localqueue_controller_test.go index 8b6e95790f..1659729790 100644 --- a/test/integration/controller/core/localqueue_controller_test.go +++ b/test/integration/controller/core/localqueue_controller_test.go @@ -91,31 +91,62 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }) ginkgo.It("Should update conditions when clusterQueues that its localQueue references are updated", func() { - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionFalse, - Reason: "ClusterQueueDoesNotExist", - Message: "Can't submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionFalse, + Reason: "ClusterQueueDoesNotExist", + Message: "Can't submit new workloads to clusterQueue", + }, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) + emptyUsage := []kueue.LocalQueueFlavorUsage{ + { + Name: flavorModelC, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + { + Name: flavorModelD, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + } + ginkgo.By("Creating a clusterQueue") gomega.Expect(k8sClient.Create(ctx, clusterQueue)).To(gomega.Succeed()) - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionFalse, - Reason: "ClusterQueueIsInactive", - Message: "Can't submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionFalse, + Reason: "ClusterQueueIsInactive", + Message: "Can't submit new workloads to clusterQueue", + }, + }, + FlavorsReservation: emptyUsage, + FlavorUsage: emptyUsage, + AvailableFlavors: []kueue.AvailableFlavor{ + {Name: flavorModelC}, + {Name: flavorModelD}, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) @@ -135,31 +166,41 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai Message: "Can admit new workloads", }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionTrue, - Reason: "Ready", - Message: "Can submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Can submit new workloads to clusterQueue", + }, + }, + FlavorsReservation: emptyUsage, + FlavorUsage: emptyUsage, + AvailableFlavors: []kueue.AvailableFlavor{ + {Name: flavorModelC}, + {Name: flavorModelD}, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Deleting a clusterQueue") gomega.Expect(k8sClient.Delete(ctx, clusterQueue)).To(gomega.Succeed()) - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionFalse, - Reason: "ClusterQueueDoesNotExist", - Message: "Can't submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionFalse, + Reason: "ClusterQueueDoesNotExist", + Message: "Can't submit new workloads to clusterQueue", + }, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }) @@ -239,6 +280,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorUsage: emptyUsage, + AvailableFlavors: []kueue.AvailableFlavor{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Setting the workloads quota reservation") @@ -289,6 +334,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorUsage: emptyUsage, + AvailableFlavors: []kueue.AvailableFlavor{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Setting the workloads admission checks") @@ -314,6 +363,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorUsage: fullUsage, + AvailableFlavors: []kueue.AvailableFlavor{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Finishing workloads") @@ -333,6 +386,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorUsage: emptyUsage, + AvailableFlavors: []kueue.AvailableFlavor{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }) })