ProvisioningRequest admission check controller

Makefile

@@ -153,7 +153,7 @@ test: generate gotestsum ## Run tests.
 	$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out
 
 .PHONY: test-integration
-test-integration: manifests generate envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd kf-training-operator-crd ## Run tests.
+test-integration: manifests generate envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd kf-training-operator-crd  cluster-autoscaler-crd ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \
 	$(GINKGO) $(GINKGO_ARGS) --junit-report=junit.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
 
@@ -337,3 +337,10 @@ JOBSETROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" sigs.k8s.io/jobset)
 jobset-operator-crd:
 	mkdir -p $(PROJECT_DIR)/dep-crds/jobset-operator/
 	cp -f $(JOBSETROOT)/config/components/crd/bases/* $(PROJECT_DIR)/dep-crds/jobset-operator/
+
+
+CAROOT = $(shell $(GO_CMD) list -m -f "{{.Dir}}" k8s.io/autoscaler/cluster-autoscaler)
+.PHONY: cluster-autoscaler-crd
+cluster-autoscaler-crd:
+	mkdir -p $(PROJECT_DIR)/dep-crds/cluster-autoscaler/
+	cp -f $(CAROOT)/config/crd/* $(PROJECT_DIR)/dep-crds/cluster-autoscaler/

apis/kueue/v1beta1/provisioningrequestconfig_types.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ProvisioningRequestConfigSpec defines the desired state of ProvisioningRequestConfig
+type ProvisioningRequestConfigSpec struct {
+	// ProvisioningClassName describes the different modes of provisioning the resources.
+	// Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
+	// +kubebuilder:validation:MaxLength=253
+	ProvisioningClassName string `json:"provisioningClassName"`
+
+	// Parameters contains all other parameters classes may require.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxProperties=100
+	Parameters map[string]Parameter `json:"parameters,omitempty"`
+
+	// managedResources contains the list of resources managed by the autoscaling.
+	//
+	// If empty, all resources are considered managed.
+	//
+	// If not empty, the ProvisioningRequest will contain only the podsets that are
+	// requesting at least one of them.
+	//
+	// If none of the workloads podsets is requesting at least a managed resource,
+	// the workload is considered ready.
+	//
+	// +optional
+	// +listType=set
+	// +kubebuilder:validation:MaxItems=100
+	ManagedResources []corev1.ResourceName `json:"managedResources,omitempty"`
+}
+
+// Parameter is limited to 255 characters.
+// +kubebuilder:validation:MaxLength=255
+type Parameter string
+
+//+genclient
+//+kubebuilder:object:root=true
+//+kubebuilder:storageversion
+//+kubebuilder:resource:scope=Cluster
+
+// ProvisioningRequestConfig is the Schema for the provisioningrequestconfig API
+type ProvisioningRequestConfig struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ProvisioningRequestConfigSpec `json:"spec,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// ProvisioningRequestConfigList contains a list of ProvisioningRequestConfig
+type ProvisioningRequestConfigList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ProvisioningRequestConfig `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ProvisioningRequestConfig{}, &ProvisioningRequestConfigList{})
+}

charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml

@@ -0,0 +1,87 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    {{- if .Values.enableCertManager }}
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "kueue.fullname" . }}-serving-cert
+    {{- end }}
+    controller-gen.kubebuilder.io/version: v0.12.0
+  name: provisioningrequestconfigs.kueue.x-k8s.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: {{ include "kueue.fullname" . }}-webhook-service
+          namespace: '{{ .Release.Namespace }}'
+          path: /convert
+      conversionReviewVersions:
+      - v1
+  group: kueue.x-k8s.io
+  names:
+    kind: ProvisioningRequestConfig
+    listKind: ProvisioningRequestConfigList
+    plural: provisioningrequestconfigs
+    singular: provisioningrequestconfig
+  scope: Cluster
+  versions:
+  - name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: ProvisioningRequestConfig is the Schema for the provisioningrequestconfig
+          API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ProvisioningRequestConfigSpec defines the desired state of
+              ProvisioningRequestConfig
+            properties:
+              managedResources:
+                description: "managedResources contains the list of resources managed
+                  by the autoscaling. \n If empty, all resources are considered managed.
+                  \n If not empty, the ProvisioningRequest will contain only the podsets
+                  that are requesting at least one of them. \n If none of the workloads
+                  podsets is requesting at least a managed resource, the workload
+                  is considered ready."
+                items:
+                  description: ResourceName is the name identifying various resources
+                    in a ResourceList.
+                  type: string
+                maxItems: 100
+                type: array
+                x-kubernetes-list-type: set
+              parameters:
+                additionalProperties:
+                  description: Parameter is limited to 255 characters.
+                  maxLength: 255
+                  type: string
+                description: Parameters contains all other parameters classes may
+                  require.
+                maxProperties: 100
+                type: object
+              provisioningClassName:
+                description: ProvisioningClassName describes the different modes of
+                  provisioning the resources. Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName
+                  for details.
+                maxLength: 253
+                pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
+                type: string
+            required:
+            - provisioningClassName
+            type: object
+        type: object
+    served: true
+    storage: true

charts/kueue/templates/rbac/role.yaml

@@ -46,6 +46,17 @@ rules:
     verbs:
       - get
       - patch
+  - apiGroups:
+      - ""
+    resources:
+      - podtemplates
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - update
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -73,6 +84,24 @@ rules:
       - list
       - update
       - watch
+  - apiGroups:
+      - autoscaling.x-k8s.io
+    resources:
+      - provisioningrequests
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - autoscaling.x-k8s.io
+    resources:
+      - provisioningrequests/status
+    verbs:
+      - get
   - apiGroups:
       - batch
     resources:
@@ -295,6 +324,14 @@ rules:
       - get
       - patch
       - update
+  - apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - provisioningrequestconfigs
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - kueue.x-k8s.io
     resources: