diff --git a/PROJECT b/PROJECT index fe90a666bc..bab428bcbd 100644 --- a/PROJECT +++ b/PROJECT @@ -38,4 +38,11 @@ resources: kind: ResourceFlavor path: sigs.k8s.io/kueue/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + domain: x-k8s.io + group: config + kind: KueueConfiguration + path: sigs.k8s.io/kueue/apis/config/v1alpha1 + version: v1alpha1 version: "3" diff --git a/apis/config/v1alpha1/configuration_types.go b/apis/config/v1alpha1/configuration_types.go new file mode 100644 index 0000000000..9db341e11e --- /dev/null +++ b/apis/config/v1alpha1/configuration_types.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" +) + +//+kubebuilder:object:root=true + +// Configuration is the Schema for the kueueconfigurations API +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // ControllerManagerConfigurationSpec returns the configurations for controllers + cfg.ControllerManagerConfigurationSpec `json:",inline"` + + // ManageJobsWithoutQueueName controls whether or not Kueue reconciles + // batch/v1.Jobs that don't set the annotation kueue.x-k8s.io/queue-name. + // If set to true, then those jobs will be suspended and never started unless + // they are assigned a queue and eventually admitted. This also applies to + // jobs created before starting the kueue controller. + // Defaults to false; therefore, those jobs are not managed and if they are created + // unsuspended, they will start immediately. + ManageJobsWithoutQueueName bool `json:"manageJobsWithoutQueueName"` +} + +func init() { + SchemeBuilder.Register(&Configuration{}) +} diff --git a/apis/config/v1alpha1/groupversion_info.go b/apis/config/v1alpha1/groupversion_info.go new file mode 100644 index 0000000000..37cc6cf825 --- /dev/null +++ b/apis/config/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the config v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=config.x-k8s.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "config.kueue.x-k8s.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/config/v1alpha1/zz_generated.deepcopy.go b/apis/config/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..6c3786ea24 --- /dev/null +++ b/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,51 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 5f4a332df5..9210442986 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -32,7 +32,7 @@ patchesStrategicMerge: # Mount the controller config file for loading manager configurations # through a ComponentConfig type -#- manager_config_patch.yaml +- manager_config_patch.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 15bb913b0f..ca55a81174 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -22,7 +22,4 @@ spec: name: https - name: manager args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" - "--zap-log-level=2" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index 6c400155cf..7937e6666f 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -10,6 +10,7 @@ spec: - name: manager args: - "--config=controller_manager_config.yaml" + - "--zap-log-level=2" volumeMounts: - name: manager-config mountPath: /controller_manager_config.yaml diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index c3aa23da1b..9a9df85b7d 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -1,11 +1,12 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfig +apiVersion: config.kueue.x-k8s.io/v1alpha1 +kind: Configuration health: healthProbeBindAddress: :8081 metrics: - bindAddress: 127.0.0.1:8080 + bindAddress: :8080 webhook: port: 9443 leaderElection: leaderElect: true resourceName: c1f6bfd2.kueue.x-k8s.io +#manageJobsWithoutQueueName: true diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 9ca1150794..c166fcdcf9 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -8,6 +8,7 @@ configMapGenerator: - files: - controller_manager_config.yaml name: manager-config + apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: diff --git a/main.go b/main.go index 4d23280aa7..508dfae21d 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,9 @@ limitations under the License. package main import ( + "bytes" "flag" + "fmt" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -26,12 +28,14 @@ import ( schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + configv1alpha1 "sigs.k8s.io/kueue/apis/config/v1alpha1" kueuev1alpha1 "sigs.k8s.io/kueue/apis/core/v1alpha1" "sigs.k8s.io/kueue/pkg/cache" "sigs.k8s.io/kueue/pkg/constants" @@ -52,32 +56,46 @@ func init() { utilruntime.Must(schedulingv1.AddToScheme(scheme)) utilruntime.Must(kueuev1alpha1.AddToScheme(scheme)) + utilruntime.Must(configv1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") + var configFile string + flag.StringVar(&configFile, "config", "", + "The controller will load its initial configuration from this file. "+ + "Omit this flag to use the default configuration values. ") + opts := zap.Options{} opts.BindFlags(flag.CommandLine) flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + options := ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, + HealthProbeBindAddress: ":8081", + MetricsBindAddress: ":8080", Port: 9443, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, LeaderElectionID: "c1f6bfd2.kueue.x-k8s.io", - }) + } + var err error + config := configv1alpha1.Configuration{} + if configFile != "" { + options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&config)) + if err != nil { + setupLog.Error(err, "unable to load the config file") + os.Exit(1) + } + cfgStr, err := encodeConfig(&config) + if err != nil { + setupLog.Error(err, "unable to encode config file") + os.Exit(1) + } + setupLog.Info("Successfully loaded config file", "config", cfgStr) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) @@ -94,8 +112,11 @@ func main() { if failedCtrl, err := core.SetupControllers(mgr, queues, cCache); err != nil { setupLog.Error(err, "Unable to create controller", "controller", failedCtrl) } - if err = job.NewReconciler(mgr.GetScheme(), mgr.GetClient(), - mgr.GetEventRecorderFor(constants.JobControllerName)).SetupWithManager(mgr); err != nil { + if err = job.NewReconciler(mgr.GetScheme(), + mgr.GetClient(), + mgr.GetEventRecorderFor(constants.JobControllerName), + job.WithManageJobsWithoutQueueName(config.ManageJobsWithoutQueueName), + ).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Job") os.Exit(1) } @@ -125,3 +146,19 @@ func main() { os.Exit(1) } } + +func encodeConfig(cfg *configv1alpha1.Configuration) (string, error) { + codecs := serializer.NewCodecFactory(scheme) + const mediaType = runtime.ContentTypeYAML + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) + if !ok { + return "", fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) + } + + encoder := codecs.EncoderForVersion(info.Serializer, configv1alpha1.GroupVersion) + buf := new(bytes.Buffer) + if err := encoder.Encode(cfg, buf); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/pkg/controller/workload/job/job_controller.go b/pkg/controller/workload/job/job_controller.go index 786a3da4f3..9798a4b2ba 100644 --- a/pkg/controller/workload/job/job_controller.go +++ b/pkg/controller/workload/job/job_controller.go @@ -46,24 +46,24 @@ var ( // JobReconciler reconciles a Job object type JobReconciler struct { - client client.Client - scheme *runtime.Scheme - record record.EventRecorder - processJobsWithoutQueueName bool + client client.Client + scheme *runtime.Scheme + record record.EventRecorder + manageJobsWithoutQueueName bool } type options struct { - processJobsWithoutQueueName bool + manageJobsWithoutQueueName bool } // Option configures the reconciler. type Option func(*options) -// WithProcessJobsWithoutQueueName indicates if the controller should reconcile +// WithManageJobsWithoutQueueName indicates if the controller should reconcile // jobs that don't set the queue name annotation. -func WithProcessJobsWithoutQueueName(f bool) Option { +func WithManageJobsWithoutQueueName(f bool) Option { return func(o *options) { - o.processJobsWithoutQueueName = f + o.manageJobsWithoutQueueName = f } } @@ -81,10 +81,10 @@ func NewReconciler( } return &JobReconciler{ - scheme: scheme, - client: client, - record: record, - processJobsWithoutQueueName: options.processJobsWithoutQueueName, + scheme: scheme, + client: client, + record: record, + manageJobsWithoutQueueName: options.manageJobsWithoutQueueName, } } @@ -133,7 +133,7 @@ func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R log := ctrl.LoggerFrom(ctx).WithValues("job", klog.KObj(&job)) ctx = ctrl.LoggerInto(ctx, log) - if queueName(&job) == "" && !r.processJobsWithoutQueueName { + if queueName(&job) == "" && !r.manageJobsWithoutQueueName { log.V(3).Info(fmt.Sprintf("%s annotation is not set, ignoring the job", constants.QueueAnnotation)) return ctrl.Result{}, nil } diff --git a/test/integration/controller/job/job_controller_test.go b/test/integration/controller/job/job_controller_test.go index c1ada5c202..ca16e586d8 100644 --- a/test/integration/controller/job/job_controller_test.go +++ b/test/integration/controller/job/job_controller_test.go @@ -64,7 +64,7 @@ var ( var _ = ginkgo.Describe("Job controller", func() { ginkgo.BeforeEach(func() { fwk = &framework.Framework{ - ManagerSetup: managerSetup(job.WithProcessJobsWithoutQueueName(true)), + ManagerSetup: managerSetup(job.WithManageJobsWithoutQueueName(true)), CRDPath: crdPath, } ctx, cfg, k8sClient = fwk.Setup()