Merge pull request #1299 from adrianchiris/cleanup-multus-conf

Cleanup multus conf on exit

cmd/thin_entrypoint/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/spf13/pflag"
 
 	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/cmdutils"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/signals"
 )
 
 // Options stores command line options
@@ -57,6 +58,7 @@ type Options struct {
 	AdditionalBinDir         string
 	ForceCNIVersion          bool
 	SkipTLSVerify            bool
+	SkipMultusConfWatch      bool
 }
 
 const (
@@ -84,7 +86,8 @@ func (o *Options) addFlags() {
 	fs.StringVar(&o.MultusLogLevel, "multus-log-level", "", "multus log level")
 	fs.StringVar(&o.MultusLogFile, "multus-log-file", "", "multus log file")
 	fs.BoolVar(&o.OverrideNetworkName, "override-network-name", false, "override network name from master cni file (used only with --multus-conf-file=auto)")
-	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit (used only with --multus-conf-file=auto)")
+	fs.BoolVar(&o.CleanupConfigOnExit, "cleanup-config-on-exit", false, "cleanup config file on exit")
+	fs.BoolVar(&o.SkipMultusConfWatch, "skip-config-watch", false, "dont watch for config (master cni and kubeconfig) changes (used only with --multus-conf-file=auto)")
 	fs.BoolVar(&o.RenameConfFile, "rename-conf-file", false, "rename master config file to invalidate (used only with --multus-conf-file=auto)")
 	fs.StringVar(&o.ReadinessIndicatorFile, "readiness-indicator-file", "", "readiness indicator file (used only with --multus-conf-file=auto)")
 	fs.StringVar(&o.AdditionalBinDir, "additional-bin-dir", "", "adds binDir option to configuration (used only with --multus-conf-file=auto)")
@@ -604,43 +607,72 @@ func main() {
 		fmt.Printf("multus config file is created.\n")
 	}
 
-	if opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" {
-		fmt.Printf("Entering watch loop...\n")
-		for {
-			// Check kubeconfig and update if different (i.e. service account updated)
-			caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
-				return
-			}
+	ctx := signals.SetupSignalHandler()
 
-			// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
-			_, err = os.Stat(masterConfigFilePath)
+	if opt.CleanupConfigOnExit {
+		defer cleanupMultusConf(&opt)
+	}
+
+	watchChanges := opt.CleanupConfigOnExit && opt.MultusConfFile == "auto" && !opt.SkipMultusConfWatch
+	if watchChanges {
+		fmt.Printf("Entering watch loop...\n")
+		masterConfigExists := true
+
+	outer:
+		for range time.Tick(1 * time.Second) {
+			select {
+			case <-ctx.Done():
+				// signal received break from loop
+				break outer
+			default:
+				// Check kubeconfig and update if different (i.e. service account updated)
+				caHash, saTokenHash, err = opt.createKubeConfig(caHash, saTokenHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to update multus kubeconfig: %v\n", err)
+					return
+				}
 
-			// if masterConfigFilePath is no longer exists
-			if os.IsNotExist(err) {
-				fmt.Printf("Master plugin @ %q has been deleted. Allowing 45 seconds for its restoration...\n", masterConfigFilePath)
-				time.Sleep(10 * time.Second)
+				// TODO: should we watch master CNI config (by fsnotify? https://github.com/fsnotify/fsnotify)
+				_, err = os.Stat(masterConfigFilePath)
 
-				for range time.Tick(1 * time.Second) {
-					_, err = os.Stat(masterConfigFilePath)
-					if !os.IsNotExist(err) {
-						fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
-						break
+				// if masterConfigFilePath is no longer exists
+				if os.IsNotExist(err) {
+					if masterConfigExists {
+						fmt.Printf("Master plugin @ %q has been deleted. waiting for its restoration...\n", masterConfigFilePath)
 					}
+					masterConfigExists = false
+					continue
+				}
+
+				if !masterConfigExists {
+					fmt.Printf("Master plugin @ %q was restored. Regenerating given configuration.\n", masterConfigFilePath)
+					masterConfigExists = true
+				}
+
+				masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
+					return
 				}
 			}
-			masterConfigFilePath, masterConfigHash, err = opt.createMultusConfig(masterConfigHash)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "failed to create multus config: %v\n", err)
-				return
-			}
-			time.Sleep(1 * time.Second)
 		}
 	} else {
-		// sleep infinitely
-		for {
-			time.Sleep(time.Duration(1<<63 - 1))
-		}
+		// wait until signal received
+		<-ctx.Done()
 	}
 }
+
+func cleanupMultusConf(opt *Options) {
+	// try remove multus conf
+	if opt.MultusConfFile == "auto" {
+		multusConfFilePath := fmt.Sprintf("%s/00-multus.conf", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+
+		multusConfFilePath = fmt.Sprintf("%s/00-multus.conflist", opt.CNIConfDir)
+		_ = os.Remove(multusConfFilePath)
+	} else {
+		confFileName := filepath.Base(opt.MultusConfFile)
+		_ = os.Remove(filepath.Join(opt.CNIConfDir, confFileName))
+	}
+
+}

docs/how-to-use.md

@@ -551,7 +551,7 @@ Typically, you'd modified the daemonset YAML itself to specify these parameters.
 For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
 
 ```
-  command: ["/entrypoint.sh"]
+  command: ["/thin_entrypoint"]
   args:
   - "--multus-conf-file=auto"
   - "--namespace-isolation=true"
@@ -590,7 +590,7 @@ The `--multus-conf-file` is one of two options; it can be set to a source file t
 
 The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
 
-In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will wait (600 seconds) until there's a CNI configuration file there, and it will take the alphabetically first configuration there, and it will wrap that configuration into a Multus configuration.
+In the case that `--multus-conf-file=auto` -- The entrypoint script will look at the `--multus-autoconfig-dir` (by default, the same as the `--cni-conf-dir`). Multus will take the alphabetically first configuration there and wrap that into a Multus configuration.
 
     --multus-autoconfig-dir=/host/etc/cni/net.d
 
@@ -619,6 +619,14 @@ In some cases, the original CNI configuration that the Multus configuration was
 
     --cleanup-config-on-exit=true
 
+When specifying `--cleanup-config-on-exit=true` the entrypoint script will delete any generated/copied Multus configuration files when entrypoint script
+exits (upon Pod termination). This allows Multus to be safely removed from the cluster when its no longer needed.
+
+In addition, when both `--cleanup-config-on-exit=true` and `--multus-conf-file=auto` are specified, the entrypoint script will watch for changes of the
+master CNI configuration and kubeconfig. when such change detected, the script will re-genereate Multus configuration. Watch can be skipped by setting:
+
+    --skip-config-watch
+
 Additionally when using CRIO, you may wish to have the CNI config file that's used as the source for `--multus-conf-file=auto` renamed. This boolean option when set to true automatically renames the file with a `.old` suffix to the original filename.
 
     --rename-conf-file=true

pkg/signals/signals.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2024 Multus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package signals provides handling for os signals.
+package signals
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+
+// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned
+// which is canceled on one of these signals. If a second signal is caught, the program
+// is terminated with exit code 1.
+func SetupSignalHandler() context.Context {
+	close(onlyOneSignalHandler) // panics when called twice
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-c
+		cancel()
+		<-c
+		os.Exit(1) // second signal. Exit directly.
+	}()
+
+	return ctx
+}