fetch current container runtime config through the command line

Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com> add default runtime binary path to runtimes field of toolkit config toml Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com> [no-relnote] Get low-level runtimes consistently We ensure that we use the same low-level runtimes regardless of the runtime engine being configured. This ensures consistent behaviour. Signed-off-by: Evan Lezar <elezar@nvidia.com>
NVIDIA · Oct 9, 2024 · 83909ce · 83909ce
1 parent 4604e3b
commit 83909ce
Show file tree

Hide file tree

Showing 20 changed files with 600 additions and 37 deletions.
diff --git a/cmd/nvidia-ctk/runtime/configure/configure.go b/cmd/nvidia-ctk/runtime/configure/configure.go
@@ -44,6 +44,10 @@ const (
 	defaultContainerdConfigFilePath = "/etc/containerd/config.toml"
 	defaultCrioConfigFilePath       = "/etc/crio/crio.conf"
 	defaultDockerConfigFilePath     = "/etc/docker/daemon.json"
+
+	defaultConfigSource = configSourceFile
+	configSourceCommand = "command"
+	configSourceFile    = "file"
 )
 
 type command struct {
@@ -64,6 +68,7 @@ type config struct {
 	dryRun         bool
 	runtime        string
 	configFilePath string
+	configSource   string
 	mode           string
 	hookFilePath   string
 
@@ -120,6 +125,12 @@ func (m command) build() *cli.Command {
 			Usage:       "the config mode for runtimes that support multiple configuration mechanisms",
 			Destination: &config.mode,
 		},
+		&cli.StringFlag{
+			Name:        "config-source",
+			Usage:       "the source to retrieve the container runtime configuration; one of [command, file]\"",
+			Destination: &config.configSource,
+			Value:       defaultConfigSource,
+		},
 		&cli.StringFlag{
 			Name:        "oci-hook-path",
 			Usage:       "the path to the OCI runtime hook to create if --config-mode=oci-hook is specified. If no path is specified, the generated hook is output to STDOUT.\n\tNote: The use of OCI hooks is deprecated.",
@@ -202,6 +213,18 @@ func (m command) validateFlags(c *cli.Context, config *config) error {
 		config.runtimeConfigOverrideJSON = ""
 	}
 
+	switch config.configSource {
+	case configSourceCommand:
+		if config.runtime == "docker" {
+			m.logger.Warningf("A %v Config Source is not supported for %v; using %v", config.configSource, config.runtime, configSourceFile)
+			config.configSource = configSourceFile
+		}
+	case configSourceFile:
+		break
+	default:
+		return fmt.Errorf("unrecognized Config Source: %v", config.configSource)
+	}
+
 	return nil
 }
 
@@ -220,20 +243,25 @@ func (m command) configureWrapper(c *cli.Context, config *config) error {
 func (m command) configureConfigFile(c *cli.Context, config *config) error {
 	configFilePath := config.resolveConfigFilePath()
 
-	var cfg engine.Interface
 	var err error
+	configSource, err := config.resolveConfigSource()
+	if err != nil {
+		return err
+	}
+
+	var cfg engine.Interface
 	switch config.runtime {
 	case "containerd":
 		cfg, err = containerd.New(
 			containerd.WithLogger(m.logger),
 			containerd.WithPath(configFilePath),
-			containerd.WithConfigSource(toml.FromFile(configFilePath)),
+			containerd.WithConfigSource(configSource),
 		)
 	case "crio":
 		cfg, err = crio.New(
 			crio.WithLogger(m.logger),
 			crio.WithPath(configFilePath),
-			crio.WithConfigSource(toml.FromFile(configFilePath)),
+			crio.WithConfigSource(configSource),
 		)
 	case "docker":
 		cfg, err = docker.New(
@@ -295,6 +323,29 @@ func (c *config) resolveConfigFilePath() string {
 	return ""
 }
 
+// resolveConfigSource returns the default config source or the user provided config source
+func (c *config) resolveConfigSource() (toml.Loader, error) {
+	switch c.configSource {
+	case configSourceCommand:
+		return c.getCommandConfigSource(), nil
+	case configSourceFile:
+		return toml.FromFile(c.configFilePath), nil
+	default:
+		return nil, fmt.Errorf("unrecognized config source: %s", c.configSource)
+	}
+}
+
+// getConfigSourceCommand returns the default cli command to fetch the current runtime config
+func (c *config) getCommandConfigSource() toml.Loader {
+	switch c.runtime {
+	case "containerd":
+		return containerd.CommandLineSource("")
+	case "crio":
+		return crio.CommandLineSource("")
+	}
+	return toml.Empty
+}
+
 // getOuputConfigPath returns the configured config path or "" if dry-run is enabled
 func (c *config) getOuputConfigPath() string {
 	if c.dryRun {

diff --git a/internal/config/toml.go b/internal/config/toml.go
@@ -170,11 +170,22 @@ func (t *Toml) Get(key string) interface{} {
 	return (*toml.Tree)(t).Get(key)
 }
 
+// GetDefault returns the value for the specified key and falls back to the default value if the Get call fails
+func (t *Toml) GetDefault(key string, def interface{}) interface{} {
+	return (*toml.Tree)(t).GetDefault(key, def)
+}
+
 // Set sets the specified key to the specified value in the TOML config.
 func (t *Toml) Set(key string, value interface{}) {
 	(*toml.Tree)(t).Set(key, value)
 }
 
+// WriteTo encode the Tree as Toml and writes it to the writer w.
+// Returns the number of bytes written in case of success, or an error if anything happened.
+func (t *Toml) WriteTo(w io.Writer) (int64, error) {
+	return (*toml.Tree)(t).WriteTo(w)
+}
+
 // commentDefaults applies the required comments for default values to the Toml.
 func (t *Toml) commentDefaults() *Toml {
 	asToml := (*toml.Tree)(t)

diff --git a/pkg/config/engine/api.go b/pkg/config/engine/api.go
@@ -23,4 +23,10 @@ type Interface interface {
 	Set(string, interface{})
 	RemoveRuntime(string) error
 	Save(string) (int64, error)
+	GetRuntimeConfig(string) (RuntimeConfig, error)
+}
+
+// RuntimeConfig defines the interface to query container runtime handler configuration
+type RuntimeConfig interface {
+	GetBinaryPath() string
 }
diff --git a/pkg/config/engine/containerd/config_v1.go b/pkg/config/engine/containerd/config_v1.go
@@ -19,9 +19,8 @@ package containerd
 import (
 	"fmt"
 
-	"github.com/pelletier/go-toml"
-
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
 // ConfigV1 represents a version 1 containerd config
@@ -39,11 +38,7 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error
 
 	config.Set("version", int64(1))
 
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
 	for _, r := range runtimeNamesForConfig {
 		options := config.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", r})
 		if options == nil {
@@ -157,3 +152,14 @@ func (c *ConfigV1) Set(key string, value interface{}) {
 func (c ConfigV1) Save(path string) (int64, error) {
 	return (Config)(c).Save(path)
 }
+
+func (c *ConfigV1) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"plugins", "cri", "containerd", "runtimes", name})
+
+	return &containerdCfgRuntime{
+		tree: runtimeData,
+	}, nil
+}
diff --git a/pkg/config/engine/containerd/config_v2.go b/pkg/config/engine/containerd/config_v2.go
@@ -19,6 +19,7 @@ package containerd
 import (
 	"fmt"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/engine"
 	"github.com/NVIDIA/nvidia-container-toolkit/pkg/config/toml"
 )
 
@@ -31,11 +32,7 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
 
 	config.Set("version", int64(2))
 
-	// By default we extract the runtime options from the runc settings; if this does not exist we get the options from the default runtime specified in the config.
-	runtimeNamesForConfig := []string{"runc"}
-	if name, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok && name != "" {
-		runtimeNamesForConfig = append(runtimeNamesForConfig, name)
-	}
+	runtimeNamesForConfig := engine.GetLowLevelRuntimes(c)
 	for _, r := range runtimeNamesForConfig {
 		options := config.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", r})
 		if options == nil {

diff --git a/pkg/config/engine/containerd/config_v2_test.go b/pkg/config/engine/containerd/config_v2_test.go
@@ -216,3 +216,99 @@ func TestAddRuntime(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRuntimeConfig(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	config := `
+	version = 2
+	[plugins]
+	[plugins."io.containerd.grpc.v1.cri"]
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "nvidia"
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      ignore_blockio_not_enabled_errors = false
+      ignore_rdt_not_enabled_errors = false
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        base_runtime_spec = ""
+        cni_conf_dir = ""
+        cni_max_conf_num = 0
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        privileged_without_host_devices_all_devices_allowed = false
+        runtime_engine = ""
+        runtime_path = ""
+        runtime_root = ""
+        runtime_type = ""
+        sandbox_mode = ""
+        snapshotter = ""
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          base_runtime_spec = ""
+          cni_conf_dir = ""
+          cni_max_conf_num = 0
+          container_annotations = []
+          pod_annotations = []
+          privileged_without_host_devices = false
+          privileged_without_host_devices_all_devices_allowed = false
+          runtime_engine = ""
+          runtime_path = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+          sandbox_mode = "podsandbox"
+          snapshotter = ""
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = "/usr/bin/runc"
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
+            IoUid = 0
+            NoNewKeyring = false
+            NoPivotRoot = false
+            Root = ""
+            ShimCgroup = ""
+            SystemdCgroup = false
+`
+	testCases := []struct {
+		description   string
+		runtime       string
+		expected      string
+		expectedError error
+	}{
+		{
+			description:   "valid runtime config, existing runtime",
+			runtime:       "runc",
+			expected:      "/usr/bin/runc",
+			expectedError: nil,
+		},
+		{
+			description:   "valid runtime config, non-existing runtime",
+			runtime:       "some-other-runtime",
+			expected:      "",
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			cfg, err := toml.Load(config)
+			require.NoError(t, err)
+
+			c := &Config{
+				Logger: logger,
+				Tree:   cfg,
+			}
+			rc, err := c.GetRuntimeConfig(tc.runtime)
+			require.Equal(t, tc.expectedError, err)
+			require.Equal(t, tc.expected, rc.GetBinaryPath())
+		})
+	}
+}
diff --git a/pkg/config/engine/containerd/containerd.go b/pkg/config/engine/containerd/containerd.go
@@ -35,6 +35,22 @@ type Config struct {
 
 var _ engine.Interface = (*Config)(nil)
 
+type containerdCfgRuntime struct {
+	tree *toml.Tree
+}
+
+var _ engine.RuntimeConfig = (*containerdCfgRuntime)(nil)
+
+// GetBinaryPath retrieves the path to the actual low-level runtime binary invoked by the runtime handler
+func (c *containerdCfgRuntime) GetBinaryPath() string {
+	if c == nil || c.tree == nil {
+		return ""
+	}
+
+	binPath, _ := c.tree.GetPath([]string{"options", "BinaryName"}).(string)
+	return binPath
+}
+
 // New creates a containerd config with the specified options
 func New(opts ...Option) (engine.Interface, error) {
 	b := &builder{
@@ -98,3 +114,27 @@ func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
 		return -1, fmt.Errorf("unsupported type for version field: %v", v)
 	}
 }
+
+func (c *Config) GetRuntimeConfig(name string) (engine.RuntimeConfig, error) {
+	if c == nil || c.Tree == nil {
+		return nil, fmt.Errorf("config is nil")
+	}
+	runtimeData := c.GetSubtreeByPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
+	return &containerdCfgRuntime{
+		tree: runtimeData,
+	}, nil
+}
+
+// CommandLineSource returns the CLI-based containerd config loader
+func CommandLineSource(hostRoot string) toml.Loader {
+	commandLine := chrootIfRequired(hostRoot, "containerd", "config", "dump")
+	return toml.FromCommandLine(commandLine[0], commandLine[1:]...)
+}
+
+func chrootIfRequired(hostRoot string, commandLine ...string) []string {
+	if hostRoot == "" || hostRoot == "/" {
+		return commandLine
+	}
+
+	return append([]string{"chroot", hostRoot}, commandLine...)
+}