From c0426acd83c48d8f86a209bb6982a9d073f93cc1 Mon Sep 17 00:00:00 2001 From: Nikolay Martyanov Date: Mon, 16 Sep 2024 19:06:36 +0200 Subject: [PATCH] pillar: Add per-domain OVMF_VARS.fd handling for FML guests. Introduce support for managing per-domain OVMF_VARS.fd files, which are essential for maintaining persistent UEFI settings for FML guests. It adds functionality to prepare and clean up individual OVMF settings files stored in the persist directory, ensuring that each virtual machine has its own dedicated NVRAM file. The VM configuration structures are updated to reference the bootloader settings file, enabling the creation of unique UEFI variable stores for each domain. Signed-off-by: Nikolay Martyanov --- pkg/pillar/hypervisor/kvm.go | 92 +++++++++++++++++++++++++++--- pkg/pillar/types/domainmgrtypes.go | 19 +++--- pkg/pillar/types/locationconsts.go | 4 ++ 3 files changed, 98 insertions(+), 17 deletions(-) diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index 6b5332ca6de..8ab71fe3ba7 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -20,6 +20,7 @@ import ( "github.com/lf-edge/eve/pkg/pillar/pubsub" "github.com/lf-edge/eve/pkg/pillar/types" "github.com/lf-edge/eve/pkg/pillar/utils" + fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file" uuid "github.com/satori/go.uuid" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -85,7 +86,9 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr kernel-irqchip = "on" {{- end -}} {{- if .DomainConfig.BootLoader }} + {{- if ne .VirtualizationMode "FML" }} firmware = "{{.DomainConfig.BootLoader}}" + {{- end }} {{- end -}} {{- if .DomainConfig.Kernel }} kernel = "{{.DomainConfig.Kernel}}" @@ -123,6 +126,21 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr driver = "intel-iommu" caching-mode = "on" {{ end }} + +{{if eq .VirtualizationMode "FML"}} +# UEFI boot using OVMF +[drive "drive-ovmf-code"] + if = "pflash" + format = "raw" + readonly = "on" + file = "{{.DomainConfig.BootLoader}}" + +[drive "drive-ovmf-vars"] + if = "pflash" + format = "raw" + file = "{{.DomainConfig.BootLoaderSettingsFile}}" + +{{end}} [realtime] mlock = "off" @@ -455,12 +473,13 @@ const vfioDriverPath = "/sys/bus/pci/drivers/vfio-pci" type KvmContext struct { ctrdContext // for now the following is statically configured and can not be changed per domain - devicemodel string - dmExec string - dmArgs []string - dmCPUArgs []string - dmFmlCPUArgs []string - capabilities *types.Capabilities + devicemodel string + virtualizationMode string + dmExec string + dmArgs []string + dmCPUArgs []string + dmFmlCPUArgs []string + capabilities *types.Capabilities } func newKvm() Hypervisor { @@ -707,6 +726,44 @@ func vmmOverhead(domainName string, domainUUID uuid.UUID, domainRAMSize int64, v return overhead, nil } +func getOVMFSettingsFile(domainName string) string { + return types.OVMFSettingsDir + "/" + domainName + "_OVMF_VARS.fd" +} + +func prepareOVMFSettings(domainName string) (string, error) { + logrus.Infof("@ohm: I Preparing OVMF settings for domain %s", domainName) + + // Create the OVMF settings directory if it does not exist + if _, err := os.Stat(types.OVMFSettingsDir); os.IsNotExist(err) { + if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil { + return "", logError("failed to create OVMF settings directory: %v", err) + } + } + // Create a copy of the OVMF_VARS.fd file in _OVMF_VARS.fd + ovmfVarsFile := getOVMFSettingsFile(domainName) + if _, err := os.Stat(ovmfVarsFile); os.IsNotExist(err) { + // Copy the OVMF_VARS.fd file, using statndard copy function + if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfVarsFile); err != nil { + return "", logError("failed to copy OVMF_VARS.fd file: %v", err) + } + } + // Set the RW permissions for the OVMF_VARS.fd file + if err := os.Chmod(ovmfVarsFile, 0666); err != nil { + return "", logError("failed to set RW permissions for OVMF_VARS.fd file: %v", err) + } + return ovmfVarsFile, nil +} + +func cleanupOVMFSettings(domainName string) error { + ovmfVarsFile := getOVMFSettingsFile(domainName) + if _, err := os.Stat(ovmfVarsFile); err == nil { + if err := os.Remove(ovmfVarsFile); err != nil { + return logError("failed to remove OVMF_VARS.fd file: %v", err) + } + } + return nil +} + // Setup sets up kvm func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig, aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error { @@ -721,6 +778,16 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig swtpmCtrlSock = fmt.Sprintf(types.SwtpmCtrlSocketPath, domainName) } + // Before we start building the domain config, we need to prepare the OVMF settings + if config.VirtualizationMode == types.FML { + ctx.virtualizationMode = "FML" + ovmfSettingsFile, err := prepareOVMFSettings(domainName) + if err != nil { + return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err) + } + config.BootLoaderSettingsFile = ovmfSettingsFile + } + // first lets build the domain config if err := ctx.CreateDomConfig(domainName, config, status, diskStatusList, aa, globalConfig, swtpmCtrlSock, file); err != nil { @@ -794,10 +861,11 @@ func (ctx KvmContext) CreateDomConfig(domainName string, diskStatusList []types.DiskStatus, aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, swtpmCtrlSock string, file *os.File) error { tmplCtx := struct { - Machine string + Machine string + VirtualizationMode string types.DomainConfig types.DomainStatus - }{ctx.devicemodel, config, status} + }{ctx.devicemodel, ctx.virtualizationMode, config, status} tmplCtx.DomainConfig.Memory = (config.Memory + 1023) / 1024 tmplCtx.DomainConfig.EnableVncShimVM = isVncShimVMEnabled(globalConfig, config) @@ -1150,6 +1218,14 @@ func (ctx KvmContext) Cleanup(domainName string) error { return fmt.Errorf("error waiting for Qmp absent for domain %s: %v", domainName, err) } + // Cleanup OVMF settings + // XXX it should be a check for FML mode based on some config/status option. But we have + // only domain name here. So we check if the OVMF settings file exists. + if _, err := os.Stat(getOVMFSettingsFile(domainName)); err == nil { + if err := cleanupOVMFSettings(domainName); err != nil { + return fmt.Errorf("failed to cleanup OVMF settings for domain %s: %v", domainName, err) + } + } return nil } diff --git a/pkg/pillar/types/domainmgrtypes.go b/pkg/pillar/types/domainmgrtypes.go index 07c18354b4f..e974794f3a3 100644 --- a/pkg/pillar/types/domainmgrtypes.go +++ b/pkg/pillar/types/domainmgrtypes.go @@ -232,15 +232,16 @@ func (config DomainConfig) LogKey() string { // so-called "fixed resources", which means that the virtual machine // must be restarted before changes to the field will take effect. type VmConfig struct { - Kernel string // default "" - Ramdisk string // default "" - Memory int // in kbytes; Rounded up to Mbytes for xen - MaxMem int // in kbytes; Default equal to 'Memory', so no ballooning for xen - VCpus int // default 1 - MaxCpus int // default VCpus - RootDev string // default "/dev/xvda1" - ExtraArgs string // added to bootargs - BootLoader string // default "" + Kernel string // default "" + Ramdisk string // default "" + Memory int // in kbytes; Rounded up to Mbytes for xen + MaxMem int // in kbytes; Default equal to 'Memory', so no ballooning for xen + VCpus int // default 1 + MaxCpus int // default VCpus + RootDev string // default "/dev/xvda1" + ExtraArgs string // added to bootargs + BootLoader string // default "" + BootLoaderSettingsFile string // used to pass bootloader settings file, for example OVMF_VARS.fd // For CPU pinning CPUs string // default "", list of "1,2" // Needed for device passthru diff --git a/pkg/pillar/types/locationconsts.go b/pkg/pillar/types/locationconsts.go index 351711d830b..8aae9af005c 100644 --- a/pkg/pillar/types/locationconsts.go +++ b/pkg/pillar/types/locationconsts.go @@ -132,6 +132,10 @@ const ( MemoryMonitorOutputDir = MemoryMonitorDir + "/output" // MemoryMonitorPSIStatsFile - file to store memory PSI (Pressure Stall Information) statistics MemoryMonitorPSIStatsFile = MemoryMonitorOutputDir + "/psi.txt" + + // OVMFSettingsDir - directory for OVMF settings, they are stored in per-domain files + OVMFSettingsDir = PersistDir + "/ovmf" + OVMFSettingsTemplate = "/usr/lib/xen/boot/ovmf_vars.bin" ) var (