pillar: Add per-domain OVMF_VARS.fd handling for FML guests.

Introduce support for managing per-domain OVMF_VARS.fd files, which are
essential for maintaining persistent UEFI settings for FML guests. It
adds functionality to prepare and clean up individual OVMF settings
files stored in the persist directory, ensuring that each virtual
machine has its own dedicated NVRAM file. The VM configuration
structures are updated to reference the bootloader settings file,
enabling the creation of unique UEFI variable stores for each domain.

Signed-off-by: Nikolay Martyanov <nikolay@zededa.com>

pkg/pillar/hypervisor/kvm.go

@@ -20,6 +20,7 @@ import (
 	"github.com/lf-edge/eve/pkg/pillar/pubsub"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/utils"
+	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
 	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -85,7 +86,9 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
   kernel-irqchip = "on"
 {{- end -}}
 {{- if .DomainConfig.BootLoader }}
+  {{- if ne .VirtualizationMode "FML" }}
   firmware = "{{.DomainConfig.BootLoader}}"
+  {{- end }}
 {{- end -}}
 {{- if .DomainConfig.Kernel }}
   kernel = "{{.DomainConfig.Kernel}}"
@@ -123,6 +126,21 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
   driver = "intel-iommu"
   caching-mode = "on"
 {{ end }}
+
+{{if eq .VirtualizationMode "FML"}}
+# UEFI boot using OVMF
+[drive "drive-ovmf-code"]
+  if = "pflash"
+  format = "raw"
+  readonly = "on"
+  file = "{{.DomainConfig.BootLoader}}"
+
+[drive "drive-ovmf-vars"]
+  if = "pflash"
+  format = "raw"
+  file = "{{.DomainConfig.BootLoaderSettingsFile}}"
+
+{{end}}
 [realtime]
   mlock = "off"
 
@@ -455,12 +473,13 @@ const vfioDriverPath = "/sys/bus/pci/drivers/vfio-pci"
 type KvmContext struct {
 	ctrdContext
 	// for now the following is statically configured and can not be changed per domain
-	devicemodel  string
-	dmExec       string
-	dmArgs       []string
-	dmCPUArgs    []string
-	dmFmlCPUArgs []string
-	capabilities *types.Capabilities
+	devicemodel        string
+	virtualizationMode string
+	dmExec             string
+	dmArgs             []string
+	dmCPUArgs          []string
+	dmFmlCPUArgs       []string
+	capabilities       *types.Capabilities
 }
 
 func newKvm() Hypervisor {
@@ -707,6 +726,44 @@ func vmmOverhead(domainName string, domainUUID uuid.UUID, domainRAMSize int64, v
 	return overhead, nil
 }
 
+func getOVMFSettingsFile(domainName string) string {
+	return types.OVMFSettingsDir + "/" + domainName + "_OVMF_VARS.fd"
+}
+
+func prepareOVMFSettings(domainName string) (string, error) {
+	logrus.Infof("@ohm: I Preparing OVMF settings for domain %s", domainName)
+
+	// Create the OVMF settings directory if it does not exist
+	if _, err := os.Stat(types.OVMFSettingsDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil {
+			return "", logError("failed to create OVMF settings directory: %v", err)
+		}
+	}
+	// Create a copy of the OVMF_VARS.fd file in <UUID>_OVMF_VARS.fd
+	ovmfVarsFile := getOVMFSettingsFile(domainName)
+	if _, err := os.Stat(ovmfVarsFile); os.IsNotExist(err) {
+		// Copy the OVMF_VARS.fd file, using statndard copy function
+		if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfVarsFile); err != nil {
+			return "", logError("failed to copy OVMF_VARS.fd file: %v", err)
+		}
+	}
+	// Set the RW permissions for the OVMF_VARS.fd file
+	if err := os.Chmod(ovmfVarsFile, 0666); err != nil {
+		return "", logError("failed to set RW permissions for OVMF_VARS.fd file: %v", err)
+	}
+	return ovmfVarsFile, nil
+}
+
+func cleanupOVMFSettings(domainName string) error {
+	ovmfVarsFile := getOVMFSettingsFile(domainName)
+	if _, err := os.Stat(ovmfVarsFile); err == nil {
+		if err := os.Remove(ovmfVarsFile); err != nil {
+			return logError("failed to remove OVMF_VARS.fd file: %v", err)
+		}
+	}
+	return nil
+}
+
 // Setup sets up kvm
 func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig,
 	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
@@ -721,6 +778,16 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig
 		swtpmCtrlSock = fmt.Sprintf(types.SwtpmCtrlSocketPath, domainName)
 	}
 
+	// Before we start building the domain config, we need to prepare the OVMF settings
+	if config.VirtualizationMode == types.FML {
+		ctx.virtualizationMode = "FML"
+		ovmfSettingsFile, err := prepareOVMFSettings(domainName)
+		if err != nil {
+			return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err)
+		}
+		config.BootLoaderSettingsFile = ovmfSettingsFile
+	}
+
 	// first lets build the domain config
 	if err := ctx.CreateDomConfig(domainName, config, status, diskStatusList,
 		aa, globalConfig, swtpmCtrlSock, file); err != nil {
@@ -794,10 +861,11 @@ func (ctx KvmContext) CreateDomConfig(domainName string,
 	diskStatusList []types.DiskStatus, aa *types.AssignableAdapters,
 	globalConfig *types.ConfigItemValueMap, swtpmCtrlSock string, file *os.File) error {
 	tmplCtx := struct {
-		Machine string
+		Machine            string
+		VirtualizationMode string
 		types.DomainConfig
 		types.DomainStatus
-	}{ctx.devicemodel, config, status}
+	}{ctx.devicemodel, ctx.virtualizationMode, config, status}
 	tmplCtx.DomainConfig.Memory = (config.Memory + 1023) / 1024
 	tmplCtx.DomainConfig.EnableVncShimVM =
 		isVncShimVMEnabled(globalConfig, config)
@@ -1150,6 +1218,14 @@ func (ctx KvmContext) Cleanup(domainName string) error {
 		return fmt.Errorf("error waiting for Qmp absent for domain %s: %v", domainName, err)
 	}
 
+	// Cleanup OVMF settings
+	// XXX it should be a check for FML mode based on some config/status option. But we have
+	// only domain name here. So we check if the OVMF settings file exists.
+	if _, err := os.Stat(getOVMFSettingsFile(domainName)); err == nil {
+		if err := cleanupOVMFSettings(domainName); err != nil {
+			return fmt.Errorf("failed to cleanup OVMF settings for domain %s: %v", domainName, err)
+		}
+	}
 	return nil
 }
 

pkg/pillar/types/domainmgrtypes.go

@@ -232,15 +232,16 @@ func (config DomainConfig) LogKey() string {
 // so-called "fixed resources", which means that the virtual machine
 // must be restarted before changes to the field will take effect.
 type VmConfig struct {
-	Kernel     string // default ""
-	Ramdisk    string // default ""
-	Memory     int    // in kbytes; Rounded up to Mbytes for xen
-	MaxMem     int    // in kbytes; Default equal to 'Memory', so no ballooning for xen
-	VCpus      int    // default 1
-	MaxCpus    int    // default VCpus
-	RootDev    string // default "/dev/xvda1"
-	ExtraArgs  string // added to bootargs
-	BootLoader string // default ""
+	Kernel                 string // default ""
+	Ramdisk                string // default ""
+	Memory                 int    // in kbytes; Rounded up to Mbytes for xen
+	MaxMem                 int    // in kbytes; Default equal to 'Memory', so no ballooning for xen
+	VCpus                  int    // default 1
+	MaxCpus                int    // default VCpus
+	RootDev                string // default "/dev/xvda1"
+	ExtraArgs              string // added to bootargs
+	BootLoader             string // default ""
+	BootLoaderSettingsFile string // used to pass bootloader settings file, for example OVMF_VARS.fd
 	// For CPU pinning
 	CPUs string // default "", list of "1,2"
 	// Needed for device passthru

pkg/pillar/types/locationconsts.go

@@ -132,6 +132,10 @@ const (
 	MemoryMonitorOutputDir = MemoryMonitorDir + "/output"
 	// MemoryMonitorPSIStatsFile - file to store memory PSI (Pressure Stall Information) statistics
 	MemoryMonitorPSIStatsFile = MemoryMonitorOutputDir + "/psi.txt"
+
+	// OVMFSettingsDir - directory for OVMF settings, they are stored in per-domain files
+	OVMFSettingsDir      = PersistDir + "/ovmf"
+	OVMFSettingsTemplate = "/usr/lib/xen/boot/ovmf_vars.bin"
 )
 
 var (