pillar: Add per-domain OVMF_VARS.fd handling for FML guests.

Introduce support for managing per-domain OVMF_VARS.fd files, which are
essential for maintaining persistent UEFI settings for FML guests. It
adds functionality to prepare and clean up individual OVMF settings
files stored in the persist directory, ensuring that each virtual
machine has its own dedicated NVRAM file. The VM configuration
structures are updated to reference the bootloader settings file,
enabling the creation of unique UEFI variable stores for each domain.

Signed-off-by: Nikolay Martyanov <nikolay@zededa.com>

pkg/pillar/hypervisor/kvm.go

@@ -17,6 +17,7 @@ import (
 	"github.com/lf-edge/eve/pkg/pillar/agentlog"
 	"github.com/lf-edge/eve/pkg/pillar/containerd"
 	"github.com/lf-edge/eve/pkg/pillar/types"
+	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
 	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
 )
@@ -72,7 +73,9 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
   kernel-irqchip = "on"
 {{- end -}}
 {{- if .DomainConfig.BootLoader }}
+  {{- if ne .VirtualizationMode "FML" }}
   firmware = "{{.DomainConfig.BootLoader}}"
+  {{- end }}
 {{- end -}}
 {{- if .DomainConfig.Kernel }}
   kernel = "{{.DomainConfig.Kernel}}"
@@ -85,7 +88,7 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
 {{- end -}}
 {{- if .DomainConfig.ExtraArgs }}
   append = "{{.DomainConfig.ExtraArgs}}"
-{{ end }}
+{{- end }}
 {{if ne .Machine "virt" }}
 [global]
   driver = "kvm-pit"
@@ -109,7 +112,22 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
 [device]
   driver = "intel-iommu"
   caching-mode = "on"
-{{ end }}
+{{- end }}
+
+{{- if eq .VirtualizationMode "FML" }}
+
+[drive "drive-ovmf-code"]
+  if = "pflash"
+  format = "raw"
+  readonly = "on"
+  file = "{{.DomainConfig.BootLoader}}"
+
+[drive "drive-ovmf-vars"]
+  if = "pflash"
+  format = "raw"
+  file = "{{.DomainConfig.BootLoaderSettingsFile}}"
+{{- end }}
+
 [realtime]
   mlock = "off"
 
@@ -621,12 +639,56 @@ func vmmOverhead(domainName string, domainUUID uuid.UUID, domainRAMSize int64, v
 	return overhead, nil
 }
 
-func (ctx kvmContext) Setup(status types.DomainStatus, config types.DomainConfig,
+func getOVMFSettingsFile(domainName string) string {
+	return types.OVMFSettingsDir + "/" + domainName + "_ovmf_vars.bin"
+}
+
+func prepareOVMFSettings(domainName string) (string, error) {
+	// Create the OVMF settings directory if it does not exist
+	if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil {
+		return "", logError("failed to create OVMF settings directory: %v", err)
+	}
+	// Create a copy of the ovmf_vars.bin file in <domainName>_ovmf_vars.bin
+	ovmfSettingsFile := getOVMFSettingsFile(domainName)
+	if _, err := os.Stat(ovmfSettingsFile); os.IsNotExist(err) {
+		if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfSettingsFile); err != nil {
+			return "", logError("failed to copy ovmf_vars.bin file: %v", err)
+		}
+	}
+	// Set the RW permissions for the OVMF settings file
+	if err := os.Chmod(ovmfSettingsFile, 0666); err != nil {
+		return "", logError("failed to set RW permissions for ovmf_vars.bin file: %v", err)
+	}
+	return ovmfSettingsFile, nil
+}
+
+func cleanupOVMFSettings(domainName string) error {
+	ovmfVarsFile := getOVMFSettingsFile(domainName)
+	if _, err := os.Stat(ovmfVarsFile); err == nil {
+		if err := os.Remove(ovmfVarsFile); err != nil {
+			return logError("failed to remove ovmf_vars.bin file: %v", err)
+		}
+	}
+	return nil
+}
+
+// Setup sets up kvm
+func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig,
 	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
 
 	diskStatusList := status.DiskStatusList
 	domainName := status.DomainName
 	domainUUID := status.UUIDandVersion.UUID
+
+	// Before we start building the domain config, we need to prepare the OVMF settings
+	if config.VirtualizationMode == types.FML {
+		ovmfSettingsFile, err := prepareOVMFSettings(domainName)
+		if err != nil {
+			return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err)
+		}
+		config.BootLoaderSettingsFile = ovmfSettingsFile
+	}
+
 	// first lets build the domain config
 	if err := ctx.CreateDomConfig(domainName, config, status, diskStatusList, aa, file); err != nil {
 		return logError("failed to build domain config: %v", err)
@@ -679,11 +741,16 @@ func (ctx kvmContext) Setup(status types.DomainStatus, config types.DomainConfig
 
 func (ctx kvmContext) CreateDomConfig(domainName string, config types.DomainConfig, status types.DomainStatus,
 	diskStatusList []types.DiskStatus, aa *types.AssignableAdapters, file *os.File) error {
+	virtualizationMode := ""
+	if config.VirtualizationMode == types.FML {
+		virtualizationMode = "FML"
+	}
 	tmplCtx := struct {
-		Machine string
+		Machine            string
+		VirtualizationMode string
 		types.DomainConfig
 		types.DomainStatus
-	}{ctx.devicemodel, config, status}
+	}{ctx.devicemodel, virtualizationMode, config, status}
 	tmplCtx.DomainConfig.Memory = (config.Memory + 1023) / 1024
 	tmplCtx.DomainConfig.DisplayName = domainName
 
@@ -973,6 +1040,14 @@ func (ctx kvmContext) Cleanup(domainName string) error {
 		return fmt.Errorf("error waiting for Qmp absent for domain %s: %v", domainName, err)
 	}
 
+	// Cleanup OVMF settings
+	// XXX it should be a check for FML mode based on some config/status option. But we have
+	// only domain name here. So we check if the OVMF settings file exists.
+	if _, err := os.Stat(getOVMFSettingsFile(domainName)); err == nil {
+		if err := cleanupOVMFSettings(domainName); err != nil {
+			return fmt.Errorf("failed to cleanup OVMF settings for domain %s: %v", domainName, err)
+		}
+	}
 	return nil
 }
 

pkg/pillar/types/domainmgrtypes.go

@@ -195,15 +195,16 @@ func (config DomainConfig) LogKey() string {
 // StorageConfigList. For example, a Target of "kernel" means to set/override
 // the Kernel attribute below.
 type VmConfig struct {
-	Kernel     string // default ""
-	Ramdisk    string // default ""
-	Memory     int    // in kbytes; Rounded up to Mbytes for xen
-	MaxMem     int    // in kbytes; Default equal to 'Memory', so no ballooning for xen
-	VCpus      int    // default 1
-	MaxCpus    int    // default VCpus
-	RootDev    string // default "/dev/xvda1"
-	ExtraArgs  string // added to bootargs
-	BootLoader string // default ""
+	Kernel                 string // default ""
+	Ramdisk                string // default ""
+	Memory                 int    // in kbytes; Rounded up to Mbytes for xen
+	MaxMem                 int    // in kbytes; Default equal to 'Memory', so no ballooning for xen
+	VCpus                  int    // default 1
+	MaxCpus                int    // default VCpus
+	RootDev                string // default "/dev/xvda1"
+	ExtraArgs              string // added to bootargs
+	BootLoader             string // default ""
+	BootLoaderSettingsFile string // used to pass bootloader settings file, for example OVMF_VARS.fd
 	// For CPU pinning
 	CPUs string // default "", list of "1,2"
 	// Needed for device passthru

pkg/pillar/types/locationconsts.go

@@ -97,6 +97,11 @@ const (
 
 	// ContainerdContentDir - path to containerd`s content store
 	ContainerdContentDir = SealedDirName + "/containerd/io.containerd.content.v1.content"
+
+	// OVMFSettingsDir - directory for OVMF settings, they are stored in per-domain files
+	OVMFSettingsDir = PersistDir + "/ovmf"
+	// OVMFSettingsTemplate - template file for OVMF settings
+	OVMFSettingsTemplate = "/usr/lib/xen/boot/ovmf_vars.bin"
 )
 
 var (