[10.4-stable] backport: Carry media type through download verify.

pkg/pillar/cmd/verifier/verifier.go

@@ -14,6 +14,7 @@ package verifier
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"os"
 	"path"
 	"strings"
@@ -308,6 +309,7 @@ func handleCreate(ctx *verifierContext,
 	status := types.VerifyImageStatus{
 		Name:        config.Name,
 		ImageSha256: config.ImageSha256,
+		MediaType:   config.MediaType,
 		PendingAdd:  true,
 		State:       types.VERIFYING,
 		RefCount:    config.RefCount,
@@ -500,12 +502,20 @@ func handleGlobalConfigDelete(ctxArg interface{}, key string,
 }
 
 // ImageVerifierFilenames - Returns pendingFilename, verifierFilename, verifiedFilename
-// for the image
-func ImageVerifierFilenames(infile, sha256, tmpID string) (string, string, string) {
+// for the image. The verifierFilename and verifiedFilename always will have an extension
+// of the media-type, e.g. abcdeff112.application-vnd.oci.image.manifest.v1+json
+// This is because we need the media-type to process the blob. Normally, we carry
+// it around in the status (DownloadStatus -> BlobStatus), but those are ephemeral and
+// lost during a reboot. We need that information to be persistent and survive reboot,
+// so we can reconstruct it. Hence, we preserve it in the filename. It is PathEscape'd
+// so it is filename-safe.
+func ImageVerifierFilenames(infile, sha256, tmpID, mediaType string) (string, string, string) {
 	verifierDirname, verifiedDirname := getVerifierDir(), getVerifiedDir()
 	// Handle names which are paths
-	verified := tmpID + "." + sha256
-	return infile, path.Join(verifierDirname, verified), path.Join(verifiedDirname, sha256)
+	mediaTypeSafe := url.PathEscape(mediaType)
+	verifierFilename := strings.Join([]string{tmpID, sha256, mediaTypeSafe}, ".")
+	verifiedFilename := strings.Join([]string{sha256, mediaTypeSafe}, ".")
+	return infile, path.Join(verifierDirname, verifierFilename), path.Join(verifiedDirname, verifiedFilename)
 }
 
 // Returns ok, size of object
@@ -514,7 +524,7 @@ func markObjectAsVerifying(ctx *verifierContext,
 	status *types.VerifyImageStatus, tmpID uuid.UUID) (bool, int64) {
 
 	verifierDirname := getVerifierDir()
-	pendingFilename, verifierFilename, _ := ImageVerifierFilenames(config.FileLocation, config.ImageSha256, tmpID.String())
+	pendingFilename, verifierFilename, _ := ImageVerifierFilenames(config.FileLocation, config.ImageSha256, tmpID.String(), config.MediaType)
 
 	// Move to verifier directory which is RO
 	// XXX should have dom0 do this and/or have RO mounts
@@ -561,7 +571,7 @@ func markObjectAsVerifying(ctx *verifierContext,
 func markObjectAsVerified(config *types.VerifyImageConfig, status *types.VerifyImageStatus, tmpID uuid.UUID) {
 
 	verifiedDirname := getVerifiedDir()
-	_, verifierFilename, verifiedFilename := ImageVerifierFilenames(config.FileLocation, config.ImageSha256, tmpID.String())
+	_, verifierFilename, verifiedFilename := ImageVerifierFilenames(config.FileLocation, config.ImageSha256, tmpID.String(), config.MediaType)
 	// Move directory from DownloadDirname/verifier to
 	// DownloadDirname/verified
 	// XXX should have dom0 do this and/or have RO mounts
@@ -606,13 +616,33 @@ func handleInitVerifiedObjects(ctx *verifierContext) {
 	}
 }
 
-func verifyImageStatusFromImageFile(imageFileName string,
+// verifyImageStatusFromVerifiedImageFile given a verified image file,
+// return a VerifyImageStatus. Note that this is for a verified file, not a
+// verifying file.
+func verifyImageStatusFromVerifiedImageFile(imageFileName string,
 	size int64, pathname string) *types.VerifyImageStatus {
 
+	// filename might have two parts, separated by '.': digest and PathEscape(mediaType)
+	var (
+		mediaType string
+		digest    string
+	)
+	parts := strings.SplitN(imageFileName, ".", 2)
+	digest = parts[0]
+	if len(parts) == 2 {
+		// just ignore the error and treat mediaType as empty
+		mediaType, _ = url.PathUnescape(parts[1])
+		log.Tracef("verifyImageStatusFromVerifiedImageFile: mediaType %s recovered from %s", mediaType, imageFileName)
+	} else {
+		// if there is no mediaType, we force the redownload process by returning nil
+		log.Warnf("verifyImageStatusFromVerifiedImageFile: no mediaType in %s", imageFileName)
+		return nil
+	}
 	status := types.VerifyImageStatus{
 		Name:         imageFileName,
 		FileLocation: pathname,
-		ImageSha256:  imageFileName,
+		ImageSha256:  digest,
+		MediaType:    mediaType,
 		Size:         size,
 		State:        types.VERIFIED,
 		RefCount:     0,
@@ -658,9 +688,22 @@ func populateInitialStatusFromVerified(ctx *verifierContext,
 			}
 			log.Tracef("populateInitialStatusFromVerified: Processing %s: %d Mbytes",
 				pathname, size/(1024*1024))
-			status := verifyImageStatusFromImageFile(
+			status := verifyImageStatusFromVerifiedImageFile(
 				location.Name(), size, pathname)
-			if status != nil {
+			if status == nil {
+				log.Warnf("populateInitialStatusFromVerified: cannot create status for %s", location.Name())
+				// If the file exists, but we cannot create a status from it, consider it as corrupted and remove it
+				_, err := os.Stat(pathname)
+				if err != nil {
+					// file does not exist, nothing to do
+					continue
+				}
+				log.Tracef("populateInitialStatusFromVerified: removing corrupted file %s", pathname)
+				err = os.Remove(pathname)
+				if err != nil {
+					log.Errorf("populateInitialStatusFromVerified: cannot remove broken file: %v", err)
+				}
+			} else {
 				imageHash, err := fileutils.ComputeShaFile(pathname)
 				if err != nil {
 					log.Errorf("populateInitialStatusFromVerified: cannot compute sha: %v", err)

pkg/pillar/cmd/verifier/verifier_test.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2024 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package verifier
+
+import (
+	"github.com/lf-edge/eve/pkg/pillar/base"
+	"github.com/sirupsen/logrus"
+	"path"
+	"testing"
+)
+
+func TestMediaTypeInStatusFromVerifiedFilename(t *testing.T) {
+	log = base.NewSourceLogObject(logrus.StandardLogger(), "verifier_test", 0)
+	mediaType := "application/vnd.docker.distribution.manifest.v2+json"
+	tmpID := "tempID"
+	sha256 := "dummySha256"
+	infile := "dummyFile"
+
+	_, _, verifiedFilename := ImageVerifierFilenames(infile, sha256, tmpID, mediaType)
+	status := verifyImageStatusFromVerifiedImageFile(verifiedFilename, 12345, "dummyPath")
+
+	if status == nil {
+		t.Fatalf("Status is nil")
+	}
+
+	if status.MediaType != mediaType {
+		t.Errorf("MediaType in status %v does not match original %v", status.MediaType, mediaType)
+	}
+}
+
+func TestMediaTypeInStatusFromVerifiedFilenameWithNoMediaType(t *testing.T) {
+	log = base.NewSourceLogObject(logrus.StandardLogger(), "verifier_test", 0)
+	dummyPath := "dummyPath"
+	dummyFilename := "someSha256"
+	verifiedFilename := path.Join(dummyPath, dummyFilename)
+
+	status := verifyImageStatusFromVerifiedImageFile(verifiedFilename, 12345, "dummyPath")
+
+	if status != nil {
+		t.Errorf("Status is not nil")
+	}
+
+}

pkg/pillar/cmd/volumemgr/handleverifier.go

@@ -71,6 +71,7 @@ func MaybeAddVerifyImageConfigBlob(ctx *volumemgrContext, blob types.BlobStatus)
 			ImageSha256:  blob.Sha256, // the sha to verify
 			Name:         blob.Sha256, // we are just going to use the sha for the verifier display
 			RefCount:     refcount,
+			MediaType:    blob.MediaType,
 		}
 		log.Tracef("MaybeAddVerifyImageConfigBlob - config: %+v", vic)
 	}

pkg/pillar/types/verifiertypes.go

@@ -21,6 +21,7 @@ import (
 type VerifyImageConfig struct {
 	ImageSha256  string // sha256 of immutable image
 	Name         string
+	MediaType    string // MIME type
 	FileLocation string // Current location; should be info about file
 	Size         int64  //FileLocation size
 	RefCount     uint
@@ -91,6 +92,7 @@ type VerifyImageStatus struct {
 	Name          string
 	FileLocation  string // Current location
 	Size          int64
+	MediaType     string // MIME type
 	PendingAdd    bool
 	PendingModify bool
 	PendingDelete bool