Make escrow of encrypted storage key more reliable

pkg/pillar/cmd/vaultmgr/vaultmgr.go

@@ -492,7 +492,8 @@ func publishVaultKey(ctx *vaultMgrContext, vaultName string) error {
 	var encryptedVaultKey []byte
 	//we try to fill EncryptedVaultKey only in case of tpm enabled
 	//otherwise we leave it empty
-	if etpm.IsTpmEnabled() {
+	isTpmEnabled := etpm.IsTpmEnabled()
+	if isTpmEnabled {
 		if !ctx.defaultVaultUnlocked {
 			log.Errorf("Vault is not yet unlocked, waiting for Controller key")
 			return nil
@@ -504,6 +505,7 @@ func publishVaultKey(ctx *vaultMgrContext, vaultName string) error {
 
 		encryptedKey, err := etpm.EncryptDecryptUsingTpm(keyBytes, true)
 		if err != nil {
+			// XXX should we still send with no key?
 			return fmt.Errorf("Failed to encrypt vault key %w", err)
 		}
 
@@ -524,6 +526,7 @@ func publishVaultKey(ctx *vaultMgrContext, vaultName string) error {
 	keyFromDevice := types.EncryptedVaultKeyFromDevice{}
 	keyFromDevice.Name = vaultName
 	keyFromDevice.EncryptedVaultKey = encryptedVaultKey
+	keyFromDevice.IsTpmEnabled = isTpmEnabled
 	key := keyFromDevice.Key()
 	log.Tracef("Publishing EncryptedVaultKeyFromDevice %s\n", key)
 	pub := ctx.pubVaultKeyFromDevice

pkg/pillar/cmd/zedagent/attesttask.go

@@ -450,13 +450,14 @@
 	}
 	// bail if V2API is not supported
 	if !zedcloud.UseV2API() {
+		attestCtx.zedagentCtx.publishedAttestEscrow = true
 		return zattest.ErrNoVerifier
 	}
 	if attestCtx.SkipEscrow {
 		log.Notice("[ATTEST] Escrow successful skipped")
 		return nil
 	}
-	if attestCtx.EscrowData == nil {
+	if attestCtx.EscrowData == nil || len(attestCtx.EscrowData) == 0 {
 		errorDescription := types.ErrorDescription{Error: "[ATTEST] No escrow data"}
 		log.Error(errorDescription.Error)
 		setAttestErrorAndTriggerInfo(ctx, errorDescription)
@@ -537,6 +538,7 @@
 		return zattest.ErrITokenMismatch
 	case attest.AttestStorageKeysResponseCode_ATTEST_STORAGE_KEYS_RESPONSE_CODE_SUCCESS:
 		log.Notice("[ATTEST] Escrow successful")
+		attestCtx.zedagentCtx.publishedAttestEscrow = true
 		ctx.ClearError()
 		triggerPublishDevInfo(attestCtx.zedagentCtx)
 		// we sent storage keys successfully
@@ -800,7 +802,7 @@
 	attestCtx := ctx.attestCtx
 	attestCtx.EscrowData = vaultKey.EncryptedVaultKey
 	attestCtx.SkipEscrow = false
-	if len(attestCtx.EscrowData) == 0 {
+	if !vaultKey.IsTpmEnabled {
 		attestCtx.SkipEscrow = true
 	}
 

pkg/pillar/cmd/zedagent/handlecertconfig.go

@@ -493,8 +493,6 @@ func publishEdgeNodeCertsToController(ctx *zedagentContext) {
 	log.Tracef("publishEdgeNodeCertsToController: after send, total elapse sec %v",
 		time.Since(startPubTime).Seconds())
 	ctx.cipherCtx.iteration++
-	// XXX remove log?
-	log.Noticef("Maybe sent EdgeNodeCerts")
 	// The getDeferredSentHandlerFunction will set ctx.publishedEdgeNodeCerts
 	// when the message has been sent.
 }

pkg/pillar/cmd/zedagent/handleconfig.go

@@ -470,6 +470,9 @@
 			if getconfigCtx.configProcessingRV != configOK {
 				log.Noticef("config processing flag is not OK: %s", getconfigCtx.configProcessingRV)
 			}
+			if ctx.publishedEdgeNodeCerts && !ctx.publishedAttestEscrow {
+				log.Warning("Not yet published AttestEscrow")
+			}
 		}
 		ctx.ps.StillRunning(wdName, warningTime, errorTime)
 	}
@@ -608,14 +611,6 @@
 	configProcessingRetval, []netdump.TracedNetRequest) {
 
 	log.Tracef("getLatestConfig(%s, %d)", url, iteration)
-	// On first boot, if we haven't yet published our certificates we defer
-	// to ensure that the controller has our certs and can add encrypted
-	// secrets to our config.
-	if getconfigCtx.zedagentCtx.bootReason == types.BootReasonFirst &&
-		!getconfigCtx.zedagentCtx.publishedEdgeNodeCerts {
-		log.Noticef("Defer fetching config until our EdgeNodeCerts have been published")
-		return deferConfig, nil
-	}
 	ctx := getconfigCtx.zedagentCtx
 	const bailOnHTTPErr = false // For 4xx and 5xx HTTP errors we try other interfaces
 	// except http.StatusForbidden(which returns error

pkg/pillar/cmd/zedagent/zedagent.go

@@ -216,9 +216,12 @@
 	// Track the counter from force.fallback.counter to detect changes
 	forceFallbackCounter int
 
-	// Interlock with controller to ensure we get the encrypted secrets
+	// Used for retry of EdgeNodeCerts
 	publishedEdgeNodeCerts bool
 
+	// Used for retry of SendAttestEscrow
+	publishedAttestEscrow bool
+
 	attestationTryCount int
 	// cli options
 	versionPtr  *bool
@@ -439,6 +442,10 @@
 	// XXX defer this until we have some config from cloud or saved copy
 	getconfigCtx.pubAppInstanceConfig.SignalRestarted()
 
+	// Initialize remote attestation context. Do this before we get events
+	// from the AttestQuote and EncryptedKeyFromDevice subscriptions
+	attestModuleInitialize(zedagentCtx)
+
 	// With device UUID, zedagent is ready to initialize and activate all subscriptions.
 	initPostOnboardSubs(zedagentCtx)
 
@@ -450,9 +457,6 @@
 	//initialize cipher processing block
 	cipherModuleInitialize(zedagentCtx)
 
-	//initialize remote attestation context
-	attestModuleInitialize(zedagentCtx)
-
 	// Pick up debug aka log level before we start real work
 	waitUntilGCReady(zedagentCtx, stillRunning)
 
@@ -2552,7 +2556,7 @@
 				ctx.publishedEdgeNodeCerts = true
 			}
 		} else {
-			if _, ok := itemType.(attest.ZAttestReqType); ok {
+			if el, ok := itemType.(attest.ZAttestReqType); ok {
 				switch result {
 				case types.SenderStatusUpgrade:
 					log.Functionf("sendAttestReqProtobuf: Controller upgrade in progress")
@@ -2566,6 +2570,13 @@
 					log.Functionf("sendAttestReqProtobuf: Controller SenderStatusNotFound")
 					potentialUUIDUpdate(ctx.getconfigCtx)
 				}
+				if el == attest.ZAttestReqType_ATTEST_REQ_CERT {
+					log.Warnf("sendAttestReqProtobuf: Failed to send EdgeNodeCerts: %s",
+						result.String())
+					// XXX should we declare maintenance mode?
+					// We get SenderStatusNotFound when a cert can
+					// not be replaced in the controller for security reasons.
+				}
 				if !ctx.publishedEdgeNodeCerts {
 					// Attestation request does not clog the send queue (issued
 					// with the `ignoreErr` set to true), but once fails has to

pkg/pillar/types/vaultmgrtypes.go

@@ -83,6 +83,7 @@ func (status VaultStatus) LogKey() string {
 type EncryptedVaultKeyFromDevice struct {
 	Name              string
 	EncryptedVaultKey []byte // empty if no TPM enabled
+	IsTpmEnabled      bool
 }
 
 // Key returns name of the vault corresponding to this object