Fix codegen validation when Runtime APIs are stripped

codegen/src/api/mod.rs

@@ -322,11 +322,14 @@ impl RuntimeGenerator {
             .collect();
         let pallet_names_len = pallet_names.len();
 
-        let metadata_hash = self
+        let runtime_api_names: Vec<_> = self
             .metadata
-            .hasher()
-            .only_these_pallets(&pallet_names)
-            .hash();
+            .runtime_api_traits()
+            .map(|api| api.name().to_string())
+            .collect();
+        let runtime_api_names_len = runtime_api_names.len();
+
+        let metadata_hash = self.metadata.hasher().hash();
 
         let modules = pallets_with_mod_names
             .iter()
@@ -542,6 +545,9 @@ impl RuntimeGenerator {
                 // Identify the pallets composing the static metadata by name.
                 pub static PALLETS: [&str; #pallet_names_len] = [ #(#pallet_names,)* ];
 
+                // Runtime APIs in the metadata by name.
+                pub static RUNTIME_APIS: [&str; #runtime_api_names_len] = [ #(#runtime_api_names,)* ];
+
                 /// The error type returned when there is a runtime issue.
                 pub type DispatchError = #types_mod_ident::sp_runtime::DispatchError;
 
@@ -621,14 +627,14 @@ impl RuntimeGenerator {
                     )*
                 }
 
-                /// check whether the Client you are using is aligned with the statically generated codegen.
-                pub fn validate_codegen<T: #crate_path::Config, C: #crate_path::client::OfflineClientT<T>>(client: &C) -> Result<(), #crate_path::error::MetadataError> {
-                    let runtime_metadata_hash = client.metadata().hasher().only_these_pallets(&PALLETS).hash();
-                    if runtime_metadata_hash != [ #(#metadata_hash,)* ] {
-                        Err(#crate_path::error::MetadataError::IncompatibleCodegen)
-                    } else {
-                        Ok(())
-                    }
+                /// check whether the metadata provided is aligned with this statically generated code.
+                pub fn is_codegen_valid_for(metadata: &#crate_path::Metadata) -> bool {
+                    let runtime_metadata_hash = metadata
+                        .hasher()
+                        .only_these_pallets(&PALLETS)
+                        .only_these_runtime_apis(&RUNTIME_APIS)
+                        .hash();
+                    runtime_metadata_hash == [ #(#metadata_hash,)* ]
                 }
 
                 #( #modules )*

metadata/src/utils/validation.rs

@@ -407,6 +407,7 @@ pub fn get_pallet_hash(pallet: PalletMetadata) -> [u8; HASH_LEN] {
 pub struct MetadataHasher<'a> {
     metadata: &'a Metadata,
     specific_pallets: Option<Vec<&'a str>>,
+    specific_runtime_apis: Option<Vec<&'a str>>,
 }
 
 impl<'a> MetadataHasher<'a> {
@@ -415,6 +416,7 @@ impl<'a> MetadataHasher<'a> {
         Self {
             metadata,
             specific_pallets: None,
+            specific_runtime_apis: None,
         }
     }
 
@@ -424,30 +426,54 @@ impl<'a> MetadataHasher<'a> {
         self
     }
 
+    /// Only hash the provided runtime APIs instead of hashing every runtime API
+    pub fn only_these_runtime_apis<S: AsRef<str>>(
+        &mut self,
+        specific_runtime_apis: &'a [S],
+    ) -> &mut Self {
+        self.specific_runtime_apis =
+            Some(specific_runtime_apis.iter().map(|n| n.as_ref()).collect());
+        self
+    }
+
     /// Hash the given metadata.
     pub fn hash(&self) -> [u8; HASH_LEN] {
         let mut visited_ids = HashSet::<u32>::new();
 
         let metadata = self.metadata;
 
         let pallet_hash = metadata.pallets().fold([0u8; HASH_LEN], |bytes, pallet| {
-            // If specific pallets are given, only include this pallet if it's
-            // in the list.
-            if let Some(specific_pallets) = &self.specific_pallets {
-                if specific_pallets.iter().all(|&p| p != pallet.name()) {
-                    return bytes;
-                }
-            }
+            // If specific pallets are given, only include this pallet if it is in the specific pallets.
+            let should_hash = self
+                .specific_pallets
+                .as_ref()
+                .map(|specific_pallets| specific_pallets.contains(&pallet.name()))
+                .unwrap_or(true);
             // We don't care what order the pallets are seen in, so XOR their
             // hashes together to be order independent.
-            xor(bytes, get_pallet_hash(pallet))
+            if should_hash {
+                xor(bytes, get_pallet_hash(pallet))
+            } else {
+                bytes
+            }
         });
 
         let apis_hash = metadata
             .runtime_api_traits()
             .fold([0u8; HASH_LEN], |bytes, api| {
-                // We don't care what order the runtime APIs are seen in, so XOR
-                xor(bytes, get_runtime_trait_hash(api))
+                // If specific runtime APIs are given, only include this pallet if it is in the specific runtime APIs.
+                let should_hash = self
+                    .specific_runtime_apis
+                    .as_ref()
+                    .map(|specific_runtime_apis| specific_runtime_apis.contains(&api.name()))
+                    .unwrap_or(true);
+                // We don't care what order the runtime APIs are seen in, so XOR their
+                // hashes together to be order independent.
+                if should_hash {
+                    xor(bytes, xor(bytes, get_runtime_trait_hash(api)))
+                } else {
+                    bytes
+                }
             });
 
         let extrinsic_hash = get_extrinsic_hash(&metadata.types, &metadata.extrinsic);

subxt/src/metadata/metadata_type.rs

@@ -59,6 +59,13 @@ impl From<subxt_metadata::Metadata> for Metadata {
     }
 }
 
+impl TryFrom<frame_metadata::RuntimeMetadataPrefixed> for Metadata {
+    type Error = subxt_metadata::TryFromError;
+    fn try_from(value: frame_metadata::RuntimeMetadataPrefixed) -> Result<Self, Self::Error> {
+        subxt_metadata::Metadata::try_from(value).map(Metadata::from)
+    }
+}
+
 impl codec::Decode for Metadata {
     fn decode<I: codec::Input>(input: &mut I) -> Result<Self, codec::Error> {
         subxt_metadata::Metadata::decode(input).map(Metadata::new)

testing/integration-tests/src/codegen/polkadot.rs

@@ -1,5 +1,6 @@
 #[allow(dead_code, unused_imports, non_camel_case_types)]
 #[allow(clippy::all)]
+#[allow(rustdoc::broken_intra_doc_links)]
 pub mod api {
     #[allow(unused_imports)]
     mod root_mod {
@@ -64,6 +65,25 @@ pub mod api {
         "Crowdloan",
         "XcmPallet",
     ];
+    pub static RUNTIME_APIS: [&str; 17usize] = [
+        "Core",
+        "Metadata",
+        "BlockBuilder",
+        "NominationPoolsApi",
+        "StakingApi",
+        "TaggedTransactionQueue",
+        "OffchainWorkerApi",
+        "ParachainHost",
+        "BeefyApi",
+        "MmrApi",
+        "GrandpaApi",
+        "BabeApi",
+        "AuthorityDiscoveryApi",
+        "SessionKeys",
+        "AccountNonceApi",
+        "TransactionPaymentApi",
+        "TransactionPaymentCallApi",
+    ];
     #[doc = r" The error type returned when there is a runtime issue."]
     pub type DispatchError = runtime_types::sp_runtime::DispatchError;
     #[derive(
@@ -4420,26 +4440,19 @@ pub mod api {
             xcm_pallet::calls::TransactionApi
         }
     }
-    #[doc = r" check whether the Client you are using is aligned with the statically generated codegen."]
-    pub fn validate_codegen<T: ::subxt::Config, C: ::subxt::client::OfflineClientT<T>>(
-        client: &C,
-    ) -> Result<(), ::subxt::error::MetadataError> {
-        let runtime_metadata_hash = client
-            .metadata()
+    #[doc = r" check whether the metadata provided is aligned with this statically generated code."]
+    pub fn is_codegen_valid_for(metadata: &::subxt::Metadata) -> bool {
+        let runtime_metadata_hash = metadata
             .hasher()
             .only_these_pallets(&PALLETS)
+            .only_these_runtime_apis(&RUNTIME_APIS)
             .hash();
-        if runtime_metadata_hash
-            != [
-                151u8, 83u8, 251u8, 44u8, 149u8, 59u8, 20u8, 183u8, 19u8, 173u8, 234u8, 48u8,
-                114u8, 104u8, 69u8, 102u8, 189u8, 208u8, 10u8, 87u8, 154u8, 252u8, 54u8, 185u8,
-                248u8, 199u8, 45u8, 173u8, 199u8, 95u8, 189u8, 253u8,
+        runtime_metadata_hash
+            == [
+                48u8, 175u8, 255u8, 171u8, 180u8, 123u8, 181u8, 54u8, 125u8, 74u8, 109u8, 140u8,
+                192u8, 208u8, 131u8, 194u8, 195u8, 232u8, 33u8, 229u8, 178u8, 181u8, 236u8, 230u8,
+                37u8, 97u8, 134u8, 144u8, 187u8, 127u8, 47u8, 237u8,
             ]
-        {
-            Err(::subxt::error::MetadataError::IncompatibleCodegen)
-        } else {
-            Ok(())
-        }
     }
     pub mod system {
         use super::root_mod;

testing/integration-tests/src/metadata/validation.rs

@@ -91,19 +91,15 @@ async fn full_metadata_check() {
     let api = ctx.client();
 
     // Runtime metadata is identical to the metadata used during API generation.
-    assert!(node_runtime::validate_codegen(&api).is_ok());
+    assert!(node_runtime::is_codegen_valid_for(&api.metadata()));
 
     // Modify the metadata.
     let metadata = modified_metadata(api.metadata(), |md| {
         md.pallets[0].name = "NewPallet".to_string();
     });
 
-    let api = metadata_to_api(metadata, &ctx).await;
-    assert_eq!(
-        node_runtime::validate_codegen(&api)
-            .expect_err("Validation should fail for incompatible metadata"),
-        ::subxt::error::MetadataError::IncompatibleCodegen
-    );
+    // It should now be invalid:
+    assert!(!node_runtime::is_codegen_valid_for(&metadata));
 }
 
 #[tokio::test]
@@ -134,7 +130,7 @@ async fn constant_values_are_not_validated() {
 
     let api = metadata_to_api(metadata, &ctx).await;
 
-    assert!(node_runtime::validate_codegen(&api).is_ok());
+    assert!(node_runtime::is_codegen_valid_for(&api.metadata()));
     assert!(api.constants().at(&deposit_addr).is_ok());
 }
 

testing/ui-tests/src/lib.rs

@@ -15,43 +15,101 @@ mod dispatch_errors;
 mod storage;
 mod utils;
 
-use crate::utils::{MetadataTestRunner, PalletMetadataTestRunner};
+use crate::utils::MetadataTestRunner;
 
 // Each of these tests leads to some rust code being compiled and
 // executed to test that compilation is successful (or errors in the
 // way that we'd expect).
 #[test]
 fn ui_tests() {
     let mut m = MetadataTestRunner::default();
-    // specify pallets we want to test the metadata for (None => all pallets, but specifying only Some(..) speeds up test)
-    let mut p = PalletMetadataTestRunner::new(Some(&["Babe", "Claims", "Grandpa", "Balances"]));
     let t = trybuild::TestCases::new();
 
     t.pass("src/correct/*.rs");
 
     // Check that storage maps with no keys are handled properly.
-    t.pass(m.path_to_ui_test_for_metadata(
-        "storage_map_no_keys",
-        storage::metadata_storage_map_no_keys(),
-    ));
+    t.pass(
+        m.new_test_case()
+            .name("storage_map_no_keys")
+            .build(storage::metadata_storage_map_no_keys()),
+    );
 
     // Test that the codegen can handle the different types of DispatchError.
-    t.pass(m.path_to_ui_test_for_metadata(
-        "named_field_dispatch_error",
-        dispatch_errors::metadata_named_field_dispatch_error(),
-    ));
-    t.pass(m.path_to_ui_test_for_metadata(
-        "legacy_dispatch_error",
-        dispatch_errors::metadata_legacy_dispatch_error(),
-    ));
-    t.pass(m.path_to_ui_test_for_metadata(
-        "array_dispatch_error",
-        dispatch_errors::metadata_array_dispatch_error(),
-    ));
-
-    // Ensure the generate per pallet metadata compiles.
-    while let Some(path) = p.path_to_next_ui_test() {
-        t.pass(path);
+    t.pass(
+        m.new_test_case()
+            .name("named_field_dispatch_error")
+            .build(dispatch_errors::metadata_named_field_dispatch_error()),
+    );
+    t.pass(
+        m.new_test_case()
+            .name("legacy_dispatch_error")
+            .build(dispatch_errors::metadata_legacy_dispatch_error()),
+    );
+    t.pass(
+        m.new_test_case()
+            .name("array_dispatch_error")
+            .build(dispatch_errors::metadata_array_dispatch_error()),
+    );
+
+    // Test retaining only specific pallets and ensure that works.
+    for pallet in ["Babe", "Claims", "Grandpa", "Balances"] {
+        let mut metadata = MetadataTestRunner::load_metadata();
+        metadata.retain(|p| p == pallet, |_| true);
+
+        t.pass(
+            m.new_test_case()
+                .name(format!("retain_pallet_{pallet}"))
+                .build(metadata),
+        );
+    }
+
+    // Test retaining only specific runtime APIs to ensure that works.
+    for runtime_api in ["Core", "Metadata"] {
+        let mut metadata = MetadataTestRunner::load_metadata();
+        metadata.retain(|_| true, |r| r == runtime_api);
+
+        t.pass(
+            m.new_test_case()
+                .name(format!("retain_runtime_api_{runtime_api}"))
+                .build(metadata),
+        );
+    }
+
+    // Validation should succeed when metadata we codegen from is stripped and
+    // client metadata is full:
+    {
+        let mut metadata = MetadataTestRunner::load_metadata();
+        metadata.retain(
+            |p| ["Babe", "Claims"].contains(&p),
+            |r| ["Core", "Metadata"].contains(&r),
+        );
+
+        t.pass(
+            m.new_test_case()
+                .name("stripped_metadata_validates_against_full")
+                .validation_metadata(MetadataTestRunner::load_metadata())
+                .build(metadata),
+        );
+    }
+
+    // Finally as a sanity check, codegen against stripped metadata should
+    // _not_ compare valid against client with differently stripped metadata.
+    {
+        let mut codegen_metadata = MetadataTestRunner::load_metadata();
+        codegen_metadata.retain(
+            |p| ["Babe", "Claims"].contains(&p),
+            |r| ["Core", "Metadata"].contains(&r),
+        );
+        let mut validation_metadata = MetadataTestRunner::load_metadata();
+        validation_metadata.retain(|p| p != "Claims", |r| r != "Metadata");
+
+        t.pass(
+            m.new_test_case()
+                .name("stripped_metadata_doesnt_validate_against_different")
+                .validation_metadata(validation_metadata)
+                .expects_invalid()
+                .build(codegen_metadata),
+        );
     }
 }