store: add proof to ViewState response

core/primitives/src/views.rs

@@ -194,9 +194,6 @@ impl From<AccessKeyView> for AccessKey {
     }
 }
 
-/// Set of serialized TrieNodes that are encoded in base64. Represent proof of inclusion of some TrieNode in the MerkleTrie.
-pub type TrieProofPath = Vec<String>;
-
 /// Item of the state, key and value are serialized in base64 and proof for inclusion of given state item.
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
@@ -205,14 +202,18 @@ pub struct StateItem {
     pub key: Vec<u8>,
     #[serde(with = "base64_format")]
     pub value: Vec<u8>,
-    pub proof: TrieProofPath,
+    /// Deprecated, always empty, eventually will be deleted.
+    // TODO(mina86): This was deprecated in 1.30.  Get rid of the field
+    // altogether at 1.33 or something.
+    #[serde(default)]
+    pub proof: Vec<()>,
 }
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct ViewStateResult {
     pub values: Vec<StateItem>,
-    pub proof: TrieProofPath,
+    pub proof: Vec<Arc<[u8]>>,
 }
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]

core/store/src/lib.rs

@@ -31,9 +31,9 @@ use crate::db::{
 pub use crate::trie::iterator::TrieIterator;
 pub use crate::trie::update::{TrieUpdate, TrieUpdateIterator, TrieUpdateValuePtr};
 pub use crate::trie::{
-    estimator, split_state, ApplyStatePartResult, KeyForStateChanges, PartialStorage, ShardTries,
-    Trie, TrieAccess, TrieCache, TrieCachingStorage, TrieChanges, TrieConfig, TrieStorage,
-    WrappedTrieChanges,
+    estimator, split_state, ApplyStatePartResult, KeyForStateChanges, NibbleSlice, PartialStorage,
+    RawTrieNode, RawTrieNodeWithSize, ShardTries, Trie, TrieAccess, TrieCache, TrieCachingStorage,
+    TrieChanges, TrieConfig, TrieStorage, WrappedTrieChanges,
 };
 
 mod columns;

core/store/src/trie/iterator.rs

@@ -41,6 +41,9 @@ pub struct TrieIterator<'a> {
     trie: &'a Trie,
     trail: Vec<Crumb>,
     pub(crate) key_nibbles: Vec<u8>,
+
+    /// If not `None`, a list of all nodes that the iterator has visited.
+    visited_nodes: Option<Vec<std::sync::Arc<[u8]>>>,
 }
 
 pub type TrieItem = (Vec<u8>, Vec<u8>);
@@ -61,6 +64,7 @@ impl<'a> TrieIterator<'a> {
             trie,
             trail: Vec::with_capacity(8),
             key_nibbles: Vec::with_capacity(64),
+            visited_nodes: None,
         };
         r.descend_into_node(&trie.root)?;
         Ok(r)
@@ -71,6 +75,24 @@ impl<'a> TrieIterator<'a> {
         self.seek_nibble_slice(NibbleSlice::new(key.as_ref()), true).map(drop)
     }
 
+    /// Configures whether the iterator should remember all the nodes its
+    /// visiting.
+    ///
+    /// Use [`Self::into_visited_nodes`] to retrieve the list.
+    pub fn remember_visited_nodes(&mut self, remember: bool) {
+        self.visited_nodes = remember.then(|| Vec::new());
+    }
+
+    /// Consumes iterator and returns list of nodes it’s visited.
+    ///
+    /// By default the iterator *doesn’t* remember nodes it visits.  To enable
+    /// that feature use [`Self::remember_visited_nodes`] method.  If the
+    /// feature is disabled, this method returns an empty list.  Otherwise
+    /// it returns list of nodes visited since the feature was enabled.
+    pub fn into_visited_nodes(self) -> Vec<std::sync::Arc<[u8]>> {
+        self.visited_nodes.unwrap_or(Vec::new())
+    }
+
     /// Returns the hash of the last node
     pub(crate) fn seek_nibble_slice(
         &mut self,
@@ -148,9 +170,15 @@ impl<'a> TrieIterator<'a> {
 
     /// Fetches block by its hash and adds it to the trail.
     ///
-    /// The node is stored as the last [`Crumb`] in the trail.
+    /// The node is stored as the last [`Crumb`] in the trail.  If iterator is
+    /// configured to remember all the nodes its visiting (which can be enabled
+    /// with [`Self::remember_visited_nodes`]), the node will be added to the
+    /// list.
     fn descend_into_node(&mut self, hash: &CryptoHash) -> Result<(), StorageError> {
-        let node = self.trie.retrieve_node(hash)?.1;
+        let (bytes, node) = self.trie.retrieve_node(hash)?;
+        if let Some(ref mut visited) = self.visited_nodes {
+            visited.push(bytes.ok_or(StorageError::TrieNodeMissing)?);
+        }
         self.trail.push(Crumb { status: CrumbStatus::Entering, node, prefix_boundary: false });
         Ok(())
     }

core/store/src/trie/mod.rs

@@ -18,7 +18,7 @@ use crate::flat_state::FlatState;
 pub use crate::trie::config::TrieConfig;
 use crate::trie::insert_delete::NodesStorage;
 use crate::trie::iterator::TrieIterator;
-use crate::trie::nibble_slice::NibbleSlice;
+pub use crate::trie::nibble_slice::NibbleSlice;
 pub use crate::trie::shard_tries::{KeyForStateChanges, ShardTries, WrappedTrieChanges};
 pub use crate::trie::trie_storage::{TrieCache, TrieCachingStorage, TrieStorage};
 use crate::trie::trie_storage::{TrieMemoryPartialStorage, TrieRecordingStorage};
@@ -310,7 +310,7 @@ impl std::fmt::Debug for TrieNode {
 
 #[derive(Debug, Eq, PartialEq)]
 #[allow(clippy::large_enum_variant)]
-enum RawTrieNode {
+pub enum RawTrieNode {
     Leaf(Vec<u8>, u32, CryptoHash),
     Branch([Option<CryptoHash>; 16], Option<(u32, CryptoHash)>),
     Extension(Vec<u8>, CryptoHash),
@@ -319,7 +319,7 @@ enum RawTrieNode {
 /// Trie node + memory cost of its subtree
 /// memory_usage is serialized, stored, and contributes to hash
 #[derive(Debug, Eq, PartialEq)]
-struct RawTrieNodeWithSize {
+pub struct RawTrieNodeWithSize {
     pub node: RawTrieNode,
     memory_usage: u64,
 }
@@ -447,7 +447,7 @@ impl RawTrieNodeWithSize {
         out
     }
 
-    fn decode(bytes: &[u8]) -> Result<Self, std::io::Error> {
+    pub fn decode(bytes: &[u8]) -> Result<Self, std::io::Error> {
         if bytes.len() < 8 {
             return Err(std::io::Error::new(std::io::ErrorKind::Other, "Wrong type"));
         }

integration-tests/src/tests/runtime/state_viewer.rs

@@ -1,8 +1,13 @@
+use std::{collections::HashMap, io, sync::Arc};
+
 use crate::runtime_utils::{get_runtime_and_trie, get_test_trie_viewer, TEST_SHARD_UID};
 use near_primitives::{
     account::Account,
     hash::hash as sha256,
     hash::CryptoHash,
+    serialize::to_base64,
+    trie_key::trie_key_parsers,
+    types::{AccountId, StateRoot},
     views::{StateItem, ViewApplyState},
 };
 use near_primitives::{
@@ -11,11 +16,87 @@ use near_primitives::{
     types::{EpochId, StateChangeCause},
     version::PROTOCOL_VERSION,
 };
-use near_store::set_account;
+use near_store::{set_account, NibbleSlice, RawTrieNode, RawTrieNodeWithSize};
 use node_runtime::state_viewer::errors;
 use node_runtime::state_viewer::*;
 use testlib::runtime_utils::{alice_account, encode_int};
 
+struct ProofVerifier {
+    nodes: HashMap<CryptoHash, RawTrieNodeWithSize>,
+}
+
+impl ProofVerifier {
+    fn new(proof: Vec<Arc<[u8]>>) -> Result<Self, io::Error> {
+        let nodes = proof
+            .into_iter()
+            .map(|bytes| {
+                let hash = CryptoHash::hash_bytes(&bytes);
+                let node = RawTrieNodeWithSize::decode(&bytes)?;
+                Ok((hash, node))
+            })
+            .collect::<Result<HashMap<_, _>, io::Error>>()?;
+        Ok(Self { nodes })
+    }
+
+    fn verify(
+        &self,
+        state_root: &StateRoot,
+        account_id: &AccountId,
+        key: &[u8],
+        expected: Option<&[u8]>,
+    ) -> bool {
+        let query = trie_key_parsers::get_raw_prefix_for_contract_data(account_id, key);
+        let mut key = NibbleSlice::new(&query);
+
+        let mut expected_hash = state_root;
+        while let Some(node) = self.nodes.get(expected_hash) {
+            match &node.node {
+                RawTrieNode::Leaf(node_key, value_length, value_hash) => {
+                    let nib = &NibbleSlice::from_encoded(&node_key).0;
+                    return if &key != nib {
+                        return expected.is_none();
+                    } else if let Some(value) = expected {
+                        if *value_length as usize != value.len() {
+                            return false;
+                        }
+                        CryptoHash::hash_bytes(value) == *value_hash
+                    } else {
+                        false
+                    };
+                }
+
+                RawTrieNode::Extension(node_key, child_hash) => {
+                    expected_hash = child_hash;
+
+                    // To avoid unnecessary copy
+                    let nib = NibbleSlice::from_encoded(&node_key).0;
+                    if !key.starts_with(&nib) {
+                        return expected.is_none();
+                    }
+                    key = key.mid(nib.len());
+                }
+                RawTrieNode::Branch(children, value) => {
+                    if key.is_empty() {
+                        return *value
+                            == expected.map(|value| {
+                                (value.len().try_into().unwrap(), CryptoHash::hash_bytes(&value))
+                            });
+                    }
+                    let index = key.at(0);
+                    match &children[index as usize] {
+                        Some(child_hash) => {
+                            key = key.mid(1);
+                            expected_hash = child_hash;
+                        }
+                        None => return expected.is_none(),
+                    }
+                }
+            }
+        }
+        false
+    }
+}
+
 #[test]
 fn test_view_call() {
     let (viewer, root) = get_test_trie_viewer();
@@ -103,8 +184,17 @@ fn test_view_call_with_args() {
     assert_eq!(view_call_result.unwrap(), 3u64.to_le_bytes().to_vec());
 }
 
+#[track_caller]
+fn assert_proof(proof: &[Arc<[u8]>], want: &[&'static str]) {
+    let got = proof.iter().map(|bytes| to_base64(bytes)).collect::<Vec<_>>();
+    let got = got.iter().map(String::as_str).collect::<Vec<_>>();
+    assert_eq!(want, &got[..]);
+}
+
 #[test]
 fn test_view_state() {
+    // in order to ensure determinism under all conditions (compiler, build output, etc)
+    // avoid deploying a test contract. See issue #7238
     let (_, tries, root) = get_runtime_and_trie();
     let shard_uid = TEST_SHARD_UID;
     let mut state_update = tries.new_trie_update(shard_uid, root);
@@ -132,7 +222,25 @@ fn test_view_state() {
     let state_update = tries.new_trie_update(shard_uid, new_root);
     let trie_viewer = TrieViewer::default();
     let result = trie_viewer.view_state(&state_update, &alice_account(), b"").unwrap();
-    assert_eq!(result.proof, Vec::<String>::new());
+    // The proof isn’t deterministic because the state contains test contracts
+    // which aren’t built hermetically.  Fortunately, only the first two items
+    // in the proof are affected.  First one is the state root which is an
+    // Extension("0x0", child_hash) node and the second one is child hash
+    // pointing to a Branch node which splits into four: 0x0 (accounts), 0x1
+    // (contract code; that’s what’s nondeterministic), 0x2 (access keys) and
+    // 0x9 (contract data; that’s what we care about).
+    assert_proof(&result.proof[2..], &[
+        "AwEAAAAQjHWWT6rXAXqUm14fjfDxo3286ApntHMI1eK0aQAJZPfJewEAAAAAAA==",
+        "AQcCSXBK8DHIYBF47dz6xB2iFKLLsPjAIAo9syJTBC0/Y1OjJNvT5izZukYCmtq/AyVTeyWFl1Ei6yFZBf5yIJ0i96eYRr8PVilJ81MgJKvV/R1SxQuTfwwmbZ6sN/TC2XfL1SCJ4WM1GZ0yMSaNpJOdsJH9kda203WM3Zh81gxz6rmVewEAAAAAAA==",
+        "AwMAAAAWFsbwm2TFX4GHLT5G1LSpF8UkG7zQV1ohXBMR/OQcUAKZ3gwDAAAAAAAA",
+        "ASAC7S1KwgLNl0HPdSo8soL8sGOmPhL7O0xTSR8sDDR5pZrzu0ty3UPYJ5UKrFGKxXoyyyNG75AF9hnJHO3xxFkf5NQCAAAAAAAA",
+        "AwEAAAAW607KPj2q3O8dF6XkfALiIrd9mqGir2UlYIcZuLNksTsvAgAAAAAAAA==",
+        "AQhAP4sMdbiWZPtV6jz8hYKzRFSgwaSlQKiGsQXogAmMcrLOl+SJfiCOXMTEZ2a1ebmQOEGkRYa30FaIlB46sLI2IPsBAAAAAAAA",
+        "AwwAAAAWUubmVhcix0ZXN0PKtrEndk0LxM+qpzp0PVtjf+xlrzz4TT0qA+hTtm6BLlYBAAAAAAAA",
+        "AQoAVWCdny7wv/M1LvZASC3Fw0D/NNhI1NYwch9Ux+KZ2qRdQXPC1rNsCGRJ7nd66SfcNmRUVVvQY6EYCbsIiugO6gwBAAAAAAAA",
+        "AAMAAAAgMjMDAAAApmWkWSBCL51Bfkhn79xPuKBKHz//H6B+mY6G9/eieuNtAAAAAAAAAA==",
+        "AAMAAAAgMjEDAAAAjSPPbIboNKeqbt7VTCbOK7LnSQNTjGG91dIZeZerL3JtAAAAAAAAAA==",
+    ][2..]);
     assert_eq!(
         result.values,
         [
@@ -147,6 +255,41 @@ fn test_view_state() {
         result.values,
         [StateItem { key: b"test123".to_vec(), value: b"123".to_vec(), proof: vec![] }]
     );
+    assert_proof(
+        &result
+            .proof[2..],
+        &[
+            "AwEAAAAQjHWWT6rXAXqUm14fjfDxo3286ApntHMI1eK0aQAJZPfJewEAAAAAAA==",
+            "AQcCSXBK8DHIYBF47dz6xB2iFKLLsPjAIAo9syJTBC0/Y1OjJNvT5izZukYCmtq/AyVTeyWFl1Ei6yFZBf5yIJ0i96eYRr8PVilJ81MgJKvV/R1SxQuTfwwmbZ6sN/TC2XfL1SCJ4WM1GZ0yMSaNpJOdsJH9kda203WM3Zh81gxz6rmVewEAAAAAAA==",
+            "AwMAAAAWFsbwm2TFX4GHLT5G1LSpF8UkG7zQV1ohXBMR/OQcUAKZ3gwDAAAAAAAA",
+            "ASAC7S1KwgLNl0HPdSo8soL8sGOmPhL7O0xTSR8sDDR5pZrzu0ty3UPYJ5UKrFGKxXoyyyNG75AF9hnJHO3xxFkf5NQCAAAAAAAA",
+            "AwEAAAAW607KPj2q3O8dF6XkfALiIrd9mqGir2UlYIcZuLNksTsvAgAAAAAAAA==",
+            "AQhAP4sMdbiWZPtV6jz8hYKzRFSgwaSlQKiGsQXogAmMcrLOl+SJfiCOXMTEZ2a1ebmQOEGkRYa30FaIlB46sLI2IPsBAAAAAAAA",
+            "AwwAAAAWUubmVhcix0ZXN0PKtrEndk0LxM+qpzp0PVtjf+xlrzz4TT0qA+hTtm6BLlYBAAAAAAAA",
+            "AQoAVWCdny7wv/M1LvZASC3Fw0D/NNhI1NYwch9Ux+KZ2qRdQXPC1rNsCGRJ7nd66SfcNmRUVVvQY6EYCbsIiugO6gwBAAAAAAAA",
+            "AAMAAAAgMjMDAAAApmWkWSBCL51Bfkhn79xPuKBKHz//H6B+mY6G9/eieuNtAAAAAAAAAA==",
+        ][2..]
+    );
+
+    let root = state_update.get_root();
+    let account = alice_account();
+    let proof_verifier =
+        ProofVerifier::new(result.proof).expect("could not create a ProofVerifier");
+    for (want, key, value) in [
+        (true, b"test123".as_ref(), Some(b"123".as_ref())),
+        (false, b"test123".as_ref(), Some(b"321".as_ref())),
+        (false, b"test123".as_ref(), Some(b"1234".as_ref())),
+        (false, b"test123".as_ref(), None),
+        // Shorter key:
+        (false, b"test12".as_ref(), Some(b"123".as_ref())),
+        (true, b"test12", None),
+        // Longer key:
+        (false, b"test1234", Some(b"123")),
+        (true, b"test1234", None),
+    ] {
+        let got = proof_verifier.verify(root, &account, key, value);
+        assert_eq!(want, got, "key: {key:x?}; value: {value:x?}");
+    }
 }
 
 #[test]

runtime/runtime/src/state_viewer/mod.rs

@@ -143,17 +143,18 @@ impl TrieViewer {
         let query = trie_key_parsers::get_raw_prefix_for_contract_data(account_id, prefix);
         let acc_sep_len = query.len() - prefix.len();
         let mut iter = state_update.trie().iter()?;
+        iter.remember_visited_nodes(true);
         iter.seek_prefix(&query)?;
-        for item in iter {
+        for item in &mut iter {
             let (key, value) = item?;
             values.push(StateItem {
                 key: key[acc_sep_len..].to_vec(),
                 value: value,
                 proof: vec![],
             });
         }
-        // TODO(2076): Add proofs for the storage items.
-        Ok(ViewStateResult { values, proof: vec![] })
+        let proof = iter.into_visited_nodes();
+        Ok(ViewStateResult { values, proof })
     }
 
     pub fn call_function(