diff --git a/core/primitives/src/views.rs b/core/primitives/src/views.rs index 969f0fde576..987418c092a 100644 --- a/core/primitives/src/views.rs +++ b/core/primitives/src/views.rs @@ -194,9 +194,6 @@ impl From for AccessKey { } } -/// Set of serialized TrieNodes that are encoded in base64. Represent proof of inclusion of some TrieNode in the MerkleTrie. -pub type TrieProofPath = Vec; - /// Item of the state, key and value are serialized in base64 and proof for inclusion of given state item. #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))] #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] @@ -205,14 +202,18 @@ pub struct StateItem { pub key: Vec, #[serde(with = "base64_format")] pub value: Vec, - pub proof: TrieProofPath, + /// Deprecated, always empty, eventually will be deleted. + // TODO(mina86): This was deprecated in 1.30. Get rid of the field + // altogether at 1.33 or something. + #[serde(default)] + pub proof: Vec<()>, } #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))] #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct ViewStateResult { pub values: Vec, - pub proof: TrieProofPath, + pub proof: Vec>, } #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))] diff --git a/core/store/src/lib.rs b/core/store/src/lib.rs index 7b20ede2cc0..f569f00c5c0 100644 --- a/core/store/src/lib.rs +++ b/core/store/src/lib.rs @@ -31,9 +31,9 @@ use crate::db::{ pub use crate::trie::iterator::TrieIterator; pub use crate::trie::update::{TrieUpdate, TrieUpdateIterator, TrieUpdateValuePtr}; pub use crate::trie::{ - estimator, split_state, ApplyStatePartResult, KeyForStateChanges, PartialStorage, ShardTries, - Trie, TrieAccess, TrieCache, TrieCachingStorage, TrieChanges, TrieConfig, TrieStorage, - WrappedTrieChanges, + estimator, split_state, ApplyStatePartResult, KeyForStateChanges, NibbleSlice, PartialStorage, + RawTrieNode, RawTrieNodeWithSize, ShardTries, Trie, TrieAccess, TrieCache, TrieCachingStorage, + TrieChanges, TrieConfig, TrieStorage, WrappedTrieChanges, }; mod columns; diff --git a/core/store/src/trie/iterator.rs b/core/store/src/trie/iterator.rs index e99250ba480..10f83fb5d78 100644 --- a/core/store/src/trie/iterator.rs +++ b/core/store/src/trie/iterator.rs @@ -41,6 +41,9 @@ pub struct TrieIterator<'a> { trie: &'a Trie, trail: Vec, pub(crate) key_nibbles: Vec, + + /// If not `None`, a list of all nodes that the iterator has visited. + visited_nodes: Option>>, } pub type TrieItem = (Vec, Vec); @@ -61,6 +64,7 @@ impl<'a> TrieIterator<'a> { trie, trail: Vec::with_capacity(8), key_nibbles: Vec::with_capacity(64), + visited_nodes: None, }; r.descend_into_node(&trie.root)?; Ok(r) @@ -71,6 +75,24 @@ impl<'a> TrieIterator<'a> { self.seek_nibble_slice(NibbleSlice::new(key.as_ref()), true).map(drop) } + /// Configures whether the iterator should remember all the nodes its + /// visiting. + /// + /// Use [`Self::into_visited_nodes`] to retrieve the list. + pub fn remember_visited_nodes(&mut self, remember: bool) { + self.visited_nodes = remember.then(|| Vec::new()); + } + + /// Consumes iterator and returns list of nodes it’s visited. + /// + /// By default the iterator *doesn’t* remember nodes it visits. To enable + /// that feature use [`Self::remember_visited_nodes`] method. If the + /// feature is disabled, this method returns an empty list. Otherwise + /// it returns list of nodes visited since the feature was enabled. + pub fn into_visited_nodes(self) -> Vec> { + self.visited_nodes.unwrap_or(Vec::new()) + } + /// Returns the hash of the last node pub(crate) fn seek_nibble_slice( &mut self, @@ -148,9 +170,15 @@ impl<'a> TrieIterator<'a> { /// Fetches block by its hash and adds it to the trail. /// - /// The node is stored as the last [`Crumb`] in the trail. + /// The node is stored as the last [`Crumb`] in the trail. If iterator is + /// configured to remember all the nodes its visiting (which can be enabled + /// with [`Self::remember_visited_nodes`]), the node will be added to the + /// list. fn descend_into_node(&mut self, hash: &CryptoHash) -> Result<(), StorageError> { - let node = self.trie.retrieve_node(hash)?.1; + let (bytes, node) = self.trie.retrieve_node(hash)?; + if let Some(ref mut visited) = self.visited_nodes { + visited.push(bytes.ok_or(StorageError::TrieNodeMissing)?); + } self.trail.push(Crumb { status: CrumbStatus::Entering, node, prefix_boundary: false }); Ok(()) } diff --git a/core/store/src/trie/mod.rs b/core/store/src/trie/mod.rs index 94e5dc6c3ff..cdb5a0b75cf 100644 --- a/core/store/src/trie/mod.rs +++ b/core/store/src/trie/mod.rs @@ -18,7 +18,7 @@ use crate::flat_state::FlatState; pub use crate::trie::config::TrieConfig; use crate::trie::insert_delete::NodesStorage; use crate::trie::iterator::TrieIterator; -use crate::trie::nibble_slice::NibbleSlice; +pub use crate::trie::nibble_slice::NibbleSlice; pub use crate::trie::shard_tries::{KeyForStateChanges, ShardTries, WrappedTrieChanges}; pub use crate::trie::trie_storage::{TrieCache, TrieCachingStorage, TrieStorage}; use crate::trie::trie_storage::{TrieMemoryPartialStorage, TrieRecordingStorage}; @@ -310,7 +310,7 @@ impl std::fmt::Debug for TrieNode { #[derive(Debug, Eq, PartialEq)] #[allow(clippy::large_enum_variant)] -enum RawTrieNode { +pub enum RawTrieNode { Leaf(Vec, u32, CryptoHash), Branch([Option; 16], Option<(u32, CryptoHash)>), Extension(Vec, CryptoHash), @@ -319,7 +319,7 @@ enum RawTrieNode { /// Trie node + memory cost of its subtree /// memory_usage is serialized, stored, and contributes to hash #[derive(Debug, Eq, PartialEq)] -struct RawTrieNodeWithSize { +pub struct RawTrieNodeWithSize { pub node: RawTrieNode, memory_usage: u64, } @@ -447,7 +447,7 @@ impl RawTrieNodeWithSize { out } - fn decode(bytes: &[u8]) -> Result { + pub fn decode(bytes: &[u8]) -> Result { if bytes.len() < 8 { return Err(std::io::Error::new(std::io::ErrorKind::Other, "Wrong type")); } diff --git a/integration-tests/src/tests/runtime/state_viewer.rs b/integration-tests/src/tests/runtime/state_viewer.rs index f8f32a807e2..83bc1ef151a 100644 --- a/integration-tests/src/tests/runtime/state_viewer.rs +++ b/integration-tests/src/tests/runtime/state_viewer.rs @@ -1,8 +1,13 @@ +use std::{collections::HashMap, io, sync::Arc}; + use crate::runtime_utils::{get_runtime_and_trie, get_test_trie_viewer, TEST_SHARD_UID}; use near_primitives::{ account::Account, hash::hash as sha256, hash::CryptoHash, + serialize::to_base64, + trie_key::trie_key_parsers, + types::{AccountId, StateRoot}, views::{StateItem, ViewApplyState}, }; use near_primitives::{ @@ -11,11 +16,87 @@ use near_primitives::{ types::{EpochId, StateChangeCause}, version::PROTOCOL_VERSION, }; -use near_store::set_account; +use near_store::{set_account, NibbleSlice, RawTrieNode, RawTrieNodeWithSize}; use node_runtime::state_viewer::errors; use node_runtime::state_viewer::*; use testlib::runtime_utils::{alice_account, encode_int}; +struct ProofVerifier { + nodes: HashMap, +} + +impl ProofVerifier { + fn new(proof: Vec>) -> Result { + let nodes = proof + .into_iter() + .map(|bytes| { + let hash = CryptoHash::hash_bytes(&bytes); + let node = RawTrieNodeWithSize::decode(&bytes)?; + Ok((hash, node)) + }) + .collect::, io::Error>>()?; + Ok(Self { nodes }) + } + + fn verify( + &self, + state_root: &StateRoot, + account_id: &AccountId, + key: &[u8], + expected: Option<&[u8]>, + ) -> bool { + let query = trie_key_parsers::get_raw_prefix_for_contract_data(account_id, key); + let mut key = NibbleSlice::new(&query); + + let mut expected_hash = state_root; + while let Some(node) = self.nodes.get(expected_hash) { + match &node.node { + RawTrieNode::Leaf(node_key, value_length, value_hash) => { + let nib = &NibbleSlice::from_encoded(&node_key).0; + return if &key != nib { + return expected.is_none(); + } else if let Some(value) = expected { + if *value_length as usize != value.len() { + return false; + } + CryptoHash::hash_bytes(value) == *value_hash + } else { + false + }; + } + + RawTrieNode::Extension(node_key, child_hash) => { + expected_hash = child_hash; + + // To avoid unnecessary copy + let nib = NibbleSlice::from_encoded(&node_key).0; + if !key.starts_with(&nib) { + return expected.is_none(); + } + key = key.mid(nib.len()); + } + RawTrieNode::Branch(children, value) => { + if key.is_empty() { + return *value + == expected.map(|value| { + (value.len().try_into().unwrap(), CryptoHash::hash_bytes(&value)) + }); + } + let index = key.at(0); + match &children[index as usize] { + Some(child_hash) => { + key = key.mid(1); + expected_hash = child_hash; + } + None => return expected.is_none(), + } + } + } + } + false + } +} + #[test] fn test_view_call() { let (viewer, root) = get_test_trie_viewer(); @@ -103,8 +184,17 @@ fn test_view_call_with_args() { assert_eq!(view_call_result.unwrap(), 3u64.to_le_bytes().to_vec()); } +#[track_caller] +fn assert_proof(proof: &[Arc<[u8]>], want: &[&'static str]) { + let got = proof.iter().map(|bytes| to_base64(bytes)).collect::>(); + let got = got.iter().map(String::as_str).collect::>(); + assert_eq!(want, &got[..]); +} + #[test] fn test_view_state() { + // in order to ensure determinism under all conditions (compiler, build output, etc) + // avoid deploying a test contract. See issue #7238 let (_, tries, root) = get_runtime_and_trie(); let shard_uid = TEST_SHARD_UID; let mut state_update = tries.new_trie_update(shard_uid, root); @@ -132,7 +222,25 @@ fn test_view_state() { let state_update = tries.new_trie_update(shard_uid, new_root); let trie_viewer = TrieViewer::default(); let result = trie_viewer.view_state(&state_update, &alice_account(), b"").unwrap(); - assert_eq!(result.proof, Vec::::new()); + // The proof isn’t deterministic because the state contains test contracts + // which aren’t built hermetically. Fortunately, only the first two items + // in the proof are affected. First one is the state root which is an + // Extension("0x0", child_hash) node and the second one is child hash + // pointing to a Branch node which splits into four: 0x0 (accounts), 0x1 + // (contract code; that’s what’s nondeterministic), 0x2 (access keys) and + // 0x9 (contract data; that’s what we care about). + assert_proof(&result.proof[2..], &[ + "AwEAAAAQjHWWT6rXAXqUm14fjfDxo3286ApntHMI1eK0aQAJZPfJewEAAAAAAA==", + "AQcCSXBK8DHIYBF47dz6xB2iFKLLsPjAIAo9syJTBC0/Y1OjJNvT5izZukYCmtq/AyVTeyWFl1Ei6yFZBf5yIJ0i96eYRr8PVilJ81MgJKvV/R1SxQuTfwwmbZ6sN/TC2XfL1SCJ4WM1GZ0yMSaNpJOdsJH9kda203WM3Zh81gxz6rmVewEAAAAAAA==", + "AwMAAAAWFsbwm2TFX4GHLT5G1LSpF8UkG7zQV1ohXBMR/OQcUAKZ3gwDAAAAAAAA", + "ASAC7S1KwgLNl0HPdSo8soL8sGOmPhL7O0xTSR8sDDR5pZrzu0ty3UPYJ5UKrFGKxXoyyyNG75AF9hnJHO3xxFkf5NQCAAAAAAAA", + "AwEAAAAW607KPj2q3O8dF6XkfALiIrd9mqGir2UlYIcZuLNksTsvAgAAAAAAAA==", + "AQhAP4sMdbiWZPtV6jz8hYKzRFSgwaSlQKiGsQXogAmMcrLOl+SJfiCOXMTEZ2a1ebmQOEGkRYa30FaIlB46sLI2IPsBAAAAAAAA", + "AwwAAAAWUubmVhcix0ZXN0PKtrEndk0LxM+qpzp0PVtjf+xlrzz4TT0qA+hTtm6BLlYBAAAAAAAA", + "AQoAVWCdny7wv/M1LvZASC3Fw0D/NNhI1NYwch9Ux+KZ2qRdQXPC1rNsCGRJ7nd66SfcNmRUVVvQY6EYCbsIiugO6gwBAAAAAAAA", + "AAMAAAAgMjMDAAAApmWkWSBCL51Bfkhn79xPuKBKHz//H6B+mY6G9/eieuNtAAAAAAAAAA==", + "AAMAAAAgMjEDAAAAjSPPbIboNKeqbt7VTCbOK7LnSQNTjGG91dIZeZerL3JtAAAAAAAAAA==", + ][2..]); assert_eq!( result.values, [ @@ -147,6 +255,41 @@ fn test_view_state() { result.values, [StateItem { key: b"test123".to_vec(), value: b"123".to_vec(), proof: vec![] }] ); + assert_proof( + &result + .proof[2..], + &[ + "AwEAAAAQjHWWT6rXAXqUm14fjfDxo3286ApntHMI1eK0aQAJZPfJewEAAAAAAA==", + "AQcCSXBK8DHIYBF47dz6xB2iFKLLsPjAIAo9syJTBC0/Y1OjJNvT5izZukYCmtq/AyVTeyWFl1Ei6yFZBf5yIJ0i96eYRr8PVilJ81MgJKvV/R1SxQuTfwwmbZ6sN/TC2XfL1SCJ4WM1GZ0yMSaNpJOdsJH9kda203WM3Zh81gxz6rmVewEAAAAAAA==", + "AwMAAAAWFsbwm2TFX4GHLT5G1LSpF8UkG7zQV1ohXBMR/OQcUAKZ3gwDAAAAAAAA", + "ASAC7S1KwgLNl0HPdSo8soL8sGOmPhL7O0xTSR8sDDR5pZrzu0ty3UPYJ5UKrFGKxXoyyyNG75AF9hnJHO3xxFkf5NQCAAAAAAAA", + "AwEAAAAW607KPj2q3O8dF6XkfALiIrd9mqGir2UlYIcZuLNksTsvAgAAAAAAAA==", + "AQhAP4sMdbiWZPtV6jz8hYKzRFSgwaSlQKiGsQXogAmMcrLOl+SJfiCOXMTEZ2a1ebmQOEGkRYa30FaIlB46sLI2IPsBAAAAAAAA", + "AwwAAAAWUubmVhcix0ZXN0PKtrEndk0LxM+qpzp0PVtjf+xlrzz4TT0qA+hTtm6BLlYBAAAAAAAA", + "AQoAVWCdny7wv/M1LvZASC3Fw0D/NNhI1NYwch9Ux+KZ2qRdQXPC1rNsCGRJ7nd66SfcNmRUVVvQY6EYCbsIiugO6gwBAAAAAAAA", + "AAMAAAAgMjMDAAAApmWkWSBCL51Bfkhn79xPuKBKHz//H6B+mY6G9/eieuNtAAAAAAAAAA==", + ][2..] + ); + + let root = state_update.get_root(); + let account = alice_account(); + let proof_verifier = + ProofVerifier::new(result.proof).expect("could not create a ProofVerifier"); + for (want, key, value) in [ + (true, b"test123".as_ref(), Some(b"123".as_ref())), + (false, b"test123".as_ref(), Some(b"321".as_ref())), + (false, b"test123".as_ref(), Some(b"1234".as_ref())), + (false, b"test123".as_ref(), None), + // Shorter key: + (false, b"test12".as_ref(), Some(b"123".as_ref())), + (true, b"test12", None), + // Longer key: + (false, b"test1234", Some(b"123")), + (true, b"test1234", None), + ] { + let got = proof_verifier.verify(root, &account, key, value); + assert_eq!(want, got, "key: {key:x?}; value: {value:x?}"); + } } #[test] diff --git a/runtime/runtime/src/state_viewer/mod.rs b/runtime/runtime/src/state_viewer/mod.rs index 4ba0d216407..b71fbff8d9a 100644 --- a/runtime/runtime/src/state_viewer/mod.rs +++ b/runtime/runtime/src/state_viewer/mod.rs @@ -143,8 +143,9 @@ impl TrieViewer { let query = trie_key_parsers::get_raw_prefix_for_contract_data(account_id, prefix); let acc_sep_len = query.len() - prefix.len(); let mut iter = state_update.trie().iter()?; + iter.remember_visited_nodes(true); iter.seek_prefix(&query)?; - for item in iter { + for item in &mut iter { let (key, value) = item?; values.push(StateItem { key: key[acc_sep_len..].to_vec(), @@ -152,8 +153,8 @@ impl TrieViewer { proof: vec![], }); } - // TODO(2076): Add proofs for the storage items. - Ok(ViewStateResult { values, proof: vec![] }) + let proof = iter.into_visited_nodes(); + Ok(ViewStateResult { values, proof }) } pub fn call_function(