Skip to content

Commit

Permalink
Mmr client gadget - support pallet reset (paritytech#12999)
Browse files Browse the repository at this point in the history 
* Remove unneeded code

* Moving some code

* Support pallet-mmr reset

* Rename update_first_mmr_block

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* Renamings

Co-authored-by: Adrian Catangiu <adrian@parity.io>
  • Loading branch information
@serban300 @acatangiu @ltfschoen
2 people authored and ltfschoen committed Feb 22, 2023
1 parent 2311184 commit 24b9997
Show file tree
Hide file tree
Showing 5 changed files with 270 additions and 148 deletions.
62 changes: 36 additions & 26 deletions client/merkle-mountain-range/src/aux_schema.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<
}
}

/// Load or initialize persistent data from backend.
pub(crate) fn load_persistent<B, BE>(backend: &BE) -> ClientResult<Option<PersistedState<B>>>
/// Load persistent data from backend.
pub(crate) fn load_state<B, BE>(backend: &BE) -> ClientResult<Option<PersistedState<B>>>
where
B: Block,
BE: AuxStore,
Expand All @@ -73,15 +73,36 @@ where
Ok(None)
}

/// Load or initialize persistent data from backend.
pub(crate) fn load_or_init_state<B, BE>(
backend: &BE,
default: NumberFor<B>,
) -> sp_blockchain::Result<NumberFor<B>>
where
B: Block,
BE: AuxStore,
{
// Initialize gadget best_canon from AUX DB or from pallet genesis.
if let Some(best) = load_state::<B, BE>(backend)? {
info!(target: LOG_TARGET, "Loading MMR best canonicalized state from db: {:?}.", best);
Ok(best)
} else {
info!(
target: LOG_TARGET,
"Loading MMR from pallet genesis on what appears to be the first startup: {:?}.",
default
);
write_current_version(backend)?;
write_gadget_state::<B, BE>(backend, &default)?;
Ok(default)
}
}

#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::test_utils::{
run_test_with_mmr_gadget_pre_post_using_client, MmrBlock, MockClient, OffchainKeyType,
};
use crate::test_utils::{run_test_with_mmr_gadget_pre_post_using_client, MmrBlock, MockClient};
use parking_lot::Mutex;
use sp_core::offchain::{DbExternalities, StorageKind};
use sp_mmr_primitives::utils::NodesUtils;
use sp_runtime::generic::BlockId;
use std::{sync::Arc, time::Duration};
use substrate_test_runtime_client::{runtime::Block, Backend};
Expand All @@ -92,15 +113,15 @@ pub(crate) mod tests {
let backend = &*client.backend;

// version not available in db -> None
assert_eq!(load_persistent::<Block, Backend>(backend).unwrap(), None);
assert_eq!(load_state::<Block, Backend>(backend).unwrap(), None);

// populate version in db
write_current_version(backend).unwrap();
// verify correct version is retrieved
assert_eq!(load_decode(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));

// version is available in db but state isn't -> None
assert_eq!(load_persistent::<Block, Backend>(backend).unwrap(), None);
assert_eq!(load_state::<Block, Backend>(backend).unwrap(), None);
}

#[test]
Expand All @@ -113,7 +134,7 @@ pub(crate) mod tests {
// version not available in db -> None
assert_eq!(load_decode::<Backend, Option<u32>>(&*backend, VERSION_KEY).unwrap(), None);
// state not available in db -> None
assert_eq!(load_persistent::<Block, Backend>(&*backend).unwrap(), None);
assert_eq!(load_state::<Block, Backend>(&*backend).unwrap(), None);
// run the gadget while importing and finalizing 3 blocks
run_test_with_mmr_gadget_pre_post_using_client(
client.clone(),
Expand All @@ -136,7 +157,7 @@ pub(crate) mod tests {
let backend = &*client.backend;
// check there is both version and best canon available in db before running gadget
assert_eq!(load_decode(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
assert_eq!(load_persistent::<Block, Backend>(backend).unwrap(), Some(3));
assert_eq!(load_state::<Block, Backend>(backend).unwrap(), Some(3));
},
|client| async move {
let a4 = client.import_block(&BlockId::Number(3), b"a4", Some(3)).await;
Expand All @@ -148,7 +169,7 @@ pub(crate) mod tests {
// a4, a5, a6 were canonicalized
client.assert_canonicalized(&[&a4, &a5, &a6]);
// check persisted best canon was updated
assert_eq!(load_persistent::<Block, Backend>(&*client.backend).unwrap(), Some(6));
assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(6));
},
);
}
Expand Down Expand Up @@ -177,19 +198,8 @@ pub(crate) mod tests {
client.assert_canonicalized(&slice);

// now manually move them back to non-canon/temp location
let mut offchain_db = client.offchain_db();
for mmr_block in slice {
for node in NodesUtils::right_branch_ending_in_leaf(mmr_block.leaf_idx.unwrap())
{
let canon_key = mmr_block.get_offchain_key(node, OffchainKeyType::Canon);
let val = offchain_db
.local_storage_get(StorageKind::PERSISTENT, &canon_key)
.unwrap();
offchain_db.local_storage_clear(StorageKind::PERSISTENT, &canon_key);

let temp_key = mmr_block.get_offchain_key(node, OffchainKeyType::Temp);
offchain_db.local_storage_set(StorageKind::PERSISTENT, &temp_key, &val);
}
client.undo_block_canonicalization(mmr_block)
}
},
);
Expand All @@ -203,7 +213,7 @@ pub(crate) mod tests {
let slice: Vec<&MmrBlock> = blocks.iter().collect();

// verify persisted state says a1, a2, a3 were canonicalized,
assert_eq!(load_persistent::<Block, Backend>(&*client.backend).unwrap(), Some(3));
assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(3));
// but actually they are NOT canon (we manually reverted them earlier).
client.assert_not_canonicalized(&slice);
},
Expand All @@ -221,7 +231,7 @@ pub(crate) mod tests {
// but a4, a5, a6 were canonicalized
client.assert_canonicalized(&[&a4, &a5, &a6]);
// check persisted best canon was updated
assert_eq!(load_persistent::<Block, Backend>(&*client.backend).unwrap(), Some(6));
assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(6));
},
);
}
Expand Down
137 changes: 74 additions & 63 deletions client/merkle-mountain-range/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ use crate::offchain_mmr::OffchainMmr;
use beefy_primitives::MmrRootHash;
use futures::StreamExt;
use log::{debug, error, trace, warn};
use sc_client_api::{Backend, BlockchainEvents, FinalityNotifications};
use sc_client_api::{Backend, BlockchainEvents, FinalityNotification, FinalityNotifications};
use sc_offchain::OffchainDb;
use sp_api::ProvideRuntimeApi;
use sp_blockchain::{HeaderBackend, HeaderMetadata};
Expand All @@ -60,6 +60,62 @@ use std::{marker::PhantomData, sync::Arc};
/// Logging target for the mmr gadget.
pub const LOG_TARGET: &str = "mmr";

/// A convenience MMR client trait that defines all the type bounds a MMR client
/// has to satisfy and defines some helper methods.
pub trait MmrClient<B, BE>:
BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>
where
B: Block,
BE: Backend<B>,
Self::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
/// Get the block number where the mmr pallet was added to the runtime.
fn first_mmr_block_num(&self, notification: &FinalityNotification<B>) -> Option<NumberFor<B>> {
let best_block = *notification.header.number();
match self.runtime_api().mmr_leaf_count(&BlockId::number(best_block)) {
Ok(Ok(mmr_leaf_count)) => {
match utils::first_mmr_block_num::<B::Header>(best_block, mmr_leaf_count) {
Ok(first_mmr_block) => {
debug!(
target: LOG_TARGET,
"pallet-mmr detected at block {:?} with genesis at block {:?}",
best_block,
first_mmr_block
);
Some(first_mmr_block)
},
Err(e) => {
error!(
target: LOG_TARGET,
"Error calculating the first mmr block: {:?}", e
);
None
},
}
},
_ => {
trace!(
target: LOG_TARGET,
"pallet-mmr not detected at block {:?} ... (best finalized {:?})",
best_block,
notification.header.number()
);
None
},
}
}
}

impl<B, BE, T> MmrClient<B, BE> for T
where
B: Block,
BE: Backend<B>,
T: BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>,
T::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
// empty
}

struct OffchainMmrBuilder<B: Block, BE: Backend<B>, C> {
backend: Arc<BE>,
client: Arc<C>,
Expand All @@ -73,74 +129,29 @@ impl<B, BE, C> OffchainMmrBuilder<B, BE, C>
where
B: Block,
BE: Backend<B>,
C: ProvideRuntimeApi<B> + HeaderBackend<B> + HeaderMetadata<B>,
C: MmrClient<B, BE>,
C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
async fn try_build(
self,
finality_notifications: &mut FinalityNotifications<B>,
) -> Option<OffchainMmr<B, BE, C>> {
while let Some(notification) = finality_notifications.next().await {
let best_block = *notification.header.number();
match self.client.runtime_api().mmr_leaf_count(&BlockId::number(best_block)) {
Ok(Ok(mmr_leaf_count)) => {
debug!(
target: LOG_TARGET,
"pallet-mmr detected at block {:?} with mmr size {:?}",
best_block,
mmr_leaf_count
);
match utils::first_mmr_block_num::<B::Header>(best_block, mmr_leaf_count) {
Ok(first_mmr_block) => {
debug!(
target: LOG_TARGET,
"pallet-mmr genesis computed at block {:?}", first_mmr_block,
);
let best_canonicalized =
match offchain_mmr::load_or_init_best_canonicalized::<B, BE>(
&*self.backend,
first_mmr_block,
) {
Ok(best) => best,
Err(e) => {
error!(
target: LOG_TARGET,
"Error loading state from aux db: {:?}", e
);
return None
},
};
let mut offchain_mmr = OffchainMmr {
backend: self.backend,
client: self.client,
offchain_db: self.offchain_db,
indexing_prefix: self.indexing_prefix,
first_mmr_block,
best_canonicalized,
};
// We need to make sure all blocks leading up to current notification
// have also been canonicalized.
offchain_mmr.canonicalize_catch_up(&notification);
// We have to canonicalize and prune the blocks in the finality
// notification that lead to building the offchain-mmr as well.
offchain_mmr.canonicalize_and_prune(notification);
return Some(offchain_mmr)
},
Err(e) => {
error!(
target: LOG_TARGET,
"Error calculating the first mmr block: {:?}", e
);
},
}
},
_ => {
trace!(
target: LOG_TARGET,
"Waiting for MMR pallet to become available... (best finalized {:?})",
notification.header.number()
);
},
if let Some(first_mmr_block_num) = self.client.first_mmr_block_num(&notification) {
let mut offchain_mmr = OffchainMmr::new(
self.backend,
self.client,
self.offchain_db,
self.indexing_prefix,
first_mmr_block_num,
)?;
// We need to make sure all blocks leading up to current notification
// have also been canonicalized.
offchain_mmr.canonicalize_catch_up(&notification);
// We have to canonicalize and prune the blocks in the finality
// notification that lead to building the offchain-mmr as well.
offchain_mmr.canonicalize_and_prune(notification);
return Some(offchain_mmr)
}
}

Expand All @@ -165,7 +176,7 @@ where
B: Block,
<B::Header as Header>::Number: Into<LeafIndex>,
BE: Backend<B>,
C: BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>,
C: MmrClient<B, BE>,
C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
{
async fn run(mut self, builder: OffchainMmrBuilder<B, BE, C>) {
Expand Down
Loading

0 comments on commit 24b9997

Please sign in to comment.