Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Create Benchmarking Setup for Identity Pallet #4695 #4818

Merged
merged 62 commits into from
Feb 10, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
7b33492
Starting
shawntabrizi Jan 16, 2020
3b12740
closer
shawntabrizi Jan 21, 2020
55cf8b6
Compiles!
shawntabrizi Jan 21, 2020
9da52bb
comments
shawntabrizi Jan 21, 2020
27127c0
Create seperate mock
shawntabrizi Jan 21, 2020
ae84e9f
Remove changes to test env
shawntabrizi Jan 21, 2020
06d1173
Fix step calculation
shawntabrizi Jan 21, 2020
9f25cf5
Add host function
shawntabrizi Jan 22, 2020
621cc0d
Merge remote-tracking branch 'upstream/master' into shawntabrizi-iden…
shawntabrizi Jan 22, 2020
ddff026
Add runtime api
shawntabrizi Jan 22, 2020
b32c805
Merge remote-tracking branch 'upstream/master' into shawntabrizi-iden…
shawntabrizi Jan 22, 2020
890ad0d
compiles
shawntabrizi Jan 22, 2020
0a1bcfc
Update to use offchain timestamp
shawntabrizi Jan 23, 2020
931702b
Gives a result
shawntabrizi Jan 27, 2020
e8c4992
added some CLI wip
shawntabrizi Jan 28, 2020
334d1e6
Merge branch 'master' of github.com:paritytech/substrate into shawnta…
arkpar Jan 28, 2020
e5657e9
make generic
shawntabrizi Jan 28, 2020
32e9e07
Update instance
shawntabrizi Jan 28, 2020
1e2630c
Remove CLI stuff
shawntabrizi Jan 28, 2020
bafbac0
Remove last cli stuff
shawntabrizi Jan 28, 2020
86b32ef
undo more changes
shawntabrizi Jan 28, 2020
dd87c93
Update benchmarks
shawntabrizi Jan 29, 2020
6814c1a
Merge remote-tracking branch 'upstream/master' into shawntabrizi-iden…
shawntabrizi Jan 29, 2020
7c089f7
Update Cargo.lock
shawntabrizi Jan 29, 2020
7bd0ae1
remove test
shawntabrizi Jan 29, 2020
862dc67
Move loop out of runtime
shawntabrizi Jan 30, 2020
6d0c8b4
Benchmarking externalities
arkpar Feb 3, 2020
1575aae
Merge branch 'shawntabrizi-identity-bench' of github.com:paritytech/s…
arkpar Feb 3, 2020
3c5dcf7
Benchmarking state
arkpar Feb 3, 2020
d7c0ae8
Implemented commit
arkpar Feb 3, 2020
38602b8
Make CLI work, move loop back into runtime
shawntabrizi Feb 4, 2020
9614ea7
Merge branch 'master' of github.com:paritytech/substrate into shawnta…
arkpar Feb 4, 2020
e6eeab8
Merge branch 'shawntabrizi-identity-bench' of github.com:paritytech/s…
arkpar Feb 4, 2020
5ee888f
Wipe resets to genesis
arkpar Feb 4, 2020
e0b48d2
Merge branch 'master' of github.com:paritytech/substrate into shawnta…
arkpar Feb 5, 2020
d73ca25
Speedup benchmarks
arkpar Feb 5, 2020
879826c
Use enum to select extrinsic within pallet
shawntabrizi Feb 5, 2020
e041347
CLI controls which module and extrinsic to call
shawntabrizi Feb 5, 2020
115290b
Select a pallet with cli
shawntabrizi Feb 5, 2020
283860c
Add steps and repeats to cli
shawntabrizi Feb 5, 2020
978a1b4
Merge branch 'shawntabrizi-identity-bench' of https://github.com/pari…
shawntabrizi Feb 6, 2020
ca890f6
Output as CSV format
shawntabrizi Feb 6, 2020
1cb8529
Introduce benchmark pallet
shawntabrizi Feb 6, 2020
1c97bd9
Append bench
shawntabrizi Feb 7, 2020
ea5c3e1
Use Results
shawntabrizi Feb 7, 2020
94bff4b
Merge remote-tracking branch 'upstream/master' into shawntabrizi-iden…
shawntabrizi Feb 7, 2020
55a2e85
fix merge
shawntabrizi Feb 7, 2020
6787808
Clear Identity benchmark
shawntabrizi Feb 7, 2020
4c2b218
Bench request judgment and cancel request
shawntabrizi Feb 8, 2020
833c377
Add final benchmarks
shawntabrizi Feb 8, 2020
3e27b42
Fix CSV output
shawntabrizi Feb 8, 2020
34838f6
Start cleaning up for PR
shawntabrizi Feb 8, 2020
93422d4
Bump numbers in `wasmtime` integration tests.
shawntabrizi Feb 8, 2020
6bcf639
Merge remote-tracking branch 'upstream/master' into shawntabrizi-iden…
shawntabrizi Feb 8, 2020
f73ca4a
More docs
shawntabrizi Feb 8, 2020
eecaac0
Add rockdb feature to bench
shawntabrizi Feb 8, 2020
6243911
Fix formatting issues
shawntabrizi Feb 8, 2020
09804e1
Add test feature to bench
shawntabrizi Feb 9, 2020
00809ad
Add test feature to bench
shawntabrizi Feb 9, 2020
8d509d0
Add rocksdb feature flag
shawntabrizi Feb 9, 2020
fe2d0a2
Update bench.rs
gavofyork Feb 9, 2020
91827e5
Merge branch 'master' into shawntabrizi-identity-bench
gavofyork Feb 10, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 23 additions & 2 deletions bin/node/runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,14 @@ use sp_core::u32_trait::{_1, _2, _3, _4};
use node_primitives::{AccountId, AccountIndex, Balance, BlockNumber, Hash, Index, Moment, Signature};
use sp_api::impl_runtime_apis;
use sp_runtime::{
Permill, Perbill, Percent, ApplyExtrinsicResult, impl_opaque_keys, generic, create_runtime_str
Permill, Perbill, Percent, ApplyExtrinsicResult, impl_opaque_keys, generic, create_runtime_str,
BenchmarkResults,
};
use sp_runtime::curve::PiecewiseLinear;
use sp_runtime::transaction_validity::TransactionValidity;
use sp_runtime::traits::{
self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, ConvertInto, OpaqueKeys,
self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion,
ConvertInto, OpaqueKeys, Benchmarking,
};
use sp_version::RuntimeVersion;
#[cfg(any(feature = "std", test))]
Expand Down Expand Up @@ -804,6 +806,25 @@ impl_runtime_apis! {
SessionKeys::decode_into_raw_public_keys(&encoded)
}
}

impl crate::Benchmark<Block> for Runtime {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should I hide the entire benchmarking pipeline behind a feature flag?

This means by default we should:

  • Not expose a runtime API
  • Not have CLI
  • Not have benchmarking code in our runtime wasm

Is it possible?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think so; we don't really want benchmarking code making it on to the chain.

fn dispatch_benchmark(module: Vec<u8>, extrinsic: Vec<u8>, steps: u32, repeat: u32)
-> Option<Vec<BenchmarkResults>>
{
match module.as_slice() {
b"pallet-identity" | b"identity" => Identity::run_benchmark(extrinsic, steps, repeat).ok(),
_ => return None,
}
}
}
}

sp_api::decl_runtime_apis! {
pub trait Benchmark
{
fn dispatch_benchmark(module: Vec<u8>, extrinsic: Vec<u8>, steps: u32, repeat: u32)
-> Option<Vec<BenchmarkResults>>;
}
}

#[cfg(test)]
Expand Down
1 change: 1 addition & 0 deletions client/cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ use params::{
pub use params::{
SharedParams, ImportParams, ExecutionStrategy, Subcommand, RunCmd, BuildSpecCmd,
ExportBlocksCmd, ImportBlocksCmd, CheckBlockCmd, PurgeChainCmd, RevertCmd,
BenchmarkCmd,
};
pub use traits::GetSharedParams;
use app_dirs::{AppInfo, AppDataType};
Expand Down
77 changes: 77 additions & 0 deletions client/cli/src/params.rs
Original file line number Diff line number Diff line change
Expand Up @@ -845,6 +845,49 @@ pub struct PurgeChainCmd {
pub shared_params: SharedParams,
}

/// The `benchmark` command used to benchmark FRAME Pallets.
#[derive(Debug, StructOpt, Clone)]
pub struct BenchmarkCmd {
/// Select a FRAME Pallet to benchmark.
#[structopt(short, long)]
pub pallet: String,

/// Select an extrinsic to benchmark.
#[structopt(short, long)]
pub extrinsic: String,

/// Select how many samples we should take across the variable components.
#[structopt(short, long, default_value = "1")]
pub steps: u32,

/// Select how many repetitions of this benchmark should run.
#[structopt(short, long, default_value = "1")]
pub repeat: u32,

#[allow(missing_docs)]
#[structopt(flatten)]
pub shared_params: SharedParams,

/// The execution strategy that should be used for benchmarks
#[structopt(
long = "execution",
value_name = "STRATEGY",
possible_values = &ExecutionStrategy::variants(),
case_insensitive = true,
)]
pub execution: Option<ExecutionStrategy>,

/// Method for executing Wasm runtime code.
#[structopt(
long = "wasm-execution",
value_name = "METHOD",
possible_values = &WasmExecutionMethod::enabled_variants(),
case_insensitive = true,
default_value = "Interpreted"
)]
pub wasm_method: WasmExecutionMethod,
}

/// All core commands that are provided by default.
///
/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From
Expand All @@ -869,6 +912,9 @@ pub enum Subcommand {

/// Remove the whole chain data.
PurgeChain(PurgeChainCmd),

/// Run runtime benchmarks.
Benchmark(BenchmarkCmd),
}

impl Subcommand {
Expand All @@ -883,6 +929,7 @@ impl Subcommand {
CheckBlock(params) => &params.shared_params,
Revert(params) => &params.shared_params,
PurgeChain(params) => &params.shared_params,
Benchmark(params) => &params.shared_params,
}
}

Expand All @@ -909,6 +956,7 @@ impl Subcommand {
Subcommand::ImportBlocks(cmd) => cmd.run(config, builder),
Subcommand::CheckBlock(cmd) => cmd.run(config, builder),
Subcommand::PurgeChain(cmd) => cmd.run(config),
Subcommand::Benchmark(cmd) => cmd.run(config, builder),
Subcommand::Revert(cmd) => cmd.run(config, builder),
}
}
Expand Down Expand Up @@ -1186,3 +1234,32 @@ impl RevertCmd {
Ok(())
}
}

impl BenchmarkCmd {
/// Runs the command and benchmarks the chain.
pub fn run<G, E, B, BC, BB>(
self,
config: Configuration<G, E>,
_builder: B,
) -> error::Result<()>
where
B: FnOnce(Configuration<G, E>) -> Result<BC, sc_service::error::Error>,
G: RuntimeGenesis,
E: ChainSpecExtension,
BC: ServiceBuilderCommand<Block = BB> + Unpin,
BB: sp_runtime::traits::Block + Debug,
<<<BB as BlockT>::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug,
<BB as BlockT>::Hash: std::str::FromStr,
{
let spec = config.chain_spec.expect("chain_spec is always Some");
let execution_strategy = self.execution.unwrap_or(ExecutionStrategy::Native).into();
let wasm_method = self.wasm_method.into();
let pallet = self.pallet;
let extrinsic = self.extrinsic;
let steps = self.steps;
let repeat = self.repeat;
sc_service::chain_ops::benchmark_runtime::<BB, BC::NativeDispatch, _, _>(spec, execution_strategy, wasm_method, pallet, extrinsic, steps, repeat)?;
Ok(())
}
}

1 change: 1 addition & 0 deletions client/db/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ license = "GPL-3.0"
[dependencies]
parking_lot = "0.10.0"
log = "0.4.8"
rand = "0.7"
kvdb = "0.4.0"
kvdb-rocksdb = { version = "0.5", optional = true }
kvdb-memorydb = "0.4.0"
Expand Down
Loading