From 164979573526b0da4d1d061e71be1ba2e449f68c Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Mon, 27 Feb 2023 13:31:05 +0000 Subject: [PATCH 1/2] Test the recent orchestra changes --- Cargo.lock | 45 ++++++++++++++++++++------------- node/core/pvf/Cargo.toml | 2 +- node/overseer/Cargo.toml | 4 +-- node/overseer/src/lib.rs | 5 ++-- node/overseer/src/tests.rs | 1 + node/subsystem-types/Cargo.toml | 2 +- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3950cd86a827..6f871aaa63c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5055,9 +5055,8 @@ dependencies = [ [[package]] name = "orchestra" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e7d5b6bb115db09390bed8842c94180893dd83df3dfce7354f2a2aa090a4ee" +version = "0.0.7" +source = "git+https://github.com/paritytech/orchestra?branch=master#58a6a595aea5d62f115cdae380aeaddeadbce325" dependencies = [ "async-trait", "dyn-clonable", @@ -5065,16 +5064,15 @@ dependencies = [ "futures-timer", "orchestra-proc-macro", "pin-project", - "prioritized-metered-channel", + "prioritized-metered-channel 0.2.0 (git+https://github.com/paritytech/orchestra?branch=master)", "thiserror", "tracing", ] [[package]] name = "orchestra-proc-macro" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2af4dabb2286b0be0e9711d2d24e25f6217048b71210cffd3daddc3b5c84e1f" +version = "0.0.7" +source = "git+https://github.com/paritytech/orchestra?branch=master#58a6a595aea5d62f115cdae380aeaddeadbce325" dependencies = [ "expander 0.0.6", "itertools", @@ -7128,7 +7126,7 @@ dependencies = [ "parity-scale-codec", "polkadot-primitives", "polkadot-test-service", - "prioritized-metered-channel", + "prioritized-metered-channel 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "prometheus-parse", "sc-cli", "sc-client-api", @@ -7268,7 +7266,7 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", - "prioritized-metered-channel", + "prioritized-metered-channel 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.8.5", "sp-application-crypto", "sp-core", @@ -7296,7 +7294,7 @@ dependencies = [ "polkadot-node-subsystem-types", "polkadot-primitives", "polkadot-primitives-test-helpers", - "prioritized-metered-channel", + "prioritized-metered-channel 0.2.0 (git+https://github.com/paritytech/orchestra?branch=master)", "sc-client-api", "sp-api", "sp-core", @@ -8113,6 +8111,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "prioritized-metered-channel" +version = "0.2.0" +source = "git+https://github.com/paritytech/orchestra?branch=master#58a6a595aea5d62f115cdae380aeaddeadbce325" +dependencies = [ + "async-channel", + "coarsetime", + "crossbeam-queue", + "derive_more", + "futures", + "futures-timer", + "nanorand", + "thiserror", + "tracing", +] + [[package]] name = "proc-macro-crate" version = "1.1.3" @@ -8436,26 +8450,23 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "autocfg", - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.9.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "lazy_static", "num_cpus", ] diff --git a/node/core/pvf/Cargo.toml b/node/core/pvf/Cargo.toml index a810c3887c96..11301d4ec3ee 100644 --- a/node/core/pvf/Cargo.toml +++ b/node/core/pvf/Cargo.toml @@ -17,7 +17,7 @@ futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } pin-project = "1.0.9" rand = "0.8.5" -rayon = "1.5.1" +rayon = "1.6.1" slotmap = "1.0" tempfile = "3.3.0" tikv-jemalloc-ctl = { version = "0.5.0", optional = true } diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 5d26c0b6e2bf..8e652941f2d1 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -15,7 +15,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-metrics = { path = "../metrics" } polkadot-primitives = { path = "../../primitives" } -orchestra = "0.0.4" +orchestra = { git = "https://github.com/paritytech/orchestra", branch = "master" } gum = { package = "tracing-gum", path = "../gum" } lru = "0.9" sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -23,7 +23,7 @@ async-trait = "0.1.57" tikv-jemalloc-ctl = { version = "0.5.0", optional = true } [dev-dependencies] -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", git = "https://github.com/paritytech/orchestra", branch = "master" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = { version = "0.3.21", features = ["thread-pool"] } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 810f08af4857..f72426b1cb64 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -661,8 +661,9 @@ where ); metrics.memory_stats_snapshot(memory_stats_snapshot); }, - Err(e) => - gum::debug!(target: LOG_TARGET, "Failed to obtain memory stats: {:?}", e), + Err(e) => { + gum::debug!(target: LOG_TARGET, "Failed to obtain memory stats: {:?}", e) + }, }), Err(_) => { gum::debug!( diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index bc26402aedea..f461196606e1 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -1034,6 +1034,7 @@ fn overseer_all_subsystems_receive_signals_and_messages() { #[test] fn context_holds_onto_message_until_enough_signals_received() { + const CHANNEL_CAPACITY: usize = 1024; let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); diff --git a/node/subsystem-types/Cargo.toml b/node/subsystem-types/Cargo.toml index 22528503ccc4..17c8523bc338 100644 --- a/node/subsystem-types/Cargo.toml +++ b/node/subsystem-types/Cargo.toml @@ -13,7 +13,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } -orchestra = "0.0.4" +orchestra = { git = "https://github.com/paritytech/orchestra", branch = "master" } sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" } From 340fe3cf2e8c482e6eb562c9c3c70864738c7d63 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Fri, 10 Mar 2023 11:51:52 +0000 Subject: [PATCH 2/2] Update orchestra and metered-channel --- Cargo.lock | 45 ++++++++++++--------------------- node/metrics/Cargo.toml | 2 +- node/overseer/Cargo.toml | 4 +-- node/subsystem-types/Cargo.toml | 2 +- node/subsystem-util/Cargo.toml | 2 +- 5 files changed, 21 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f871aaa63c7..d7dbed688b3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5055,8 +5055,9 @@ dependencies = [ [[package]] name = "orchestra" -version = "0.0.7" -source = "git+https://github.com/paritytech/orchestra?branch=master#58a6a595aea5d62f115cdae380aeaddeadbce325" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecae97f74ddeb5c6ff00c98ef368423b4ec9a4d5ab48c0d909e84b7594b09718" dependencies = [ "async-trait", "dyn-clonable", @@ -5064,15 +5065,16 @@ dependencies = [ "futures-timer", "orchestra-proc-macro", "pin-project", - "prioritized-metered-channel 0.2.0 (git+https://github.com/paritytech/orchestra?branch=master)", + "prioritized-metered-channel", "thiserror", "tracing", ] [[package]] name = "orchestra-proc-macro" -version = "0.0.7" -source = "git+https://github.com/paritytech/orchestra?branch=master#58a6a595aea5d62f115cdae380aeaddeadbce325" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a9846c70b1df02657dbe5b54fadbad0ea3b944623ebf3fcbf7f33ef859dcba" dependencies = [ "expander 0.0.6", "itertools", @@ -7126,7 +7128,7 @@ dependencies = [ "parity-scale-codec", "polkadot-primitives", "polkadot-test-service", - "prioritized-metered-channel 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "prioritized-metered-channel", "prometheus-parse", "sc-cli", "sc-client-api", @@ -7266,7 +7268,7 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", - "prioritized-metered-channel 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "prioritized-metered-channel", "rand 0.8.5", "sp-application-crypto", "sp-core", @@ -7294,7 +7296,7 @@ dependencies = [ "polkadot-node-subsystem-types", "polkadot-primitives", "polkadot-primitives-test-helpers", - "prioritized-metered-channel 0.2.0 (git+https://github.com/paritytech/orchestra?branch=master)", + "prioritized-metered-channel", "sc-client-api", "sp-api", "sp-core", @@ -8097,24 +8099,9 @@ dependencies = [ [[package]] name = "prioritized-metered-channel" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382698e48a268c832d0b181ed438374a6bb708a82a8ca273bb0f61c74cf209c4" -dependencies = [ - "coarsetime", - "crossbeam-queue", - "derive_more", - "futures", - "futures-timer", - "nanorand", - "thiserror", - "tracing", -] - -[[package]] -name = "prioritized-metered-channel" -version = "0.2.0" -source = "git+https://github.com/paritytech/orchestra?branch=master#58a6a595aea5d62f115cdae380aeaddeadbce325" +checksum = "a3caef72a78ca8e77cbdfa87dd516ebb79d4cbe5b42e3b8435b463a8261339ff" dependencies = [ "async-channel", "coarsetime", @@ -8450,9 +8437,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -8460,9 +8447,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", diff --git a/node/metrics/Cargo.toml b/node/metrics/Cargo.toml index 8a6938190887..63864c15a24d 100644 --- a/node/metrics/Cargo.toml +++ b/node/metrics/Cargo.toml @@ -10,7 +10,7 @@ futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../gum" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.4.0" } # Both `sc-service` and `sc-cli` are required by runtime metrics `logger_hook()`. sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 8e652941f2d1..7f668ec20ad6 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -15,7 +15,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-metrics = { path = "../metrics" } polkadot-primitives = { path = "../../primitives" } -orchestra = { git = "https://github.com/paritytech/orchestra", branch = "master" } +orchestra = "0.2.0" gum = { package = "tracing-gum", path = "../gum" } lru = "0.9" sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -23,7 +23,7 @@ async-trait = "0.1.57" tikv-jemalloc-ctl = { version = "0.5.0", optional = true } [dev-dependencies] -metered = { package = "prioritized-metered-channel", git = "https://github.com/paritytech/orchestra", branch = "master" } +metered = { package = "prioritized-metered-channel", version = "0.4.0" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = { version = "0.3.21", features = ["thread-pool"] } diff --git a/node/subsystem-types/Cargo.toml b/node/subsystem-types/Cargo.toml index 17c8523bc338..9ed095622cb5 100644 --- a/node/subsystem-types/Cargo.toml +++ b/node/subsystem-types/Cargo.toml @@ -13,7 +13,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } -orchestra = { git = "https://github.com/paritytech/orchestra", branch = "master" } +orchestra = "0.2.0" sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/subsystem-util/Cargo.toml b/node/subsystem-util/Cargo.toml index 9fd4af14cde1..20314afed06f 100644 --- a/node/subsystem-util/Cargo.toml +++ b/node/subsystem-util/Cargo.toml @@ -27,7 +27,7 @@ polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } polkadot-node-primitives = { path = "../primitives" } polkadot-overseer = { path = "../overseer" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.4.0" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }