Move transactions protocol to its own crate (#12264)

* Move transaction protocol to its own crate

* Update Cargo.lock

* Fix binaries

* Update client/network/transactions/src/lib.rs

Co-authored-by: Dmitry Markin <dmitry@markin.tech>

* Update client/service/src/builder.rs

Co-authored-by: Bastian Köcher <info@kchr.de>

* Apply review comments

* Revert one change and apply cargo-fmt

* Remove Transaction from Message

* Add array-bytes

* trigger CI

* Add comment about codec index

Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Bastian Köcher <info@kchr.de>

bin/node-template/node/src/service.rs

@@ -191,7 +191,7 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 		Vec::default(),
 	));
 
-	let (network, system_rpc_tx, network_starter) =
+	let (network, system_rpc_tx, tx_handler_controller, network_starter) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
 			config: &config,
 			client: client.clone(),
@@ -238,6 +238,7 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 		rpc_builder: rpc_extensions_builder,
 		backend,
 		system_rpc_tx,
+		tx_handler_controller,
 		config,
 		telemetry: telemetry.as_mut(),
 	})?;

bin/node/cli/src/service.rs

@@ -354,7 +354,7 @@ pub fn new_full_base(
 		Vec::default(),
 	));
 
-	let (network, system_rpc_tx, network_starter) =
+	let (network, system_rpc_tx, tx_handler_controller, network_starter) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
 			config: &config,
 			client: client.clone(),
@@ -392,6 +392,7 @@ pub fn new_full_base(
 		transaction_pool: transaction_pool.clone(),
 		task_manager: &mut task_manager,
 		system_rpc_tx,
+		tx_handler_controller,
 		telemetry: telemetry.as_mut(),
 	})?;
 

client/beefy/Cargo.toml

@@ -27,6 +27,7 @@ sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" }
 sc-finality-grandpa = { version = "0.10.0-dev", path = "../../client/finality-grandpa" }
 sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
 sc-network = { version = "0.10.0-dev", path = "../network" }
+sc-network-common = { version = "0.10.0-dev", path = "../network/common" }
 sc-network-gossip = { version = "0.10.0-dev", path = "../network-gossip" }
 sc-utils = { version = "4.0.0-dev", path = "../utils" }
 sp-api = { version = "4.0.0-dev", path = "../../primitives/api" }

client/beefy/src/lib.rs

@@ -83,8 +83,8 @@ pub(crate) mod beefy_protocol_name {
 /// For standard protocol name see [`beefy_protocol_name::standard_name`].
 pub fn beefy_peers_set_config(
 	protocol_name: ProtocolName,
-) -> sc_network::config::NonDefaultSetConfig {
-	let mut cfg = sc_network::config::NonDefaultSetConfig::new(protocol_name, 1024 * 1024);
+) -> sc_network_common::config::NonDefaultSetConfig {
+	let mut cfg = sc_network_common::config::NonDefaultSetConfig::new(protocol_name, 1024 * 1024);
 
 	cfg.allow_non_reserved(25, 25);
 	cfg.add_fallback_names(beefy_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect());

client/cli/Cargo.toml

@@ -34,6 +34,7 @@ sc-client-api = { version = "4.0.0-dev", path = "../api" }
 sc-client-db = { version = "0.10.0-dev", default-features = false, path = "../db" }
 sc-keystore = { version = "4.0.0-dev", path = "../keystore" }
 sc-network = { version = "0.10.0-dev", path = "../network" }
+sc-network-common = { version = "0.10.0-dev", path = "../network/common" }
 sc-service = { version = "0.10.0-dev", default-features = false, path = "../service" }
 sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" }
 sc-tracing = { version = "4.0.0-dev", path = "../tracing" }

client/cli/src/params/network_params.rs

@@ -19,11 +19,10 @@
 use crate::{arg_enums::SyncMode, params::node_key_params::NodeKeyParams};
 use clap::Args;
 use sc_network::{
-	config::{
-		NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, SetConfig, TransportConfig,
-	},
+	config::{NetworkConfiguration, NodeKeyConfig},
 	multiaddr::Protocol,
 };
+use sc_network_common::config::{NonReservedPeerMode, SetConfig, TransportConfig};
 use sc_service::{
 	config::{Multiaddr, MultiaddrWithPeerId},
 	ChainSpec, ChainType,

client/finality-grandpa/src/lib.rs

@@ -688,18 +688,18 @@ pub struct GrandpaParams<Block: BlockT, C, N, SC, VR> {
 /// For standard protocol name see [`crate::protocol_standard_name`].
 pub fn grandpa_peers_set_config(
 	protocol_name: ProtocolName,
-) -> sc_network::config::NonDefaultSetConfig {
+) -> sc_network_common::config::NonDefaultSetConfig {
 	use communication::grandpa_protocol_name;
-	sc_network::config::NonDefaultSetConfig {
+	sc_network_common::config::NonDefaultSetConfig {
 		notifications_protocol: protocol_name,
 		fallback_names: grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(),
 		// Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot.
 		max_notification_size: 1024 * 1024,
-		set_config: sc_network::config::SetConfig {
+		set_config: sc_network_common::config::SetConfig {
 			in_peers: 0,
 			out_peers: 0,
 			reserved_nodes: Vec::new(),
-			non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny,
+			non_reserved_mode: sc_network_common::config::NonReservedPeerMode::Deny,
 		},
 	}
 }

client/network/common/Cargo.toml

@@ -24,7 +24,10 @@ codec = { package = "parity-scale-codec", version = "3.0.0", features = [
 	"derive",
 ] }
 futures = "0.3.21"
+futures-timer = "3.0.2"
 libp2p = "0.46.1"
+linked_hash_set = "0.1.3"
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../../utils/prometheus" }
 smallvec = "1.8.0"
 sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" }
 sc-peerset = { version = "4.0.0-dev", path = "../../peerset" }

client/network/common/src/config.rs

@@ -18,6 +18,8 @@
 
 //! Configuration of the networking layer.
 
+use crate::protocol;
+
 use libp2p::{multiaddr, Multiaddr, PeerId};
 use std::{fmt, str, str::FromStr};
 
@@ -171,3 +173,129 @@ impl From<multiaddr::Error> for ParseErr {
 		Self::MultiaddrParse(err)
 	}
 }
+
+/// Configuration for a set of nodes.
+#[derive(Clone, Debug)]
+pub struct SetConfig {
+	/// Maximum allowed number of incoming substreams related to this set.
+	pub in_peers: u32,
+	/// Number of outgoing substreams related to this set that we're trying to maintain.
+	pub out_peers: u32,
+	/// List of reserved node addresses.
+	pub reserved_nodes: Vec<MultiaddrWithPeerId>,
+	/// Whether nodes that aren't in [`SetConfig::reserved_nodes`] are accepted or automatically
+	/// refused.
+	pub non_reserved_mode: NonReservedPeerMode,
+}
+
+impl Default for SetConfig {
+	fn default() -> Self {
+		Self {
+			in_peers: 25,
+			out_peers: 75,
+			reserved_nodes: Vec::new(),
+			non_reserved_mode: NonReservedPeerMode::Accept,
+		}
+	}
+}
+
+/// Extension to [`SetConfig`] for sets that aren't the default set.
+///
+/// > **Note**: As new fields might be added in the future, please consider using the `new` method
+/// >			and modifiers instead of creating this struct manually.
+#[derive(Clone, Debug)]
+pub struct NonDefaultSetConfig {
+	/// Name of the notifications protocols of this set. A substream on this set will be
+	/// considered established once this protocol is open.
+	///
+	/// > **Note**: This field isn't present for the default set, as this is handled internally
+	/// > by the networking code.
+	pub notifications_protocol: protocol::ProtocolName,
+	/// If the remote reports that it doesn't support the protocol indicated in the
+	/// `notifications_protocol` field, then each of these fallback names will be tried one by
+	/// one.
+	///
+	/// If a fallback is used, it will be reported in
+	/// `sc_network::protocol::event::Event::NotificationStreamOpened::negotiated_fallback`
+	pub fallback_names: Vec<protocol::ProtocolName>,
+	/// Maximum allowed size of single notifications.
+	pub max_notification_size: u64,
+	/// Base configuration.
+	pub set_config: SetConfig,
+}
+
+impl NonDefaultSetConfig {
+	/// Creates a new [`NonDefaultSetConfig`]. Zero slots and accepts only reserved nodes.
+	pub fn new(notifications_protocol: protocol::ProtocolName, max_notification_size: u64) -> Self {
+		Self {
+			notifications_protocol,
+			max_notification_size,
+			fallback_names: Vec::new(),
+			set_config: SetConfig {
+				in_peers: 0,
+				out_peers: 0,
+				reserved_nodes: Vec::new(),
+				non_reserved_mode: NonReservedPeerMode::Deny,
+			},
+		}
+	}
+
+	/// Modifies the configuration to allow non-reserved nodes.
+	pub fn allow_non_reserved(&mut self, in_peers: u32, out_peers: u32) {
+		self.set_config.in_peers = in_peers;
+		self.set_config.out_peers = out_peers;
+		self.set_config.non_reserved_mode = NonReservedPeerMode::Accept;
+	}
+
+	/// Add a node to the list of reserved nodes.
+	pub fn add_reserved(&mut self, peer: MultiaddrWithPeerId) {
+		self.set_config.reserved_nodes.push(peer);
+	}
+
+	/// Add a list of protocol names used for backward compatibility.
+	///
+	/// See the explanations in [`NonDefaultSetConfig::fallback_names`].
+	pub fn add_fallback_names(&mut self, fallback_names: Vec<protocol::ProtocolName>) {
+		self.fallback_names.extend(fallback_names);
+	}
+}
+
+/// Configuration for the transport layer.
+#[derive(Clone, Debug)]
+pub enum TransportConfig {
+	/// Normal transport mode.
+	Normal {
+		/// If true, the network will use mDNS to discover other libp2p nodes on the local network
+		/// and connect to them if they support the same chain.
+		enable_mdns: bool,
+
+		/// If true, allow connecting to private IPv4 addresses (as defined in
+		/// [RFC1918](https://tools.ietf.org/html/rfc1918)). Irrelevant for addresses that have
+		/// been passed in `::sc_network::config::NetworkConfiguration::boot_nodes`.
+		allow_private_ipv4: bool,
+	},
+
+	/// Only allow connections within the same process.
+	/// Only addresses of the form `/memory/...` will be supported.
+	MemoryOnly,
+}
+
+/// The policy for connections to non-reserved peers.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum NonReservedPeerMode {
+	/// Accept them. This is the default.
+	Accept,
+	/// Deny them.
+	Deny,
+}
+
+impl NonReservedPeerMode {
+	/// Attempt to parse the peer mode from a string.
+	pub fn parse(s: &str) -> Option<Self> {
+		match s {
+			"accept" => Some(Self::Accept),
+			"deny" => Some(Self::Deny),
+			_ => None,
+		}
+	}
+}

client/network/src/error.rs → client/network/common/src/error.rs

@@ -18,9 +18,8 @@
 
 //! Substrate network possible errors.
 
-use crate::config::TransportConfig;
+use crate::{config::TransportConfig, protocol::ProtocolName};
 use libp2p::{Multiaddr, PeerId};
-use sc_network_common::protocol::ProtocolName;
 
 use std::fmt;
 

client/network/common/src/lib.rs

@@ -19,8 +19,16 @@
 //! Common data structures of the networking layer.
 
 pub mod config;
+pub mod error;
 pub mod message;
 pub mod protocol;
 pub mod request_responses;
 pub mod service;
 pub mod sync;
+pub mod utils;
+
+/// Minimum Requirements for a Hash within Networking
+pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {}
+
+impl<T> ExHashT for T where T: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static
+{}

client/network/common/src/service.rs

@@ -604,35 +604,6 @@ where
 	}
 }
 
-/// Provides ability to propagate transactions over the network.
-pub trait NetworkTransaction<H> {
-	/// You may call this when new transactions are imported by the transaction pool.
-	///
-	/// All transactions will be fetched from the `TransactionPool` that was passed at
-	/// initialization as part of the configuration and propagated to peers.
-	fn trigger_repropagate(&self);
-
-	/// You must call when new transaction is imported by the transaction pool.
-	///
-	/// This transaction will be fetched from the `TransactionPool` that was passed at
-	/// initialization as part of the configuration and propagated to peers.
-	fn propagate_transaction(&self, hash: H);
-}
-
-impl<T, H> NetworkTransaction<H> for Arc<T>
-where
-	T: ?Sized,
-	T: NetworkTransaction<H>,
-{
-	fn trigger_repropagate(&self) {
-		T::trigger_repropagate(self)
-	}
-
-	fn propagate_transaction(&self, hash: H) {
-		T::propagate_transaction(self, hash)
-	}
-}
-
 /// Provides ability to announce blocks to the network.
 pub trait NetworkBlock<BlockHash, BlockNumber> {
 	/// Make sure an important block is propagated to peers.