From 994421c57c582d1dfeda68ab45ce6e2dd771d716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Wed, 29 Mar 2023 09:17:50 +0200 Subject: [PATCH] Support stable rust for compiling the runtime (#13580) * Support stable rust for compiling the runtime This pull request brings support for compiling the runtime with stable Rust. This requires at least rust 1.68.0 to work on stable. The code is written in a way that it is backwards compatible and should automatically work when someone compiles with 1.68.0+ stable. * We always support nightlies! * :facepalm: * Sort by version * Review feedback * Review feedback * Fix version parsing * Apply suggestions from code review Co-authored-by: Koute --------- Co-authored-by: Koute --- Cargo.lock | 1 + primitives/io/Cargo.toml | 4 + primitives/io/build.rs | 27 ++++ primitives/io/src/lib.rs | 4 +- utils/wasm-builder/README.md | 11 +- utils/wasm-builder/src/lib.rs | 124 ++++++++++----- utils/wasm-builder/src/prerequisites.rs | 9 +- utils/wasm-builder/src/version.rs | 198 ++++++++++++++++++++++++ 8 files changed, 327 insertions(+), 51 deletions(-) create mode 100644 primitives/io/build.rs create mode 100644 utils/wasm-builder/src/version.rs diff --git a/Cargo.lock b/Cargo.lock index 33bca4005483c..cf30a6a94ae8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10474,6 +10474,7 @@ dependencies = [ "libsecp256k1", "log", "parity-scale-codec", + "rustversion", "secp256k1", "sp-core", "sp-externalities", diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index e56bfcf56041a..c6e716396aea4 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "I/O for Substrate runtimes" documentation = "https://docs.rs/sp-io" readme = "README.md" +build = "build.rs" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] @@ -37,6 +38,9 @@ ed25519-dalek = { version = "1.0.1", default-features = false, optional = true } # Force the usage of ed25519, this is being used in `ed25519-dalek`. ed25519 = { version = "1.5.2", optional = true } +[build-dependencies] +rustversion = "1.0.6" + [features] default = ["std"] std = [ diff --git a/primitives/io/build.rs b/primitives/io/build.rs new file mode 100644 index 0000000000000..8a9c0b6420b29 --- /dev/null +++ b/primitives/io/build.rs @@ -0,0 +1,27 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#[rustversion::before(1.68)] +fn main() { + if !cfg!(feature = "std") { + println!("cargo:rustc-cfg=enable_alloc_error_handler"); + } +} + +#[rustversion::since(1.68)] +fn main() {} diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 306ac8e60c529..72300eb102177 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -19,7 +19,7 @@ #![warn(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] -#![cfg_attr(not(feature = "std"), feature(alloc_error_handler))] +#![cfg_attr(enable_alloc_error_handler, feature(alloc_error_handler))] #![cfg_attr( feature = "std", doc = "Substrate runtime standard library as compiled when linked with Rust's standard library." @@ -1643,7 +1643,7 @@ pub fn panic(info: &core::panic::PanicInfo) -> ! { } /// A default OOM handler for WASM environment. -#[cfg(all(not(feature = "disable_oom"), not(feature = "std")))] +#[cfg(all(not(feature = "disable_oom"), enable_alloc_error_handler))] #[alloc_error_handler] pub fn oom(_: core::alloc::Layout) -> ! { #[cfg(feature = "improved_panic_error_reporting")] diff --git a/utils/wasm-builder/README.md b/utils/wasm-builder/README.md index a10611cc26a00..b1ccb1b35b10e 100644 --- a/utils/wasm-builder/README.md +++ b/utils/wasm-builder/README.md @@ -77,8 +77,13 @@ Wasm builder requires the following prerequisites for building the Wasm binary: - rust nightly + `wasm32-unknown-unknown` toolchain -If a specific rust nightly is installed with `rustup`, it is important that the wasm target is installed -as well. For example if installing the rust nightly from 20.02.2020 using `rustup install nightly-2020-02-20`, -the wasm target needs to be installed as well `rustup target add wasm32-unknown-unknown --toolchain nightly-2020-02-20`. +or + +- rust stable and version at least 1.68.0 + `wasm32-unknown-unknown` toolchain + +If a specific rust is installed with `rustup`, it is important that the wasm target is +installed as well. For example if installing the rust from 20.02.2020 using `rustup +install nightly-2020-02-20`, the wasm target needs to be installed as well `rustup target add +wasm32-unknown-unknown --toolchain nightly-2020-02-20`. License: Apache-2.0 diff --git a/utils/wasm-builder/src/lib.rs b/utils/wasm-builder/src/lib.rs index 659b955256af7..8405b5a0bda9e 100644 --- a/utils/wasm-builder/src/lib.rs +++ b/utils/wasm-builder/src/lib.rs @@ -100,8 +100,12 @@ //! //! - rust nightly + `wasm32-unknown-unknown` toolchain //! -//! If a specific rust nightly is installed with `rustup`, it is important that the wasm target is -//! installed as well. For example if installing the rust nightly from 20.02.2020 using `rustup +//! or +//! +//! - rust stable and version at least 1.68.0 + `wasm32-unknown-unknown` toolchain +//! +//! If a specific rust is installed with `rustup`, it is important that the wasm target is +//! installed as well. For example if installing the rust from 20.02.2020 using `rustup //! install nightly-2020-02-20`, the wasm target needs to be installed as well `rustup target add //! wasm32-unknown-unknown --toolchain nightly-2020-02-20`. @@ -111,9 +115,11 @@ use std::{ path::{Path, PathBuf}, process::Command, }; +use version::Version; mod builder; mod prerequisites; +mod version; mod wasm_project; pub use builder::{WasmBuilder, WasmBuilderSelectProject}; @@ -172,53 +178,59 @@ fn copy_file_if_changed(src: PathBuf, dst: PathBuf) { } } -/// Get a cargo command that compiles with nightly -fn get_nightly_cargo() -> CargoCommand { +/// Get a cargo command that should be used to invoke the compilation. +fn get_cargo_command() -> CargoCommand { let env_cargo = CargoCommand::new(&env::var("CARGO").expect("`CARGO` env variable is always set by cargo")); let default_cargo = CargoCommand::new("cargo"); - let rustup_run_nightly = CargoCommand::new_with_args("rustup", &["run", "nightly", "cargo"]); let wasm_toolchain = env::var(WASM_BUILD_TOOLCHAIN).ok(); // First check if the user requested a specific toolchain - if let Some(cmd) = wasm_toolchain.and_then(|t| get_rustup_nightly(Some(t))) { + if let Some(cmd) = + wasm_toolchain.map(|t| CargoCommand::new_with_args("rustup", &["run", &t, "cargo"])) + { cmd - } else if env_cargo.is_nightly() { + } else if env_cargo.supports_substrate_wasm_env() { env_cargo - } else if default_cargo.is_nightly() { + } else if default_cargo.supports_substrate_wasm_env() { default_cargo - } else if rustup_run_nightly.is_nightly() { - rustup_run_nightly } else { - // If no command before provided us with a nightly compiler, we try to search one - // with rustup. If that fails as well, we return the default cargo and let the prequisities - // check fail. - get_rustup_nightly(None).unwrap_or(default_cargo) + // If no command before provided us with a cargo that supports our Substrate wasm env, we + // try to search one with rustup. If that fails as well, we return the default cargo and let + // the prequisities check fail. + get_rustup_command().unwrap_or(default_cargo) } } -/// Get a nightly from rustup. If `selected` is `Some(_)`, a `CargoCommand` using the given -/// nightly is returned. -fn get_rustup_nightly(selected: Option) -> Option { +/// Get the newest rustup command that supports our Substrate wasm env. +/// +/// Stable versions are always favored over nightly versions even if the nightly versions are +/// newer. +fn get_rustup_command() -> Option { let host = format!("-{}", env::var("HOST").expect("`HOST` is always set by cargo")); - let version = match selected { - Some(selected) => selected, - None => { - let output = Command::new("rustup").args(&["toolchain", "list"]).output().ok()?.stdout; - let lines = output.as_slice().lines(); + let output = Command::new("rustup").args(&["toolchain", "list"]).output().ok()?.stdout; + let lines = output.as_slice().lines(); + + let mut versions = Vec::new(); + for line in lines.filter_map(|l| l.ok()) { + let rustup_version = line.trim_end_matches(&host); + + let cmd = CargoCommand::new_with_args("rustup", &["run", &rustup_version, "cargo"]); - let mut latest_nightly = None; - for line in lines.filter_map(|l| l.ok()) { - if line.starts_with("nightly-") && line.ends_with(&host) { - // Rustup prints them sorted - latest_nightly = Some(line.clone()); - } - } + if !cmd.supports_substrate_wasm_env() { + continue + } + + let Some(cargo_version) = cmd.version() else { continue; }; - latest_nightly?.trim_end_matches(&host).into() - }, - }; + versions.push((cargo_version, rustup_version.to_string())); + } + + // Sort by the parsed version to get the latest version (greatest version) at the end of the + // vec. + versions.sort_by_key(|v| v.0); + let version = &versions.last()?.1; Some(CargoCommand::new_with_args("rustup", &["run", &version, "cargo"])) } @@ -228,17 +240,23 @@ fn get_rustup_nightly(selected: Option) -> Option { struct CargoCommand { program: String, args: Vec, + version: Option, } impl CargoCommand { fn new(program: &str) -> Self { - CargoCommand { program: program.into(), args: Vec::new() } + let version = Self::extract_version(program, &[]); + + CargoCommand { program: program.into(), args: Vec::new(), version } } fn new_with_args(program: &str, args: &[&str]) -> Self { + let version = Self::extract_version(program, args); + CargoCommand { program: program.into(), args: args.iter().map(ToString::to_string).collect(), + version, } } @@ -248,20 +266,40 @@ impl CargoCommand { cmd } - /// Check if the supplied cargo command is a nightly version - fn is_nightly(&self) -> bool { + fn extract_version(program: &str, args: &[&str]) -> Option { + let version = Command::new(program) + .args(args) + .arg("--version") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok())?; + + Version::extract(&version) + } + + /// Returns the version of this cargo command or `None` if it failed to extract the version. + fn version(&self) -> Option { + self.version + } + + /// Check if the supplied cargo command supports our Substrate wasm environment. + /// + /// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo. + /// + /// Assumes that cargo version matches the rustc version. + fn supports_substrate_wasm_env(&self) -> bool { // `RUSTC_BOOTSTRAP` tells a stable compiler to behave like a nightly. So, when this env // variable is set, we can assume that whatever rust compiler we have, it is a nightly // compiler. For "more" information, see: // https://github.com/rust-lang/rust/blob/fa0f7d0080d8e7e9eb20aa9cbf8013f96c81287f/src/libsyntax/feature_gate/check.rs#L891 - env::var("RUSTC_BOOTSTRAP").is_ok() || - self.command() - .arg("--version") - .output() - .map_err(|_| ()) - .and_then(|o| String::from_utf8(o.stdout).map_err(|_| ())) - .unwrap_or_default() - .contains("-nightly") + if env::var("RUSTC_BOOTSTRAP").is_ok() { + return true + } + + let Some(version) = self.version() else { return false }; + + // Check if major and minor are greater or equal than 1.68 or this is a nightly. + version.major > 1 || (version.major == 1 && version.minor >= 68) || version.is_nightly } } diff --git a/utils/wasm-builder/src/prerequisites.rs b/utils/wasm-builder/src/prerequisites.rs index ca07a029281a8..f5a985ab92b5d 100644 --- a/utils/wasm-builder/src/prerequisites.rs +++ b/utils/wasm-builder/src/prerequisites.rs @@ -35,10 +35,13 @@ fn print_error_message(message: &str) -> String { /// /// Returns the versioned cargo command on success. pub(crate) fn check() -> Result { - let cargo_command = crate::get_nightly_cargo(); + let cargo_command = crate::get_cargo_command(); - if !cargo_command.is_nightly() { - return Err(print_error_message("Rust nightly not installed, please install it!")) + if !cargo_command.supports_substrate_wasm_env() { + return Err(print_error_message( + "Cannot compile the WASM runtime: no compatible Rust compiler found!\n\ + Install at least Rust 1.68.0 or a recent nightly version.", + )) } check_wasm_toolchain_installed(cargo_command) diff --git a/utils/wasm-builder/src/version.rs b/utils/wasm-builder/src/version.rs new file mode 100644 index 0000000000000..77e62b394bd55 --- /dev/null +++ b/utils/wasm-builder/src/version.rs @@ -0,0 +1,198 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cmp::Ordering; + +/// The version of rustc/cargo. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Version { + pub major: u32, + pub minor: u32, + pub patch: u32, + pub is_nightly: bool, + pub year: u32, + pub month: u32, + pub day: u32, +} + +impl Version { + /// Returns if `self` is a stable version. + pub fn is_stable(&self) -> bool { + !self.is_nightly + } + + /// Return if `self` is a nightly version. + pub fn is_nightly(&self) -> bool { + self.is_nightly + } + + /// Extract from the given `version` string. + pub fn extract(version: &str) -> Option { + let mut is_nightly = false; + let version_parts = version + .trim() + .split(" ") + .nth(1)? + .split(".") + .filter_map(|v| { + if let Some(rest) = v.strip_suffix("-nightly") { + is_nightly = true; + rest.parse().ok() + } else { + v.parse().ok() + } + }) + .collect::>(); + + if version_parts.len() != 3 { + return None + } + + let date = version.split(" ").nth(3)?; + + let date_parts = date + .split("-") + .filter_map(|v| v.trim().strip_suffix(")").unwrap_or(v).parse().ok()) + .collect::>(); + + if date_parts.len() != 3 { + return None + } + + Some(Version { + major: version_parts[0], + minor: version_parts[1], + patch: version_parts[2], + is_nightly, + year: date_parts[0], + month: date_parts[1], + day: date_parts[2], + }) + } +} + +/// Ordering is done in the following way: +/// +/// 1. `stable` > `nightly` +/// 2. Then compare major, minor and patch. +/// 3. Last compare the date. +impl Ord for Version { + fn cmp(&self, other: &Self) -> Ordering { + if self == other { + return Ordering::Equal + } + + // Ensure that `stable > nightly` + if self.is_stable() && other.is_nightly() { + return Ordering::Greater + } else if self.is_nightly() && other.is_stable() { + return Ordering::Less + } + + let to_compare = [ + (self.major, other.major), + (self.minor, other.minor), + (self.patch, other.patch), + (self.year, other.year), + (self.month, other.month), + (self.day, other.day), + ]; + + to_compare + .iter() + .find_map(|(l, r)| if l != r { l.partial_cmp(&r) } else { None }) + // We already checked this right at the beginning, so we should never return here + // `Equal`. + .unwrap_or(Ordering::Equal) + } +} + +impl PartialOrd for Version { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn version_compare_and_extract_works() { + let version_1_66_0 = Version::extract("cargo 1.66.0 (d65d197ad 2022-11-15)").unwrap(); + let version_1_66_1 = Version::extract("cargo 1.66.1 (d65d197ad 2022-11-15)").unwrap(); + let version_1_66_0_nightly = + Version::extract("cargo 1.66.0-nightly (d65d197ad 2022-10-15)").unwrap(); + let version_1_66_0_nightly_older_date = + Version::extract("cargo 1.66.0-nightly (d65d197ad 2022-10-14)").unwrap(); + let version_1_65_0 = Version::extract("cargo 1.65.0 (d65d197ad 2022-10-15)").unwrap(); + let version_1_65_0_older_date = + Version::extract("cargo 1.65.0 (d65d197ad 2022-10-14)").unwrap(); + + assert!(version_1_66_1 > version_1_66_0); + assert!(version_1_66_1 > version_1_65_0); + assert!(version_1_66_1 > version_1_66_0_nightly); + assert!(version_1_66_1 > version_1_66_0_nightly_older_date); + assert!(version_1_66_1 > version_1_65_0_older_date); + + assert!(version_1_66_0 > version_1_65_0); + assert!(version_1_66_0 > version_1_66_0_nightly); + assert!(version_1_66_0 > version_1_66_0_nightly_older_date); + assert!(version_1_66_0 > version_1_65_0_older_date); + + assert!(version_1_65_0 > version_1_66_0_nightly); + assert!(version_1_65_0 > version_1_66_0_nightly_older_date); + assert!(version_1_65_0 > version_1_65_0_older_date); + + let mut versions = vec![ + version_1_66_0, + version_1_66_0_nightly, + version_1_66_0_nightly_older_date, + version_1_65_0_older_date, + version_1_65_0, + version_1_66_1, + ]; + versions.sort_by(|a, b| b.cmp(a)); + + let expected_versions_order = vec![ + version_1_66_1, + version_1_66_0, + version_1_65_0, + version_1_65_0_older_date, + version_1_66_0_nightly, + version_1_66_0_nightly_older_date, + ]; + assert_eq!(expected_versions_order, versions); + } + + #[test] + fn parse_with_newline() { + let version_1_66_0 = Version::extract("cargo 1.66.0 (d65d197ad 2022-11-15)\n").unwrap(); + assert_eq!( + Version { + major: 1, + minor: 66, + patch: 0, + is_nightly: false, + year: 2022, + month: 11, + day: 15 + }, + version_1_66_0 + ); + } +}