From 8c5c9e398d22cf4f7e859a3e7144f2aed8c57a7c Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 13 Apr 2018 21:11:41 +0300 Subject: [PATCH 01/11] Extract mtime calculation --- src/cargo/core/compiler/fingerprint.rs | 14 +++++--------- src/cargo/sources/path.rs | 5 ++--- src/cargo/util/paths.rs | 8 ++++++++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs index d4b5e4ee7df..5e734f1ae36 100644 --- a/src/cargo/core/compiler/fingerprint.rs +++ b/src/cargo/core/compiler/fingerprint.rs @@ -220,9 +220,7 @@ impl Fingerprint { match *local { LocalFingerprint::MtimeBased(ref slot, ref path) => { let path = root.join(path); - let meta = fs::metadata(&path) - .chain_err(|| internal(format!("failed to stat `{}`", path.display())))?; - let mtime = FileTime::from_last_modification_time(&meta); + let mtime = paths::mtime(&path)?; *slot.0.lock().unwrap() = Some(mtime); } LocalFingerprint::EnvBased(..) | LocalFingerprint::Precalculated(..) => continue, @@ -718,22 +716,20 @@ where I: IntoIterator, I::Item: AsRef, { - let meta = match fs::metadata(output) { - Ok(meta) => meta, + let mtime = match paths::mtime(output) { + Ok(mtime) => mtime, Err(..) => return None, }; - let mtime = FileTime::from_last_modification_time(&meta); let any_stale = paths.into_iter().any(|path| { let path = path.as_ref(); - let meta = match fs::metadata(path) { - Ok(meta) => meta, + let mtime2 = match paths::mtime(path) { + Ok(mtime) => mtime, Err(..) => { info!("stale: {} -- missing", path.display()); return true; } }; - let mtime2 = FileTime::from_last_modification_time(&meta); if mtime2 > mtime { info!("stale: {} -- {} vs {}", path.display(), mtime2, mtime); true diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index cadb0cffa17..aa654abee60 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -11,6 +11,7 @@ use ignore::gitignore::GitignoreBuilder; use core::{Dependency, Package, PackageId, Registry, Source, SourceId, Summary}; use ops; use util::{self, internal, CargoResult}; +use util::paths; use util::Config; pub struct PathSource<'cfg> { @@ -529,9 +530,7 @@ impl<'cfg> Source for PathSource<'cfg> { // condition where this path was rm'ed - either way, // we can ignore the error and treat the path's mtime // as 0. - let mtime = fs::metadata(&file) - .map(|meta| FileTime::from_last_modification_time(&meta)) - .unwrap_or(FileTime::zero()); + let mtime = paths::mtime(&file).unwrap_or(FileTime::zero()); warn!("{} {}", mtime, file.display()); if mtime > max { max = mtime; diff --git a/src/cargo/util/paths.rs b/src/cargo/util/paths.rs index 0cd231e709a..07296880719 100644 --- a/src/cargo/util/paths.rs +++ b/src/cargo/util/paths.rs @@ -5,6 +5,8 @@ use std::io; use std::io::prelude::*; use std::path::{Component, Path, PathBuf}; +use filetime::FileTime; + use util::{internal, CargoResult}; use util::errors::{CargoError, CargoResultExt, Internal}; @@ -129,6 +131,12 @@ pub fn append(path: &Path, contents: &[u8]) -> CargoResult<()> { Ok(()) } +pub fn mtime(path: &Path) -> CargoResult { + let meta = fs::metadata(path) + .chain_err(|| internal(format!("failed to stat `{}`", path.display())))?; + Ok(FileTime::from_last_modification_time(&meta)) +} + #[cfg(unix)] pub fn path2bytes(path: &Path) -> CargoResult<&[u8]> { use std::os::unix::prelude::*; From f5f23c2136a280aa2564ec90c94800452365a7c8 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 14 Apr 2018 11:49:13 +0300 Subject: [PATCH 02/11] Associated Rustc with a build config We want to create `rustc` only when we already have workspace, so that we could use the `target` dir to store cached info about the compiler. --- src/bin/cli.rs | 2 +- src/cargo/core/compiler/compilation.rs | 8 ++++++-- .../core/compiler/context/compilation_files.rs | 4 +--- src/cargo/core/compiler/context/mod.rs | 5 +++-- src/cargo/core/compiler/context/target_info.rs | 2 +- src/cargo/core/compiler/custom_build.rs | 2 +- src/cargo/core/compiler/fingerprint.rs | 2 +- src/cargo/core/compiler/mod.rs | 15 +++++++++------ src/cargo/ops/cargo_test.rs | 2 +- src/cargo/util/config.rs | 12 +++++------- 10 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/bin/cli.rs b/src/bin/cli.rs index 54855739547..768012a4445 100644 --- a/src/bin/cli.rs +++ b/src/bin/cli.rs @@ -47,7 +47,7 @@ Run with 'cargo -Z [FLAG] [SUBCOMMAND]'" } if let Some(ref code) = args.value_of("explain") { - let mut procss = config.rustc()?.process(); + let mut procss = config.new_rustc()?.process(); procss.arg("--explain").arg(code).exec()?; return Ok(()); } diff --git a/src/cargo/core/compiler/compilation.rs b/src/cargo/core/compiler/compilation.rs index b8dcdb5ec1c..71bddf20fae 100644 --- a/src/cargo/core/compiler/compilation.rs +++ b/src/cargo/core/compiler/compilation.rs @@ -57,15 +57,17 @@ pub struct Compilation<'cfg> { /// Flags to pass to rustdoc when invoked from cargo test, per package. pub rustdocflags: HashMap>, + pub host: String, pub target: String, config: &'cfg Config, + rustc_process: ProcessBuilder, target_runner: LazyCell)>>, } impl<'cfg> Compilation<'cfg> { - pub fn new(config: &'cfg Config) -> Compilation<'cfg> { + pub fn new(config: &'cfg Config, rustc_process: ProcessBuilder) -> Compilation<'cfg> { Compilation { libraries: HashMap::new(), native_dirs: BTreeSet::new(), // TODO: deprecated, remove @@ -81,6 +83,8 @@ impl<'cfg> Compilation<'cfg> { cfgs: HashMap::new(), rustdocflags: HashMap::new(), config, + rustc_process, + host: String::new(), target: String::new(), target_runner: LazyCell::new(), } @@ -88,7 +92,7 @@ impl<'cfg> Compilation<'cfg> { /// See `process`. pub fn rustc_process(&self, pkg: &Package) -> CargoResult { - self.fill_env(self.config.rustc()?.process(), pkg, true) + self.fill_env(self.rustc_process.clone(), pkg, true) } /// See `process`. diff --git a/src/cargo/core/compiler/context/compilation_files.rs b/src/cargo/core/compiler/context/compilation_files.rs index da49868c3de..88aa1cfc761 100644 --- a/src/cargo/core/compiler/context/compilation_files.rs +++ b/src/cargo/core/compiler/context/compilation_files.rs @@ -434,9 +434,7 @@ fn compute_metadata<'a, 'cfg>( unit.target.name().hash(&mut hasher); unit.target.kind().hash(&mut hasher); - if let Ok(rustc) = cx.config.rustc() { - rustc.verbose_version.hash(&mut hasher); - } + cx.build_config.rustc.verbose_version.hash(&mut hasher); // Seed the contents of __CARGO_DEFAULT_LIB_METADATA to the hasher if present. // This should be the release channel, to get a different hash for each channel. diff --git a/src/cargo/core/compiler/context/mod.rs b/src/cargo/core/compiler/context/mod.rs index e6fab8ed0d6..f5f279a2d94 100644 --- a/src/cargo/core/compiler/context/mod.rs +++ b/src/cargo/core/compiler/context/mod.rs @@ -130,7 +130,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { let _p = profile::start("Context::probe_target_info"); debug!("probe_target_info"); let host_target_same = match build_config.requested_target { - Some(ref s) if s != &config.rustc()?.host => false, + Some(ref s) if s != &build_config.host_triple => false, _ => true, }; @@ -150,7 +150,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { config, target_info, host_info, - compilation: Compilation::new(config), + compilation: Compilation::new(config, build_config.rustc.process()), build_state: Arc::new(BuildState::new(&build_config)), build_config, fingerprints: HashMap::new(), @@ -302,6 +302,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { self.compilation.native_dirs.insert(dir.clone()); } } + self.compilation.host = self.build_config.host_triple.clone(); self.compilation.target = self.build_config.target_triple().to_string(); Ok(self.compilation) } diff --git a/src/cargo/core/compiler/context/target_info.rs b/src/cargo/core/compiler/context/target_info.rs index 40a84010375..78a2220ca53 100644 --- a/src/cargo/core/compiler/context/target_info.rs +++ b/src/cargo/core/compiler/context/target_info.rs @@ -53,7 +53,7 @@ impl FileType { impl TargetInfo { pub fn new(config: &Config, build_config: &BuildConfig, kind: Kind) -> CargoResult { let rustflags = env_args(config, build_config, None, kind, "RUSTFLAGS")?; - let mut process = config.rustc()?.process(); + let mut process = build_config.rustc.process(); process .arg("-") .arg("--crate-name") diff --git a/src/cargo/core/compiler/custom_build.rs b/src/cargo/core/compiler/custom_build.rs index 3c94e1e0028..578e2312cb4 100644 --- a/src/cargo/core/compiler/custom_build.rs +++ b/src/cargo/core/compiler/custom_build.rs @@ -142,7 +142,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes }, ) .env("HOST", &cx.build_config.host_triple) - .env("RUSTC", &cx.config.rustc()?.path) + .env("RUSTC", &cx.build_config.rustc.path) .env("RUSTDOC", &*cx.config.rustdoc()?) .inherit_jobserver(&cx.jobserver); diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs index 5e734f1ae36..8387fd2fdef 100644 --- a/src/cargo/core/compiler/fingerprint.rs +++ b/src/cargo/core/compiler/fingerprint.rs @@ -455,7 +455,7 @@ fn calculate<'a, 'cfg>( cx.rustflags_args(unit)? }; let fingerprint = Arc::new(Fingerprint { - rustc: util::hash_u64(&cx.config.rustc()?.verbose_version), + rustc: util::hash_u64(&cx.build_config.rustc.verbose_version), target: util::hash_u64(&unit.target), profile: util::hash_u64(&(&unit.profile, cx.incremental_args(unit)?)), // Note that .0 is hashed here, not .1 which is the cwd. That doesn't diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 127e85e92d9..f626637c74e 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -12,11 +12,10 @@ use serde_json; use core::{Feature, PackageId, Profile, Target}; use core::manifest::Lto; use core::shell::ColorChoice; -use util::{self, machine_message, Config, ProcessBuilder}; +use util::{self, machine_message, Config, ProcessBuilder, Rustc, Freshness}; use util::{internal, join_paths, profile}; use util::paths; use util::errors::{CargoResult, CargoResultExt, Internal}; -use util::Freshness; use self::job::{Job, Work}; use self::job_queue::JobQueue; @@ -48,8 +47,8 @@ pub enum Kind { } /// Configuration information for a rustc build. -#[derive(Default, Clone)] pub struct BuildConfig { + pub rustc: Rustc, /// The host arch triple /// /// e.g. x86_64-unknown-linux-gnu, would be @@ -125,20 +124,24 @@ impl BuildConfig { None => None, }; let jobs = jobs.or(cfg_jobs).unwrap_or(::num_cpus::get() as u32); - - let host_triple = config.rustc()?.host.clone(); + let rustc = config.new_rustc()?; + let host_triple = rustc.host.clone(); let host_config = TargetConfig::new(config, &host_triple)?; let target_config = match target.as_ref() { Some(triple) => TargetConfig::new(config, triple)?, None => host_config.clone(), }; Ok(BuildConfig { + rustc, host_triple, requested_target: target, jobs, host: host_config, target: target_config, - ..Default::default() + release: false, + test: false, + doc_all: false, + json_messages: false, }) } diff --git a/src/cargo/ops/cargo_test.rs b/src/cargo/ops/cargo_test.rs index de0bb07b626..c65cc1970d4 100644 --- a/src/cargo/ops/cargo_test.rs +++ b/src/cargo/ops/cargo_test.rs @@ -150,7 +150,7 @@ fn run_doc_tests( let config = options.compile_opts.config; // We don't build/rust doctests if target != host - if config.rustc()?.host != compilation.target { + if compilation.host != compilation.target { return Ok((Test::Doc, errors)); } diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index 281165ec209..a95c56c7e0a 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -158,13 +158,11 @@ impl Config { } /// Get the path to the `rustc` executable - pub fn rustc(&self) -> CargoResult<&Rustc> { - self.rustc.try_borrow_with(|| { - Rustc::new( - self.get_tool("rustc")?, - self.maybe_get_tool("rustc_wrapper")?, - ) - }) + pub fn new_rustc(&self) -> CargoResult { + Rustc::new( + self.get_tool("rustc")?, + self.maybe_get_tool("rustc_wrapper")?, + ) } /// Get the path to the `cargo` executable From 52c9f025328eca86f71539f6f4daf8b8d612aff3 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 14 Apr 2018 11:54:33 +0300 Subject: [PATCH 03/11] Hide BuildConfig.host_triple behind a getter --- src/cargo/core/compiler/context/mod.rs | 9 +++++---- src/cargo/core/compiler/custom_build.rs | 4 ++-- src/cargo/core/compiler/mod.rs | 26 +++++++++++++------------ src/cargo/util/paths.rs | 4 ++-- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/cargo/core/compiler/context/mod.rs b/src/cargo/core/compiler/context/mod.rs index f5f279a2d94..1620aad4a12 100644 --- a/src/cargo/core/compiler/context/mod.rs +++ b/src/cargo/core/compiler/context/mod.rs @@ -130,7 +130,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { let _p = profile::start("Context::probe_target_info"); debug!("probe_target_info"); let host_target_same = match build_config.requested_target { - Some(ref s) if s != &build_config.host_triple => false, + Some(ref s) if s != &build_config.host_triple() => false, _ => true, }; @@ -302,7 +302,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { self.compilation.native_dirs.insert(dir.clone()); } } - self.compilation.host = self.build_config.host_triple.clone(); + self.compilation.host = self.build_config.host_triple().to_string(); self.compilation.target = self.build_config.target_triple().to_string(); Ok(self.compilation) } @@ -441,7 +441,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { None => return true, }; let (name, info) = match kind { - Kind::Host => (self.build_config.host_triple.as_ref(), &self.host_info), + Kind::Host => (self.build_config.host_triple(), &self.host_info), Kind::Target => (self.build_config.target_triple(), &self.target_info), }; platform.matches(name, info.cfg()) @@ -653,7 +653,8 @@ fn env_args( let target = build_config .requested_target .as_ref() - .unwrap_or(&build_config.host_triple); + .map(|s| s.as_str()) + .unwrap_or(build_config.host_triple()); let key = format!("target.{}.{}", target, name); if let Some(args) = config.get_list_or_split_string(&key)? { let args = args.val.into_iter(); diff --git a/src/cargo/core/compiler/custom_build.rs b/src/cargo/core/compiler/custom_build.rs index 578e2312cb4..a219ddab5cd 100644 --- a/src/cargo/core/compiler/custom_build.rs +++ b/src/cargo/core/compiler/custom_build.rs @@ -127,7 +127,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes .env( "TARGET", &match unit.kind { - Kind::Host => &cx.build_config.host_triple, + Kind::Host => &cx.build_config.host_triple(), Kind::Target => cx.build_config.target_triple(), }, ) @@ -141,7 +141,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes "debug" }, ) - .env("HOST", &cx.build_config.host_triple) + .env("HOST", &cx.build_config.host_triple()) .env("RUSTC", &cx.build_config.rustc.path) .env("RUSTDOC", &*cx.config.rustdoc()?) .inherit_jobserver(&cx.jobserver); diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index f626637c74e..7f9c503b960 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -12,7 +12,7 @@ use serde_json; use core::{Feature, PackageId, Profile, Target}; use core::manifest::Lto; use core::shell::ColorChoice; -use util::{self, machine_message, Config, ProcessBuilder, Rustc, Freshness}; +use util::{self, machine_message, Config, Freshness, ProcessBuilder, Rustc}; use util::{internal, join_paths, profile}; use util::paths; use util::errors::{CargoResult, CargoResultExt, Internal}; @@ -49,13 +49,6 @@ pub enum Kind { /// Configuration information for a rustc build. pub struct BuildConfig { pub rustc: Rustc, - /// The host arch triple - /// - /// e.g. x86_64-unknown-linux-gnu, would be - /// - machine: x86_64 - /// - hardware-platform: unknown - /// - operating system: linux-gnu - pub host_triple: String, /// Build information for the host arch pub host: TargetConfig, /// The target arch triple, defaults to host arch @@ -125,15 +118,13 @@ impl BuildConfig { }; let jobs = jobs.or(cfg_jobs).unwrap_or(::num_cpus::get() as u32); let rustc = config.new_rustc()?; - let host_triple = rustc.host.clone(); - let host_config = TargetConfig::new(config, &host_triple)?; + let host_config = TargetConfig::new(config, &rustc.host)?; let target_config = match target.as_ref() { Some(triple) => TargetConfig::new(config, triple)?, None => host_config.clone(), }; Ok(BuildConfig { rustc, - host_triple, requested_target: target, jobs, host: host_config, @@ -145,10 +136,21 @@ impl BuildConfig { }) } + /// The host arch triple + /// + /// e.g. x86_64-unknown-linux-gnu, would be + /// - machine: x86_64 + /// - hardware-platform: unknown + /// - operating system: linux-gnu + pub fn host_triple(&self) -> &str { + &self.rustc.host + } + pub fn target_triple(&self) -> &str { self.requested_target .as_ref() - .unwrap_or_else(|| &self.host_triple) + .map(|s| s.as_str()) + .unwrap_or(self.host_triple()) } } diff --git a/src/cargo/util/paths.rs b/src/cargo/util/paths.rs index 07296880719..b08a589d4b5 100644 --- a/src/cargo/util/paths.rs +++ b/src/cargo/util/paths.rs @@ -132,8 +132,8 @@ pub fn append(path: &Path, contents: &[u8]) -> CargoResult<()> { } pub fn mtime(path: &Path) -> CargoResult { - let meta = fs::metadata(path) - .chain_err(|| internal(format!("failed to stat `{}`", path.display())))?; + let meta = + fs::metadata(path).chain_err(|| internal(format!("failed to stat `{}`", path.display())))?; Ok(FileTime::from_last_modification_time(&meta)) } From 6b1dc52b86fabb278e7bcede19428f359e52282a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 14 Apr 2018 13:39:59 +0300 Subject: [PATCH 04/11] Speedup no-op builds by caching rustc invocations --- src/bin/cli.rs | 2 +- .../core/compiler/context/target_info.rs | 11 +- src/cargo/core/compiler/mod.rs | 3 +- src/cargo/ops/cargo_clean.rs | 2 +- src/cargo/ops/cargo_compile.rs | 3 +- src/cargo/util/config.rs | 23 +-- src/cargo/util/paths.rs | 18 ++ src/cargo/util/rustc.rs | 180 +++++++++++++++++- tests/testsuite/cargotest/mod.rs | 2 +- tests/testsuite/main.rs | 1 + tests/testsuite/rustc_info_cache.rs | 120 ++++++++++++ 11 files changed, 327 insertions(+), 38 deletions(-) create mode 100644 tests/testsuite/rustc_info_cache.rs diff --git a/src/bin/cli.rs b/src/bin/cli.rs index 768012a4445..0161a0f6bf9 100644 --- a/src/bin/cli.rs +++ b/src/bin/cli.rs @@ -47,7 +47,7 @@ Run with 'cargo -Z [FLAG] [SUBCOMMAND]'" } if let Some(ref code) = args.value_of("explain") { - let mut procss = config.new_rustc()?.process(); + let mut procss = config.rustc(None)?.process(); procss.arg("--explain").arg(code).exec()?; return Ok(()); } diff --git a/src/cargo/core/compiler/context/target_info.rs b/src/cargo/core/compiler/context/target_info.rs index 78a2220ca53..ff691ff2a30 100644 --- a/src/cargo/core/compiler/context/target_info.rs +++ b/src/cargo/core/compiler/context/target_info.rs @@ -78,20 +78,19 @@ impl TargetInfo { with_cfg.arg("--print=cfg"); let mut has_cfg_and_sysroot = true; - let output = with_cfg - .exec_with_output() + let (output, error) = build_config + .rustc + .cached_output(&with_cfg) .or_else(|_| { has_cfg_and_sysroot = false; - process.exec_with_output() + build_config.rustc.cached_output(&process) }) .chain_err(|| "failed to run `rustc` to learn about target-specific information")?; - let error = str::from_utf8(&output.stderr).unwrap(); - let output = str::from_utf8(&output.stdout).unwrap(); let mut lines = output.lines(); let mut map = HashMap::new(); for crate_type in KNOWN_CRATE_TYPES { - let out = parse_crate_type(crate_type, error, &mut lines)?; + let out = parse_crate_type(crate_type, &error, &mut lines)?; map.insert(crate_type.to_string(), out); } diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 7f9c503b960..c414cafd8b4 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -80,6 +80,7 @@ impl BuildConfig { config: &Config, jobs: Option, requested_target: &Option, + rustc_info_cache: Option, ) -> CargoResult { if let &Some(ref s) = requested_target { if s.trim().is_empty() { @@ -117,7 +118,7 @@ impl BuildConfig { None => None, }; let jobs = jobs.or(cfg_jobs).unwrap_or(::num_cpus::get() as u32); - let rustc = config.new_rustc()?; + let rustc = config.rustc(rustc_info_cache)?; let host_config = TargetConfig::new(config, &rustc.host)?; let target_config = match target.as_ref() { Some(triple) => TargetConfig::new(config, triple)?, diff --git a/src/cargo/ops/cargo_clean.rs b/src/cargo/ops/cargo_clean.rs index 43a89ea6423..a8c388752dd 100644 --- a/src/cargo/ops/cargo_clean.rs +++ b/src/cargo/ops/cargo_clean.rs @@ -84,7 +84,7 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> { } } - let mut build_config = BuildConfig::new(config, Some(1), &opts.target)?; + let mut build_config = BuildConfig::new(config, Some(1), &opts.target, None)?; build_config.release = opts.release; let mut cx = Context::new(ws, &resolve, &packages, opts.config, build_config, profiles)?; cx.prepare_units(None, &units)?; diff --git a/src/cargo/ops/cargo_compile.rs b/src/cargo/ops/cargo_compile.rs index e734c4cb801..485f2b46af0 100644 --- a/src/cargo/ops/cargo_compile.rs +++ b/src/cargo/ops/cargo_compile.rs @@ -260,7 +260,8 @@ pub fn compile_ws<'a>( bail!("jobs must be at least 1") } - let mut build_config = BuildConfig::new(config, jobs, &target)?; + let rustc_info_cache = ws.target_dir().join("rustc_info.json").into_path_unlocked(); + let mut build_config = BuildConfig::new(config, jobs, &target, Some(rustc_info_cache))?; build_config.release = release; build_config.test = mode == CompileMode::Test || mode == CompileMode::Bench; build_config.json_messages = message_format == MessageFormat::Json; diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index a95c56c7e0a..4596f07c143 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -158,10 +158,11 @@ impl Config { } /// Get the path to the `rustc` executable - pub fn new_rustc(&self) -> CargoResult { + pub fn rustc(&self, cache_location: Option) -> CargoResult { Rustc::new( self.get_tool("rustc")?, self.maybe_get_tool("rustc_wrapper")?, + cache_location, ) } @@ -191,25 +192,7 @@ impl Config { .map(PathBuf::from) .next() .ok_or(format_err!("no argv[0]"))?; - if argv0.components().count() == 1 { - probe_path(argv0) - } else { - Ok(argv0.canonicalize()?) - } - } - - fn probe_path(argv0: PathBuf) -> CargoResult { - let paths = env::var_os("PATH").ok_or(format_err!("no PATH"))?; - for path in env::split_paths(&paths) { - let candidate = PathBuf::from(path).join(&argv0); - if candidate.is_file() { - // PATH may have a component like "." in it, so we still need to - // canonicalize. - return Ok(candidate.canonicalize()?); - } - } - - bail!("no cargo executable candidate found in PATH") + paths::resolve_executable(&argv0) } let exe = from_current_exe() diff --git a/src/cargo/util/paths.rs b/src/cargo/util/paths.rs index b08a589d4b5..eeee00ddde9 100644 --- a/src/cargo/util/paths.rs +++ b/src/cargo/util/paths.rs @@ -85,6 +85,24 @@ pub fn without_prefix<'a>(long_path: &'a Path, prefix: &'a Path) -> Option<&'a P } } +pub fn resolve_executable(exec: &Path) -> CargoResult { + if exec.components().count() == 1 { + let paths = env::var_os("PATH").ok_or(format_err!("no PATH"))?; + for path in env::split_paths(&paths) { + let candidate = PathBuf::from(path).join(&exec); + if candidate.is_file() { + // PATH may have a component like "." in it, so we still need to + // canonicalize. + return Ok(candidate.canonicalize()?); + } + } + + bail!("no executable for `{}` found in PATH", exec.display()) + } else { + Ok(exec.canonicalize()?) + } +} + pub fn read(path: &Path) -> CargoResult { match String::from_utf8(read_bytes(path)?) { Ok(s) => Ok(s), diff --git a/src/cargo/util/rustc.rs b/src/cargo/util/rustc.rs index d9e8fe657dd..b9ec492c20a 100644 --- a/src/cargo/util/rustc.rs +++ b/src/cargo/util/rustc.rs @@ -1,6 +1,15 @@ -use std::path::PathBuf; +#![allow(deprecated)] // for SipHasher + +use std::path::{Path, PathBuf}; +use std::hash::{Hash, Hasher, SipHasher}; +use std::collections::hash_map::{Entry, HashMap}; +use std::sync::Mutex; +use std::env; + +use serde_json; use util::{self, internal, profile, CargoResult, ProcessBuilder}; +use util::paths; /// Information on the `rustc` executable #[derive(Debug)] @@ -14,6 +23,7 @@ pub struct Rustc { pub verbose_version: String, /// The host triple (arch-platform-OS), this comes from verbose_version. pub host: String, + cache: Mutex, } impl Rustc { @@ -22,16 +32,18 @@ impl Rustc { /// /// If successful this function returns a description of the compiler along /// with a list of its capabilities. - pub fn new(path: PathBuf, wrapper: Option) -> CargoResult { + pub fn new( + path: PathBuf, + wrapper: Option, + cache_location: Option, + ) -> CargoResult { let _p = profile::start("Rustc::new"); + let mut cache = Cache::load(&path, cache_location); + let mut cmd = util::process(&path); cmd.arg("-vV"); - - let output = cmd.exec_with_output()?; - - let verbose_version = String::from_utf8(output.stdout) - .map_err(|_| internal("rustc -v didn't return utf8 output"))?; + let verbose_version = cache.cached_output(&cmd)?.0; let host = { let triple = verbose_version @@ -47,6 +59,7 @@ impl Rustc { wrapper, verbose_version, host, + cache: Mutex::new(cache), }) } @@ -62,4 +75,157 @@ impl Rustc { util::process(&self.path) } } + + pub fn cached_output(&self, cmd: &ProcessBuilder) -> CargoResult<(String, String)> { + self.cache.lock().unwrap().cached_output(cmd) + } +} + +/// It is a well known that `rustc` is not the fastest compiler in the world. +/// What is less known is that even `rustc --version --verbose` takes about a +/// hundred milliseconds! Because we need compiler version info even for no-op +/// builds, we cache it here, based on compiler's mtime and rustup's current +/// toolchain. +#[derive(Debug)] +struct Cache { + cache_location: Option, + dirty: bool, + data: CacheData, +} + +#[derive(Serialize, Deserialize, Debug, Default)] +struct CacheData { + rustc_fingerprint: u64, + outputs: HashMap, +} + +impl Cache { + fn load(rustc: &Path, cache_location: Option) -> Cache { + match (cache_location, rustc_fingerprint(rustc)) { + (Some(cache_location), Ok(rustc_fingerprint)) => { + let empty = CacheData { + rustc_fingerprint, + outputs: HashMap::new(), + }; + let mut dirty = true; + let data = match read(&cache_location) { + Ok(data) => { + if data.rustc_fingerprint == rustc_fingerprint { + info!("reusing existing rustc info cache"); + dirty = false; + data + } else { + info!("different compiler, creating new rustc info cache"); + empty + } + } + Err(e) => { + info!("failed to read rustc info cache: {}", e); + empty + } + }; + return Cache { + cache_location: Some(cache_location), + dirty, + data, + }; + + fn read(path: &Path) -> CargoResult { + let json = paths::read(path)?; + Ok(serde_json::from_str(&json)?) + } + } + (_, fingerprint) => { + if let Err(e) = fingerprint { + warn!("failed to calculate rustc fingerprint: {}", e); + } + info!("rustc info cache disabled"); + Cache { + cache_location: None, + dirty: false, + data: CacheData::default(), + } + } + } + } + + fn cached_output(&mut self, cmd: &ProcessBuilder) -> CargoResult<(String, String)> { + let calculate = || { + let output = cmd.exec_with_output()?; + let stdout = String::from_utf8(output.stdout) + .map_err(|_| internal("rustc didn't return utf8 output"))?; + let stderr = String::from_utf8(output.stderr) + .map_err(|_| internal("rustc didn't return utf8 output"))?; + Ok((stdout, stderr)) + }; + if self.cache_location.is_none() { + info!("rustc info uncached"); + return calculate(); + } + + let key = process_fingerprint(cmd); + match self.data.outputs.entry(key) { + Entry::Occupied(entry) => { + info!("rustc info cache hit"); + Ok(entry.get().clone()) + } + Entry::Vacant(entry) => { + info!("rustc info cache miss"); + let output = calculate()?; + entry.insert(output.clone()); + self.dirty = true; + Ok(output) + } + } + } +} + +impl Drop for Cache { + fn drop(&mut self) { + match (&self.cache_location, self.dirty) { + (&Some(ref path), true) => { + let json = serde_json::to_string(&self.data).unwrap(); + match paths::write(path, json.as_bytes()) { + Ok(()) => info!("updated rustc info cache"), + Err(e) => warn!("failed to update rustc info cache: {}", e), + } + } + _ => (), + } + } +} + +fn rustc_fingerprint(path: &Path) -> CargoResult { + let mut hasher = SipHasher::new_with_keys(0, 0); + + let path = paths::resolve_executable(path)?; + path.hash(&mut hasher); + + paths::mtime(&path)?.hash(&mut hasher); + + match (env::var("RUSTUP_HOME"), env::var("RUSTUP_TOOLCHAIN")) { + (Ok(rustup_home), Ok(rustup_toolchain)) => { + debug!("adding rustup info to rustc fingerprint"); + rustup_toolchain.hash(&mut hasher); + rustup_home.hash(&mut hasher); + let rustup_rustc = Path::new(&rustup_home) + .join("toolchains") + .join(rustup_toolchain) + .join("bin") + .join("rustc"); + paths::mtime(&rustup_rustc)?.hash(&mut hasher); + } + _ => (), + } + + Ok(hasher.finish()) +} + +fn process_fingerprint(cmd: &ProcessBuilder) -> u64 { + let mut hasher = SipHasher::new_with_keys(0, 0); + cmd.get_args().hash(&mut hasher); + let mut env = cmd.get_envs().iter().collect::>(); + env.sort(); + env.hash(&mut hasher); + hasher.finish() } diff --git a/tests/testsuite/cargotest/mod.rs b/tests/testsuite/cargotest/mod.rs index 73d5c370bd7..23efab70360 100644 --- a/tests/testsuite/cargotest/mod.rs +++ b/tests/testsuite/cargotest/mod.rs @@ -10,7 +10,7 @@ pub mod support; pub mod install; -thread_local!(pub static RUSTC: Rustc = Rustc::new(PathBuf::from("rustc"), None).unwrap()); +thread_local!(pub static RUSTC: Rustc = Rustc::new(PathBuf::from("rustc"), None, None).unwrap()); pub fn rustc_host() -> String { RUSTC.with(|r| r.host.clone()) diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs index da6ef5ea04a..e1d3181cd08 100644 --- a/tests/testsuite/main.rs +++ b/tests/testsuite/main.rs @@ -78,6 +78,7 @@ mod required_features; mod resolve; mod run; mod rustc; +mod rustc_info_cache; mod rustdocflags; mod rustdoc; mod rustflags; diff --git a/tests/testsuite/rustc_info_cache.rs b/tests/testsuite/rustc_info_cache.rs new file mode 100644 index 00000000000..7d3bcb5e17d --- /dev/null +++ b/tests/testsuite/rustc_info_cache.rs @@ -0,0 +1,120 @@ +use cargotest::support::{execs, project}; +use cargotest::support::paths::CargoPathExt; +use hamcrest::assert_that; +use std::env; + +#[test] +fn rustc_info_cache() { + let p = project("foo") + .file( + "Cargo.toml", + r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + "#, + ) + .file("src/main.rs", r#"fn main() { println!("hello"); }"#) + .build(); + + let miss = "[..] rustc info cache miss[..]"; + let hit = "[..] rustc info cache hit[..]"; + + assert_that( + p.cargo("build").env("RUST_LOG", "cargo::util::rustc=info"), + execs() + .with_status(0) + .with_stderr_contains("[..]failed to read rustc info cache[..]") + .with_stderr_contains(miss) + .with_stderr_does_not_contain(hit), + ); + + assert_that( + p.cargo("build").env("RUST_LOG", "cargo::util::rustc=info"), + execs() + .with_status(0) + .with_stderr_contains("[..]reusing existing rustc info cache[..]") + .with_stderr_contains(hit) + .with_stderr_does_not_contain(miss), + ); + + let other_rustc = { + let p = project("compiler") + .file( + "Cargo.toml", + r#" + [package] + name = "compiler" + version = "0.1.0" + "#, + ) + .file( + "src/main.rs", + r#" + use std::process::Command; + use std::env; + + fn main() { + let mut cmd = Command::new("rustc"); + for arg in env::args_os().skip(1) { + cmd.arg(arg); + } + std::process::exit(cmd.status().unwrap().code().unwrap()); + } + "#, + ) + .build(); + assert_that(p.cargo("build"), execs().with_status(0)); + + p.root() + .join("target/debug/compiler") + .with_extension(env::consts::EXE_EXTENSION) + }; + + assert_that( + p.cargo("build") + .env("RUST_LOG", "cargo::util::rustc=info") + .env("RUSTC", other_rustc.display().to_string()), + execs() + .with_status(0) + .with_stderr_contains("[..]different compiler, creating new rustc info cache[..]") + .with_stderr_contains(miss) + .with_stderr_does_not_contain(hit), + ); + + assert_that( + p.cargo("build") + .env("RUST_LOG", "cargo::util::rustc=info") + .env("RUSTC", other_rustc.display().to_string()), + execs() + .with_status(0) + .with_stderr_contains("[..]reusing existing rustc info cache[..]") + .with_stderr_contains(hit) + .with_stderr_does_not_contain(miss), + ); + + other_rustc.move_into_the_future(); + + assert_that( + p.cargo("build") + .env("RUST_LOG", "cargo::util::rustc=info") + .env("RUSTC", other_rustc.display().to_string()), + execs() + .with_status(0) + .with_stderr_contains("[..]different compiler, creating new rustc info cache[..]") + .with_stderr_contains(miss) + .with_stderr_does_not_contain(hit), + ); + + assert_that( + p.cargo("build") + .env("RUST_LOG", "cargo::util::rustc=info") + .env("RUSTC", other_rustc.display().to_string()), + execs() + .with_status(0) + .with_stderr_contains("[..]reusing existing rustc info cache[..]") + .with_stderr_contains(hit) + .with_stderr_does_not_contain(miss), + ); +} From c473479ab0459f04adf9d4ddbfe062497d03e3d4 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 14 Apr 2018 14:09:23 +0300 Subject: [PATCH 05/11] Add env-var to suppress rustc caching --- src/cargo/util/config.rs | 15 ++++++++- .../src/reference/environment-variables.md | 18 +++++----- tests/testsuite/rustc_info_cache.rs | 33 +++++++++++++++---- 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index 4596f07c143..1dfefb8c8ae 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -66,6 +66,9 @@ pub struct Config { easy: LazyCell>, /// Cache of the `SourceId` for crates.io crates_io_source_id: LazyCell, + /// If false, don't cache `rustc --version --verbose` invocations + cache_rustc_info: bool, + /// Creation time of this config, used to output the total build time creation_time: Instant, } @@ -82,6 +85,11 @@ impl Config { } }); + let cache_rustc_info = match env::var("CARGO_CACHE_RUSTC_INFO") { + Ok(cache) => cache != "0", + _ => true, + }; + Config { home_path: Filesystem::new(homedir), shell: RefCell::new(shell), @@ -103,6 +111,7 @@ impl Config { cli_flags: CliUnstable::default(), easy: LazyCell::new(), crates_io_source_id: LazyCell::new(), + cache_rustc_info, creation_time: Instant::now(), } } @@ -162,7 +171,11 @@ impl Config { Rustc::new( self.get_tool("rustc")?, self.maybe_get_tool("rustc_wrapper")?, - cache_location, + if self.cache_rustc_info { + cache_location + } else { + None + }, ) } diff --git a/src/doc/src/reference/environment-variables.md b/src/doc/src/reference/environment-variables.md index b5f81e2d224..3b505faa2e3 100644 --- a/src/doc/src/reference/environment-variables.md +++ b/src/doc/src/reference/environment-variables.md @@ -9,29 +9,31 @@ with them: You can override these environment variables to change Cargo's behavior on your system: -* `CARGO_HOME` - Cargo maintains a local cache of the registry index and of git +* `CARGO_HOME` — Cargo maintains a local cache of the registry index and of git checkouts of crates. By default these are stored under `$HOME/.cargo`, but this variable overrides the location of this directory. Once a crate is cached it is not removed by the clean command. -* `CARGO_TARGET_DIR` - Location of where to place all generated artifacts, +* `CARGO_TARGET_DIR` — Location of where to place all generated artifacts, relative to the current working directory. -* `RUSTC` - Instead of running `rustc`, Cargo will execute this specified +* `RUSTC` — Instead of running `rustc`, Cargo will execute this specified compiler instead. -* `RUSTC_WRAPPER` - Instead of simply running `rustc`, Cargo will execute this +* `RUSTC_WRAPPER` — Instead of simply running `rustc`, Cargo will execute this specified wrapper instead, passing as its commandline arguments the rustc invocation, with the first argument being rustc. -* `RUSTDOC` - Instead of running `rustdoc`, Cargo will execute this specified +* `RUSTDOC` — Instead of running `rustdoc`, Cargo will execute this specified `rustdoc` instance instead. -* `RUSTDOCFLAGS` - A space-separated list of custom flags to pass to all `rustdoc` +* `RUSTDOCFLAGS` — A space-separated list of custom flags to pass to all `rustdoc` invocations that Cargo performs. In contrast with `cargo rustdoc`, this is useful for passing a flag to *all* `rustdoc` instances. -* `RUSTFLAGS` - A space-separated list of custom flags to pass to all compiler +* `RUSTFLAGS` — A space-separated list of custom flags to pass to all compiler invocations that Cargo performs. In contrast with `cargo rustc`, this is useful for passing a flag to *all* compiler instances. -* `CARGO_INCREMENTAL` - If this is set to 1 then Cargo will force incremental +* `CARGO_INCREMENTAL` — If this is set to 1 then Cargo will force incremental compilation to be enabled for the current compilation, and when set to 0 it will force disabling it. If this env var isn't present then cargo's defaults will otherwise be used. +* `CARGO_CACHE_RUSTC_INFO` — If this is set to 0 then Cargo will not try to cache + compiler version information. Note that Cargo will also read environment variables for `.cargo/config` configuration values, as described in [that documentation][config-env] diff --git a/tests/testsuite/rustc_info_cache.rs b/tests/testsuite/rustc_info_cache.rs index 7d3bcb5e17d..28d2c5dc631 100644 --- a/tests/testsuite/rustc_info_cache.rs +++ b/tests/testsuite/rustc_info_cache.rs @@ -19,7 +19,9 @@ fn rustc_info_cache() { .build(); let miss = "[..] rustc info cache miss[..]"; - let hit = "[..] rustc info cache hit[..]"; + let hit = "[..]rustc info cache hit[..]"; + let uncached = "[..]rustc info uncached[..]"; + let update = "[..]updated rustc info cache[..]"; assert_that( p.cargo("build").env("RUST_LOG", "cargo::util::rustc=info"), @@ -27,7 +29,8 @@ fn rustc_info_cache() { .with_status(0) .with_stderr_contains("[..]failed to read rustc info cache[..]") .with_stderr_contains(miss) - .with_stderr_does_not_contain(hit), + .with_stderr_does_not_contain(hit) + .with_stderr_contains(update), ); assert_that( @@ -36,7 +39,19 @@ fn rustc_info_cache() { .with_status(0) .with_stderr_contains("[..]reusing existing rustc info cache[..]") .with_stderr_contains(hit) - .with_stderr_does_not_contain(miss), + .with_stderr_does_not_contain(miss) + .with_stderr_does_not_contain(update), + ); + + assert_that( + p.cargo("build") + .env("RUST_LOG", "cargo::util::rustc=info") + .env("CARGO_CACHE_RUSTC_INFO", "0"), + execs() + .with_status(0) + .with_stderr_contains("[..]rustc info cache disabled[..]") + .with_stderr_contains(uncached) + .with_stderr_does_not_contain(update), ); let other_rustc = { @@ -80,7 +95,8 @@ fn rustc_info_cache() { .with_status(0) .with_stderr_contains("[..]different compiler, creating new rustc info cache[..]") .with_stderr_contains(miss) - .with_stderr_does_not_contain(hit), + .with_stderr_does_not_contain(hit) + .with_stderr_contains(update), ); assert_that( @@ -91,7 +107,8 @@ fn rustc_info_cache() { .with_status(0) .with_stderr_contains("[..]reusing existing rustc info cache[..]") .with_stderr_contains(hit) - .with_stderr_does_not_contain(miss), + .with_stderr_does_not_contain(miss) + .with_stderr_does_not_contain(update), ); other_rustc.move_into_the_future(); @@ -104,7 +121,8 @@ fn rustc_info_cache() { .with_status(0) .with_stderr_contains("[..]different compiler, creating new rustc info cache[..]") .with_stderr_contains(miss) - .with_stderr_does_not_contain(hit), + .with_stderr_does_not_contain(hit) + .with_stderr_contains(update), ); assert_that( @@ -115,6 +133,7 @@ fn rustc_info_cache() { .with_status(0) .with_stderr_contains("[..]reusing existing rustc info cache[..]") .with_stderr_contains(hit) - .with_stderr_does_not_contain(miss), + .with_stderr_does_not_contain(miss) + .with_stderr_does_not_contain(update), ); } From 42f347a5ab92fada99857ce764aa6e75a4ec48e4 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 14 Apr 2018 17:06:05 +0300 Subject: [PATCH 06/11] Correctly resolve executables in paths on windows --- src/cargo/util/paths.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cargo/util/paths.rs b/src/cargo/util/paths.rs index eeee00ddde9..eb5ada5cbf5 100644 --- a/src/cargo/util/paths.rs +++ b/src/cargo/util/paths.rs @@ -4,6 +4,7 @@ use std::fs::{self, File, OpenOptions}; use std::io; use std::io::prelude::*; use std::path::{Component, Path, PathBuf}; +use std::iter; use filetime::FileTime; @@ -88,8 +89,16 @@ pub fn without_prefix<'a>(long_path: &'a Path, prefix: &'a Path) -> Option<&'a P pub fn resolve_executable(exec: &Path) -> CargoResult { if exec.components().count() == 1 { let paths = env::var_os("PATH").ok_or(format_err!("no PATH"))?; - for path in env::split_paths(&paths) { + let candidates = env::split_paths(&paths).flat_map(|path| { let candidate = PathBuf::from(path).join(&exec); + let with_exe = if env::consts::EXE_EXTENSION == "" { + None + } else { + Some(candidate.with_extension(env::consts::EXE_EXTENSION)) + }; + iter::once(candidate).chain(with_exe) + }); + for candidate in candidates { if candidate.is_file() { // PATH may have a component like "." in it, so we still need to // canonicalize. From e2f33787f0cfda6e7dfd19e2b0bceb2f3849126f Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 16 Apr 2018 17:57:25 +0300 Subject: [PATCH 07/11] Remove `internal` from error handling --- src/cargo/util/paths.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cargo/util/paths.rs b/src/cargo/util/paths.rs index eb5ada5cbf5..350663620ae 100644 --- a/src/cargo/util/paths.rs +++ b/src/cargo/util/paths.rs @@ -8,8 +8,7 @@ use std::iter; use filetime::FileTime; -use util::{internal, CargoResult}; -use util::errors::{CargoError, CargoResultExt, Internal}; +use util::errors::{CargoError, CargoResult, CargoResultExt, Internal}; pub fn join_paths>(paths: &[T], env: &str) -> CargoResult { let err = match env::join_paths(paths.iter()) { @@ -154,13 +153,13 @@ pub fn append(path: &Path, contents: &[u8]) -> CargoResult<()> { f.write_all(contents)?; Ok(()) })() - .chain_err(|| internal(format!("failed to write `{}`", path.display())))?; + .chain_err(|| format!("failed to write `{}`", path.display()))?; Ok(()) } pub fn mtime(path: &Path) -> CargoResult { let meta = - fs::metadata(path).chain_err(|| internal(format!("failed to stat `{}`", path.display())))?; + fs::metadata(path).chain_err(|| format!("failed to stat `{}`", path.display()))?; Ok(FileTime::from_last_modification_time(&meta)) } From 501823b0bbf9128acf70a36bf27eca1dbc9adbb5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 16 Apr 2018 18:50:25 +0300 Subject: [PATCH 08/11] Handle non-rustup cargo + rustup rustc in rustc cache --- src/cargo/util/config.rs | 1 + src/cargo/util/rustc.rs | 31 +++++++++++++++++++++++++------ tests/testsuite/cargotest/mod.rs | 11 +++++++++-- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index 1dfefb8c8ae..c7a37e744e2 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -171,6 +171,7 @@ impl Config { Rustc::new( self.get_tool("rustc")?, self.maybe_get_tool("rustc_wrapper")?, + &self.home().join("bin").join("rustc").into_path_unlocked(), if self.cache_rustc_info { cache_location } else { diff --git a/src/cargo/util/rustc.rs b/src/cargo/util/rustc.rs index b9ec492c20a..53f74e130b2 100644 --- a/src/cargo/util/rustc.rs +++ b/src/cargo/util/rustc.rs @@ -35,11 +35,12 @@ impl Rustc { pub fn new( path: PathBuf, wrapper: Option, + rustup_rustc: &Path, cache_location: Option, ) -> CargoResult { let _p = profile::start("Rustc::new"); - let mut cache = Cache::load(&path, cache_location); + let mut cache = Cache::load(&path, rustup_rustc, cache_location); let mut cmd = util::process(&path); cmd.arg("-vV"); @@ -86,6 +87,9 @@ impl Rustc { /// hundred milliseconds! Because we need compiler version info even for no-op /// builds, we cache it here, based on compiler's mtime and rustup's current /// toolchain. +/// +/// https://github.com/rust-lang/cargo/issues/5315 +/// https://github.com/rust-lang/rust/issues/49761 #[derive(Debug)] struct Cache { cache_location: Option, @@ -100,8 +104,8 @@ struct CacheData { } impl Cache { - fn load(rustc: &Path, cache_location: Option) -> Cache { - match (cache_location, rustc_fingerprint(rustc)) { + fn load(rustc: &Path, rustup_rustc: &Path, cache_location: Option) -> Cache { + match (cache_location, rustc_fingerprint(rustc, rustup_rustc)) { (Some(cache_location), Ok(rustc_fingerprint)) => { let empty = CacheData { rustc_fingerprint, @@ -195,7 +199,7 @@ impl Drop for Cache { } } -fn rustc_fingerprint(path: &Path) -> CargoResult { +fn rustc_fingerprint(path: &Path, rustup_rustc: &Path) -> CargoResult { let mut hasher = SipHasher::new_with_keys(0, 0); let path = paths::resolve_executable(path)?; @@ -203,8 +207,20 @@ fn rustc_fingerprint(path: &Path) -> CargoResult { paths::mtime(&path)?.hash(&mut hasher); - match (env::var("RUSTUP_HOME"), env::var("RUSTUP_TOOLCHAIN")) { - (Ok(rustup_home), Ok(rustup_toolchain)) => { + // Rustup can change the effective compiler without touching + // the `rustc` binary, so we try to account for this here. + // If we see rustup's env vars, we mix them into the fingerprint, + // but we also mix in the mtime of the actual compiler (and not + // the rustup shim at `~/.cargo/bin/rustup`), because `RUSTUP_TOOLCHAIN` + // could be just `stable-x86_64-unknown-linux-gnu`, i.e, it could + // not mention the version of Rust at all, which changes after + // `rustup update`. + // + // If we don't see rustup env vars, but it looks like the compiler + // is managed by rustup, we conservatively bail out. + let maybe_rustup = rustup_rustc == path; + match (maybe_rustup, env::var("RUSTUP_HOME"), env::var("RUSTUP_TOOLCHAIN")) { + (_, Ok(rustup_home), Ok(rustup_toolchain)) => { debug!("adding rustup info to rustc fingerprint"); rustup_toolchain.hash(&mut hasher); rustup_home.hash(&mut hasher); @@ -215,6 +231,9 @@ fn rustc_fingerprint(path: &Path) -> CargoResult { .join("rustc"); paths::mtime(&rustup_rustc)?.hash(&mut hasher); } + (true, _, _) => { + bail!("probably rustup rustc, but without rustup's env vars") + } _ => (), } diff --git a/tests/testsuite/cargotest/mod.rs b/tests/testsuite/cargotest/mod.rs index 23efab70360..c2852ef1433 100644 --- a/tests/testsuite/cargotest/mod.rs +++ b/tests/testsuite/cargotest/mod.rs @@ -3,14 +3,21 @@ use std::time::Duration; use cargo::util::Rustc; use cargo; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; #[macro_use] pub mod support; pub mod install; -thread_local!(pub static RUSTC: Rustc = Rustc::new(PathBuf::from("rustc"), None, None).unwrap()); +thread_local!( +pub static RUSTC: Rustc = Rustc::new( + PathBuf::from("rustc"), + None, + Path::new("should be path to rustup rustc, but we don't care in tests"), + None, +).unwrap() +); pub fn rustc_host() -> String { RUSTC.with(|r| r.host.clone()) From 9e5aaf9305e1f9c7ed8aba7421dcbf6152afd2ea Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 16 Apr 2018 18:52:35 +0300 Subject: [PATCH 09/11] Make rustc_info a hidden file --- src/cargo/ops/cargo_compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cargo/ops/cargo_compile.rs b/src/cargo/ops/cargo_compile.rs index 485f2b46af0..fbf6acb3024 100644 --- a/src/cargo/ops/cargo_compile.rs +++ b/src/cargo/ops/cargo_compile.rs @@ -260,7 +260,7 @@ pub fn compile_ws<'a>( bail!("jobs must be at least 1") } - let rustc_info_cache = ws.target_dir().join("rustc_info.json").into_path_unlocked(); + let rustc_info_cache = ws.target_dir().join(".rustc_info.json").into_path_unlocked(); let mut build_config = BuildConfig::new(config, jobs, &target, Some(rustc_info_cache))?; build_config.release = release; build_config.test = mode == CompileMode::Test || mode == CompileMode::Bench; From c0e326d75a32765560a4cbfdb6ce6050afad2a18 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 16 Apr 2018 18:53:12 +0300 Subject: [PATCH 10/11] Simplify --- src/cargo/util/rustc.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cargo/util/rustc.rs b/src/cargo/util/rustc.rs index 53f74e130b2..25bb9b5ba95 100644 --- a/src/cargo/util/rustc.rs +++ b/src/cargo/util/rustc.rs @@ -186,15 +186,15 @@ impl Cache { impl Drop for Cache { fn drop(&mut self) { - match (&self.cache_location, self.dirty) { - (&Some(ref path), true) => { - let json = serde_json::to_string(&self.data).unwrap(); - match paths::write(path, json.as_bytes()) { - Ok(()) => info!("updated rustc info cache"), - Err(e) => warn!("failed to update rustc info cache: {}", e), - } + if !self.dirty { + return; + } + if let Some(ref path) = self.cache_location { + let json = serde_json::to_string(&self.data).unwrap(); + match paths::write(path, json.as_bytes()) { + Ok(()) => info!("updated rustc info cache"), + Err(e) => warn!("failed to update rustc info cache: {}", e), } - _ => (), } } } From 76b0a8a02ee07e9e80df3b4ac1fb5192f4ef2044 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 16 Apr 2018 19:22:53 +0300 Subject: [PATCH 11/11] rustc is rustc.exe on windows --- src/cargo/ops/cargo_compile.rs | 4 +++- src/cargo/util/config.rs | 6 +++++- src/cargo/util/paths.rs | 3 +-- src/cargo/util/rustc.rs | 17 ++++++++++------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/cargo/ops/cargo_compile.rs b/src/cargo/ops/cargo_compile.rs index fbf6acb3024..32d2cd2003c 100644 --- a/src/cargo/ops/cargo_compile.rs +++ b/src/cargo/ops/cargo_compile.rs @@ -260,7 +260,9 @@ pub fn compile_ws<'a>( bail!("jobs must be at least 1") } - let rustc_info_cache = ws.target_dir().join(".rustc_info.json").into_path_unlocked(); + let rustc_info_cache = ws.target_dir() + .join(".rustc_info.json") + .into_path_unlocked(); let mut build_config = BuildConfig::new(config, jobs, &target, Some(rustc_info_cache))?; build_config.release = release; build_config.test = mode == CompileMode::Test || mode == CompileMode::Bench; diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index c7a37e744e2..ac8bf21f6a4 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -171,7 +171,11 @@ impl Config { Rustc::new( self.get_tool("rustc")?, self.maybe_get_tool("rustc_wrapper")?, - &self.home().join("bin").join("rustc").into_path_unlocked(), + &self.home() + .join("bin") + .join("rustc") + .into_path_unlocked() + .with_extension(env::consts::EXE_EXTENSION), if self.cache_rustc_info { cache_location } else { diff --git a/src/cargo/util/paths.rs b/src/cargo/util/paths.rs index 350663620ae..de87a6a3cb0 100644 --- a/src/cargo/util/paths.rs +++ b/src/cargo/util/paths.rs @@ -158,8 +158,7 @@ pub fn append(path: &Path, contents: &[u8]) -> CargoResult<()> { } pub fn mtime(path: &Path) -> CargoResult { - let meta = - fs::metadata(path).chain_err(|| format!("failed to stat `{}`", path.display()))?; + let meta = fs::metadata(path).chain_err(|| format!("failed to stat `{}`", path.display()))?; Ok(FileTime::from_last_modification_time(&meta)) } diff --git a/src/cargo/util/rustc.rs b/src/cargo/util/rustc.rs index 25bb9b5ba95..883563d2fd9 100644 --- a/src/cargo/util/rustc.rs +++ b/src/cargo/util/rustc.rs @@ -219,21 +219,24 @@ fn rustc_fingerprint(path: &Path, rustup_rustc: &Path) -> CargoResult { // If we don't see rustup env vars, but it looks like the compiler // is managed by rustup, we conservatively bail out. let maybe_rustup = rustup_rustc == path; - match (maybe_rustup, env::var("RUSTUP_HOME"), env::var("RUSTUP_TOOLCHAIN")) { + match ( + maybe_rustup, + env::var("RUSTUP_HOME"), + env::var("RUSTUP_TOOLCHAIN"), + ) { (_, Ok(rustup_home), Ok(rustup_toolchain)) => { debug!("adding rustup info to rustc fingerprint"); rustup_toolchain.hash(&mut hasher); rustup_home.hash(&mut hasher); - let rustup_rustc = Path::new(&rustup_home) + let real_rustc = Path::new(&rustup_home) .join("toolchains") .join(rustup_toolchain) .join("bin") - .join("rustc"); - paths::mtime(&rustup_rustc)?.hash(&mut hasher); - } - (true, _, _) => { - bail!("probably rustup rustc, but without rustup's env vars") + .join("rustc") + .with_extension(env::consts::EXE_EXTENSION); + paths::mtime(&real_rustc)?.hash(&mut hasher); } + (true, _, _) => bail!("probably rustup rustc, but without rustup's env vars"), _ => (), }