Auto merge of #5359 - matklad:rustc-cache, r=<try>

Rustc cache

This implements rustc caching, to speed-up no-op builds.

The cache is per-project, and stored in `target` directory. To implement this, I had to move `rustc` from `Config` down to `BuildConfig`.

closes #5315

src/bin/cli.rs

@@ -47,7 +47,7 @@ Run with 'cargo -Z [FLAG] [SUBCOMMAND]'"
     }
 
     if let Some(ref code) = args.value_of("explain") {
-        let mut procss = config.rustc()?.process();
+        let mut procss = config.rustc(None)?.process();
         procss.arg("--explain").arg(code).exec()?;
         return Ok(());
     }

src/cargo/core/compiler/compilation.rs

@@ -57,15 +57,17 @@ pub struct Compilation<'cfg> {
     /// Flags to pass to rustdoc when invoked from cargo test, per package.
     pub rustdocflags: HashMap<PackageId, Vec<String>>,
 
+    pub host: String,
     pub target: String,
 
     config: &'cfg Config,
+    rustc_process: ProcessBuilder,
 
     target_runner: LazyCell<Option<(PathBuf, Vec<String>)>>,
 }
 
 impl<'cfg> Compilation<'cfg> {
-    pub fn new(config: &'cfg Config) -> Compilation<'cfg> {
+    pub fn new(config: &'cfg Config, rustc_process: ProcessBuilder) -> Compilation<'cfg> {
         Compilation {
             libraries: HashMap::new(),
             native_dirs: BTreeSet::new(), // TODO: deprecated, remove
@@ -81,14 +83,16 @@ impl<'cfg> Compilation<'cfg> {
             cfgs: HashMap::new(),
             rustdocflags: HashMap::new(),
             config,
+            rustc_process,
+            host: String::new(),
             target: String::new(),
             target_runner: LazyCell::new(),
         }
     }
 
     /// See `process`.
     pub fn rustc_process(&self, pkg: &Package) -> CargoResult<ProcessBuilder> {
-        self.fill_env(self.config.rustc()?.process(), pkg, true)
+        self.fill_env(self.rustc_process.clone(), pkg, true)
     }
 
     /// See `process`.

src/cargo/core/compiler/context/compilation_files.rs

@@ -434,9 +434,7 @@ fn compute_metadata<'a, 'cfg>(
     unit.target.name().hash(&mut hasher);
     unit.target.kind().hash(&mut hasher);
 
-    if let Ok(rustc) = cx.config.rustc() {
-        rustc.verbose_version.hash(&mut hasher);
-    }
+    cx.build_config.rustc.verbose_version.hash(&mut hasher);
 
     // Seed the contents of __CARGO_DEFAULT_LIB_METADATA to the hasher if present.
     // This should be the release channel, to get a different hash for each channel.

src/cargo/core/compiler/context/mod.rs

@@ -130,7 +130,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             let _p = profile::start("Context::probe_target_info");
             debug!("probe_target_info");
             let host_target_same = match build_config.requested_target {
-                Some(ref s) if s != &config.rustc()?.host => false,
+                Some(ref s) if s != &build_config.host_triple() => false,
                 _ => true,
             };
 
@@ -150,7 +150,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             config,
             target_info,
             host_info,
-            compilation: Compilation::new(config),
+            compilation: Compilation::new(config, build_config.rustc.process()),
             build_state: Arc::new(BuildState::new(&build_config)),
             build_config,
             fingerprints: HashMap::new(),
@@ -302,6 +302,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
                 self.compilation.native_dirs.insert(dir.clone());
             }
         }
+        self.compilation.host = self.build_config.host_triple().to_string();
         self.compilation.target = self.build_config.target_triple().to_string();
         Ok(self.compilation)
     }
@@ -440,7 +441,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             None => return true,
         };
         let (name, info) = match kind {
-            Kind::Host => (self.build_config.host_triple.as_ref(), &self.host_info),
+            Kind::Host => (self.build_config.host_triple(), &self.host_info),
             Kind::Target => (self.build_config.target_triple(), &self.target_info),
         };
         platform.matches(name, info.cfg())
@@ -652,7 +653,8 @@ fn env_args(
     let target = build_config
         .requested_target
         .as_ref()
-        .unwrap_or(&build_config.host_triple);
+        .map(|s| s.as_str())
+        .unwrap_or(build_config.host_triple());
     let key = format!("target.{}.{}", target, name);
     if let Some(args) = config.get_list_or_split_string(&key)? {
         let args = args.val.into_iter();

src/cargo/core/compiler/context/target_info.rs

@@ -53,7 +53,7 @@ impl FileType {
 impl TargetInfo {
     pub fn new(config: &Config, build_config: &BuildConfig, kind: Kind) -> CargoResult<TargetInfo> {
         let rustflags = env_args(config, build_config, None, kind, "RUSTFLAGS")?;
-        let mut process = config.rustc()?.process();
+        let mut process = build_config.rustc.process();
         process
             .arg("-")
             .arg("--crate-name")
@@ -78,20 +78,19 @@ impl TargetInfo {
         with_cfg.arg("--print=cfg");
 
         let mut has_cfg_and_sysroot = true;
-        let output = with_cfg
-            .exec_with_output()
+        let (output, error) = build_config
+            .rustc
+            .cached_output(&with_cfg)
             .or_else(|_| {
                 has_cfg_and_sysroot = false;
-                process.exec_with_output()
+                build_config.rustc.cached_output(&process)
             })
             .chain_err(|| "failed to run `rustc` to learn about target-specific information")?;
 
-        let error = str::from_utf8(&output.stderr).unwrap();
-        let output = str::from_utf8(&output.stdout).unwrap();
         let mut lines = output.lines();
         let mut map = HashMap::new();
         for crate_type in KNOWN_CRATE_TYPES {
-            let out = parse_crate_type(crate_type, error, &mut lines)?;
+            let out = parse_crate_type(crate_type, &error, &mut lines)?;
             map.insert(crate_type.to_string(), out);
         }
 

src/cargo/core/compiler/custom_build.rs

@@ -127,7 +127,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes
         .env(
             "TARGET",
             &match unit.kind {
-                Kind::Host => &cx.build_config.host_triple,
+                Kind::Host => &cx.build_config.host_triple(),
                 Kind::Target => cx.build_config.target_triple(),
             },
         )
@@ -141,8 +141,8 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes
                 "debug"
             },
         )
-        .env("HOST", &cx.build_config.host_triple)
-        .env("RUSTC", &cx.config.rustc()?.path)
+        .env("HOST", &cx.build_config.host_triple())
+        .env("RUSTC", &cx.build_config.rustc.path)
         .env("RUSTDOC", &*cx.config.rustdoc()?)
         .inherit_jobserver(&cx.jobserver);
 

src/cargo/core/compiler/fingerprint.rs

@@ -220,9 +220,7 @@ impl Fingerprint {
             match *local {
                 LocalFingerprint::MtimeBased(ref slot, ref path) => {
                     let path = root.join(path);
-                    let meta = fs::metadata(&path)
-                        .chain_err(|| internal(format!("failed to stat `{}`", path.display())))?;
-                    let mtime = FileTime::from_last_modification_time(&meta);
+                    let mtime = paths::mtime(&path)?;
                     *slot.0.lock().unwrap() = Some(mtime);
                 }
                 LocalFingerprint::EnvBased(..) | LocalFingerprint::Precalculated(..) => continue,
@@ -457,7 +455,7 @@ fn calculate<'a, 'cfg>(
         cx.rustflags_args(unit)?
     };
     let fingerprint = Arc::new(Fingerprint {
-        rustc: util::hash_u64(&cx.config.rustc()?.verbose_version),
+        rustc: util::hash_u64(&cx.build_config.rustc.verbose_version),
         target: util::hash_u64(&unit.target),
         profile: util::hash_u64(&(&unit.profile, cx.incremental_args(unit)?)),
         // Note that .0 is hashed here, not .1 which is the cwd. That doesn't
@@ -718,22 +716,20 @@ where
     I: IntoIterator,
     I::Item: AsRef<Path>,
 {
-    let meta = match fs::metadata(output) {
-        Ok(meta) => meta,
+    let mtime = match paths::mtime(output) {
+        Ok(mtime) => mtime,
         Err(..) => return None,
     };
-    let mtime = FileTime::from_last_modification_time(&meta);
 
     let any_stale = paths.into_iter().any(|path| {
         let path = path.as_ref();
-        let meta = match fs::metadata(path) {
-            Ok(meta) => meta,
+        let mtime2 = match paths::mtime(path) {
+            Ok(mtime) => mtime,
             Err(..) => {
                 info!("stale: {} -- missing", path.display());
                 return true;
             }
         };
-        let mtime2 = FileTime::from_last_modification_time(&meta);
         if mtime2 > mtime {
             info!("stale: {} -- {} vs {}", path.display(), mtime2, mtime);
             true

src/cargo/core/compiler/mod.rs

@@ -12,11 +12,10 @@ use serde_json;
 use core::{Feature, PackageId, Profile, Target};
 use core::manifest::Lto;
 use core::shell::ColorChoice;
-use util::{self, machine_message, Config, ProcessBuilder};
+use util::{self, machine_message, Config, Freshness, ProcessBuilder, Rustc};
 use util::{internal, join_paths, profile};
 use util::paths;
 use util::errors::{CargoResult, CargoResultExt, Internal};
-use util::Freshness;
 
 use self::job::{Job, Work};
 use self::job_queue::JobQueue;
@@ -48,15 +47,8 @@ pub enum Kind {
 }
 
 /// Configuration information for a rustc build.
-#[derive(Default, Clone)]
 pub struct BuildConfig {
-    /// The host arch triple
-    ///
-    /// e.g. x86_64-unknown-linux-gnu, would be
-    ///  - machine: x86_64
-    ///  - hardware-platform: unknown
-    ///  - operating system: linux-gnu
-    pub host_triple: String,
+    pub rustc: Rustc,
     /// Build information for the host arch
     pub host: TargetConfig,
     /// The target arch triple, defaults to host arch
@@ -88,6 +80,7 @@ impl BuildConfig {
         config: &Config,
         jobs: Option<u32>,
         requested_target: &Option<String>,
+        rustc_info_cache: Option<PathBuf>,
     ) -> CargoResult<BuildConfig> {
         if let &Some(ref s) = requested_target {
             if s.trim().is_empty() {
@@ -125,27 +118,40 @@ impl BuildConfig {
             None => None,
         };
         let jobs = jobs.or(cfg_jobs).unwrap_or(::num_cpus::get() as u32);
-
-        let host_triple = config.rustc()?.host.clone();
-        let host_config = TargetConfig::new(config, &host_triple)?;
+        let rustc = config.rustc(rustc_info_cache)?;
+        let host_config = TargetConfig::new(config, &rustc.host)?;
         let target_config = match target.as_ref() {
             Some(triple) => TargetConfig::new(config, triple)?,
             None => host_config.clone(),
         };
         Ok(BuildConfig {
-            host_triple,
+            rustc,
             requested_target: target,
             jobs,
             host: host_config,
             target: target_config,
-            ..Default::default()
+            release: false,
+            test: false,
+            doc_all: false,
+            json_messages: false,
         })
     }
 
+    /// The host arch triple
+    ///
+    /// e.g. x86_64-unknown-linux-gnu, would be
+    ///  - machine: x86_64
+    ///  - hardware-platform: unknown
+    ///  - operating system: linux-gnu
+    pub fn host_triple(&self) -> &str {
+        &self.rustc.host
+    }
+
     pub fn target_triple(&self) -> &str {
         self.requested_target
             .as_ref()
-            .unwrap_or_else(|| &self.host_triple)
+            .map(|s| s.as_str())
+            .unwrap_or(self.host_triple())
     }
 }
 

src/cargo/ops/cargo_clean.rs

@@ -84,7 +84,7 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> {
         }
     }
 
-    let mut build_config = BuildConfig::new(config, Some(1), &opts.target)?;
+    let mut build_config = BuildConfig::new(config, Some(1), &opts.target, None)?;
     build_config.release = opts.release;
     let mut cx = Context::new(ws, &resolve, &packages, opts.config, build_config, profiles)?;
     cx.prepare_units(None, &units)?;

src/cargo/ops/cargo_compile.rs

@@ -260,7 +260,8 @@ pub fn compile_ws<'a>(
         bail!("jobs must be at least 1")
     }
 
-    let mut build_config = BuildConfig::new(config, jobs, &target)?;
+    let rustc_info_cache = ws.target_dir().join("rustc_info.json").into_path_unlocked();
+    let mut build_config = BuildConfig::new(config, jobs, &target, Some(rustc_info_cache))?;
     build_config.release = release;
     build_config.test = mode == CompileMode::Test || mode == CompileMode::Bench;
     build_config.json_messages = message_format == MessageFormat::Json;

src/cargo/ops/cargo_test.rs

@@ -150,7 +150,7 @@ fn run_doc_tests(
     let config = options.compile_opts.config;
 
     // We don't build/rust doctests if target != host
-    if config.rustc()?.host != compilation.target {
+    if compilation.host != compilation.target {
         return Ok((Test::Doc, errors));
     }
 

src/cargo/sources/path.rs

@@ -11,6 +11,7 @@ use ignore::gitignore::GitignoreBuilder;
 use core::{Dependency, Package, PackageId, Registry, Source, SourceId, Summary};
 use ops;
 use util::{self, internal, CargoResult};
+use util::paths;
 use util::Config;
 
 pub struct PathSource<'cfg> {
@@ -529,9 +530,7 @@ impl<'cfg> Source for PathSource<'cfg> {
             // condition where this path was rm'ed - either way,
             // we can ignore the error and treat the path's mtime
             // as 0.
-            let mtime = fs::metadata(&file)
-                .map(|meta| FileTime::from_last_modification_time(&meta))
-                .unwrap_or(FileTime::zero());
+            let mtime = paths::mtime(&file).unwrap_or(FileTime::zero());
             warn!("{} {}", mtime, file.display());
             if mtime > max {
                 max = mtime;