Auto merge of #36758 - michaelwoerister:incr-comp-file-headers, r=eddyb

incr.comp.: Let the compiler ignore incompatible incr. comp. cache artifacts

Implements #35720.

cc @nikomatsakis

src/librustc_incremental/persist/file_format.rs

@@ -0,0 +1,122 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! This module defines a generic file format that allows to check if a given
+//! file generated by incremental compilation was generated by a compatible
+//! compiler version. This file format is used for the on-disk version of the
+//! dependency graph and the exported metadata hashes.
+//!
+//! In practice "compatible compiler version" means "exactly the same compiler
+//! version", since the header encodes the git commit hash of the compiler.
+//! Since we can always just ignore the incremental compilation cache and
+//! compiler versions don't change frequently for the typical user, being
+//! conservative here practically has no downside.
+
+use std::io::{self, Read};
+use std::path::Path;
+use std::fs::File;
+use std::env;
+
+use rustc::session::config::nightly_options;
+
+/// The first few bytes of files generated by incremental compilation
+const FILE_MAGIC: &'static [u8] = b"RSIC";
+
+/// Change this if the header format changes
+const HEADER_FORMAT_VERSION: u16 = 0;
+
+/// A version string that hopefully is always different for compiler versions
+/// with different encodings of incremental compilation artifacts. Contains
+/// the git commit hash.
+const RUSTC_VERSION: Option<&'static str> = option_env!("CFG_VERSION");
+
+pub fn write_file_header<W: io::Write>(stream: &mut W) -> io::Result<()> {
+    stream.write_all(FILE_MAGIC)?;
+    stream.write_all(&[(HEADER_FORMAT_VERSION >> 0) as u8,
+                       (HEADER_FORMAT_VERSION >> 8) as u8])?;
+
+    let rustc_version = rustc_version();
+    assert_eq!(rustc_version.len(), (rustc_version.len() as u8) as usize);
+    stream.write_all(&[rustc_version.len() as u8])?;
+    stream.write_all(rustc_version.as_bytes())?;
+
+    Ok(())
+}
+
+/// Reads the contents of a file with a file header as defined in this module.
+///
+/// - Returns `Ok(Some(data))` if the file existed and was generated by a
+///   compatible compiler version. `data` is the entire contents of the file
+///   *after* the header.
+/// - Returns `Ok(None)` if the file did not exist or was generated by an
+///   incompatible version of the compiler.
+/// - Returns `Err(..)` if some kind of IO error occurred while reading the
+///   file.
+pub fn read_file(path: &Path) -> io::Result<Option<Vec<u8>>> {
+    if !path.exists() {
+        return Ok(None);
+    }
+
+    let mut file = File::open(path)?;
+
+    // Check FILE_MAGIC
+    {
+        debug_assert!(FILE_MAGIC.len() == 4);
+        let mut file_magic = [0u8; 4];
+        file.read_exact(&mut file_magic)?;
+        if file_magic != FILE_MAGIC {
+            return Ok(None)
+        }
+    }
+
+    // Check HEADER_FORMAT_VERSION
+    {
+        debug_assert!(::std::mem::size_of_val(&HEADER_FORMAT_VERSION) == 2);
+        let mut header_format_version = [0u8; 2];
+        file.read_exact(&mut header_format_version)?;
+        let header_format_version = (header_format_version[0] as u16) |
+                                    ((header_format_version[1] as u16) << 8);
+
+        if header_format_version != HEADER_FORMAT_VERSION {
+            return Ok(None)
+        }
+    }
+
+    // Check RUSTC_VERSION
+    {
+        let mut rustc_version_str_len = [0u8; 1];
+        file.read_exact(&mut rustc_version_str_len)?;
+        let rustc_version_str_len = rustc_version_str_len[0] as usize;
+        let mut buffer = Vec::with_capacity(rustc_version_str_len);
+        buffer.resize(rustc_version_str_len, 0);
+        file.read_exact(&mut buffer[..])?;
+
+        if &buffer[..] != rustc_version().as_bytes() {
+            return Ok(None);
+        }
+    }
+
+    let mut data = vec![];
+    file.read_to_end(&mut data)?;
+
+    Ok(Some(data))
+}
+
+fn rustc_version() -> String {
+    if nightly_options::is_nightly_build() {
+        if let Some(val) = env::var_os("RUSTC_FORCE_INCR_COMP_ARTIFACT_HEADER") {
+            return val.to_string_lossy().into_owned()
+        }
+    }
+
+    RUSTC_VERSION.expect("Cannot use rustc without explicit version for \
+                          incremental compilation")
+                 .to_string()
+}

src/librustc_incremental/persist/fs.rs

@@ -345,6 +345,15 @@ pub fn finalize_session_directory(sess: &Session, svh: Svh) {
     let _ = garbage_collect_session_directories(sess);
 }
 
+pub fn delete_all_session_dir_contents(sess: &Session) -> io::Result<()> {
+    let sess_dir_iterator = sess.incr_comp_session_dir().read_dir()?;
+    for entry in sess_dir_iterator {
+        let entry = entry?;
+        safe_remove_file(&entry.path())?
+    }
+    Ok(())
+}
+
 fn copy_files(target_dir: &Path,
               source_dir: &Path,
               print_stats_on_success: bool)

src/librustc_incremental/persist/hash.rs

@@ -16,12 +16,11 @@ use rustc_data_structures::fnv::FnvHashMap;
 use rustc_data_structures::flock;
 use rustc_serialize::Decodable;
 use rustc_serialize::opaque::Decoder;
-use std::io::{ErrorKind, Read};
-use std::fs::File;
 
 use IncrementalHashesMap;
 use super::data::*;
 use super::fs::*;
+use super::file_format;
 
 pub struct HashContext<'a, 'tcx: 'a> {
     pub tcx: TyCtxt<'a, 'tcx, 'tcx>,
@@ -153,12 +152,9 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
 
             let hashes_file_path = metadata_hash_import_path(&session_dir);
 
-            let mut data = vec![];
-            match
-                File::open(&hashes_file_path)
-                     .and_then(|mut file| file.read_to_end(&mut data))
+            match file_format::read_file(&hashes_file_path)
             {
-                Ok(_) => {
+                Ok(Some(data)) => {
                     match self.load_from_data(cnum, &data, svh) {
                         Ok(()) => { }
                         Err(err) => {
@@ -167,18 +163,13 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
                         }
                     }
                 }
+                Ok(None) => {
+                    // If the file is not found, that's ok.
+                }
                 Err(err) => {
-                    match err.kind() {
-                        ErrorKind::NotFound => {
-                            // If the file is not found, that's ok.
-                        }
-                        _ => {
-                            self.tcx.sess.err(
-                                &format!("could not load dep information from `{}`: {}",
-                                         hashes_file_path.display(), err));
-                            return;
-                        }
-                    }
+                    self.tcx.sess.err(
+                        &format!("could not load dep information from `{}`: {}",
+                                 hashes_file_path.display(), err));
                 }
             }
         }

src/librustc_incremental/persist/load.rs

@@ -18,8 +18,7 @@ use rustc::ty::TyCtxt;
 use rustc_data_structures::fnv::{FnvHashSet, FnvHashMap};
 use rustc_serialize::Decodable as RustcDecodable;
 use rustc_serialize::opaque::Decoder;
-use std::io::Read;
-use std::fs::{self, File};
+use std::fs;
 use std::path::{Path};
 
 use IncrementalHashesMap;
@@ -28,6 +27,7 @@ use super::directory::*;
 use super::dirty_clean;
 use super::hash::*;
 use super::fs::*;
+use super::file_format;
 
 pub type DirtyNodes = FnvHashSet<DepNode<DefPathIndex>>;
 
@@ -94,25 +94,26 @@ fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 }
 
 fn load_data(sess: &Session, path: &Path) -> Option<Vec<u8>> {
-    if !path.exists() {
-        return None;
-    }
-
-    let mut data = vec![];
-    match
-        File::open(path)
-        .and_then(|mut file| file.read_to_end(&mut data))
-    {
-        Ok(_) => {
-            Some(data)
+    match file_format::read_file(path) {
+        Ok(Some(data)) => return Some(data),
+        Ok(None) => {
+            // The file either didn't exist or was produced by an incompatible
+            // compiler version. Neither is an error.
         }
         Err(err) => {
             sess.err(
                 &format!("could not load dep-graph from `{}`: {}",
                          path.display(), err));
-            None
         }
     }
+
+    if let Err(err) = delete_all_session_dir_contents(sess) {
+        sess.err(&format!("could not clear incompatible incremental \
+                           compilation session directory `{}`: {}",
+                          path.display(), err));
+    }
+
+    None
 }
 
 /// Decode the dep graph and load the edges/nodes that are still clean
@@ -331,16 +332,22 @@ fn load_prev_metadata_hashes(tcx: TyCtxt,
 
     debug!("load_prev_metadata_hashes() - File: {}", file_path.display());
 
-    let mut data = vec![];
-    if !File::open(&file_path)
-             .and_then(|mut file| file.read_to_end(&mut data)).is_ok() {
-        debug!("load_prev_metadata_hashes() - Couldn't read file containing \
-                hashes at `{}`", file_path.display());
-        return
-    }
+    let data = match file_format::read_file(&file_path) {
+        Ok(Some(data)) => data,
+        Ok(None) => {
+            debug!("load_prev_metadata_hashes() - File produced by incompatible \
+                    compiler version: {}", file_path.display());
+            return
+        }
+        Err(err) => {
+            debug!("load_prev_metadata_hashes() - Error reading file `{}`: {}",
+                   file_path.display(), err);
+            return
+        }
+    };
 
     debug!("load_prev_metadata_hashes() - Decoding hashes");
-    let mut decoder = Decoder::new(&mut data, 0);
+    let mut decoder = Decoder::new(&data, 0);
     let _ = Svh::decode(&mut decoder).unwrap();
     let serialized_hashes = SerializedMetadataHashes::decode(&mut decoder).unwrap();
 
@@ -358,3 +365,4 @@ fn load_prev_metadata_hashes(tcx: TyCtxt,
     debug!("load_prev_metadata_hashes() - successfully loaded {} hashes",
            serialized_hashes.index_map.len());
 }
+

src/librustc_incremental/persist/mod.rs

@@ -21,6 +21,7 @@ mod load;
 mod preds;
 mod save;
 mod work_product;
+mod file_format;
 
 pub use self::fs::finalize_session_directory;
 pub use self::fs::in_incr_comp_dir;

src/librustc_incremental/persist/save.rs

@@ -28,6 +28,7 @@ use super::hash::*;
 use super::preds::*;
 use super::fs::*;
 use super::dirty_clean;
+use super::file_format;
 
 pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                                 incremental_hashes_map: &IncrementalHashesMap,
@@ -102,6 +103,7 @@ fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)
 
     // generate the data in a memory buffer
     let mut wr = Cursor::new(Vec::new());
+    file_format::write_file_header(&mut wr).unwrap();
     match encode(&mut Encoder::new(&mut wr)) {
         Ok(()) => {}
         Err(err) => {

src/test/incremental/cache_file_headers.rs

@@ -0,0 +1,29 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// This test case makes sure that the compiler does not try to re-use anything
+// from the incremental compilation cache if the cache was produced by a
+// different compiler version. This is tested by artificially forcing the
+// emission of a different compiler version in the header of rpass1 artifacts,
+// and then making sure that the only object file of the test program gets
+// re-translated although the program stays unchanged.
+
+// The `l33t haxx0r` Rust compiler is known to produce incr. comp. artifacts
+// that are outrageously incompatible with just about anything, even itself:
+//[rpass1] rustc-env:RUSTC_FORCE_INCR_COMP_ARTIFACT_HEADER="l33t haxx0r rustc 2.1 LTS"
+
+// revisions:rpass1 rpass2
+
+#![feature(rustc_attrs)]
+#![rustc_partition_translated(module="cache_file_headers", cfg="rpass2")]
+
+fn main() {
+    // empty
+}