diff --git a/src/librustc_incremental/persist/file_format.rs b/src/librustc_incremental/persist/file_format.rs new file mode 100644 index 0000000000000..7c2b69e762b93 --- /dev/null +++ b/src/librustc_incremental/persist/file_format.rs @@ -0,0 +1,122 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! This module defines a generic file format that allows to check if a given +//! file generated by incremental compilation was generated by a compatible +//! compiler version. This file format is used for the on-disk version of the +//! dependency graph and the exported metadata hashes. +//! +//! In practice "compatible compiler version" means "exactly the same compiler +//! version", since the header encodes the git commit hash of the compiler. +//! Since we can always just ignore the incremental compilation cache and +//! compiler versions don't change frequently for the typical user, being +//! conservative here practically has no downside. + +use std::io::{self, Read}; +use std::path::Path; +use std::fs::File; +use std::env; + +use rustc::session::config::nightly_options; + +/// The first few bytes of files generated by incremental compilation +const FILE_MAGIC: &'static [u8] = b"RSIC"; + +/// Change this if the header format changes +const HEADER_FORMAT_VERSION: u16 = 0; + +/// A version string that hopefully is always different for compiler versions +/// with different encodings of incremental compilation artifacts. Contains +/// the git commit hash. +const RUSTC_VERSION: Option<&'static str> = option_env!("CFG_VERSION"); + +pub fn write_file_header(stream: &mut W) -> io::Result<()> { + stream.write_all(FILE_MAGIC)?; + stream.write_all(&[(HEADER_FORMAT_VERSION >> 0) as u8, + (HEADER_FORMAT_VERSION >> 8) as u8])?; + + let rustc_version = rustc_version(); + assert_eq!(rustc_version.len(), (rustc_version.len() as u8) as usize); + stream.write_all(&[rustc_version.len() as u8])?; + stream.write_all(rustc_version.as_bytes())?; + + Ok(()) +} + +/// Reads the contents of a file with a file header as defined in this module. +/// +/// - Returns `Ok(Some(data))` if the file existed and was generated by a +/// compatible compiler version. `data` is the entire contents of the file +/// *after* the header. +/// - Returns `Ok(None)` if the file did not exist or was generated by an +/// incompatible version of the compiler. +/// - Returns `Err(..)` if some kind of IO error occurred while reading the +/// file. +pub fn read_file(path: &Path) -> io::Result>> { + if !path.exists() { + return Ok(None); + } + + let mut file = File::open(path)?; + + // Check FILE_MAGIC + { + debug_assert!(FILE_MAGIC.len() == 4); + let mut file_magic = [0u8; 4]; + file.read_exact(&mut file_magic)?; + if file_magic != FILE_MAGIC { + return Ok(None) + } + } + + // Check HEADER_FORMAT_VERSION + { + debug_assert!(::std::mem::size_of_val(&HEADER_FORMAT_VERSION) == 2); + let mut header_format_version = [0u8; 2]; + file.read_exact(&mut header_format_version)?; + let header_format_version = (header_format_version[0] as u16) | + ((header_format_version[1] as u16) << 8); + + if header_format_version != HEADER_FORMAT_VERSION { + return Ok(None) + } + } + + // Check RUSTC_VERSION + { + let mut rustc_version_str_len = [0u8; 1]; + file.read_exact(&mut rustc_version_str_len)?; + let rustc_version_str_len = rustc_version_str_len[0] as usize; + let mut buffer = Vec::with_capacity(rustc_version_str_len); + buffer.resize(rustc_version_str_len, 0); + file.read_exact(&mut buffer[..])?; + + if &buffer[..] != rustc_version().as_bytes() { + return Ok(None); + } + } + + let mut data = vec![]; + file.read_to_end(&mut data)?; + + Ok(Some(data)) +} + +fn rustc_version() -> String { + if nightly_options::is_nightly_build() { + if let Some(val) = env::var_os("RUSTC_FORCE_INCR_COMP_ARTIFACT_HEADER") { + return val.to_string_lossy().into_owned() + } + } + + RUSTC_VERSION.expect("Cannot use rustc without explicit version for \ + incremental compilation") + .to_string() +} diff --git a/src/librustc_incremental/persist/fs.rs b/src/librustc_incremental/persist/fs.rs index c9cfaf4f6613f..2d28afeaebf2d 100644 --- a/src/librustc_incremental/persist/fs.rs +++ b/src/librustc_incremental/persist/fs.rs @@ -345,6 +345,15 @@ pub fn finalize_session_directory(sess: &Session, svh: Svh) { let _ = garbage_collect_session_directories(sess); } +pub fn delete_all_session_dir_contents(sess: &Session) -> io::Result<()> { + let sess_dir_iterator = sess.incr_comp_session_dir().read_dir()?; + for entry in sess_dir_iterator { + let entry = entry?; + safe_remove_file(&entry.path())? + } + Ok(()) +} + fn copy_files(target_dir: &Path, source_dir: &Path, print_stats_on_success: bool) diff --git a/src/librustc_incremental/persist/hash.rs b/src/librustc_incremental/persist/hash.rs index 5a4716e45f6e1..ca173db15fcac 100644 --- a/src/librustc_incremental/persist/hash.rs +++ b/src/librustc_incremental/persist/hash.rs @@ -16,12 +16,11 @@ use rustc_data_structures::fnv::FnvHashMap; use rustc_data_structures::flock; use rustc_serialize::Decodable; use rustc_serialize::opaque::Decoder; -use std::io::{ErrorKind, Read}; -use std::fs::File; use IncrementalHashesMap; use super::data::*; use super::fs::*; +use super::file_format; pub struct HashContext<'a, 'tcx: 'a> { pub tcx: TyCtxt<'a, 'tcx, 'tcx>, @@ -153,12 +152,9 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> { let hashes_file_path = metadata_hash_import_path(&session_dir); - let mut data = vec![]; - match - File::open(&hashes_file_path) - .and_then(|mut file| file.read_to_end(&mut data)) + match file_format::read_file(&hashes_file_path) { - Ok(_) => { + Ok(Some(data)) => { match self.load_from_data(cnum, &data, svh) { Ok(()) => { } Err(err) => { @@ -167,18 +163,13 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> { } } } + Ok(None) => { + // If the file is not found, that's ok. + } Err(err) => { - match err.kind() { - ErrorKind::NotFound => { - // If the file is not found, that's ok. - } - _ => { - self.tcx.sess.err( - &format!("could not load dep information from `{}`: {}", - hashes_file_path.display(), err)); - return; - } - } + self.tcx.sess.err( + &format!("could not load dep information from `{}`: {}", + hashes_file_path.display(), err)); } } } diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index ba15529c81a57..db8d3125e510b 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -18,8 +18,7 @@ use rustc::ty::TyCtxt; use rustc_data_structures::fnv::{FnvHashSet, FnvHashMap}; use rustc_serialize::Decodable as RustcDecodable; use rustc_serialize::opaque::Decoder; -use std::io::Read; -use std::fs::{self, File}; +use std::fs; use std::path::{Path}; use IncrementalHashesMap; @@ -28,6 +27,7 @@ use super::directory::*; use super::dirty_clean; use super::hash::*; use super::fs::*; +use super::file_format; pub type DirtyNodes = FnvHashSet>; @@ -94,25 +94,26 @@ fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } fn load_data(sess: &Session, path: &Path) -> Option> { - if !path.exists() { - return None; - } - - let mut data = vec![]; - match - File::open(path) - .and_then(|mut file| file.read_to_end(&mut data)) - { - Ok(_) => { - Some(data) + match file_format::read_file(path) { + Ok(Some(data)) => return Some(data), + Ok(None) => { + // The file either didn't exist or was produced by an incompatible + // compiler version. Neither is an error. } Err(err) => { sess.err( &format!("could not load dep-graph from `{}`: {}", path.display(), err)); - None } } + + if let Err(err) = delete_all_session_dir_contents(sess) { + sess.err(&format!("could not clear incompatible incremental \ + compilation session directory `{}`: {}", + path.display(), err)); + } + + None } /// Decode the dep graph and load the edges/nodes that are still clean @@ -331,16 +332,22 @@ fn load_prev_metadata_hashes(tcx: TyCtxt, debug!("load_prev_metadata_hashes() - File: {}", file_path.display()); - let mut data = vec![]; - if !File::open(&file_path) - .and_then(|mut file| file.read_to_end(&mut data)).is_ok() { - debug!("load_prev_metadata_hashes() - Couldn't read file containing \ - hashes at `{}`", file_path.display()); - return - } + let data = match file_format::read_file(&file_path) { + Ok(Some(data)) => data, + Ok(None) => { + debug!("load_prev_metadata_hashes() - File produced by incompatible \ + compiler version: {}", file_path.display()); + return + } + Err(err) => { + debug!("load_prev_metadata_hashes() - Error reading file `{}`: {}", + file_path.display(), err); + return + } + }; debug!("load_prev_metadata_hashes() - Decoding hashes"); - let mut decoder = Decoder::new(&mut data, 0); + let mut decoder = Decoder::new(&data, 0); let _ = Svh::decode(&mut decoder).unwrap(); let serialized_hashes = SerializedMetadataHashes::decode(&mut decoder).unwrap(); @@ -358,3 +365,4 @@ fn load_prev_metadata_hashes(tcx: TyCtxt, debug!("load_prev_metadata_hashes() - successfully loaded {} hashes", serialized_hashes.index_map.len()); } + diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs index ba0f71971bb45..26fcde05868b9 100644 --- a/src/librustc_incremental/persist/mod.rs +++ b/src/librustc_incremental/persist/mod.rs @@ -21,6 +21,7 @@ mod load; mod preds; mod save; mod work_product; +mod file_format; pub use self::fs::finalize_session_directory; pub use self::fs::in_incr_comp_dir; diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index 896e8a9845e94..e6fb1da1982c1 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -28,6 +28,7 @@ use super::hash::*; use super::preds::*; use super::fs::*; use super::dirty_clean; +use super::file_format; pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, incremental_hashes_map: &IncrementalHashesMap, @@ -102,6 +103,7 @@ fn save_in(sess: &Session, path_buf: PathBuf, encode: F) // generate the data in a memory buffer let mut wr = Cursor::new(Vec::new()); + file_format::write_file_header(&mut wr).unwrap(); match encode(&mut Encoder::new(&mut wr)) { Ok(()) => {} Err(err) => { diff --git a/src/test/incremental/cache_file_headers.rs b/src/test/incremental/cache_file_headers.rs new file mode 100644 index 0000000000000..274a3920be8d4 --- /dev/null +++ b/src/test/incremental/cache_file_headers.rs @@ -0,0 +1,29 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// This test case makes sure that the compiler does not try to re-use anything +// from the incremental compilation cache if the cache was produced by a +// different compiler version. This is tested by artificially forcing the +// emission of a different compiler version in the header of rpass1 artifacts, +// and then making sure that the only object file of the test program gets +// re-translated although the program stays unchanged. + +// The `l33t haxx0r` Rust compiler is known to produce incr. comp. artifacts +// that are outrageously incompatible with just about anything, even itself: +//[rpass1] rustc-env:RUSTC_FORCE_INCR_COMP_ARTIFACT_HEADER="l33t haxx0r rustc 2.1 LTS" + +// revisions:rpass1 rpass2 + +#![feature(rustc_attrs)] +#![rustc_partition_translated(module="cache_file_headers", cfg="rpass2")] + +fn main() { + // empty +}