Auto merge of #25387 - eddyb:syn-file-loader, r=nikomatsakis

This allows compiling entire crates from memory or preprocessing source files before they are tokenized.

Minor API refactoring included, which is a [breaking-change] for libsyntax users:
* `ParseSess::{next_node_id, reserve_node_ids}` moved to rustc's `Session`
* `new_parse_sess` -> `ParseSess::new`
* `new_parse_sess_special_handler` -> `ParseSess::with_span_handler`
* `mk_span_handler` -> `SpanHandler::new`
* `default_handler` -> `Handler::new`
* `mk_handler` -> `Handler::with_emitter`
* `string_to_filemap(sess source, path)` -> `sess.codemap().new_filemap(path, source)`

src/grammar/verify.rs

@@ -287,11 +287,9 @@ fn main() {
     let options = config::basic_options();
     let session = session::build_session(options, None,
                                          syntax::diagnostics::registry::Registry::new(&[]));
-    let filemap = parse::string_to_filemap(&session.parse_sess,
-                                           code,
-                                           String::from_str("<n/a>"));
+    let filemap = session.parse_sess.codemap().new_filemap(String::from_str("<n/a>"), code);
     let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
-    let ref cm = lexer.span_diagnostic.cm;
+    let cm = session.codemap();
 
     // ANTLR
     let mut token_file = File::open(&Path::new(&args.next().unwrap())).unwrap();

src/librustc/middle/astencode.rs

@@ -1866,7 +1866,7 @@ impl FakeExtCtxt for parse::ParseSess {
 
 #[cfg(test)]
 fn mk_ctxt() -> parse::ParseSess {
-    parse::new_parse_sess()
+    parse::ParseSess::new()
 }
 
 #[cfg(test)]

src/librustc/session/config.rs

@@ -852,7 +852,7 @@ pub fn parse_cfgspecs(cfgspecs: Vec<String> ) -> ast::CrateConfig {
         parse::parse_meta_from_source_str("cfgspec".to_string(),
                                           s.to_string(),
                                           Vec::new(),
-                                          &parse::new_parse_sess())
+                                          &parse::ParseSess::new())
     }).collect::<ast::CrateConfig>()
 }
 

src/librustc/session/mod.rs

@@ -64,7 +64,9 @@ pub struct Session {
     /// operations such as auto-dereference and monomorphization.
     pub recursion_limit: Cell<usize>,
 
-    pub can_print_warnings: bool
+    pub can_print_warnings: bool,
+
+    next_node_id: Cell<ast::NodeId>
 }
 
 impl Session {
@@ -213,16 +215,23 @@ impl Session {
         lints.insert(id, vec!((lint_id, sp, msg)));
     }
     pub fn next_node_id(&self) -> ast::NodeId {
-        self.parse_sess.next_node_id()
+        self.reserve_node_ids(1)
     }
     pub fn reserve_node_ids(&self, count: ast::NodeId) -> ast::NodeId {
-        self.parse_sess.reserve_node_ids(count)
+        let id = self.next_node_id.get();
+
+        match id.checked_add(count) {
+            Some(next) => self.next_node_id.set(next),
+            None => self.bug("Input too large, ran out of node ids!")
+        }
+
+        id
     }
     pub fn diagnostic<'a>(&'a self) -> &'a diagnostic::SpanHandler {
         &self.parse_sess.span_diagnostic
     }
     pub fn codemap<'a>(&'a self) -> &'a codemap::CodeMap {
-        &self.parse_sess.span_diagnostic.cm
+        self.parse_sess.codemap()
     }
     // This exists to help with refactoring to eliminate impossible
     // cases later on
@@ -359,9 +368,9 @@ pub fn build_session(sopts: config::Options,
 
     let codemap = codemap::CodeMap::new();
     let diagnostic_handler =
-        diagnostic::default_handler(sopts.color, Some(registry), can_print_warnings);
+        diagnostic::Handler::new(sopts.color, Some(registry), can_print_warnings);
     let span_diagnostic_handler =
-        diagnostic::mk_span_handler(diagnostic_handler, codemap);
+        diagnostic::SpanHandler::new(diagnostic_handler, codemap);
 
     build_session_(sopts, local_crate_source_file, span_diagnostic_handler)
 }
@@ -378,7 +387,7 @@ pub fn build_session_(sopts: config::Options,
     }
     };
     let target_cfg = config::build_target_config(&sopts, &span_diagnostic);
-    let p_s = parse::new_parse_sess_special_handler(span_diagnostic);
+    let p_s = parse::ParseSess::with_span_handler(span_diagnostic);
     let default_sysroot = match sopts.maybe_sysroot {
         Some(_) => None,
         None => Some(filesearch::get_or_default_sysroot())
@@ -421,7 +430,8 @@ pub fn build_session_(sopts: config::Options,
         delayed_span_bug: RefCell::new(None),
         features: RefCell::new(feature_gate::Features::new()),
         recursion_limit: Cell::new(64),
-        can_print_warnings: can_print_warnings
+        can_print_warnings: can_print_warnings,
+        next_node_id: Cell::new(1)
     };
 
     sess

src/librustc_back/target/mod.rs

@@ -218,7 +218,7 @@ impl Target {
         // this is 1. ugly, 2. error prone.
 
 
-        let handler = diagnostic::default_handler(diagnostic::Auto, None, true);
+        let handler = diagnostic::Handler::new(diagnostic::Auto, None, true);
 
         let get_req_field = |name: &str| {
             match obj.find(name)

src/librustc_driver/test.rs

@@ -105,9 +105,9 @@ fn test_env<F>(source_string: &str,
     let codemap =
         CodeMap::new();
     let diagnostic_handler =
-        diagnostic::mk_handler(true, emitter);
+        diagnostic::Handler::with_emitter(true, emitter);
     let span_diagnostic_handler =
-        diagnostic::mk_span_handler(diagnostic_handler, codemap);
+        diagnostic::SpanHandler::new(diagnostic_handler, codemap);
 
     let sess = session::build_session_(options, None, span_diagnostic_handler);
     rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));

src/librustc_trans/back/write.rs

@@ -21,7 +21,7 @@ use util::common::time;
 use util::common::path2cstr;
 use syntax::codemap;
 use syntax::diagnostic;
-use syntax::diagnostic::{Emitter, Handler, Level, mk_handler};
+use syntax::diagnostic::{Emitter, Handler, Level};
 
 use std::ffi::{CStr, CString};
 use std::fs;
@@ -928,7 +928,7 @@ fn run_work_multithreaded(sess: &Session,
         futures.push(rx);
 
         thread::Builder::new().name(format!("codegen-{}", i)).spawn(move || {
-            let diag_handler = mk_handler(true, box diag_emitter);
+            let diag_handler = Handler::with_emitter(true, box diag_emitter);
 
             // Must construct cgcx inside the proc because it has non-Send
             // fields.

src/librustdoc/core.rs

@@ -110,9 +110,9 @@ pub fn run_core(search_paths: SearchPaths, cfgs: Vec<String>, externs: Externs,
     };
 
     let codemap = codemap::CodeMap::new();
-    let diagnostic_handler = diagnostic::default_handler(diagnostic::Auto, None, true);
+    let diagnostic_handler = diagnostic::Handler::new(diagnostic::Auto, None, true);
     let span_diagnostic_handler =
-        diagnostic::mk_span_handler(diagnostic_handler, codemap);
+        diagnostic::SpanHandler::new(diagnostic_handler, codemap);
 
     let sess = session::build_session_(sessopts, cpath,
                                        span_diagnostic_handler);

src/librustdoc/html/highlight.rs

@@ -24,10 +24,8 @@ use syntax::parse;
 /// Highlights some source code, returning the HTML output.
 pub fn highlight(src: &str, class: Option<&str>, id: Option<&str>) -> String {
     debug!("highlighting: ================\n{}\n==============", src);
-    let sess = parse::new_parse_sess();
-    let fm = parse::string_to_filemap(&sess,
-                                      src.to_string(),
-                                      "<stdin>".to_string());
+    let sess = parse::ParseSess::new();
+    let fm = sess.codemap().new_filemap("<stdin>".to_string(), src.to_string());
 
     let mut out = Vec::new();
     doit(&sess,
@@ -62,7 +60,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
     loop {
         let next = lexer.next_token();
 
-        let snip = |sp| sess.span_diagnostic.cm.span_to_snippet(sp).unwrap();
+        let snip = |sp| sess.codemap().span_to_snippet(sp).unwrap();
 
         if next.tok == token::Eof { break }
 
@@ -178,7 +176,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
 
         // as mentioned above, use the original source code instead of
         // stringifying this token
-        let snip = sess.span_diagnostic.cm.span_to_snippet(next.sp).unwrap();
+        let snip = sess.codemap().span_to_snippet(next.sp).unwrap();
         if klass == "" {
             try!(write!(out, "{}", Escape(&snip)));
         } else {

src/librustdoc/test.rs

@@ -65,9 +65,9 @@ pub fn run(input: &str,
     };
 
     let codemap = CodeMap::new();
-    let diagnostic_handler = diagnostic::default_handler(diagnostic::Auto, None, true);
+    let diagnostic_handler = diagnostic::Handler::new(diagnostic::Auto, None, true);
     let span_diagnostic_handler =
-    diagnostic::mk_span_handler(diagnostic_handler, codemap);
+    diagnostic::SpanHandler::new(diagnostic_handler, codemap);
 
     let sess = session::build_session_(sessopts,
                                       Some(input_path.clone()),
@@ -184,7 +184,7 @@ fn runtest(test: &str, cratename: &str, libs: SearchPaths,
     // it with a sink that is also passed to rustc itself. When this function
     // returns the output of the sink is copied onto the output of our own thread.
     //
-    // The basic idea is to not use a default_handler() for rustc, and then also
+    // The basic idea is to not use a default Handler for rustc, and then also
     // not print things by default to the actual stderr.
     struct Sink(Arc<Mutex<Vec<u8>>>);
     impl Write for Sink {
@@ -206,9 +206,9 @@ fn runtest(test: &str, cratename: &str, libs: SearchPaths,
 
     // Compile the code
     let codemap = CodeMap::new();
-    let diagnostic_handler = diagnostic::mk_handler(true, box emitter);
+    let diagnostic_handler = diagnostic::Handler::with_emitter(true, box emitter);
     let span_diagnostic_handler =
-        diagnostic::mk_span_handler(diagnostic_handler, codemap);
+        diagnostic::SpanHandler::new(diagnostic_handler, codemap);
 
     let sess = session::build_session_(sessopts,
                                        None,

src/libsyntax/codemap.rs

@@ -21,9 +21,11 @@ pub use self::MacroFormat::*;
 
 use std::cell::RefCell;
 use std::ops::{Add, Sub};
+use std::path::Path;
 use std::rc::Rc;
 
-use std::fmt;
+use std::{fmt, fs};
+use std::io::{self, Read};
 
 use serialize::{Encodable, Decodable, Encoder, Decoder};
 
@@ -527,24 +529,66 @@ impl FileMap {
     }
 }
 
+/// An abstraction over the fs operations used by the Parser.
+pub trait FileLoader {
+    /// Query the existence of a file.
+    fn file_exists(&self, path: &Path) -> bool;
+
+    /// Read the contents of an UTF-8 file into memory.
+    fn read_file(&self, path: &Path) -> io::Result<String>;
+}
+
+/// A FileLoader that uses std::fs to load real files.
+pub struct RealFileLoader;
+
+impl FileLoader for RealFileLoader {
+    fn file_exists(&self, path: &Path) -> bool {
+        fs::metadata(path).is_ok()
+    }
+
+    fn read_file(&self, path: &Path) -> io::Result<String> {
+        let mut src = String::new();
+        try!(try!(fs::File::open(path)).read_to_string(&mut src));
+        Ok(src)
+    }
+}
 
 // _____________________________________________________________________________
 // CodeMap
 //
 
 pub struct CodeMap {
     pub files: RefCell<Vec<Rc<FileMap>>>,
-    expansions: RefCell<Vec<ExpnInfo>>
+    expansions: RefCell<Vec<ExpnInfo>>,
+    file_loader: Box<FileLoader>
 }
 
 impl CodeMap {
     pub fn new() -> CodeMap {
         CodeMap {
             files: RefCell::new(Vec::new()),
             expansions: RefCell::new(Vec::new()),
+            file_loader: Box::new(RealFileLoader)
+        }
+    }
+
+    pub fn with_file_loader(file_loader: Box<FileLoader>) -> CodeMap {
+        CodeMap {
+            files: RefCell::new(Vec::new()),
+            expansions: RefCell::new(Vec::new()),
+            file_loader: file_loader
         }
     }
 
+    pub fn file_exists(&self, path: &Path) -> bool {
+        self.file_loader.file_exists(path)
+    }
+
+    pub fn load_file(&self, path: &Path) -> io::Result<Rc<FileMap>> {
+        let src = try!(self.file_loader.read_file(path));
+        Ok(self.new_filemap(path.to_str().unwrap().to_string(), src))
+    }
+
     pub fn new_filemap(&self, filename: FileName, mut src: String) -> Rc<FileMap> {
         let mut files = self.files.borrow_mut();
         let start_pos = match files.last() {

src/libsyntax/diagnostic.rs

@@ -122,6 +122,12 @@ pub struct SpanHandler {
 }
 
 impl SpanHandler {
+    pub fn new(handler: Handler, cm: codemap::CodeMap) -> SpanHandler {
+        SpanHandler {
+            handler: handler,
+            cm: cm,
+        }
+    }
     pub fn span_fatal(&self, sp: Span, msg: &str) -> FatalError {
         self.handler.emit(Some((&self.cm, sp)), msg, Fatal);
         return FatalError;
@@ -187,6 +193,19 @@ pub struct Handler {
 }
 
 impl Handler {
+    pub fn new(color_config: ColorConfig,
+               registry: Option<diagnostics::registry::Registry>,
+               can_emit_warnings: bool) -> Handler {
+        let emitter = Box::new(EmitterWriter::stderr(color_config, registry));
+        Handler::with_emitter(can_emit_warnings, emitter)
+    }
+    pub fn with_emitter(can_emit_warnings: bool, e: Box<Emitter + Send>) -> Handler {
+        Handler {
+            err_count: Cell::new(0),
+            emit: RefCell::new(e),
+            can_emit_warnings: can_emit_warnings
+        }
+    }
     pub fn fatal(&self, msg: &str) -> ! {
         self.emit.borrow_mut().emit(None, msg, None, Fatal);
         panic!(FatalError);
@@ -254,27 +273,6 @@ impl Handler {
     }
 }
 
-pub fn mk_span_handler(handler: Handler, cm: codemap::CodeMap) -> SpanHandler {
-    SpanHandler {
-        handler: handler,
-        cm: cm,
-    }
-}
-
-pub fn default_handler(color_config: ColorConfig,
-                       registry: Option<diagnostics::registry::Registry>,
-                       can_emit_warnings: bool) -> Handler {
-    mk_handler(can_emit_warnings, Box::new(EmitterWriter::stderr(color_config, registry)))
-}
-
-pub fn mk_handler(can_emit_warnings: bool, e: Box<Emitter + Send>) -> Handler {
-    Handler {
-        err_count: Cell::new(0),
-        emit: RefCell::new(e),
-        can_emit_warnings: can_emit_warnings
-    }
-}
-
 #[derive(Copy, PartialEq, Clone, Debug)]
 pub enum Level {
     Bug,

src/libsyntax/ext/base.rs

@@ -648,7 +648,7 @@ impl<'a> ExtCtxt<'a> {
         parse::tts_to_parser(self.parse_sess, tts.to_vec(), self.cfg())
     }
 
-    pub fn codemap(&self) -> &'a CodeMap { &self.parse_sess.span_diagnostic.cm }
+    pub fn codemap(&self) -> &'a CodeMap { self.parse_sess.codemap() }
     pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }
     pub fn cfg(&self) -> ast::CrateConfig { self.cfg.clone() }
     pub fn call_site(&self) -> Span {