Refactor TokenStream

mk/crates.mk

@@ -60,7 +60,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_data_structures rustc_platform_intrinsics rustc_errors \
                 rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
                 rustc_const_eval rustc_const_math rustc_incremental proc_macro
-HOST_CRATES := syntax syntax_ext proc_macro_tokens proc_macro_plugin syntax_pos $(RUSTC_CRATES) \
+HOST_CRATES := syntax syntax_ext proc_macro_plugin syntax_pos $(RUSTC_CRATES) \
 		rustdoc fmt_macros flate arena graphviz log serialize
 TOOLS := compiletest rustdoc rustc rustbook error_index_generator
 
@@ -102,8 +102,7 @@ DEPS_syntax := std term serialize log arena libc rustc_bitflags std_unicode rust
 DEPS_syntax_ext := syntax syntax_pos rustc_errors fmt_macros proc_macro
 DEPS_proc_macro := syntax syntax_pos rustc_plugin log
 DEPS_syntax_pos := serialize
-DEPS_proc_macro_tokens := syntax syntax_pos log
-DEPS_proc_macro_plugin := syntax syntax_pos rustc_plugin log proc_macro_tokens
+DEPS_proc_macro_plugin := syntax syntax_pos rustc_plugin
 
 DEPS_rustc_const_math := std syntax log serialize rustc_i128
 DEPS_rustc_const_eval := rustc_const_math rustc syntax log serialize \

src/libproc_macro/lib.rs

@@ -82,14 +82,15 @@ pub mod __internal {
     use syntax::ast;
     use syntax::ptr::P;
     use syntax::parse::{self, token, ParseSess};
-    use syntax::tokenstream::TokenStream as TokenStream_;
+    use syntax::tokenstream::{TokenTree, TokenStream as TokenStream_};
 
     use super::{TokenStream, LexError};
 
     pub fn new_token_stream(item: P<ast::Item>) -> TokenStream {
-        TokenStream { inner: TokenStream_::from_tokens(vec![
-            token::Interpolated(Rc::new(token::NtItem(item)))
-        ])}
+        TokenStream {
+            inner: TokenTree::Token(item.span, token::Interpolated(Rc::new(token::NtItem(item))))
+                .into()
+        }
     }
 
     pub fn token_stream_wrap(inner: TokenStream_) -> TokenStream {
@@ -175,7 +176,7 @@ impl FromStr for TokenStream {
             let tts = try!(parse::parse_tts_from_source_str(name, src, sess)
                 .map_err(parse_to_lex_err));
 
-            Ok(__internal::token_stream_wrap(TokenStream_::from_tts(tts)))
+            Ok(__internal::token_stream_wrap(tts.into_iter().collect()))
         })
     }
 }

src/libproc_macro_plugin/Cargo.toml

@@ -8,7 +8,6 @@ path = "lib.rs"
 crate-type = ["dylib"]
 
 [dependencies]
-log = { path = "../liblog" }
 rustc_plugin = { path = "../librustc_plugin" }
 syntax = { path = "../libsyntax" }
-proc_macro_tokens = { path = "../libproc_macro_tokens" }
+syntax_pos = { path = "../libsyntax_pos" }

src/libproc_macro_plugin/lib.rs

@@ -15,11 +15,8 @@
 //! ## Usage
 //! This crate provides the `qquote!` macro for syntax creation.
 //!
-//! The `qquote!` macro imports `syntax::ext::proc_macro_shim::prelude::*`, so you
-//! will need to `extern crate syntax` for usage. (This is a temporary solution until more
-//! of the external API in libproc_macro_tokens is stabilized to support the token construction
-//! operations that the qausiquoter relies on.) The shim file also provides additional
-//! operations, such as `build_block_emitter` (as used in the `cond` example below).
+//! The `qquote!` macro uses the crate `syntax`, so users must declare `extern crate syntax;`
+//! at the crate root. This is a temporary solution until we have better hygiene.
 //!
 //! ## Quasiquotation
 //!
@@ -88,19 +85,20 @@
 
 extern crate rustc_plugin;
 extern crate syntax;
-extern crate proc_macro_tokens;
-#[macro_use] extern crate log;
+extern crate syntax_pos;
 
 mod qquote;
-
 use qquote::qquote;
 
 use rustc_plugin::Registry;
+use syntax::ext::base::SyntaxExtension;
+use syntax::symbol::Symbol;
 
 // ____________________________________________________________________________________________
 // Main macro definition
 
 #[plugin_registrar]
 pub fn plugin_registrar(reg: &mut Registry) {
-    reg.register_macro("qquote", qquote);
+    reg.register_syntax_extension(Symbol::intern("qquote"),
+                                  SyntaxExtension::ProcMacro(Box::new(qquote)));
 }