rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Auto merge of #29582 - oli-obk:token_tree, r=sfackler

Browse Source
This commit is contained in:
bors
2015-11-06 21:11:06 +00:00
parent b14dc5bc1c fcc7067904
commit 475f91f46e
15 changed files with 219 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/libsyntax/ext/concat_idents.rs
View File

@@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use ast;
use ast::{self, TokenTree};
use codemap::Span;
use ext::base::*;
use ext::base;
@@ -17,7 +17,7 @@ use parse::token;
use parse::token::str_to_ident;
use ptr::P;
pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree])
pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
-> Box<base::MacResult+'cx> {
if !cx.ecfg.enable_concat_idents() {
feature_gate::emit_feature_err(&cx.parse_sess.span_diagnostic,
@@ -32,7 +32,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]
for (i, e) in tts.iter().enumerate() {
if i & 1 == 1 {
match *e {
ast::TtToken(_, token::Comma) => {},
TokenTree::Token(_, token::Comma) => {},
_ => {
cx.span_err(sp, "concat_idents! expecting comma.");
return DummyResult::expr(sp);
@@ -40,7 +40,7 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]
}
} else {
match *e {
ast::TtToken(_, token::Ident(ident, _)) => {
TokenTree::Token(_, token::Ident(ident, _)) => {
res_str.push_str(&ident.name.as_str())
},
_ => {

99
src/libsyntax/ext/quote.rs
View File

@@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use ast;
use ast::{self, TokenTree};
use codemap::Span;
use ext::base::ExtCtxt;
use ext::base;
@@ -71,67 +71,69 @@ pub mod rt {
impl ToTokens for ast::Ident {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(DUMMY_SP, token::Ident(*self, token::Plain))]
vec![TokenTree::Token(DUMMY_SP, token::Ident(*self, token::Plain))]
}
}
impl ToTokens for ast::Path {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtPath(Box::new(self.clone()))))]
vec![TokenTree::Token(DUMMY_SP,
token::Interpolated(token::NtPath(Box::new(self.clone()))))]
}
}
impl ToTokens for ast::Ty {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtTy(P(self.clone()))))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtTy(P(self.clone()))))]
}
}
impl ToTokens for ast::Block {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtBlock(P(self.clone()))))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtBlock(P(self.clone()))))]
}
}
impl ToTokens for ast::Generics {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtGenerics(self.clone())))]
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtGenerics(self.clone())))]
}
}
impl ToTokens for ast::WhereClause {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtWhereClause(self.clone())))]
vec![TokenTree::Token(DUMMY_SP,
token::Interpolated(token::NtWhereClause(self.clone())))]
}
}
impl ToTokens for P<ast::Item> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtItem(self.clone())))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtItem(self.clone())))]
}
}
impl ToTokens for P<ast::ImplItem> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtImplItem(self.clone())))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtImplItem(self.clone())))]
}
}
impl ToTokens for P<ast::TraitItem> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtTraitItem(self.clone())))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtTraitItem(self.clone())))]
}
}
impl ToTokens for P<ast::Stmt> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
let mut tts = vec![
ast::TtToken(self.span, token::Interpolated(token::NtStmt(self.clone())))
TokenTree::Token(self.span, token::Interpolated(token::NtStmt(self.clone())))
];
// Some statements require a trailing semicolon.
if classify::stmt_ends_with_semi(&self.node) {
tts.push(ast::TtToken(self.span, token::Semi));
tts.push(TokenTree::Token(self.span, token::Semi));
}
tts
@@ -140,19 +142,19 @@ pub mod rt {
impl ToTokens for P<ast::Expr> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtExpr(self.clone())))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtExpr(self.clone())))]
}
}
impl ToTokens for P<ast::Pat> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(self.span, token::Interpolated(token::NtPat(self.clone())))]
vec![TokenTree::Token(self.span, token::Interpolated(token::NtPat(self.clone())))]
}
}
impl ToTokens for ast::Arm {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtArm(self.clone())))]
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtArm(self.clone())))]
}
}
@@ -173,12 +175,12 @@ pub mod rt {
};
}
impl_to_tokens_slice! { ast::Ty, [ast::TtToken(DUMMY_SP, token::Comma)] }
impl_to_tokens_slice! { ast::Ty, [TokenTree::Token(DUMMY_SP, token::Comma)] }
impl_to_tokens_slice! { P<ast::Item>, [] }
impl ToTokens for P<ast::MetaItem> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtMeta(self.clone())))]
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtMeta(self.clone())))]
}
}
@@ -186,11 +188,11 @@ pub mod rt {
fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
let mut r = vec![];
// FIXME: The spans could be better
r.push(ast::TtToken(self.span, token::Pound));
r.push(TokenTree::Token(self.span, token::Pound));
if self.node.style == ast::AttrStyle::Inner {
r.push(ast::TtToken(self.span, token::Not));
r.push(TokenTree::Token(self.span, token::Not));
}
r.push(ast::TtDelimited(self.span, Rc::new(ast::Delimited {
r.push(TokenTree::Delimited(self.span, Rc::new(ast::Delimited {
delim: token::Bracket,
open_span: self.span,
tts: self.node.value.to_tokens(cx),
@@ -210,7 +212,7 @@ pub mod rt {
impl ToTokens for () {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![ast::TtDelimited(DUMMY_SP, Rc::new(ast::Delimited {
vec![TokenTree::Delimited(DUMMY_SP, Rc::new(ast::Delimited {
delim: token::Paren,
open_span: DUMMY_SP,
tts: vec![],
@@ -278,7 +280,7 @@ pub mod rt {
fn parse_item(&self, s: String) -> P<ast::Item>;
fn parse_expr(&self, s: String) -> P<ast::Expr>;
fn parse_stmt(&self, s: String) -> P<ast::Stmt>;
fn parse_tts(&self, s: String) -> Vec<ast::TokenTree>;
fn parse_tts(&self, s: String) -> Vec<TokenTree>;
}
impl<'a> ExtParseUtils for ExtCtxt<'a> {
@@ -305,7 +307,7 @@ pub mod rt {
self.parse_sess())
}
fn parse_tts(&self, s: String) -> Vec<ast::TokenTree> {
fn parse_tts(&self, s: String) -> Vec<TokenTree> {
parse::parse_tts_from_source_str("<quote expansion>".to_string(),
s,
self.cfg(),
@@ -316,7 +318,7 @@ pub mod rt {
pub fn expand_quote_tokens<'cx>(cx: &'cx mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'cx> {
let (cx_expr, expr) = expand_tts(cx, sp, tts);
let expanded = expand_wrapper(cx, sp, cx_expr, expr, &[&["syntax", "ext", "quote", "rt"]]);
@@ -325,7 +327,7 @@ pub fn expand_quote_tokens<'cx>(cx: &'cx mut ExtCtxt,
pub fn expand_quote_expr<'cx>(cx: &'cx mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'cx> {
let expanded = expand_parse_call(cx, sp, "parse_expr_panic", vec!(), tts);
base::MacEager::expr(expanded)
@@ -333,7 +335,7 @@ pub fn expand_quote_expr<'cx>(cx: &'cx mut ExtCtxt,
pub fn expand_quote_item<'cx>(cx: &mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'cx> {
let expanded = expand_parse_call(cx, sp, "parse_item_panic", vec!(), tts);
base::MacEager::expr(expanded)
@@ -341,7 +343,7 @@ pub fn expand_quote_item<'cx>(cx: &mut ExtCtxt,
pub fn expand_quote_pat<'cx>(cx: &'cx mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'cx> {
let expanded = expand_parse_call(cx, sp, "parse_pat_panic", vec!(), tts);
base::MacEager::expr(expanded)
@@ -349,7 +351,7 @@ pub fn expand_quote_pat<'cx>(cx: &'cx mut ExtCtxt,
pub fn expand_quote_arm(cx: &mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'static> {
let expanded = expand_parse_call(cx, sp, "parse_arm_panic", vec!(), tts);
base::MacEager::expr(expanded)
@@ -357,7 +359,7 @@ pub fn expand_quote_arm(cx: &mut ExtCtxt,
pub fn expand_quote_ty(cx: &mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'static> {
let expanded = expand_parse_call(cx, sp, "parse_ty_panic", vec!(), tts);
base::MacEager::expr(expanded)
@@ -365,7 +367,7 @@ pub fn expand_quote_ty(cx: &mut ExtCtxt,
pub fn expand_quote_stmt(cx: &mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'static> {
let expanded = expand_parse_call(cx, sp, "parse_stmt_panic", vec!(), tts);
base::MacEager::expr(expanded)
@@ -373,7 +375,7 @@ pub fn expand_quote_stmt(cx: &mut ExtCtxt,
pub fn expand_quote_attr(cx: &mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'static> {
let expanded = expand_parse_call(cx, sp, "parse_attribute_panic",
vec!(cx.expr_bool(sp, true)), tts);
@@ -383,7 +385,7 @@ pub fn expand_quote_attr(cx: &mut ExtCtxt,
pub fn expand_quote_matcher(cx: &mut ExtCtxt,
sp: Span,
tts: &[ast::TokenTree])
tts: &[TokenTree])
-> Box<base::MacResult+'static> {
let (cx_expr, tts) = parse_arguments_to_quote(cx, tts);
let mut vector = mk_stmts_let(cx, sp);
@@ -423,6 +425,11 @@ fn mk_name(cx: &ExtCtxt, sp: Span, ident: ast::Ident) -> P<ast::Expr> {
vec!(e_str))
}
fn mk_tt_path(cx: &ExtCtxt, sp: Span, name: &str) -> P<ast::Expr> {
let idents = vec!(id_ext("syntax"), id_ext("ast"), id_ext("TokenTree"), id_ext(name));
cx.expr_path(cx.path_global(sp, idents))
}
fn mk_ast_path(cx: &ExtCtxt, sp: Span, name: &str) -> P<ast::Expr> {
let idents = vec!(id_ext("syntax"), id_ext("ast"), id_ext(name));
cx.expr_path(cx.path_global(sp, idents))
@@ -591,9 +598,9 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
mk_token_path(cx, sp, name)
}
fn statements_mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree, matcher: bool) -> Vec<P<ast::Stmt>> {
fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, matcher: bool) -> Vec<P<ast::Stmt>> {
match *tt {
ast::TtToken(sp, SubstNt(ident, _)) => {
TokenTree::Token(sp, SubstNt(ident, _)) => {
// tt.extend($ident.to_tokens(ext_cx))
let e_to_toks =
@@ -612,17 +619,17 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree, matcher: bool) -> Vec<P<a
vec!(cx.stmt_expr(e_push))
}
ref tt @ ast::TtToken(_, MatchNt(..)) if !matcher => {
ref tt @ TokenTree::Token(_, MatchNt(..)) if !matcher => {
let mut seq = vec![];
for i in 0..tt.len() {
seq.push(tt.get_tt(i));
}
statements_mk_tts(cx, &seq[..], matcher)
}
ast::TtToken(sp, ref tok) => {
TokenTree::Token(sp, ref tok) => {
let e_sp = cx.expr_ident(sp, id_ext("_sp"));
let e_tok = cx.expr_call(sp,
mk_ast_path(cx, sp, "TtToken"),
mk_tt_path(cx, sp, "Token"),
vec!(e_sp, expr_mk_token(cx, sp, tok)));
let e_push =
cx.expr_method_call(sp,
@@ -631,16 +638,16 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree, matcher: bool) -> Vec<P<a
vec!(e_tok));
vec!(cx.stmt_expr(e_push))
},
ast::TtDelimited(_, ref delimed) => {
TokenTree::Delimited(_, ref delimed) => {
statements_mk_tt(cx, &delimed.open_tt(), matcher).into_iter()
.chain(delimed.tts.iter()
.flat_map(|tt| statements_mk_tt(cx, tt, matcher)))
.chain(statements_mk_tt(cx, &delimed.close_tt(), matcher))
.collect()
},
ast::TtSequence(sp, ref seq) => {
TokenTree::Sequence(sp, ref seq) => {
if !matcher {
panic!("TtSequence in quote!");
panic!("TokenTree::Sequence in quote!");
}
let e_sp = cx.expr_ident(sp, id_ext("_sp"));
@@ -671,7 +678,7 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree, matcher: bool) -> Vec<P<a
id_ext("new")],
vec![e_seq_struct]);
let e_tok = cx.expr_call(sp,
mk_ast_path(cx, sp, "TtSequence"),
mk_tt_path(cx, sp, "Sequence"),
vec!(e_sp, e_rc_new));
let e_push =
cx.expr_method_call(sp,
@@ -683,8 +690,8 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &ast::TokenTree, matcher: bool) -> Vec<P<a
}
}
fn parse_arguments_to_quote(cx: &ExtCtxt, tts: &[ast::TokenTree])
-> (P<ast::Expr>, Vec<ast::TokenTree>) {
fn parse_arguments_to_quote(cx: &ExtCtxt, tts: &[TokenTree])
-> (P<ast::Expr>, Vec<TokenTree>) {
// NB: It appears that the main parser loses its mind if we consider
// $foo as a SubstNt during the main parse, so we have to re-parse
// under quote_depth > 0. This is silly and should go away; the _guess_ is
@@ -746,7 +753,7 @@ fn mk_stmts_let(cx: &ExtCtxt, sp: Span) -> Vec<P<ast::Stmt>> {
vec!(stmt_let_sp, stmt_let_tt)
}
fn statements_mk_tts(cx: &ExtCtxt, tts: &[ast::TokenTree], matcher: bool) -> Vec<P<ast::Stmt>> {
fn statements_mk_tts(cx: &ExtCtxt, tts: &[TokenTree], matcher: bool) -> Vec<P<ast::Stmt>> {
let mut ss = Vec::new();
for tt in tts {
ss.extend(statements_mk_tt(cx, tt, matcher));
@@ -754,7 +761,7 @@ fn statements_mk_tts(cx: &ExtCtxt, tts: &[ast::TokenTree], matcher: bool) -> Vec
ss
}
fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree])
fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[TokenTree])
-> (P<ast::Expr>, P<ast::Expr>) {
let (cx_expr, tts) = parse_arguments_to_quote(cx, tts);
@@ -790,7 +797,7 @@ fn expand_parse_call(cx: &ExtCtxt,
sp: Span,
parse_method: &str,
arg_exprs: Vec<P<ast::Expr>> ,
tts: &[ast::TokenTree]) -> P<ast::Expr> {
tts: &[TokenTree]) -> P<ast::Expr> {
let (cx_expr, tts_expr) = expand_tts(cx, sp, tts);
let cfg_call = || cx.expr_method_call(

8
src/libsyntax/ext/trace_macros.rs
View File

@@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use ast;
use ast::TokenTree;
use codemap::Span;
use ext::base::ExtCtxt;
use ext::base;
@@ -18,7 +18,7 @@ use parse::token::keywords;
pub fn expand_trace_macros(cx: &mut ExtCtxt,
sp: Span,
tt: &[ast::TokenTree])
tt: &[TokenTree])
-> Box<base::MacResult+'static> {
if !cx.ecfg.enable_trace_macros() {
feature_gate::emit_feature_err(&cx.parse_sess.span_diagnostic,
@@ -30,10 +30,10 @@ pub fn expand_trace_macros(cx: &mut ExtCtxt,
}
match (tt.len(), tt.first()) {
(1, Some(&ast::TtToken(_, ref tok))) if tok.is_keyword(keywords::True) => {
(1, Some(&TokenTree::Token(_, ref tok))) if tok.is_keyword(keywords::True) => {
cx.set_trace_macros(true);
}
(1, Some(&ast::TtToken(_, ref tok))) if tok.is_keyword(keywords::False) => {
(1, Some(&TokenTree::Token(_, ref tok))) if tok.is_keyword(keywords::False) => {
cx.set_trace_macros(false);
}
_ => cx.span_err(sp, "trace_macros! accepts only `true` or `false`"),

35
src/libsyntax/ext/tt/macro_parser.rs
View File

@@ -80,7 +80,6 @@ use self::TokenTreeOrTokenTreeVec::*;
use ast;
use ast::{TokenTree, Name};
use ast::{TtDelimited, TtSequence, TtToken};
use codemap::{BytePos, mk_sp, Span};
use codemap;
use parse::lexer::*; //resolve bug?
@@ -146,16 +145,16 @@ pub struct MatcherPos {
pub fn count_names(ms: &[TokenTree]) -> usize {
ms.iter().fold(0, |count, elt| {
count + match elt {
&TtSequence(_, ref seq) => {
&TokenTree::Sequence(_, ref seq) => {
seq.num_captures
}
&TtDelimited(_, ref delim) => {
&TokenTree::Delimited(_, ref delim) => {
count_names(&delim.tts)
}
&TtToken(_, MatchNt(..)) => {
&TokenTree::Token(_, MatchNt(..)) => {
1
}
&TtToken(_, _) => 0,
&TokenTree::Token(_, _) => 0,
}
})
}
@@ -205,17 +204,17 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
fn n_rec(p_s: &ParseSess, m: &TokenTree, res: &[Rc<NamedMatch>],
ret_val: &mut HashMap<Name, Rc<NamedMatch>>, idx: &mut usize) {
match m {
&TtSequence(_, ref seq) => {
&TokenTree::Sequence(_, ref seq) => {
for next_m in &seq.tts {
n_rec(p_s, next_m, res, ret_val, idx)
}
}
&TtDelimited(_, ref delim) => {
&TokenTree::Delimited(_, ref delim) => {
for next_m in &delim.tts {
n_rec(p_s, next_m, res, ret_val, idx)
}
}
&TtToken(sp, MatchNt(bind_name, _, _, _)) => {
&TokenTree::Token(sp, MatchNt(bind_name, _, _, _)) => {
match ret_val.entry(bind_name.name) {
Vacant(spot) => {
spot.insert(res[*idx].clone());
@@ -229,8 +228,8 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
}
}
}
&TtToken(_, SubstNt(..)) => panic!("Cannot fill in a NT"),
&TtToken(_, _) => (),
&TokenTree::Token(_, SubstNt(..)) => panic!("Cannot fill in a NT"),
&TokenTree::Token(_, _) => (),
}
}
let mut ret_val = HashMap::new();
@@ -362,7 +361,7 @@ pub fn parse(sess: &ParseSess,
} else {
match ei.top_elts.get_tt(idx) {
/* need to descend into sequence */
TtSequence(sp, seq) => {
TokenTree::Sequence(sp, seq) => {
if seq.op == ast::ZeroOrMore {
let mut new_ei = ei.clone();
new_ei.match_cur += seq.num_captures;
@@ -388,10 +387,10 @@ pub fn parse(sess: &ParseSess,
match_hi: ei_t.match_cur + seq.num_captures,
up: Some(ei_t),
sp_lo: sp.lo,
top_elts: Tt(TtSequence(sp, seq)),
top_elts: Tt(TokenTree::Sequence(sp, seq)),
}));
}
TtToken(_, MatchNt(..)) => {
TokenTree::Token(_, MatchNt(..)) => {
// Built-in nonterminals never start with these tokens,
// so we can eliminate them from consideration.
match tok {
@@ -399,10 +398,10 @@ pub fn parse(sess: &ParseSess,
_ => bb_eis.push(ei),
}
}
TtToken(sp, SubstNt(..)) => {
TokenTree::Token(sp, SubstNt(..)) => {
return Error(sp, "missing fragment specifier".to_string())
}
seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) => {
seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, DocComment(..)) => {
let lower_elts = mem::replace(&mut ei.top_elts, Tt(seq));
let idx = ei.idx;
ei.stack.push(MatcherTtFrame {
@@ -412,7 +411,7 @@ pub fn parse(sess: &ParseSess,
ei.idx = 0;
cur_eis.push(ei);
}
TtToken(_, ref t) => {
TokenTree::Token(_, ref t) => {
let mut ei_t = ei.clone();
if token_name_eq(t,&tok) {
ei_t.idx += 1;
@@ -440,7 +439,7 @@ pub fn parse(sess: &ParseSess,
if (!bb_eis.is_empty() && !next_eis.is_empty())
|| bb_eis.len() > 1 {
let nts = bb_eis.iter().map(|ei| match ei.top_elts.get_tt(ei.idx) {
TtToken(_, MatchNt(bind, name, _, _)) => {
TokenTree::Token(_, MatchNt(bind, name, _, _)) => {
format!("{} ('{}')", name, bind)
}
_ => panic!()
@@ -468,7 +467,7 @@ pub fn parse(sess: &ParseSess,
let mut ei = bb_eis.pop().unwrap();
match ei.top_elts.get_tt(ei.idx) {
TtToken(span, MatchNt(_, ident, _, _)) => {
TokenTree::Token(span, MatchNt(_, ident, _, _)) => {
let match_cur = ei.match_cur;
(&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
parse_nt(&mut rust_parser, span, &ident.name.as_str()))));

62
src/libsyntax/ext/tt/macro_rules.rs
View File

@@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use ast::{self, TokenTree, TtDelimited, TtSequence, TtToken};
use ast::{self, TokenTree};
use codemap::{Span, DUMMY_SP};
use ext::base::{ExtCtxt, MacResult, SyntaxExtension};
use ext::base::{NormalTT, TTMacroExpander};
@@ -26,6 +26,7 @@ use util::small_vector::SmallVector;
use std::cell::RefCell;
use std::rc::Rc;
use std::iter::once;
struct ParserAnyMacro<'a> {
parser: RefCell<Parser<'a>>,
@@ -171,7 +172,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
match **lhs {
MatchedNonterminal(NtTT(ref lhs_tt)) => {
let lhs_tt = match **lhs_tt {
TtDelimited(_, ref delim) => &delim.tts[..],
TokenTree::Delimited(_, ref delim) => &delim.tts[..],
_ => panic!(cx.span_fatal(sp, "malformed macro lhs"))
};
@@ -182,7 +183,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
MatchedNonterminal(NtTT(ref tt)) => {
match **tt {
// ignore delimiters
TtDelimited(_, ref delimed) => delimed.tts.clone(),
TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(),
_ => panic!(cx.span_fatal(sp, "macro rhs must be delimited")),
}
},
@@ -243,21 +244,21 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain);
let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain);
let argument_gram = vec!(
TtSequence(DUMMY_SP,
TokenTree::Sequence(DUMMY_SP,
Rc::new(ast::SequenceRepetition {
tts: vec![
TtToken(DUMMY_SP, match_lhs_tok),
TtToken(DUMMY_SP, token::FatArrow),
TtToken(DUMMY_SP, match_rhs_tok)],
TokenTree::Token(DUMMY_SP, match_lhs_tok),
TokenTree::Token(DUMMY_SP, token::FatArrow),
TokenTree::Token(DUMMY_SP, match_rhs_tok)],
separator: Some(token::Semi),
op: ast::OneOrMore,
num_captures: 2
})),
//to phase into semicolon-termination instead of
//semicolon-separation
TtSequence(DUMMY_SP,
TokenTree::Sequence(DUMMY_SP,
Rc::new(ast::SequenceRepetition {
tts: vec![TtToken(DUMMY_SP, token::Semi)],
tts: vec![TokenTree::Token(DUMMY_SP, token::Semi)],
separator: None,
op: ast::ZeroOrMore,
num_captures: 0
@@ -307,14 +308,14 @@ pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
}
fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &NamedMatch, sp: Span) {
// lhs is going to be like MatchedNonterminal(NtTT(TtDelimited(...))), where the entire lhs is
// those tts. Or, it can be a "bare sequence", not wrapped in parens.
// lhs is going to be like MatchedNonterminal(NtTT(TokenTree::Delimited(...))), where the
// entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
match lhs {
&MatchedNonterminal(NtTT(ref inner)) => match &**inner {
&TtDelimited(_, ref tts) => {
&TokenTree::Delimited(_, ref tts) => {
check_matcher(cx, tts.tts.iter(), &Eof);
},
tt @ &TtSequence(..) => {
tt @ &TokenTree::Sequence(..) => {
check_matcher(cx, Some(tt).into_iter(), &Eof);
},
_ => cx.span_err(sp, "Invalid macro matcher; matchers must be contained \
@@ -327,7 +328,7 @@ fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &NamedMatch, sp: Span) {
// after parsing/expansion. we can report every error in every macro this way.
}
// returns the last token that was checked, for TtSequence. this gets used later on.
// returns the last token that was checked, for TokenTree::Sequence. this gets used later on.
fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
-> Option<(Span, Token)> where I: Iterator<Item=&'a TokenTree> {
use print::pprust::token_to_string;
@@ -338,7 +339,7 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
let mut tokens = matcher.peekable();
while let Some(token) = tokens.next() {
last = match *token {
TtToken(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
TokenTree::Token(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
// ii. If T is a simple NT, look ahead to the next token T' in
// M. If T' is in the set FOLLOW(NT), continue. Else; reject.
if can_be_followed_by_any(&frag_spec.name.as_str()) {
@@ -346,9 +347,9 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
} else {
let next_token = match tokens.peek() {
// If T' closes a complex NT, replace T' with F
Some(&&TtToken(_, CloseDelim(_))) => follow.clone(),
Some(&&TtToken(_, ref tok)) => tok.clone(),
Some(&&TtSequence(sp, _)) => {
Some(&&TokenTree::Token(_, CloseDelim(_))) => follow.clone(),
Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
Some(&&TokenTree::Sequence(sp, _)) => {
// Be conservative around sequences: to be
// more specific, we would need to
// consider FIRST sets, but also the
@@ -366,12 +367,16 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
Eof
},
// die next iteration
Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
Some(&&TokenTree::Delimited(_, ref delim)) => delim.close_token(),
// else, we're at the end of the macro or sequence
None => follow.clone()
};
let tok = if let TtToken(_, ref tok) = *token { tok } else { unreachable!() };
let tok = if let TokenTree::Token(_, ref tok) = *token {
tok
} else {
unreachable!()
};
// If T' is in the set FOLLOW(NT), continue. Else, reject.
match (&next_token, is_in_follow(cx, &next_token, &frag_spec.name.as_str())) {
@@ -391,7 +396,7 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
}
}
},
TtSequence(sp, ref seq) => {
TokenTree::Sequence(sp, ref seq) => {
// iii. Else, T is a complex NT.
match seq.separator {
// If T has the form $(...)U+ or $(...)U* for some token U,
@@ -408,8 +413,9 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
// but conservatively correct.
Some((span, tok)) => {
let fol = match tokens.peek() {
Some(&&TtToken(_, ref tok)) => tok.clone(),
Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
Some(&&TokenTree::Delimited(_, ref delim)) =>
delim.close_token(),
Some(_) => {
cx.span_err(sp, "sequence repetition followed by \
another sequence repetition, which is not allowed");
@@ -417,7 +423,7 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
},
None => Eof
};
check_matcher(cx, Some(&TtToken(span, tok.clone())).into_iter(),
check_matcher(cx, once(&TokenTree::Token(span, tok.clone())),
&fol)
},
None => last,
@@ -428,8 +434,8 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
// sequence. If it accepts, continue, else, reject.
None => {
let fol = match tokens.peek() {
Some(&&TtToken(_, ref tok)) => tok.clone(),
Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
Some(&&TokenTree::Delimited(_, ref delim)) => delim.close_token(),
Some(_) => {
cx.span_err(sp, "sequence repetition followed by another \
sequence repetition, which is not allowed");
@@ -441,11 +447,11 @@ fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
}
}
},
TtToken(..) => {
TokenTree::Token(..) => {
// i. If T is not an NT, continue.
continue
},
TtDelimited(_, ref tts) => {
TokenTree::Delimited(_, ref tts) => {
// if we don't pass in that close delimiter, we'll incorrectly consider the matcher
// `{ $foo:ty }` as having a follow that isn't `RBrace`
check_matcher(cx, tts.tts.iter(), &tts.close_token())

42
src/libsyntax/ext/tt/transcribe.rs
View File

@@ -10,7 +10,7 @@
use self::LockstepIterSize::*;
use ast;
use ast::{TokenTree, TtDelimited, TtToken, TtSequence, Ident, Name};
use ast::{TokenTree, Ident, Name};
use codemap::{Span, DUMMY_SP};
use diagnostic::SpanHandler;
use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
@@ -53,7 +53,7 @@ pub struct TtReader<'a> {
}
/// This can do Macro-By-Example transcription. On the other hand, if
/// `src` contains no `TtSequence`s, `MatchNt`s or `SubstNt`s, `interp` can
/// `src` contains no `TokenTree::Sequence`s, `MatchNt`s or `SubstNt`s, `interp` can
/// (and should) be None.
pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler,
interp: Option<HashMap<Name, Rc<NamedMatch>>>,
@@ -67,7 +67,7 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler,
/// like any other attribute which consists of `meta` and surrounding #[ ] tokens.
///
/// This can do Macro-By-Example transcription. On the other hand, if
/// `src` contains no `TtSequence`s, `MatchNt`s or `SubstNt`s, `interp` can
/// `src` contains no `TokenTree::Sequence`s, `MatchNt`s or `SubstNt`s, `interp` can
/// (and should) be None.
pub fn new_tt_reader_with_doc_flag<'a>(sp_diag: &'a SpanHandler,
interp: Option<HashMap<Name, Rc<NamedMatch>>>,
@@ -78,7 +78,7 @@ pub fn new_tt_reader_with_doc_flag<'a>(sp_diag: &'a SpanHandler,
let mut r = TtReader {
sp_diag: sp_diag,
stack: vec!(TtFrame {
forest: TtSequence(DUMMY_SP, Rc::new(ast::SequenceRepetition {
forest: TokenTree::Sequence(DUMMY_SP, Rc::new(ast::SequenceRepetition {
tts: src,
// doesn't matter. This merely holds the root unzipping.
separator: None, op: ast::ZeroOrMore, num_captures: 0
@@ -151,17 +151,17 @@ impl Add for LockstepIterSize {
fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
match *t {
TtDelimited(_, ref delimed) => {
TokenTree::Delimited(_, ref delimed) => {
delimed.tts.iter().fold(LisUnconstrained, |size, tt| {
size + lockstep_iter_size(tt, r)
})
},
TtSequence(_, ref seq) => {
TokenTree::Sequence(_, ref seq) => {
seq.tts.iter().fold(LisUnconstrained, |size, tt| {
size + lockstep_iter_size(tt, r)
})
},
TtToken(_, SubstNt(name, _)) | TtToken(_, MatchNt(name, _, _, _)) =>
TokenTree::Token(_, SubstNt(name, _)) | TokenTree::Token(_, MatchNt(name, _, _, _)) =>
match lookup_cur_matched(r, name) {
Some(matched) => match *matched {
MatchedNonterminal(_) => LisUnconstrained,
@@ -169,7 +169,7 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
},
_ => LisUnconstrained
},
TtToken(..) => LisUnconstrained,
TokenTree::Token(..) => LisUnconstrained,
}
}
@@ -232,17 +232,17 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
}
}
}
loop { /* because it's easiest, this handles `TtDelimited` not starting
with a `TtToken`, even though it won't happen */
loop { /* because it's easiest, this handles `TokenTree::Delimited` not starting
with a `TokenTree::Token`, even though it won't happen */
let t = {
let frame = r.stack.last().unwrap();
// FIXME(pcwalton): Bad copy.
frame.forest.get_tt(frame.idx)
};
match t {
TtSequence(sp, seq) => {
TokenTree::Sequence(sp, seq) => {
// FIXME(pcwalton): Bad copy.
match lockstep_iter_size(&TtSequence(sp, seq.clone()),
match lockstep_iter_size(&TokenTree::Sequence(sp, seq.clone()),
r) {
LisUnconstrained => {
panic!(r.sp_diag.span_fatal(
@@ -272,20 +272,20 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
idx: 0,
dotdotdoted: true,
sep: seq.separator.clone(),
forest: TtSequence(sp, seq),
forest: TokenTree::Sequence(sp, seq),
});
}
}
}
// FIXME #2887: think about span stuff here
TtToken(sp, SubstNt(ident, namep)) => {
TokenTree::Token(sp, SubstNt(ident, namep)) => {
r.stack.last_mut().unwrap().idx += 1;
match lookup_cur_matched(r, ident) {
None => {
r.cur_span = sp;
r.cur_tok = SubstNt(ident, namep);
return ret_val;
// this can't be 0 length, just like TtDelimited
// this can't be 0 length, just like TokenTree::Delimited
}
Some(cur_matched) => {
match *cur_matched {
@@ -313,8 +313,8 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
}
}
}
// TtDelimited or any token that can be unzipped
seq @ TtDelimited(..) | seq @ TtToken(_, MatchNt(..)) => {
// TokenTree::Delimited or any token that can be unzipped
seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, MatchNt(..)) => {
// do not advance the idx yet
r.stack.push(TtFrame {
forest: seq,
@@ -324,15 +324,15 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
});
// if this could be 0-length, we'd need to potentially recur here
}
TtToken(sp, DocComment(name)) if r.desugar_doc_comments => {
TokenTree::Token(sp, DocComment(name)) if r.desugar_doc_comments => {
r.stack.push(TtFrame {
forest: TtToken(sp, DocComment(name)),
forest: TokenTree::Token(sp, DocComment(name)),
idx: 0,
dotdotdoted: false,
sep: None
});
}
TtToken(sp, token::SpecialVarNt(SpecialMacroVar::CrateMacroVar)) => {
TokenTree::Token(sp, token::SpecialVarNt(SpecialMacroVar::CrateMacroVar)) => {
r.stack.last_mut().unwrap().idx += 1;
if r.imported_from.is_some() {
@@ -344,7 +344,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
// otherwise emit nothing and proceed to the next token
}
TtToken(sp, tok) => {
TokenTree::Token(sp, tok) => {
r.cur_span = sp;
r.cur_tok = tok;
r.stack.last_mut().unwrap().idx += 1;