rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Require passing an AttrWrapper to collect_tokens_trailing_token

Browse Source 
This is a pure refactoring split out from #80689.
It represents the most invasive part of that PR, requiring changes in
every caller of `parse_outer_attributes`

In order to eagerly expand `#[cfg]` attributes while preserving the
original `TokenStream`, we need to know the range of tokens that
corresponds to every attribute target. This is accomplished by making
`parse_outer_attributes` return an opaque `AttrWrapper` struct. An
`AttrWrapper` must be converted to a plain `AttrVec` by passing it to
`collect_tokens_trailing_token`. This makes it difficult to accidentally
construct an AST node with attributes without calling `collect_tokens_trailing_token`,
since AST nodes store an `AttrVec`, not an `AttrWrapper`.

As a result, we now call `collect_tokens_trailing_token` for attribute
targets which only support inert attributes, such as generic arguments
and struct fields. Currently, the constructed `LazyTokenStream` is
simply discarded. Future PRs will record the token range corresponding
to the attribute target, allowing those tokens to be removed from an
enclosing `collect_tokens_trailing_token` call if necessary.
This commit is contained in:
Aaron Hill
2021-01-22 13:28:08 -05:00
parent 7e0241c637
commit 0b411f56e1
9 changed files with 621 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
compiler/rustc_parse/src/parser/item.rs
View File

@@ -1,8 +1,6 @@
use super::diagnostics::{dummy_arg, ConsumeClosingDelim, Error};
use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
use super::{FollowedByType, ForceCollect, Parser, PathStyle, TrailingToken};
use crate::{maybe_collect_tokens, maybe_whole};
use super::{AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, TrailingToken};
use rustc_ast::ast::*;
use rustc_ast::ptr::P;
@@ -108,25 +106,40 @@ impl<'a> Parser<'a> {
pub(super) fn parse_item_common(
&mut self,
mut attrs: Vec<Attribute>,
attrs: AttrWrapper,
mac_allowed: bool,
attrs_allowed: bool,
req_name: ReqName,
force_collect: ForceCollect,
) -> PResult<'a, Option<Item>> {
maybe_whole!(self, NtItem, |item| {
let mut item = item;
mem::swap(&mut item.attrs, &mut attrs);
item.attrs.extend(attrs);
Some(item.into_inner())
});
// Don't use `maybe_whole` so that we have precise control
// over when we bump the parser
if let token::Interpolated(nt) = &self.token.kind {
if let token::NtItem(item) = &**nt {
let item = item.clone();
return self.collect_tokens_trailing_token(
attrs,
force_collect,
|this, mut attrs| {
let mut item = item;
mem::swap(&mut item.attrs, &mut attrs);
item.attrs.extend(attrs);
// Bump the parser so the we capture the token::Interpolated
this.bump();
Ok((Some(item.into_inner()), TrailingToken::None))
},
);
}
};
let mut unclosed_delims = vec![];
let item = maybe_collect_tokens!(self, force_collect, &attrs, |this: &mut Self| {
let item = this.parse_item_common_(attrs, mac_allowed, attrs_allowed, req_name);
unclosed_delims.append(&mut this.unclosed_delims);
Ok((item?, TrailingToken::None))
})?;
let item =
self.collect_tokens_trailing_token(attrs, force_collect, |this: &mut Self, attrs| {
let item = this.parse_item_common_(attrs, mac_allowed, attrs_allowed, req_name);
unclosed_delims.append(&mut this.unclosed_delims);
Ok((item?, TrailingToken::None))
})?;
self.unclosed_delims.append(&mut unclosed_delims);
Ok(item)
@@ -1109,39 +1122,45 @@ impl<'a> Parser<'a> {
fn parse_enum_variant(&mut self) -> PResult<'a, Option<Variant>> {
let variant_attrs = self.parse_outer_attributes()?;
let vlo = self.token.span;
self.collect_tokens_trailing_token(
variant_attrs,
ForceCollect::No,
|this, variant_attrs| {
let vlo = this.token.span;
let vis = self.parse_visibility(FollowedByType::No)?;
if !self.recover_nested_adt_item(kw::Enum)? {
return Ok(None);
}
let ident = self.parse_ident()?;
let vis = this.parse_visibility(FollowedByType::No)?;
if !this.recover_nested_adt_item(kw::Enum)? {
return Ok((None, TrailingToken::None));
}
let ident = this.parse_ident()?;
let struct_def = if self.check(&token::OpenDelim(token::Brace)) {
// Parse a struct variant.
let (fields, recovered) = self.parse_record_struct_body()?;
VariantData::Struct(fields, recovered)
} else if self.check(&token::OpenDelim(token::Paren)) {
VariantData::Tuple(self.parse_tuple_struct_body()?, DUMMY_NODE_ID)
} else {
VariantData::Unit(DUMMY_NODE_ID)
};
let struct_def = if this.check(&token::OpenDelim(token::Brace)) {
// Parse a struct variant.
let (fields, recovered) = this.parse_record_struct_body()?;
VariantData::Struct(fields, recovered)
} else if this.check(&token::OpenDelim(token::Paren)) {
VariantData::Tuple(this.parse_tuple_struct_body()?, DUMMY_NODE_ID)
} else {
VariantData::Unit(DUMMY_NODE_ID)
};
let disr_expr =
if self.eat(&token::Eq) { Some(self.parse_anon_const_expr()?) } else { None };
let disr_expr =
if this.eat(&token::Eq) { Some(this.parse_anon_const_expr()?) } else { None };
let vr = ast::Variant {
ident,
vis,
id: DUMMY_NODE_ID,
attrs: variant_attrs,
data: struct_def,
disr_expr,
span: vlo.to(self.prev_token.span),
is_placeholder: false,
};
let vr = ast::Variant {
ident,
vis,
id: DUMMY_NODE_ID,
attrs: variant_attrs,
data: struct_def,
disr_expr,
span: vlo.to(this.prev_token.span),
is_placeholder: false,
};
Ok(Some(vr))
Ok((Some(vr), TrailingToken::MaybeComma))
},
)
}
/// Parses `struct Foo { ... }`.
@@ -1262,17 +1281,23 @@ impl<'a> Parser<'a> {
// Unit like structs are handled in parse_item_struct function
self.parse_paren_comma_seq(|p| {
let attrs = p.parse_outer_attributes()?;
let lo = p.token.span;
let vis = p.parse_visibility(FollowedByType::Yes)?;
let ty = p.parse_ty()?;
Ok(StructField {
span: lo.to(ty.span),
vis,
ident: None,
id: DUMMY_NODE_ID,
ty,
attrs,
is_placeholder: false,
p.collect_tokens_trailing_token(attrs, ForceCollect::No, |p, attrs| {
let lo = p.token.span;
let vis = p.parse_visibility(FollowedByType::Yes)?;
let ty = p.parse_ty()?;
Ok((
StructField {
span: lo.to(ty.span),
vis,
ident: None,
id: DUMMY_NODE_ID,
ty,
attrs,
is_placeholder: false,
},
TrailingToken::MaybeComma,
))
})
})
.map(|(r, _)| r)
@@ -1281,9 +1306,11 @@ impl<'a> Parser<'a> {
/// Parses an element of a struct declaration.
fn parse_struct_decl_field(&mut self) -> PResult<'a, StructField> {
let attrs = self.parse_outer_attributes()?;
let lo = self.token.span;
let vis = self.parse_visibility(FollowedByType::No)?;
self.parse_single_struct_field(lo, vis, attrs)
self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
let lo = this.token.span;
let vis = this.parse_visibility(FollowedByType::No)?;
Ok((this.parse_single_struct_field(lo, vis, attrs)?, TrailingToken::None))
})
}
/// Parses a structure field declaration.
@@ -1736,74 +1763,79 @@ impl<'a> Parser<'a> {
fn parse_param_general(&mut self, req_name: ReqName, first_param: bool) -> PResult<'a, Param> {
let lo = self.token.span;
let attrs = self.parse_outer_attributes()?;
self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
// Possibly parse `self`. Recover if we parsed it and it wasn't allowed here.
if let Some(mut param) = this.parse_self_param()? {
param.attrs = attrs.into();
let res = if first_param { Ok(param) } else { this.recover_bad_self_param(param) };
return Ok((res?, TrailingToken::None));
}
// Possibly parse `self`. Recover if we parsed it and it wasn't allowed here.
if let Some(mut param) = self.parse_self_param()? {
param.attrs = attrs.into();
return if first_param { Ok(param) } else { self.recover_bad_self_param(param) };
}
let is_name_required = match this.token.kind {
token::DotDotDot => false,
_ => req_name(this.token.span.edition()),
};
let (pat, ty) = if is_name_required || this.is_named_param() {
debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required);
let is_name_required = match self.token.kind {
token::DotDotDot => false,
_ => req_name(self.token.span.edition()),
};
let (pat, ty) = if is_name_required || self.is_named_param() {
debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required);
let pat = this.parse_fn_param_pat()?;
if let Err(mut err) = this.expect(&token::Colon) {
return if let Some(ident) =
this.parameter_without_type(&mut err, pat, is_name_required, first_param)
{
err.emit();
Ok((dummy_arg(ident), TrailingToken::None))
} else {
Err(err)
};
}
let pat = self.parse_fn_param_pat()?;
if let Err(mut err) = self.expect(&token::Colon) {
return if let Some(ident) =
self.parameter_without_type(&mut err, pat, is_name_required, first_param)
this.eat_incorrect_doc_comment_for_param_type();
(pat, this.parse_ty_for_param()?)
} else {
debug!("parse_param_general ident_to_pat");
let parser_snapshot_before_ty = this.clone();
this.eat_incorrect_doc_comment_for_param_type();
let mut ty = this.parse_ty_for_param();
if ty.is_ok()
&& this.token != token::Comma
&& this.token != token::CloseDelim(token::Paren)
{
err.emit();
Ok(dummy_arg(ident))
} else {
Err(err)
};
}
self.eat_incorrect_doc_comment_for_param_type();
(pat, self.parse_ty_for_param()?)
} else {
debug!("parse_param_general ident_to_pat");
let parser_snapshot_before_ty = self.clone();
self.eat_incorrect_doc_comment_for_param_type();
let mut ty = self.parse_ty_for_param();
if ty.is_ok()
&& self.token != token::Comma
&& self.token != token::CloseDelim(token::Paren)
{
// This wasn't actually a type, but a pattern looking like a type,
// so we are going to rollback and re-parse for recovery.
ty = self.unexpected();
}
match ty {
Ok(ty) => {
let ident = Ident::new(kw::Empty, self.prev_token.span);
let bm = BindingMode::ByValue(Mutability::Not);
let pat = self.mk_pat_ident(ty.span, bm, ident);
(pat, ty)
// This wasn't actually a type, but a pattern looking like a type,
// so we are going to rollback and re-parse for recovery.
ty = this.unexpected();
}
// If this is a C-variadic argument and we hit an error, return the error.
Err(err) if self.token == token::DotDotDot => return Err(err),
// Recover from attempting to parse the argument as a type without pattern.
Err(mut err) => {
err.cancel();
*self = parser_snapshot_before_ty;
self.recover_arg_parse()?
match ty {
Ok(ty) => {
let ident = Ident::new(kw::Empty, this.prev_token.span);
let bm = BindingMode::ByValue(Mutability::Not);
let pat = this.mk_pat_ident(ty.span, bm, ident);
(pat, ty)
}
// If this is a C-variadic argument and we hit an error, return the error.
Err(err) if this.token == token::DotDotDot => return Err(err),
// Recover from attempting to parse the argument as a type without pattern.
Err(mut err) => {
err.cancel();
*this = parser_snapshot_before_ty;
this.recover_arg_parse()?
}
}
}
};
};
let span = lo.until(self.token.span);
let span = lo.until(this.token.span);
Ok(Param {
attrs: attrs.into(),
id: ast::DUMMY_NODE_ID,
is_placeholder: false,
pat,
span,
ty,
Ok((
Param {
attrs: attrs.into(),
id: ast::DUMMY_NODE_ID,
is_placeholder: false,
pat,
span,
ty,
},
TrailingToken::None,
))
})
}