rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Implement token-based handling of attributes during expansion

Browse Source 
This PR modifies the macro expansion infrastructure to handle attributes
in a fully token-based manner. As a result:

* Derives macros no longer lose spans when their input is modified
  by eager cfg-expansion. This is accomplished by performing eager
  cfg-expansion on the token stream that we pass to the derive
  proc-macro
* Inner attributes now preserve spans in all cases, including when we
  have multiple inner attributes in a row.

This is accomplished through the following changes:

* New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced.
  These are very similar to a normal `TokenTree`, but they also track
  the position of attributes and attribute targets within the stream.
  They are built when we collect tokens during parsing.
  An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when
  we invoke a macro.
* Token capturing and `LazyTokenStream` are modified to work with
  `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which
  is created during the parsing of a nested AST node to make the 'outer'
  AST node aware of the attributes and attribute target stored deeper in the token stream.
* When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`),
we tokenize and reparse our target, capturing additional information about the locations of
`#[cfg]` and `#[cfg_attr]` attributes at any depth within the target.
This is a performance optimization, allowing us to perform less work
in the typical case where captured tokens never have eager cfg-expansion run.
This commit is contained in:
Aaron Hill
2020-11-28 18:33:17 -05:00
parent 25ea6be13e
commit a93c4f05de
33 changed files with 2046 additions and 1192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
compiler/rustc_ast/src/ast_like.rs
View File

@@ -1,20 +1,32 @@
use super::ptr::P;
use super::token::Nonterminal;
use super::tokenstream::LazyTokenStream;
use super::{Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
use super::{AssocItem, Expr, ForeignItem, Item, Local};
use super::{AssocItem, Expr, ForeignItem, Item, Local, MacCallStmt};
use super::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
use super::{AttrVec, Attribute, Stmt, StmtKind};
use std::fmt::Debug;
/// An `AstLike` represents an AST node (or some wrapper around
/// and AST node) which stores some combination of attributes
/// and tokens.
pub trait AstLike: Sized {
pub trait AstLike: Sized + Debug {
/// This is `true` if this `AstLike` might support 'custom' (proc-macro) inner
/// attributes. Attributes like `#![cfg]` and `#![cfg_attr]` are not
/// considered 'custom' attributes
///
/// If this is `false`, then this `AstLike` definitely does
/// not support 'custom' inner attributes, which enables some optimizations
/// during token collection.
const SUPPORTS_CUSTOM_INNER_ATTRS: bool;
fn attrs(&self) -> &[Attribute];
fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>));
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
}
impl<T: AstLike + 'static> AstLike for P<T> {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
fn attrs(&self) -> &[Attribute] {
(**self).attrs()
}
@@ -26,6 +38,55 @@ impl<T: AstLike + 'static> AstLike for P<T> {
}
}
impl AstLike for crate::token::Nonterminal {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
fn attrs(&self) -> &[Attribute] {
match self {
Nonterminal::NtItem(item) => item.attrs(),
Nonterminal::NtStmt(stmt) => stmt.attrs(),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.attrs(),
Nonterminal::NtPat(_)
| Nonterminal::NtTy(_)
| Nonterminal::NtMeta(_)
| Nonterminal::NtPath(_)
| Nonterminal::NtVis(_)
| Nonterminal::NtTT(_)
| Nonterminal::NtBlock(_)
| Nonterminal::NtIdent(..)
| Nonterminal::NtLifetime(_) => &[],
}
}
fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
match self {
Nonterminal::NtItem(item) => item.visit_attrs(f),
Nonterminal::NtStmt(stmt) => stmt.visit_attrs(f),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.visit_attrs(f),
Nonterminal::NtPat(_)
| Nonterminal::NtTy(_)
| Nonterminal::NtMeta(_)
| Nonterminal::NtPath(_)
| Nonterminal::NtVis(_)
| Nonterminal::NtTT(_)
| Nonterminal::NtBlock(_)
| Nonterminal::NtIdent(..)
| Nonterminal::NtLifetime(_) => {}
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
match self {
Nonterminal::NtItem(item) => item.tokens_mut(),
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
Nonterminal::NtPat(pat) => pat.tokens_mut(),
Nonterminal::NtTy(ty) => ty.tokens_mut(),
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
Nonterminal::NtPath(path) => path.tokens_mut(),
Nonterminal::NtVis(vis) => vis.tokens_mut(),
_ => panic!("Called tokens_mut on {:?}", self),
}
}
}
fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
crate::mut_visit::visit_clobber(attrs, |attrs| {
let mut vec = attrs.into();
@@ -35,6 +96,10 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec<Attribute>)) {
}
impl AstLike for StmtKind {
// This might be an `StmtKind::Item`, which contains
// an item that supports inner attrs
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
fn attrs(&self) -> &[Attribute] {
match self {
StmtKind::Local(local) => local.attrs(),
@@ -66,6 +131,8 @@ impl AstLike for StmtKind {
}
impl AstLike for Stmt {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = StmtKind::SUPPORTS_CUSTOM_INNER_ATTRS;
fn attrs(&self) -> &[Attribute] {
self.kind.attrs()
}
@@ -79,6 +146,8 @@ impl AstLike for Stmt {
}
impl AstLike for Attribute {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
fn attrs(&self) -> &[Attribute] {
&[]
}
@@ -94,6 +163,8 @@ impl AstLike for Attribute {
}
impl<T: AstLike> AstLike for Option<T> {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS;
fn attrs(&self) -> &[Attribute] {
self.as_ref().map(|inner| inner.attrs()).unwrap_or(&[])
}
@@ -127,8 +198,13 @@ impl VecOrAttrVec for AttrVec {
}
macro_rules! derive_has_tokens_and_attrs {
($($ty:path),*) => { $(
(
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs:literal;
$($ty:path),*
) => { $(
impl AstLike for $ty {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = $inner_attrs;
fn attrs(&self) -> &[Attribute] {
&self.attrs
}
@@ -140,6 +216,7 @@ macro_rules! derive_has_tokens_and_attrs {
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
Some(&mut self.tokens)
}
}
)* }
}
@@ -147,6 +224,8 @@ macro_rules! derive_has_tokens_and_attrs {
macro_rules! derive_has_attrs_no_tokens {
($($ty:path),*) => { $(
impl AstLike for $ty {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
fn attrs(&self) -> &[Attribute] {
&self.attrs
}
@@ -165,12 +244,13 @@ macro_rules! derive_has_attrs_no_tokens {
macro_rules! derive_has_tokens_no_attrs {
($($ty:path),*) => { $(
impl AstLike for $ty {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
fn attrs(&self) -> &[Attribute] {
&[]
}
fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec<Attribute>)) {}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
Some(&mut self.tokens)
}
@@ -178,10 +258,18 @@ macro_rules! derive_has_tokens_no_attrs {
)* }
}
// These AST nodes support both inert and active
// attributes, so they also have tokens.
// These ast nodes support both active and inert attributes,
// so they have tokens collected to pass to proc macros
derive_has_tokens_and_attrs! {
Item, Expr, Local, AssocItem, ForeignItem
// Both `Item` and `AssocItem` can have bodies, which
// can contain inner attributes
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = true;
Item, AssocItem, ForeignItem
}
derive_has_tokens_and_attrs! {
const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
Local, MacCallStmt, Expr
}
// These ast nodes only support inert attributes, so they don't