Implement token-based handling of attributes during expansion

This PR modifies the macro expansion infrastructure to handle attributes
in a fully token-based manner. As a result:

* Derives macros no longer lose spans when their input is modified
  by eager cfg-expansion. This is accomplished by performing eager
  cfg-expansion on the token stream that we pass to the derive
  proc-macro
* Inner attributes now preserve spans in all cases, including when we
  have multiple inner attributes in a row.

This is accomplished through the following changes:

* New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced.
  These are very similar to a normal `TokenTree`, but they also track
  the position of attributes and attribute targets within the stream.
  They are built when we collect tokens during parsing.
  An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when
  we invoke a macro.
* Token capturing and `LazyTokenStream` are modified to work with
  `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which
  is created during the parsing of a nested AST node to make the 'outer'
  AST node aware of the attributes and attribute target stored deeper in the token stream.
* When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`),
we tokenize and reparse our target, capturing additional information about the locations of
`#[cfg]` and `#[cfg_attr]` attributes at any depth within the target.
This is a performance optimization, allowing us to perform less work
in the typical case where captured tokens never have eager cfg-expansion run.

compiler/rustc_expand/src/base.rs

@@ -3,7 +3,7 @@ use crate::module::DirOwnership;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Nonterminal};
-use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, LazyTokenStream, TokenStream};
+use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream};
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_ast::{self as ast, AstLike, Attribute, Item, NodeId, PatKind};
 use rustc_attr::{self as attr, Deprecation, Stability};
@@ -46,62 +46,6 @@ pub enum Annotatable {
     Variant(ast::Variant),
 }
 
-impl AstLike for Annotatable {
-    fn attrs(&self) -> &[Attribute] {
-        match *self {
-            Annotatable::Item(ref item) => &item.attrs,
-            Annotatable::TraitItem(ref trait_item) => &trait_item.attrs,
-            Annotatable::ImplItem(ref impl_item) => &impl_item.attrs,
-            Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs,
-            Annotatable::Stmt(ref stmt) => stmt.attrs(),
-            Annotatable::Expr(ref expr) => &expr.attrs,
-            Annotatable::Arm(ref arm) => &arm.attrs,
-            Annotatable::ExprField(ref field) => &field.attrs,
-            Annotatable::PatField(ref fp) => &fp.attrs,
-            Annotatable::GenericParam(ref gp) => &gp.attrs,
-            Annotatable::Param(ref p) => &p.attrs,
-            Annotatable::FieldDef(ref sf) => &sf.attrs,
-            Annotatable::Variant(ref v) => &v.attrs(),
-        }
-    }
-
-    fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
-        match self {
-            Annotatable::Item(item) => item.visit_attrs(f),
-            Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f),
-            Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f),
-            Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f),
-            Annotatable::Stmt(stmt) => stmt.visit_attrs(f),
-            Annotatable::Expr(expr) => expr.visit_attrs(f),
-            Annotatable::Arm(arm) => arm.visit_attrs(f),
-            Annotatable::ExprField(field) => field.visit_attrs(f),
-            Annotatable::PatField(fp) => fp.visit_attrs(f),
-            Annotatable::GenericParam(gp) => gp.visit_attrs(f),
-            Annotatable::Param(p) => p.visit_attrs(f),
-            Annotatable::FieldDef(sf) => sf.visit_attrs(f),
-            Annotatable::Variant(v) => v.visit_attrs(f),
-        }
-    }
-
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
-        match self {
-            Annotatable::Item(item) => item.tokens_mut(),
-            Annotatable::TraitItem(trait_item) => trait_item.tokens_mut(),
-            Annotatable::ImplItem(impl_item) => impl_item.tokens_mut(),
-            Annotatable::ForeignItem(foreign_item) => foreign_item.tokens_mut(),
-            Annotatable::Stmt(stmt) => stmt.tokens_mut(),
-            Annotatable::Expr(expr) => expr.tokens_mut(),
-            Annotatable::Arm(arm) => arm.tokens_mut(),
-            Annotatable::ExprField(field) => field.tokens_mut(),
-            Annotatable::PatField(fp) => fp.tokens_mut(),
-            Annotatable::GenericParam(gp) => gp.tokens_mut(),
-            Annotatable::Param(p) => p.tokens_mut(),
-            Annotatable::FieldDef(sf) => sf.tokens_mut(),
-            Annotatable::Variant(v) => v.tokens_mut(),
-        }
-    }
-}
-
 impl Annotatable {
     pub fn span(&self) -> Span {
         match *self {
@@ -121,6 +65,24 @@ impl Annotatable {
         }
     }
 
+    pub fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
+        match self {
+            Annotatable::Item(item) => item.visit_attrs(f),
+            Annotatable::TraitItem(trait_item) => trait_item.visit_attrs(f),
+            Annotatable::ImplItem(impl_item) => impl_item.visit_attrs(f),
+            Annotatable::ForeignItem(foreign_item) => foreign_item.visit_attrs(f),
+            Annotatable::Stmt(stmt) => stmt.visit_attrs(f),
+            Annotatable::Expr(expr) => expr.visit_attrs(f),
+            Annotatable::Arm(arm) => arm.visit_attrs(f),
+            Annotatable::ExprField(field) => field.visit_attrs(f),
+            Annotatable::PatField(fp) => fp.visit_attrs(f),
+            Annotatable::GenericParam(gp) => gp.visit_attrs(f),
+            Annotatable::Param(p) => p.visit_attrs(f),
+            Annotatable::FieldDef(sf) => sf.visit_attrs(f),
+            Annotatable::Variant(v) => v.visit_attrs(f),
+        }
+    }
+
     pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
         match self {
             Annotatable::Item(item) => visitor.visit_item(item),
@@ -139,7 +101,7 @@ impl Annotatable {
         }
     }
 
-    crate fn into_nonterminal(self) -> Nonterminal {
+    pub fn into_nonterminal(self) -> Nonterminal {
         match self {
             Annotatable::Item(item) => token::NtItem(item),
             Annotatable::TraitItem(item) | Annotatable::ImplItem(item) => {
@@ -161,10 +123,7 @@ impl Annotatable {
     }
 
     crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
-        // Tokens of an attribute target may be invalidated by some outer `#[derive]` performing
-        // "full configuration" (attributes following derives on the same item should be the most
-        // common case), that's why synthesizing tokens is allowed.
-        nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::Yes)
+        nt_to_tokenstream(&self.into_nonterminal(), sess, CanSynthesizeMissingTokens::No)
     }
 
     pub fn expect_item(self) -> P<ast::Item> {