hygiene: Introduce a helper method for creating new expansions

Creating a fresh expansion and immediately generating a span from it is the most common scenario.

Also avoid allocating `allow_internal_unstable` lists for derive markers repeatedly.
And rename `ExpnInfo::with_unstable` to `ExpnInfo::allow_unstable`, seems to be a better fitting name.

src/librustc/hir/lowering.rs

@@ -60,7 +60,7 @@ use syntax::attr;
 use syntax::ast;
 use syntax::ast::*;
 use syntax::errors;
-use syntax::ext::hygiene::{Mark, SyntaxContext};
+use syntax::ext::hygiene::Mark;
 use syntax::print::pprust;
 use syntax::source_map::{respan, ExpnInfo, ExpnKind, DesugaringKind, Spanned};
 use syntax::std_inject;
@@ -875,13 +875,11 @@ impl<'a> LoweringContext<'a> {
         span: Span,
         allow_internal_unstable: Option<Lrc<[Symbol]>>,
     ) -> Span {
-        let mark = Mark::fresh(Mark::root());
+        span.fresh_expansion(Mark::root(), ExpnInfo {
-        mark.set_expn_info(ExpnInfo {
             def_site: span,
             allow_internal_unstable,
             ..ExpnInfo::default(ExpnKind::Desugaring(reason), span, self.sess.edition())
-        });
+        })
-        span.with_ctxt(SyntaxContext::empty().apply_mark(mark))
     }
 
     fn with_anonymous_lifetime_mode<R>(

src/librustc/ty/query/on_disk_cache.rs

@@ -588,41 +588,40 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for CacheDecoder<'a, 'tcx> {
 
         let expn_info_tag = u8::decode(self)?;
 
-        let ctxt = match expn_info_tag {
+        // FIXME(mw): This method does not restore `MarkData::parent` or
+        // `SyntaxContextData::prev_ctxt` or `SyntaxContextData::opaque`. These things
+        // don't seem to be used after HIR lowering, so everything should be fine
+        // as long as incremental compilation does not kick in before that.
+        let location = || Span::new(lo, hi, SyntaxContext::empty());
+        let recover_from_expn_info = |this: &Self, expn_info, pos| {
+            let span = location().fresh_expansion(Mark::root(), expn_info);
+            this.synthetic_expansion_infos.borrow_mut().insert(pos, span.ctxt());
+            span
+        };
+        Ok(match expn_info_tag {
             TAG_NO_EXPANSION_INFO => {
-                SyntaxContext::empty()
+                location()
             }
             TAG_EXPANSION_INFO_INLINE => {
-                let pos = AbsoluteBytePos::new(self.opaque.position());
+                let expn_info = Decodable::decode(self)?;
-                let expn_info: ExpnInfo = Decodable::decode(self)?;
+                recover_from_expn_info(
-                let ctxt = SyntaxContext::allocate_directly(expn_info);
+                    self, expn_info, AbsoluteBytePos::new(self.opaque.position())
-                self.synthetic_expansion_infos.borrow_mut().insert(pos, ctxt);
+                )
-                ctxt
             }
             TAG_EXPANSION_INFO_SHORTHAND => {
                 let pos = AbsoluteBytePos::decode(self)?;
-                let cached_ctxt = self.synthetic_expansion_infos
+                if let Some(cached_ctxt) = self.synthetic_expansion_infos.borrow().get(&pos) {
-                                      .borrow()
+                    Span::new(lo, hi, *cached_ctxt)
-                                      .get(&pos)
-                                      .cloned();
-
-                if let Some(ctxt) = cached_ctxt {
-                    ctxt
                 } else {
-                    let expn_info = self.with_position(pos.to_usize(), |this| {
+                    let expn_info =
-                         ExpnInfo::decode(this)
+                        self.with_position(pos.to_usize(), |this| ExpnInfo::decode(this))?;
-                    })?;
+                    recover_from_expn_info(self, expn_info, pos)
-                    let ctxt = SyntaxContext::allocate_directly(expn_info);
-                    self.synthetic_expansion_infos.borrow_mut().insert(pos, ctxt);
-                    ctxt
                 }
             }
             _ => {
                 unreachable!()
             }
-        };
+        })
-
-        Ok(Span::new(lo, hi, ctxt))
     }
 }
 

src/librustc_allocator/expand.rs

@@ -14,7 +14,7 @@ use syntax::{
         base::{ExtCtxt, MacroKind, Resolver},
         build::AstBuilder,
         expand::ExpansionConfig,
-        hygiene::{Mark, SyntaxContext},
+        hygiene::Mark,
     },
     mut_visit::{self, MutVisitor},
     parse::ParseSess,
@@ -84,16 +84,12 @@ impl MutVisitor for ExpandAllocatorDirectives<'_> {
         }
         self.found = true;
 
-        // Create a fresh Mark for the new macro expansion we are about to do
+        // Create a new expansion for the generated allocator code.
-        let mark = Mark::fresh(Mark::root());
+        let span = item.span.fresh_expansion(Mark::root(), ExpnInfo::allow_unstable(
-        mark.set_expn_info(ExpnInfo::with_unstable(
             ExpnKind::Macro(MacroKind::Attr, sym::global_allocator), item.span, self.sess.edition,
-            &[sym::rustc_attrs],
+            [sym::rustc_attrs][..].into(),
         ));
 
-        // Tie the span to the macro expansion info we just created
-        let span = item.span.with_ctxt(SyntaxContext::empty().apply_mark(mark));
-
         // Create an expansion config
         let ecfg = ExpansionConfig::default(self.crate_name.take().unwrap());
 

src/librustc_resolve/macros.rs

@@ -17,7 +17,7 @@ use syntax::errors::DiagnosticBuilder;
 use syntax::ext::base::{self, Indeterminate};
 use syntax::ext::base::{MacroKind, SyntaxExtension};
 use syntax::ext::expand::{AstFragment, Invocation, InvocationKind};
-use syntax::ext::hygiene::{self, Mark};
+use syntax::ext::hygiene::{self, Mark, ExpnInfo, ExpnKind};
 use syntax::ext::tt::macro_rules;
 use syntax::feature_gate::{feature_err, emit_feature_err, is_builtin_attr_name};
 use syntax::feature_gate::{AttributeGate, GateIssue, Stability, BUILTIN_ATTRIBUTES};
@@ -148,7 +148,10 @@ impl<'a> base::Resolver for Resolver<'a> {
     }
 
     fn get_module_scope(&mut self, id: ast::NodeId) -> Mark {
-        let mark = Mark::fresh(Mark::root());
+        let span = DUMMY_SP.fresh_expansion(Mark::root(), ExpnInfo::default(
+            ExpnKind::Macro(MacroKind::Attr, sym::test_case), DUMMY_SP, self.session.edition()
+        ));
+        let mark = span.ctxt().outer();
         let module = self.module_map[&self.definitions.local_def_id(id)];
         self.definitions.set_invocation_parent(mark, module.def_id().unwrap().index);
         self.invocations.insert(mark, self.arenas.alloc_invocation_data(InvocationData {

src/libsyntax/ext/base.rs

@@ -721,6 +721,7 @@ pub struct ExtCtxt<'a> {
     pub resolver: &'a mut dyn Resolver,
     pub current_expansion: ExpansionData,
     pub expansions: FxHashMap<Span, Vec<String>>,
+    pub allow_derive_markers: Lrc<[Symbol]>,
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -740,6 +741,7 @@ impl<'a> ExtCtxt<'a> {
                 directory_ownership: DirectoryOwnership::Owned { relative: None },
             },
             expansions: FxHashMap::default(),
+            allow_derive_markers: [sym::rustc_attrs, sym::structural_match][..].into(),
         }
     }
 

src/libsyntax/ext/derive.rs

@@ -8,7 +8,6 @@ use crate::symbol::{Symbol, sym};
 use crate::errors::Applicability;
 
 use syntax_pos::Span;
-use syntax_pos::hygiene::{Mark, SyntaxContext};
 use rustc_data_structures::fx::FxHashSet;
 
 pub fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> {
@@ -55,13 +54,11 @@ pub fn add_derived_markers<T>(cx: &mut ExtCtxt<'_>, span: Span, traits: &[ast::P
         names.insert(unwrap_or!(path.segments.get(0), continue).ident.name);
     }
 
-    let mark = Mark::fresh(cx.current_expansion.mark);
+    let span = span.fresh_expansion(cx.current_expansion.mark, ExpnInfo::allow_unstable(
-    mark.set_expn_info(ExpnInfo::with_unstable(
         ExpnKind::Macro(MacroKind::Derive, Symbol::intern(&pretty_name)), span,
-        cx.parse_sess.edition, &[sym::rustc_attrs, sym::structural_match],
+        cx.parse_sess.edition, cx.allow_derive_markers.clone(),
     ));
 
-    let span = span.with_ctxt(SyntaxContext::empty().apply_mark(mark));
     item.visit_attrs(|attrs| {
         if names.contains(&sym::Eq) && names.contains(&sym::PartialEq) {
             let meta = cx.meta_word(span, sym::structural_match);

src/libsyntax/ext/expand.rs

@@ -362,7 +362,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 derives.reserve(traits.len());
                 invocations.reserve(traits.len());
                 for path in traits {
-                    let mark = Mark::fresh(self.cx.current_expansion.mark);
+                    let mark = Mark::fresh(self.cx.current_expansion.mark, None);
                     derives.push(mark);
                     invocations.push(Invocation {
                         kind: InvocationKind::Derive {
@@ -847,7 +847,7 @@ struct InvocationCollector<'a, 'b> {
 
 impl<'a, 'b> InvocationCollector<'a, 'b> {
     fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment {
-        let mark = Mark::fresh(self.cx.current_expansion.mark);
+        let mark = Mark::fresh(self.cx.current_expansion.mark, None);
         self.invocations.push(Invocation {
             kind,
             fragment_kind,

src/libsyntax/std_inject.rs

@@ -1,7 +1,7 @@
 use crate::ast;
 use crate::attr;
 use crate::edition::Edition;
-use crate::ext::hygiene::{Mark, SyntaxContext, MacroKind};
+use crate::ext::hygiene::{Mark, MacroKind};
 use crate::symbol::{Ident, Symbol, kw, sym};
 use crate::source_map::{ExpnInfo, ExpnKind, dummy_spanned, respan};
 use crate::ptr::P;
@@ -9,19 +9,7 @@ use crate::tokenstream::TokenStream;
 
 use std::cell::Cell;
 use std::iter;
-use syntax_pos::{DUMMY_SP, Span};
+use syntax_pos::DUMMY_SP;
-
-/// Craft a span that will be ignored by the stability lint's
-/// call to source_map's `is_internal` check.
-/// The expanded code uses the unstable `#[prelude_import]` attribute.
-fn ignored_span(sp: Span, edition: Edition) -> Span {
-    let mark = Mark::fresh(Mark::root());
-    mark.set_expn_info(ExpnInfo::with_unstable(
-        ExpnKind::Macro(MacroKind::Attr, Symbol::intern("std_inject")), sp, edition,
-        &[sym::prelude_import],
-    ));
-    sp.with_ctxt(SyntaxContext::empty().apply_mark(mark))
-}
 
 pub fn injected_crate_name() -> Option<&'static str> {
     INJECTED_CRATE_NAME.with(|name| name.get())
@@ -87,7 +75,11 @@ pub fn maybe_inject_crates_ref(
 
     INJECTED_CRATE_NAME.with(|opt_name| opt_name.set(Some(name)));
 
-    let span = ignored_span(DUMMY_SP, edition);
+    let span = DUMMY_SP.fresh_expansion(Mark::root(), ExpnInfo::allow_unstable(
+        ExpnKind::Macro(MacroKind::Attr, sym::std_inject), DUMMY_SP, edition,
+        [sym::prelude_import][..].into(),
+    ));
+
     krate.module.items.insert(0, P(ast::Item {
         attrs: vec![ast::Attribute {
             style: ast::AttrStyle::Outer,

src/libsyntax/test.rs

@@ -43,7 +43,6 @@ struct TestCtxt<'a> {
     test_cases: Vec<Test>,
     reexport_test_harness_main: Option<Symbol>,
     is_libtest: bool,
-    ctxt: SyntaxContext,
     features: &'a Features,
     test_runner: Option<ast::Path>,
 
@@ -259,8 +258,6 @@ fn generate_test_harness(sess: &ParseSess,
     let mut cleaner = EntryPointCleaner { depth: 0 };
     cleaner.visit_crate(krate);
 
-    let mark = Mark::fresh(Mark::root());
-
     let mut econfig = ExpansionConfig::default("test".to_string());
     econfig.features = Some(features);
 
@@ -274,16 +271,10 @@ fn generate_test_harness(sess: &ParseSess,
         is_libtest: attr::find_crate_name(&krate.attrs)
             .map(|s| s == sym::test).unwrap_or(false),
         toplevel_reexport: None,
-        ctxt: SyntaxContext::empty().apply_mark(mark),
         features,
         test_runner
     };
 
-    mark.set_expn_info(ExpnInfo::with_unstable(
-        ExpnKind::Macro(MacroKind::Attr, sym::test_case), DUMMY_SP, sess.edition,
-        &[sym::main, sym::test, sym::rustc_attrs],
-    ));
-
     TestHarnessGenerator {
         cx,
         tests: Vec::new(),
@@ -291,13 +282,6 @@ fn generate_test_harness(sess: &ParseSess,
     }.visit_crate(krate);
 }
 
-/// Craft a span that will be ignored by the stability lint's
-/// call to source_map's `is_internal` check.
-/// The expanded code calls some unstable functions in the test crate.
-fn ignored_span(cx: &TestCtxt<'_>, sp: Span) -> Span {
-    sp.with_ctxt(cx.ctxt)
-}
-
 enum HasTestSignature {
     Yes,
     No(BadTestSignature),
@@ -314,12 +298,15 @@ enum BadTestSignature {
 /// Creates a function item for use as the main function of a test build.
 /// This function will call the `test_runner` as specified by the crate attribute
 fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
-    // Writing this out by hand with 'ignored_span':
+    // Writing this out by hand:
     //        pub fn main() {
     //            #![main]
     //            test::test_main_static(&[..tests]);
     //        }
-    let sp = ignored_span(cx, DUMMY_SP);
+    let sp = DUMMY_SP.fresh_expansion(Mark::root(), ExpnInfo::allow_unstable(
+        ExpnKind::Macro(MacroKind::Attr, sym::test_case), DUMMY_SP, cx.ext_cx.parse_sess.edition,
+        [sym::main, sym::test, sym::rustc_attrs][..].into(),
+    ));
     let ecx = &cx.ext_cx;
     let test_id = Ident::with_empty_ctxt(sym::test);
 

src/libsyntax_ext/proc_macro_decls.rs

@@ -346,12 +346,10 @@ fn mk_decls(
     custom_attrs: &[ProcMacroDef],
     custom_macros: &[ProcMacroDef],
 ) -> P<ast::Item> {
-    let mark = Mark::fresh(Mark::root());
+    let span = DUMMY_SP.fresh_expansion(Mark::root(), ExpnInfo::allow_unstable(
-    mark.set_expn_info(ExpnInfo::with_unstable(
         ExpnKind::Macro(MacroKind::Attr, sym::proc_macro), DUMMY_SP, cx.parse_sess.edition,
-        &[sym::rustc_attrs, Symbol::intern("proc_macro_internals")],
+        [sym::rustc_attrs, sym::proc_macro_internals][..].into(),
     ));
-    let span = DUMMY_SP.apply_mark(mark);
 
     let hidden = cx.meta_list_item_word(span, sym::hidden);
     let doc = cx.meta_list(span, sym::doc, vec![hidden]);

src/libsyntax_pos/hygiene.rs

@@ -82,11 +82,8 @@ pub enum Transparency {
 }
 
 impl Mark {
-    pub fn fresh(parent: Mark) -> Self {
+    pub fn fresh(parent: Mark, expn_info: Option<ExpnInfo>) -> Self {
-        HygieneData::with(|data| {
+        HygieneData::with(|data| data.fresh_mark(parent, expn_info))
-            data.marks.push(MarkData { parent, expn_info: None });
-            Mark(data.marks.len() as u32 - 1)
-        })
     }
 
     /// The mark of the theoretical expansion that generates freshly parsed, unexpanded AST.
@@ -180,6 +177,11 @@ impl HygieneData {
         GLOBALS.with(|globals| f(&mut *globals.hygiene_data.borrow_mut()))
     }
 
+    fn fresh_mark(&mut self, parent: Mark, expn_info: Option<ExpnInfo>) -> Mark {
+        self.marks.push(MarkData { parent, expn_info });
+        Mark(self.marks.len() as u32 - 1)
+    }
+
     fn expn_info(&self, mark: Mark) -> Option<&ExpnInfo> {
         self.marks[mark.0 as usize].expn_info.as_ref()
     }
@@ -396,33 +398,6 @@ impl SyntaxContext {
         SyntaxContext(raw)
     }
 
-    // Allocate a new SyntaxContext with the given ExpnInfo. This is used when
-    // deserializing Spans from the incr. comp. cache.
-    // FIXME(mw): This method does not restore MarkData::parent or
-    // SyntaxContextData::prev_ctxt or SyntaxContextData::opaque. These things
-    // don't seem to be used after HIR lowering, so everything should be fine
-    // as long as incremental compilation does not kick in before that.
-    pub fn allocate_directly(expansion_info: ExpnInfo) -> Self {
-        HygieneData::with(|data| {
-            data.marks.push(MarkData {
-                parent: Mark::root(),
-                expn_info: Some(expansion_info),
-            });
-
-            let mark = Mark(data.marks.len() as u32 - 1);
-
-            data.syntax_contexts.push(SyntaxContextData {
-                outer_mark: mark,
-                transparency: Transparency::SemiTransparent,
-                prev_ctxt: SyntaxContext::empty(),
-                opaque: SyntaxContext::empty(),
-                opaque_and_semitransparent: SyntaxContext::empty(),
-                dollar_crate_name: kw::DollarCrate,
-            });
-            SyntaxContext(data.syntax_contexts.len() as u32 - 1)
-        })
-    }
-
     /// Extend a syntax context with a given mark and default transparency for that mark.
     pub fn apply_mark(self, mark: Mark) -> SyntaxContext {
         HygieneData::with(|data| data.apply_mark(self, mark))
@@ -615,6 +590,20 @@ impl fmt::Debug for SyntaxContext {
     }
 }
 
+impl Span {
+    /// Creates a fresh expansion with given properties.
+    /// Expansions are normally created by macros, but in some cases expansions are created for
+    /// other compiler-generated code to set per-span properties like allowed unstable features.
+    /// The returned span belongs to the created expansion and has the new properties,
+    /// but its location is inherited from the current span.
+    pub fn fresh_expansion(self, parent: Mark, expn_info: ExpnInfo) -> Span {
+        HygieneData::with(|data| {
+            let mark = data.fresh_mark(parent, Some(expn_info));
+            self.with_ctxt(data.apply_mark(SyntaxContext::empty(), mark))
+        })
+    }
+}
+
 /// A subset of properties from both macro definition and macro call available through global data.
 /// Avoid using this if you have access to the original definition or call structures.
 #[derive(Clone, Debug, RustcEncodable, RustcDecodable)]
@@ -669,10 +658,10 @@ impl ExpnInfo {
         }
     }
 
-    pub fn with_unstable(kind: ExpnKind, call_site: Span, edition: Edition,
+    pub fn allow_unstable(kind: ExpnKind, call_site: Span, edition: Edition,
-                         allow_internal_unstable: &[Symbol]) -> ExpnInfo {
+                          allow_internal_unstable: Lrc<[Symbol]>) -> ExpnInfo {
         ExpnInfo {
-            allow_internal_unstable: Some(allow_internal_unstable.into()),
+            allow_internal_unstable: Some(allow_internal_unstable),
             ..ExpnInfo::default(kind, call_site, edition)
         }
     }

src/libsyntax_pos/symbol.rs

@@ -508,6 +508,7 @@ symbols! {
         proc_macro_expr,
         proc_macro_gen,
         proc_macro_hygiene,
+        proc_macro_internals,
         proc_macro_mod,
         proc_macro_non_items,
         proc_macro_path_invoc,
@@ -631,6 +632,7 @@ symbols! {
         static_nobundle,
         static_recursion,
         std,
+        std_inject,
         str,
         stringify,
         stmt,