Implement RFC 3503: frontmatters

Supercedes #137193

compiler/rustc_lexer/src/cursor.rs

@@ -1,5 +1,10 @@
 use std::str::Chars;
 
+pub enum FrontmatterAllowed {
+    Yes,
+    No,
+}
+
 /// Peekable iterator over a char sequence.
 ///
 /// Next characters can be peeked via `first` method,
@@ -8,6 +13,7 @@ pub struct Cursor<'a> {
     len_remaining: usize,
     /// Iterator over chars. Slightly faster than a &str.
     chars: Chars<'a>,
+    pub(crate) frontmatter_allowed: FrontmatterAllowed,
     #[cfg(debug_assertions)]
     prev: char,
 }
@@ -15,10 +21,11 @@ pub struct Cursor<'a> {
 pub(crate) const EOF_CHAR: char = '\0';
 
 impl<'a> Cursor<'a> {
-    pub fn new(input: &'a str) -> Cursor<'a> {
+    pub fn new(input: &'a str, frontmatter_allowed: FrontmatterAllowed) -> Cursor<'a> {
         Cursor {
             len_remaining: input.len(),
             chars: input.chars(),
+            frontmatter_allowed,
             #[cfg(debug_assertions)]
             prev: EOF_CHAR,
         }
@@ -95,6 +102,11 @@ impl<'a> Cursor<'a> {
         Some(c)
     }
 
+    /// Moves to a substring by a number of bytes.
+    pub(crate) fn bump_bytes(&mut self, n: usize) {
+        self.chars = self.as_str()[n..].chars();
+    }
+
     /// Eats symbols while predicate returns true or until the end of file is reached.
     pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
         // It was tried making optimized version of this for eg. line comments, but