Add Span and separate open/close delims to TTDelim

This came up when working [on the gl-rs generator extension](990383de80/src/gl_generator/lib.rs (L135-L146)).

The new definition of  `TTDelim` adds an associated `Span` that covers the whole token tree and enforces the invariant that a delimited sequence of token trees must have an opening and closing delimiter.

A `get_span` method has also been added to `TokenTree` type to make it easier to implement better error messages for syntax extensions.

src/libsyntax/ast.rs

@@ -592,6 +592,20 @@ pub enum CaptureClause {
     CaptureByRef,
 }
 
+/// A token that delimits a sequence of token trees
+#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
+pub struct Delimiter {
+    pub span: Span,
+    pub token: ::parse::token::Token,
+}
+
+impl Delimiter {
+    /// Convert the delimiter to a `TTTok`
+    pub fn to_tt(&self) -> TokenTree {
+        TTTok(self.span, self.token.clone())
+    }
+}
+
 /// When the main rust parser encounters a syntax-extension invocation, it
 /// parses the arguments to the invocation as a token-tree. This is a very
 /// loose structure, such that all sorts of different AST-fragments can
@@ -611,10 +625,9 @@ pub enum CaptureClause {
 pub enum TokenTree {
     /// A single token
     TTTok(Span, ::parse::token::Token),
-    /// A delimited sequence (the delimiters appear as the first
-    /// and last elements of the vector)
+    /// A delimited sequence of token trees
     // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST.
-    TTDelim(Rc<Vec<TokenTree>>),
+    TTDelim(Span, Delimiter, Rc<Vec<TokenTree>>, Delimiter),
 
     // These only make sense for right-hand-sides of MBE macros:
 
@@ -628,6 +641,18 @@ pub enum TokenTree {
     TTNonterminal(Span, Ident)
 }
 
+impl TokenTree {
+    /// Returns the `Span` corresponding to this token tree.
+    pub fn get_span(&self) -> Span {
+        match *self {
+            TTTok(span, _)         => span,
+            TTDelim(span, _, _, _) => span,
+            TTSeq(span, _, _, _)   => span,
+            TTNonterminal(span, _) => span,
+        }
+    }
+}
+
 // Matchers are nodes defined-by and recognized-by the main rust parser and
 // language, but they're only ever found inside syntax-extension invocations;
 // indeed, the only thing that ever _activates_ the rules in the rust parser