Remove trivia tokens

This commit is contained in:
Aleksey Kladov
2020-09-01 11:24:52 +02:00
parent 8f24c2ec9d
commit 5326361fc0
5 changed files with 5 additions and 28 deletions

View File

@@ -303,7 +303,7 @@ const UNICODE_ARRAY: &[(char, &str, char)] = &[
// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
// fancier error recovery to it, as there will be less overall work to do this way.
const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[
(' ', "Space", Some(token::Whitespace)),
(' ', "Space", None),
('_', "Underscore", Some(token::Ident(kw::Underscore, false))),
('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))),
(',', "Comma", Some(token::Comma)),