Split float literal tokens at the .
This commit is contained in:
@@ -318,7 +318,7 @@ fn name_ref(p: &mut Parser) {
|
||||
}
|
||||
|
||||
fn name_ref_or_index(p: &mut Parser) {
|
||||
assert!(p.at(IDENT) || p.at(INT_NUMBER));
|
||||
assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART));
|
||||
let m = p.start();
|
||||
p.bump_any();
|
||||
m.complete(p, NAME_REF);
|
||||
|
||||
@@ -475,11 +475,8 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
|
||||
assert!(p.at(T![.]));
|
||||
let m = lhs.precede(p);
|
||||
p.bump(T![.]);
|
||||
if p.at(IDENT) || p.at(INT_NUMBER) {
|
||||
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) {
|
||||
name_ref_or_index(p);
|
||||
} else if p.at(FLOAT_NUMBER) {
|
||||
// FIXME: How to recover and instead parse INT + T![.]?
|
||||
p.bump_any();
|
||||
} else {
|
||||
p.error("expected field name or number");
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
|
||||
T![true],
|
||||
T![false],
|
||||
INT_NUMBER,
|
||||
FLOAT_NUMBER,
|
||||
FLOAT_NUMBER_PART,
|
||||
BYTE,
|
||||
CHAR,
|
||||
STRING,
|
||||
@@ -29,11 +29,19 @@ pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
|
||||
return None;
|
||||
}
|
||||
let m = p.start();
|
||||
if p.at(FLOAT_NUMBER) {
|
||||
if p.at(FLOAT_NUMBER_PART) {
|
||||
// Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT`
|
||||
let f = p.start();
|
||||
p.bump(FLOAT_NUMBER);
|
||||
p.bump(FLOAT_NUMBER_PART);
|
||||
if p.at(DOT) {
|
||||
p.bump(DOT);
|
||||
if p.at(FLOAT_NUMBER_PART) {
|
||||
p.bump(FLOAT_NUMBER_PART);
|
||||
}
|
||||
}
|
||||
f.complete(p, FLOAT_LITERAL);
|
||||
} else {
|
||||
// Everything else is just one token.
|
||||
p.bump_any();
|
||||
}
|
||||
Some(m.complete(p, LITERAL))
|
||||
|
||||
@@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
|
||||
}
|
||||
|
||||
fn is_literal_pat_start(p: &Parser) -> bool {
|
||||
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
|
||||
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART)
|
||||
|| p.at_ts(expressions::LITERAL_FIRST)
|
||||
}
|
||||
|
||||
|
||||
@@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
|
||||
|
||||
rustc_lexer::TokenKind::RawIdent => IDENT,
|
||||
rustc_lexer::TokenKind::Literal { kind, .. } => {
|
||||
self.extend_literal(token_text.len(), kind);
|
||||
self.extend_literal(token_text, kind);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
|
||||
self.push(syntax_kind, token_text.len(), err);
|
||||
}
|
||||
|
||||
fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
|
||||
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) {
|
||||
let mut err = "";
|
||||
|
||||
let syntax_kind = match *kind {
|
||||
@@ -237,7 +237,22 @@ impl<'a> Converter<'a> {
|
||||
if empty_exponent {
|
||||
err = "Missing digits after the exponent symbol";
|
||||
}
|
||||
FLOAT_NUMBER
|
||||
|
||||
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
|
||||
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
|
||||
if let Some((before, after)) = token_text.split_once('.') {
|
||||
let err = if err.is_empty() { None } else { Some(err) };
|
||||
if !before.is_empty() {
|
||||
self.push(FLOAT_NUMBER_PART, before.len(), None);
|
||||
}
|
||||
self.push(DOT, 1, None);
|
||||
if !after.is_empty() {
|
||||
self.push(FLOAT_NUMBER_PART, after.len(), err);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
FLOAT_NUMBER_PART
|
||||
}
|
||||
rustc_lexer::LiteralKind::Char { terminated } => {
|
||||
if !terminated {
|
||||
@@ -295,6 +310,6 @@ impl<'a> Converter<'a> {
|
||||
};
|
||||
|
||||
let err = if err.is_empty() { None } else { Some(err) };
|
||||
self.push(syntax_kind, len, err);
|
||||
self.push(syntax_kind, token_text.len(), err);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user