refactor lexer to use idiomatic borrowing
This commit is contained in:
@@ -321,33 +321,29 @@ impl<'a> StringReader<'a> {
|
||||
(pos - self.source_file.start_pos).to_usize()
|
||||
}
|
||||
|
||||
/// Calls `f` with a string slice of the source text spanning from `start`
|
||||
/// up to but excluding `self.pos`, meaning the slice does not include
|
||||
/// the character `self.ch`.
|
||||
fn with_str_from<T, F>(&self, start: BytePos, f: F) -> T
|
||||
where F: FnOnce(&str) -> T
|
||||
/// Slice of the source text from `start` up to but excluding `self.pos`,
|
||||
/// meaning the slice does not include the character `self.ch`.
|
||||
fn str_from(&self, start: BytePos) -> &str
|
||||
{
|
||||
self.with_str_from_to(start, self.pos, f)
|
||||
self.str_from_to(start, self.pos)
|
||||
}
|
||||
|
||||
/// Creates a Name from a given offset to the current offset.
|
||||
fn name_from(&self, start: BytePos) -> ast::Name {
|
||||
debug!("taking an ident from {:?} to {:?}", start, self.pos);
|
||||
self.with_str_from(start, Symbol::intern)
|
||||
Symbol::intern(self.str_from(start))
|
||||
}
|
||||
|
||||
/// As name_from, with an explicit endpoint.
|
||||
fn name_from_to(&self, start: BytePos, end: BytePos) -> ast::Name {
|
||||
debug!("taking an ident from {:?} to {:?}", start, end);
|
||||
self.with_str_from_to(start, end, Symbol::intern)
|
||||
Symbol::intern(self.str_from_to(start, end))
|
||||
}
|
||||
|
||||
/// Calls `f` with a string slice of the source text spanning from `start`
|
||||
/// up to but excluding `end`.
|
||||
fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T
|
||||
where F: FnOnce(&str) -> T
|
||||
/// Slice of the source text spanning from `start` up to but excluding `end`.
|
||||
fn str_from_to(&self, start: BytePos, end: BytePos) -> &str
|
||||
{
|
||||
f(&self.src[self.src_index(start)..self.src_index(end)])
|
||||
&self.src[self.src_index(start)..self.src_index(end)]
|
||||
}
|
||||
|
||||
/// Converts CRLF to LF in the given string, raising an error on bare CR.
|
||||
@@ -456,8 +452,8 @@ impl<'a> StringReader<'a> {
|
||||
self.bump();
|
||||
}
|
||||
|
||||
self.with_str_from(start, |string| {
|
||||
if string == "_" {
|
||||
match self.str_from(start) {
|
||||
"_" => {
|
||||
self.sess.span_diagnostic
|
||||
.struct_span_warn(self.mk_sp(start, self.pos),
|
||||
"underscore literal suffix is not allowed")
|
||||
@@ -468,10 +464,9 @@ impl<'a> StringReader<'a> {
|
||||
<https://github.com/rust-lang/rust/issues/42326>")
|
||||
.emit();
|
||||
None
|
||||
} else {
|
||||
Some(Symbol::intern(string))
|
||||
}
|
||||
})
|
||||
name => Some(Symbol::intern(name))
|
||||
}
|
||||
}
|
||||
|
||||
/// PRECONDITION: self.ch is not whitespace
|
||||
@@ -513,9 +508,7 @@ impl<'a> StringReader<'a> {
|
||||
}
|
||||
|
||||
let kind = if doc_comment {
|
||||
self.with_str_from(start_bpos, |string| {
|
||||
token::DocComment(Symbol::intern(string))
|
||||
})
|
||||
token::DocComment(self.name_from(start_bpos))
|
||||
} else {
|
||||
token::Comment
|
||||
};
|
||||
@@ -615,23 +608,22 @@ impl<'a> StringReader<'a> {
|
||||
self.bump();
|
||||
}
|
||||
|
||||
self.with_str_from(start_bpos, |string| {
|
||||
// but comments with only "*"s between two "/"s are not
|
||||
let kind = if is_block_doc_comment(string) {
|
||||
let string = if has_cr {
|
||||
self.translate_crlf(start_bpos,
|
||||
string,
|
||||
"bare CR not allowed in block doc-comment")
|
||||
} else {
|
||||
string.into()
|
||||
};
|
||||
token::DocComment(Symbol::intern(&string[..]))
|
||||
let string = self.str_from(start_bpos);
|
||||
// but comments with only "*"s between two "/"s are not
|
||||
let kind = if is_block_doc_comment(string) {
|
||||
let string = if has_cr {
|
||||
self.translate_crlf(start_bpos,
|
||||
string,
|
||||
"bare CR not allowed in block doc-comment")
|
||||
} else {
|
||||
token::Comment
|
||||
string.into()
|
||||
};
|
||||
token::DocComment(Symbol::intern(&string[..]))
|
||||
} else {
|
||||
token::Comment
|
||||
};
|
||||
|
||||
Some(Token::new(kind, self.mk_sp(start_bpos, self.pos)))
|
||||
})
|
||||
Some(Token::new(kind, self.mk_sp(start_bpos, self.pos)))
|
||||
}
|
||||
|
||||
/// Scan through any digits (base `scan_radix`) or underscores,
|
||||
@@ -838,20 +830,17 @@ impl<'a> StringReader<'a> {
|
||||
self.bump();
|
||||
}
|
||||
|
||||
return Ok(self.with_str_from(start, |string| {
|
||||
// FIXME: perform NFKC normalization here. (Issue #2253)
|
||||
let name = ast::Name::intern(string);
|
||||
|
||||
if is_raw_ident {
|
||||
let span = self.mk_sp(raw_start, self.pos);
|
||||
if !name.can_be_raw() {
|
||||
self.err_span(span, &format!("`{}` cannot be a raw identifier", name));
|
||||
}
|
||||
self.sess.raw_identifier_spans.borrow_mut().push(span);
|
||||
// FIXME: perform NFKC normalization here. (Issue #2253)
|
||||
let name = self.name_from(start);
|
||||
if is_raw_ident {
|
||||
let span = self.mk_sp(raw_start, self.pos);
|
||||
if !name.can_be_raw() {
|
||||
self.err_span(span, &format!("`{}` cannot be a raw identifier", name));
|
||||
}
|
||||
self.sess.raw_identifier_spans.borrow_mut().push(span);
|
||||
}
|
||||
|
||||
token::Ident(name, is_raw_ident)
|
||||
}));
|
||||
return Ok(token::Ident(name, is_raw_ident));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1300,101 +1289,95 @@ impl<'a> StringReader<'a> {
|
||||
}
|
||||
|
||||
fn validate_char_escape(&self, start_with_quote: BytePos) {
|
||||
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
|
||||
if let Err((off, err)) = unescape::unescape_char(lit) {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::Char,
|
||||
0..off,
|
||||
err,
|
||||
)
|
||||
}
|
||||
});
|
||||
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
|
||||
if let Err((off, err)) = unescape::unescape_char(lit) {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::Char,
|
||||
0..off,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_byte_escape(&self, start_with_quote: BytePos) {
|
||||
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
|
||||
if let Err((off, err)) = unescape::unescape_byte(lit) {
|
||||
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
|
||||
if let Err((off, err)) = unescape::unescape_byte(lit) {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::Byte,
|
||||
0..off,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_str_escape(&self, start_with_quote: BytePos) {
|
||||
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
|
||||
unescape::unescape_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::Byte,
|
||||
0..off,
|
||||
unescape::Mode::Str,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn validate_str_escape(&self, start_with_quote: BytePos) {
|
||||
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
|
||||
unescape::unescape_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::Str,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_raw_str_escape(&self, content_start: BytePos, content_end: BytePos) {
|
||||
self.with_str_from_to(content_start, content_end, |lit: &str| {
|
||||
unescape::unescape_raw_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
|
||||
unescape::Mode::Str,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
});
|
||||
let lit = self.str_from_to(content_start, content_end);
|
||||
unescape::unescape_raw_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
|
||||
unescape::Mode::Str,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_raw_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) {
|
||||
self.with_str_from_to(content_start, content_end, |lit: &str| {
|
||||
unescape::unescape_raw_byte_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
|
||||
unescape::Mode::ByteStr,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
});
|
||||
let lit = self.str_from_to(content_start, content_end);
|
||||
unescape::unescape_raw_byte_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)),
|
||||
unescape::Mode::ByteStr,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_byte_str_escape(&self, start_with_quote: BytePos) {
|
||||
self.with_str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1), |lit| {
|
||||
unescape::unescape_byte_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::ByteStr,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
});
|
||||
let lit = self.str_from_to(start_with_quote + BytePos(1), self.pos - BytePos(1));
|
||||
unescape::unescape_byte_str(lit, &mut |range, c| {
|
||||
if let Err(err) = c {
|
||||
emit_unescape_error(
|
||||
&self.sess.span_diagnostic,
|
||||
lit,
|
||||
self.mk_sp(start_with_quote, self.pos),
|
||||
unescape::Mode::ByteStr,
|
||||
range,
|
||||
err,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user