This commit is contained in:
Aleksey Kladov
2019-02-21 15:24:42 +03:00
parent 2fa2805887
commit 412ac63ff5
11 changed files with 143 additions and 98 deletions

View File

@@ -5,6 +5,26 @@ use crate::{
parsing::lexer::Token,
};
pub(crate) struct ParserInput<'t> {
text: &'t str,
/// start position of each token(expect whitespace and comment)
/// ```non-rust
/// struct Foo;
/// ^------^---
/// | | ^-
/// 0 7 10
/// ```
/// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]`
start_offsets: Vec<TextUnit>,
/// non-whitespace/comment tokens
/// ```non-rust
/// struct Foo {}
/// ^^^^^^ ^^^ ^^
/// ```
/// tokens: `[struct, Foo, {, }]`
tokens: Vec<Token>,
}
impl<'t> TokenSource for ParserInput<'t> {
fn token_kind(&self, pos: usize) -> SyntaxKind {
if !(pos < self.tokens.len()) {
@@ -28,26 +48,6 @@ impl<'t> TokenSource for ParserInput<'t> {
}
}
pub(crate) struct ParserInput<'t> {
text: &'t str,
/// start position of each token(expect whitespace and comment)
/// ```non-rust
/// struct Foo;
/// ^------^---
/// | | ^-
/// 0 7 10
/// ```
/// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]`
start_offsets: Vec<TextUnit>,
/// non-whitespace/comment tokens
/// ```non-rust
/// struct Foo {}
/// ^^^^^^ ^^^ ^^
/// ```
/// tokens: `[struct, Foo, {, }]`
tokens: Vec<Token>,
}
impl<'t> ParserInput<'t> {
/// Generate input from tokens(expect comment and whitespace).
pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> {