Reimplemented lexer with vectors instead of iterators

2020-01-26 20:44:49 +02:00
parent ad24976da3
commit ac37a11f04
10 changed files with 254 additions and 200 deletions
--- a/crates/ra_syntax/src/lib.rs
+++ b/crates/ra_syntax/src/lib.rs
@@ -41,7 +41,7 @@ use crate::syntax_node::GreenNode;
 pub use crate::{
    algo::InsertPosition,
    ast::{AstNode, AstToken},
-    parsing::{classify_literal, tokenize, Token},
+    parsing::{first_token, single_token, tokenize, tokenize_append, Token, TokenizeError},
    ptr::{AstPtr, SyntaxNodePtr},
    syntax_error::{Location, SyntaxError, SyntaxErrorKind},
    syntax_node::{