Reimplemented lexer with vectors instead of iterators

2020-01-26 20:44:49 +02:00
parent ad24976da3
commit ac37a11f04
10 changed files with 254 additions and 200 deletions
--- a/crates/ra_syntax/src/tests.rs
+++ b/crates/ra_syntax/src/tests.rs
@@ -10,7 +10,8 @@ use crate::{fuzz, SourceFile};
 #[test]
 fn lexer_tests() {
    dir_tests(&test_data_dir(), &["lexer"], |text, _| {
-        let tokens = crate::tokenize(text);
+        // FIXME: add tests for errors (their format is up to discussion)
+        let tokens = crate::tokenize(text).tokens;
        dump_tokens(&tokens, text)
    })
 }