2019-08-02 00:26:40 +03:00
|
|
|
use crate::tests::string_to_stream;
|
2019-10-10 10:26:10 +02:00
|
|
|
|
2020-02-29 20:37:32 +03:00
|
|
|
use rustc_ast::token;
|
Remove `TreeAndSpacing`.
A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is
not quite right. `Spacing` makes sense for `TokenTree::Token`, but does
not make sense for `TokenTree::Delimited`, because a
`TokenTree::Delimited` cannot be joined with another `TokenTree`.
This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`,
changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the
`TreeAndSpacing` typedef.
The commit removes these two impls:
- `impl From<TokenTree> for TokenStream`
- `impl From<TokenTree> for TreeAndSpacing`
These were useful, but also resulted in code with many `.into()` calls
that was hard to read, particularly for anyone not highly familiar with
the relevant types. This commit makes some other changes to compensate:
- `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`.
- `TokenStream::token_{alone,joint}()` are added.
- `TokenStream::delimited` is added.
This results in things like this:
```rust
TokenTree::token(token::Semi, stmt.span).into()
```
changing to this:
```rust
TokenStream::token_alone(token::Semi, stmt.span)
```
This makes the type of the result, and its spacing, clearer.
These changes also simplifies `Cursor` and `CursorRef`, because they no longer
need to distinguish between `next` and `next_with_spacing`.
2022-07-28 10:31:04 +10:00
|
|
|
use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder};
|
2021-05-05 21:31:25 +02:00
|
|
|
use rustc_span::create_default_session_globals_then;
|
2020-04-25 21:13:56 +02:00
|
|
|
use rustc_span::{BytePos, Span, Symbol};
|
2019-08-02 00:26:40 +03:00
|
|
|
|
|
|
|
|
fn string_to_ts(string: &str) -> TokenStream {
|
|
|
|
|
string_to_stream(string.to_owned())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn sp(a: u32, b: u32) -> Span {
|
2019-08-11 01:44:55 +03:00
|
|
|
Span::with_root_ctxt(BytePos(a), BytePos(b))
|
2019-08-02 00:26:40 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_concat() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_res = string_to_ts("foo::bar::baz");
|
|
|
|
|
let test_fst = string_to_ts("foo::bar");
|
|
|
|
|
let test_snd = string_to_ts("::baz");
|
2022-06-20 09:33:08 +10:00
|
|
|
let mut builder = TokenStreamBuilder::new();
|
|
|
|
|
builder.push(test_fst);
|
|
|
|
|
builder.push(test_snd);
|
|
|
|
|
let eq_res = builder.build();
|
2019-08-02 00:26:40 +03:00
|
|
|
assert_eq!(test_res.trees().count(), 5);
|
|
|
|
|
assert_eq!(eq_res.trees().count(), 5);
|
|
|
|
|
assert_eq!(test_res.eq_unspanned(&eq_res), true);
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_to_from_bijection() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_start = string_to_ts("foo::bar(baz)");
|
2022-05-16 18:58:15 +03:00
|
|
|
let test_end = test_start.trees().cloned().collect();
|
2019-08-02 00:26:40 +03:00
|
|
|
assert_eq!(test_start, test_end)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_eq_0() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_res = string_to_ts("foo");
|
|
|
|
|
let test_eqs = string_to_ts("foo");
|
|
|
|
|
assert_eq!(test_res, test_eqs)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_eq_1() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_res = string_to_ts("::bar::baz");
|
|
|
|
|
let test_eqs = string_to_ts("::bar::baz");
|
|
|
|
|
assert_eq!(test_res, test_eqs)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_eq_3() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_res = string_to_ts("");
|
|
|
|
|
let test_eqs = string_to_ts("");
|
|
|
|
|
assert_eq!(test_res, test_eqs)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_diseq_0() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_res = string_to_ts("::bar::baz");
|
|
|
|
|
let test_eqs = string_to_ts("bar::baz");
|
|
|
|
|
assert_eq!(test_res == test_eqs, false)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_diseq_1() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let test_res = string_to_ts("(bar,baz)");
|
|
|
|
|
let test_eqs = string_to_ts("bar,baz");
|
|
|
|
|
assert_eq!(test_res == test_eqs, false)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_is_empty() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
Remove `TreeAndSpacing`.
A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is
not quite right. `Spacing` makes sense for `TokenTree::Token`, but does
not make sense for `TokenTree::Delimited`, because a
`TokenTree::Delimited` cannot be joined with another `TokenTree`.
This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`,
changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the
`TreeAndSpacing` typedef.
The commit removes these two impls:
- `impl From<TokenTree> for TokenStream`
- `impl From<TokenTree> for TreeAndSpacing`
These were useful, but also resulted in code with many `.into()` calls
that was hard to read, particularly for anyone not highly familiar with
the relevant types. This commit makes some other changes to compensate:
- `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`.
- `TokenStream::token_{alone,joint}()` are added.
- `TokenStream::delimited` is added.
This results in things like this:
```rust
TokenTree::token(token::Semi, stmt.span).into()
```
changing to this:
```rust
TokenStream::token_alone(token::Semi, stmt.span)
```
This makes the type of the result, and its spacing, clearer.
These changes also simplifies `Cursor` and `CursorRef`, because they no longer
need to distinguish between `next` and `next_with_spacing`.
2022-07-28 10:31:04 +10:00
|
|
|
let test0 = TokenStream::default();
|
|
|
|
|
let test1 = TokenStream::token_alone(token::Ident(Symbol::intern("a"), false), sp(0, 1));
|
2019-08-02 00:26:40 +03:00
|
|
|
let test2 = string_to_ts("foo(bar::baz)");
|
|
|
|
|
|
|
|
|
|
assert_eq!(test0.is_empty(), true);
|
|
|
|
|
assert_eq!(test1.is_empty(), false);
|
|
|
|
|
assert_eq!(test2.is_empty(), false);
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn test_dotdotdot() {
|
2021-05-05 21:31:25 +02:00
|
|
|
create_default_session_globals_then(|| {
|
2019-08-02 00:26:40 +03:00
|
|
|
let mut builder = TokenStreamBuilder::new();
|
Remove `TreeAndSpacing`.
A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is
not quite right. `Spacing` makes sense for `TokenTree::Token`, but does
not make sense for `TokenTree::Delimited`, because a
`TokenTree::Delimited` cannot be joined with another `TokenTree`.
This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`,
changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the
`TreeAndSpacing` typedef.
The commit removes these two impls:
- `impl From<TokenTree> for TokenStream`
- `impl From<TokenTree> for TreeAndSpacing`
These were useful, but also resulted in code with many `.into()` calls
that was hard to read, particularly for anyone not highly familiar with
the relevant types. This commit makes some other changes to compensate:
- `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`.
- `TokenStream::token_{alone,joint}()` are added.
- `TokenStream::delimited` is added.
This results in things like this:
```rust
TokenTree::token(token::Semi, stmt.span).into()
```
changing to this:
```rust
TokenStream::token_alone(token::Semi, stmt.span)
```
This makes the type of the result, and its spacing, clearer.
These changes also simplifies `Cursor` and `CursorRef`, because they no longer
need to distinguish between `next` and `next_with_spacing`.
2022-07-28 10:31:04 +10:00
|
|
|
builder.push(TokenStream::token_joint(token::Dot, sp(0, 1)));
|
|
|
|
|
builder.push(TokenStream::token_joint(token::Dot, sp(1, 2)));
|
|
|
|
|
builder.push(TokenStream::token_alone(token::Dot, sp(2, 3)));
|
2019-08-02 00:26:40 +03:00
|
|
|
let stream = builder.build();
|
|
|
|
|
assert!(stream.eq_unspanned(&string_to_ts("...")));
|
|
|
|
|
assert_eq!(stream.trees().count(), 1);
|
|
|
|
|
})
|
|
|
|
|
}
|