Rollup merge of #89082 - smoelius:master, r=kennytm

Implement #85440 (Random test ordering)

This PR adds `--shuffle` and `--shuffle-seed` options to `libtest`. The options are similar to the [`-shuffle` option](c894b442d1/src/testing/testing.go (L1482-L1499)) that was recently added to Go.

Here are the relevant parts of the help message:
```
        --shuffle       Run tests in random order
        --shuffle-seed SEED
                        Run tests in random order; seed the random number
                        generator with SEED
...
By default, the tests are run in alphabetical order. Use --shuffle or set
RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated
"shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the
tests in the same order again. Note that --shuffle and --shuffle-seed do not
affect whether the tests are run in parallel.
```
Is an RFC needed for this?
This commit is contained in:
Jubilee
2021-10-07 20:26:12 -07:00
committed by GitHub
14 changed files with 313 additions and 40 deletions

View File

@@ -91,6 +91,7 @@ mod tests;
use event::{CompletedTest, TestEvent};
use helpers::concurrency::get_concurrency;
use helpers::exit_code::get_exit_code;
use helpers::shuffle::{get_shuffle_seed, shuffle_tests};
use options::{Concurrent, RunStrategy};
use test_result::*;
use time::TestExecTime;
@@ -247,7 +248,9 @@ where
let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
let event = TestEvent::TeFiltered(filtered_descs);
let shuffle_seed = get_shuffle_seed(opts);
let event = TestEvent::TeFiltered(filtered_descs, shuffle_seed);
notify_about_test_event(event)?;
let (filtered_tests, filtered_benchs): (Vec<_>, _) = filtered_tests
@@ -259,7 +262,11 @@ where
let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
let mut remaining = filtered_tests;
remaining.reverse();
if let Some(shuffle_seed) = shuffle_seed {
shuffle_tests(shuffle_seed, &mut remaining);
} else {
remaining.reverse();
}
let mut pending = 0;
let (tx, rx) = channel::<CompletedTest>();