rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1338 from bjorn3/build_system_rework5

Browse Source 
Avoid clobbering build_system/ and ~/.cargo/bin
This commit is contained in:
bjorn3
2023-01-05 19:57:10 +01:00
committed by GitHub
parent 571405deea e14e5c2af1
commit 20429c00b2
16 changed files with 130 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.cirrus.yml
View File

@@ -6,8 +6,6 @@ task:
- pkg install -y curl git bash
- curl https://sh.rustup.rs -sSf --output rustup.sh
- sh rustup.sh --default-toolchain none -y --profile=minimal
cargo_bin_cache:
folder: ~/.cargo/bin
target_cache:
folder: target
prepare_script:

6
.github/workflows/main.yml vendored
View File

@@ -62,12 +62,6 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Cache cargo installed crates
uses: actions/cache@v3
with:
path: ~/.cargo/bin
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-installed-crates
- name: Cache cargo registry and index
uses: actions/cache@v3
with:

6
.github/workflows/nightly-cranelift.yml vendored
View File

@@ -13,12 +13,6 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Cache cargo installed crates
uses: actions/cache@v3
with:
path: ~/.cargo/bin
key: ubuntu-latest-cargo-installed-crates
- name: Prepare dependencies
run: |
git config --global user.email "user@example.com"

12
.github/workflows/rustc.yml vendored
View File

@@ -10,12 +10,6 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Cache cargo installed crates
uses: actions/cache@v3
with:
path: ~/.cargo/bin
key: ${{ runner.os }}-cargo-installed-crates
- name: Cache cargo registry and index
uses: actions/cache@v3
with:
@@ -44,12 +38,6 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Cache cargo installed crates
uses: actions/cache@v3
with:
path: ~/.cargo/bin
key: ${{ runner.os }}-cargo-installed-crates
- name: Cache cargo registry and index
uses: actions/cache@v3
with:

5
.gitignore vendored
View File

@@ -1,4 +1,4 @@
target
/target
**/*.rs.bk
*.rlib
*.o
@@ -11,9 +11,6 @@ perf.data.old
/y.exe
/y.pdb
/build
/build_sysroot/sysroot_src
/build_sysroot/compiler-builtins
/build_sysroot/rustc_version
/dist
/rust
/download

2
.vscode/settings.json vendored
View File

@@ -30,7 +30,7 @@
]
},
{
"sysroot_src": "./build_sysroot/sysroot_src/library",
"sysroot_src": "./download/sysroot/sysroot_src/library",
"crates": [
{
"root_module": "./example/std_example.rs",

79
build_system/bench.rs Normal file
View File

@@ -0,0 +1,79 @@
use std::env;
use std::fs;
use std::path::Path;
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::{get_file_name, get_wrapper_file_name};
use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject};
pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
"ebobby",
"simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
"<none>",
);
pub(crate) static SIMPLE_RAYTRACER: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
pub(crate) fn benchmark(dirs: &Dirs) {
benchmark_simple_raytracer(dirs);
}
fn benchmark_simple_raytracer(dirs: &Dirs) {
if std::process::Command::new("hyperfine").output().is_err() {
eprintln!("Hyperfine not installed");
eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
std::process::exit(1);
}
let run_runs = env::var("RUN_RUNS")
.unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
.parse()
.unwrap();
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
let cargo_clif = RelPath::DIST.to_path(dirs).join(get_wrapper_file_name("cargo-clif", "bin"));
let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
let clean_cmd = format!(
"cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let llvm_build_cmd = format!(
"cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let clif_build_cmd = format!(
"{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
cargo_clif = cargo_clif.display(),
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let bench_compile =
hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
spawn_and_wait(bench_compile);
eprintln!("[BENCH RUN] ebobby/simple-raytracer");
fs::copy(
target_dir.join("debug").join(get_file_name("main", "bin")),
RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")),
)
.unwrap();
let mut bench_run = hyperfine_command(
0,
run_runs,
None,
Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
);
bench_run.current_dir(RelPath::BUILD.to_path(dirs));
spawn_and_wait(bench_run);
}

9
build_system/build_sysroot.rs
View File

@@ -146,10 +146,11 @@ pub(crate) fn build_sysroot(
}
}
// FIXME move to download/ or dist/
pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version");
pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src");
static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot");
pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
static STANDARD_LIBRARY: CargoProject = CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
fn build_clif_sysroot_for_triple(
dirs: &Dirs,

17
build_system/mod.rs
View File

@@ -5,6 +5,7 @@ use std::process;
use self::utils::is_ci;
mod abi_cafe;
mod bench;
mod build_backend;
mod build_sysroot;
mod config;
@@ -20,6 +21,7 @@ USAGE:
./y.rs prepare [--out-dir DIR]
./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
OPTIONS:
--sysroot none|clif|llvm
@@ -54,6 +56,7 @@ enum Command {
Prepare,
Build,
Test,
Bench,
}
#[derive(Copy, Clone, Debug)]
@@ -67,7 +70,7 @@ pub fn main() {
if env::var("RUST_BACKTRACE").is_err() {
env::set_var("RUST_BACKTRACE", "1");
}
env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); // FIXME disable this by default
env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
if is_ci() {
@@ -83,6 +86,7 @@ pub fn main() {
Some("prepare") => Command::Prepare,
Some("build") => Command::Build,
Some("test") => Command::Test,
Some("bench") => Command::Bench,
Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
Some(command) => arg_error!("Unknown command {}", command),
None => {
@@ -198,5 +202,16 @@ pub fn main() {
&target_triple,
);
}
Command::Bench => {
build_sysroot::build_sysroot(
&dirs,
channel,
sysroot_kind,
&cg_clif_dylib,
&host_triple,
&target_triple,
);
bench::benchmark(&dirs);
}
}
}

1
build_system/path.rs
View File

@@ -42,7 +42,6 @@ impl RelPath {
pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist);
pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts");
pub(crate) const BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches");
pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath {

30
build_system/prepare.rs
View File

@@ -3,9 +3,11 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use super::build_sysroot::{SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
use crate::build_system::rustc_info::get_default_sysroot;
use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
use super::rustc_info::{get_file_name, get_rustc_version};
use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler};
pub(crate) fn prepare(dirs: &Dirs) {
@@ -31,14 +33,14 @@ pub(crate) fn prepare(dirs: &Dirs) {
super::tests::RAND_REPO.fetch(dirs);
super::tests::REGEX_REPO.fetch(dirs);
super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs);
super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
eprintln!("[LLVM BUILD] simple-raytracer");
let host_compiler = Compiler::host();
let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
let build_cmd = super::bench::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
spawn_and_wait(build_cmd);
fs::copy(
super::tests::SIMPLE_RAYTRACER
super::bench::SIMPLE_RAYTRACER
.target_dir(dirs)
.join(&host_compiler.triple)
.join("debug")
@@ -49,27 +51,27 @@ pub(crate) fn prepare(dirs: &Dirs) {
}
fn prepare_sysroot(dirs: &Dirs) {
let rustc_path = get_rustc_path();
let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
let sysroot_src = SYSROOT_SRC;
let sysroot_src_orig = get_default_sysroot().join("lib/rustlib/src/rust");
assert!(sysroot_src_orig.exists());
sysroot_src.ensure_fresh(dirs);
fs::create_dir_all(sysroot_src.to_path(dirs).join("library")).unwrap();
eprintln!("[COPY] sysroot src");
BUILD_SYSROOT.ensure_fresh(dirs);
copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs));
fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap();
copy_dir_recursively(
&sysroot_src_orig.join("library"),
&sysroot_src.to_path(dirs).join("library"),
&SYSROOT_SRC.to_path(dirs).join("library"),
);
let rustc_version = get_rustc_version();
fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
eprintln!("[GIT] init");
init_git_repo(&sysroot_src.to_path(dirs));
init_git_repo(&SYSROOT_SRC.to_path(dirs));
apply_patches(dirs, "sysroot", &sysroot_src.to_path(dirs));
apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs));
}
pub(crate) struct GitRepo {

84
build_system/tests.rs
View File

@@ -1,11 +1,9 @@
use super::build_sysroot;
use super::bench::SIMPLE_RAYTRACER;
use super::build_sysroot::{self, SYSROOT_SRC};
use super::config;
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::{get_file_name, get_wrapper_file_name};
use super::utils::{
hyperfine_command, is_ci, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler,
};
use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
use super::SysrootKind;
use std::env;
use std::ffi::OsStr;
@@ -251,18 +249,8 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
static PORTABLE_SIMD: CargoProject =
CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
"ebobby",
"simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
"<none>",
);
pub(crate) static SIMPLE_RAYTRACER: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
static LIBCORE_TESTS: CargoProject =
CargoProject::new(&RelPath::BUILD_SYSROOT.join("sysroot_src/library/core/tests"), "core_tests");
CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
TestCase::new("test.rust-random/rand", &|runner| {
@@ -280,67 +268,9 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
spawn_and_wait(build_cmd);
}
}),
TestCase::new("bench.simple-raytracer", &|runner| {
let run_runs = env::var("RUN_RUNS")
.unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
.parse()
.unwrap();
if runner.is_native {
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
let cargo_clif = RelPath::DIST
.to_path(&runner.dirs)
.join(get_wrapper_file_name("cargo-clif", "bin"));
let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs);
let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs);
let clean_cmd = format!(
"cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let llvm_build_cmd = format!(
"cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let clif_build_cmd = format!(
"{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
cargo_clif = cargo_clif.display(),
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let bench_compile =
hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
spawn_and_wait(bench_compile);
eprintln!("[BENCH RUN] ebobby/simple-raytracer");
fs::copy(
target_dir.join("debug").join(get_file_name("main", "bin")),
RelPath::BUILD
.to_path(&runner.dirs)
.join(get_file_name("raytracer_cg_clif", "bin")),
)
.unwrap();
let mut bench_run = hyperfine_command(
0,
run_runs,
None,
Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
);
bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs));
spawn_and_wait(bench_run);
} else {
spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs));
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)");
eprintln!("[COMPILE] ebobby/simple-raytracer");
spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)");
}
TestCase::new("test.simple-raytracer", &|runner| {
spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.host_compiler.cargo, &runner.dirs));
spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
}),
TestCase::new("test.libcore", &|runner| {
spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs));

5
clean_all.sh
View File

@@ -1,10 +1,9 @@
#!/usr/bin/env bash
set -e
rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
rm -rf target/ build/ dist/ perf.data{,.old} y.bin
rm -rf download/
rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb
# Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
# FIXME remove at some point in the future
rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/
rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}

2
config.txt
View File

@@ -44,7 +44,7 @@ aot.issue-72793
testsuite.extended_sysroot
test.rust-random/rand
bench.simple-raytracer
test.simple-raytracer
test.libcore
test.regex-shootout-regex-dna
test.regex

2
scripts/rustup.sh
View File

@@ -17,10 +17,10 @@ case $1 in
done
./clean_all.sh
./y.rs prepare
(cd build_sysroot && cargo update)
./y.rs prepare
;;
"commit")
git add rust-toolchain build_sysroot/Cargo.lock

2
scripts/setup_rust_fork.sh
View File

@@ -51,7 +51,7 @@ popd
# FIXME remove once inline asm is fully supported
export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc"
export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build_sysroot/sysroot_src; pwd)"
export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)"
# Allow the testsuite to use llvm tools
host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")