Auto merge of #51459 - kennytm:dist-at-stage-0, r=Mark-Simulacrum

Miscellaneous changes to rustbuild and CI.

1. Don't build LLVM when running rust-installer.
2. If toolstate is unchanged, don't push a commit to the toolstate repo.
3. Allow `./x.py build src/librustc_codegen_llvm`
4. Added log to track #50887.

.travis.yml

@@ -284,6 +284,9 @@ after_failure:
       -exec head -750 {} \;
       -exec echo travis_fold":"end:crashlog \; || true
 
+  # see #50887
+  - head -30 ./obj/build/x86_64-unknown-linux-gnu/native/asan/build/lib/asan/clang_rt.asan-dynamic-i386.vers || true
+
   # attempt to debug anything killed by the oom killer on linux, just to see if
   # it happened
   - dmesg | grep -i kill

src/bootstrap/builder.rs

@@ -339,6 +339,7 @@ impl<'a> Builder<'a> {
                 compile::Std,
                 compile::Test,
                 compile::Rustc,
+                compile::CodegenBackend,
                 compile::StartupObjects,
                 tool::BuildManifest,
                 tool::Rustbook,

src/bootstrap/tool.rs

@@ -254,7 +254,7 @@ pub fn prepare_tool_cargo(
 }
 
 macro_rules! tool {
-    ($($name:ident, $path:expr, $tool_name:expr, $mode:expr;)+) => {
+    ($($name:ident, $path:expr, $tool_name:expr, $mode:expr $(,llvm_tools = $llvm:expr)*;)+) => {
         #[derive(Copy, Clone)]
         pub enum Tool {
             $(
@@ -269,6 +269,13 @@ macro_rules! tool {
                 };
                 mode
             }
+
+            /// Whether this tool requires LLVM to run
+            pub fn uses_llvm_tools(&self) -> bool {
+                match self {
+                    $(Tool::$name => true $(&& $llvm)*,)+
+                }
+            }
         }
 
         impl<'a> Builder<'a> {
@@ -333,6 +340,9 @@ macro_rules! tool {
     }
 }
 
+// FIXME(#51459): We have only checked that RustInstaller does not require
+// the LLVM binaries when running. We should go through all tools to determine
+// if they really need LLVM binaries, and make `llvm_tools` a required argument.
 tool!(
     Rustbook, "src/tools/rustbook", "rustbook", Mode::ToolRustc;
     ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::ToolRustc;
@@ -343,7 +353,7 @@ tool!(
     Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolTest;
     BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolStd;
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolStd;
-    RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolStd;
+    RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolStd, llvm_tools = false;
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::ToolStd;
 );
 
@@ -586,7 +596,7 @@ impl<'a> Builder<'a> {
     pub fn tool_cmd(&self, tool: Tool) -> Command {
         let mut cmd = Command::new(self.tool_exe(tool));
         let compiler = self.compiler(self.tool_default_stage(tool), self.config.build);
-        self.prepare_tool_cmd(compiler, tool.get_mode(), &mut cmd);
+        self.prepare_tool_cmd(compiler, tool, &mut cmd);
         cmd
     }
 
@@ -594,11 +604,11 @@ impl<'a> Builder<'a> {
     ///
     /// Notably this munges the dynamic library lookup path to point to the
     /// right location to run `compiler`.
-    fn prepare_tool_cmd(&self, compiler: Compiler, mode: Mode, cmd: &mut Command) {
+    fn prepare_tool_cmd(&self, compiler: Compiler, tool: Tool, cmd: &mut Command) {
         let host = &compiler.host;
         let mut lib_paths: Vec<PathBuf> = vec![
             PathBuf::from(&self.sysroot_libdir(compiler, compiler.host)),
-            self.cargo_out(compiler, mode, *host).join("deps"),
+            self.cargo_out(compiler, tool.get_mode(), *host).join("deps"),
         ];
 
         // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make
@@ -621,17 +631,19 @@ impl<'a> Builder<'a> {
 
         // Add the llvm/bin directory to PATH since it contains lots of
         // useful, platform-independent tools
-        if let Some(llvm_bin_path) = self.llvm_bin_path() {
+        if tool.uses_llvm_tools() {
-            if host.contains("windows") {
+            if let Some(llvm_bin_path) = self.llvm_bin_path() {
-                // On Windows, PATH and the dynamic library path are the same,
+                if host.contains("windows") {
-                // so we just add the LLVM bin path to lib_path
+                    // On Windows, PATH and the dynamic library path are the same,
-                lib_paths.push(llvm_bin_path);
+                    // so we just add the LLVM bin path to lib_path
-            } else {
+                    lib_paths.push(llvm_bin_path);
-                let old_path = env::var_os("PATH").unwrap_or_default();
+                } else {
-                let new_path = env::join_paths(iter::once(llvm_bin_path)
+                    let old_path = env::var_os("PATH").unwrap_or_default();
-                        .chain(env::split_paths(&old_path)))
+                    let new_path = env::join_paths(iter::once(llvm_bin_path)
-                    .expect("Could not add LLVM bin path to PATH");
+                            .chain(env::split_paths(&old_path)))
-                cmd.env("PATH", new_path);
+                        .expect("Could not add LLVM bin path to PATH");
+                    cmd.env("PATH", new_path);
+                }
             }
         }
 

src/ci/docker/x86_64-gnu-tools/checkregression.py

@@ -18,6 +18,7 @@ if __name__ == '__main__':
     os_name = sys.argv[1]
     toolstate_file = sys.argv[2]
     current_state = sys.argv[3]
+    verb = sys.argv[4] # 'regressed' or 'changed'
 
     with open(toolstate_file, 'r') as f:
         toolstate = json.load(f)
@@ -29,10 +30,17 @@ if __name__ == '__main__':
         tool = cur['tool']
         state = cur[os_name]
         new_state = toolstate.get(tool, '')
-        if new_state < state:
+        if verb == 'regressed':
+            updated = new_state < state
+        elif verb == 'changed':
+            updated = new_state != state
+        else:
+            print('Unknown verb {}'.format(updated))
+            sys.exit(2)
+        if updated:
             print(
-                'Error: The state of "{}" has regressed from "{}" to "{}"'
+                'The state of "{}" has {} from "{}" to "{}"'
-                .format(tool, state, new_state)
+                .format(tool, verb, state, new_state)
             )
             regressed = True
 

src/ci/docker/x86_64-gnu-tools/checktools.sh

@@ -91,19 +91,26 @@ status_check() {
 
 status_check "submodule_changed"
 
+CHECK_NOT="$(dirname $0)/checkregression.py"
+change_toolstate() {
+    # only update the history
+    if python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" changed; then
+        echo 'Toolstate is not changed. Not updating.'
+    else
+        if [ $SIX_WEEK_CYCLE -eq 5 ]; then
+            python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" regressed
+        fi
+        sed -i "1 a\\
+$COMMIT\t$(cat "$TOOLSTATE_FILE")
+" "history/$OS.tsv"
+    fi
+}
+
 if [ "$RUST_RELEASE_CHANNEL" = nightly -a -n "${TOOLSTATE_REPO_ACCESS_TOKEN+is_set}" ]; then
     . "$(dirname $0)/repo.sh"
     MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
     echo "($OS CI update)" > "$MESSAGE_FILE"
-    commit_toolstate_change "$MESSAGE_FILE" \
+    commit_toolstate_change "$MESSAGE_FILE" change_toolstate
-        sed -i "1 a\\
-$COMMIT\t$(cat "$TOOLSTATE_FILE")
-" "history/$OS.tsv"
-    # if we are at the last week in the 6-week release cycle, reject any kind of regression.
-    if [ $SIX_WEEK_CYCLE -eq 5 ]; then
-        python2.7 "$(dirname $0)/checkregression.py" \
-            "$OS" "$TOOLSTATE_FILE" "rust-toolstate/_data/latest.json"
-    fi
     rm -f "$MESSAGE_FILE"
     exit 0
 fi