rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Auto merge of #42081 - ishitatsuyuki:submodule-better, r=aidanhs

Browse Source 
Use the improved submodule handling

r? @alexcrichton

That was a crap...
```
Updating submodules
Traceback (most recent call last):
  File "./x.py", line 20, in <module>
    bootstrap.main()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 684, in main
    bootstrap()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 662, in bootstrap
    rb.update_submodules()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 566, in update_submodules
    path = line[1:].split(' ')[1]
TypeError: a bytes-like object is required, not 'str'
```

Maybe we need to confirm the compatibility of git options, such as `git config` or `git -C` (I believe they existed long before, though). This is tested locally.
This commit is contained in:
bors
2017-05-26 18:11:54 +00:00
parent 256e497fe6 15c26c92cc
commit 557967766b
1 changed files with 77 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
src/bootstrap/bootstrap.py
View File

@@ -40,7 +40,8 @@ def get(url, path, verbose=False):
return return
else: else:
if verbose: if verbose:
print("ignoring already-download file " + path + " due to failed verification") print("ignoring already-download file " +
path + " due to failed verification")
os.unlink(path) os.unlink(path)
download(temp_path, url, True, verbose) download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose): if not verify(temp_path, sha_path, verbose):
@@ -100,8 +101,8 @@ def verify(path, sha_path, verbose):
verified = found == expected verified = found == expected
if not verified: if not verified:
print("invalid checksum:\n" print("invalid checksum:\n"
" found: {}\n" " found: {}\n"
" expected: {}".format(found, expected)) " expected: {}".format(found, expected))
return verified return verified
@@ -127,13 +128,13 @@ def unpack(tarball, dst, verbose=False, match=None):
shutil.move(tp, fp) shutil.move(tp, fp)
shutil.rmtree(os.path.join(dst, fname)) shutil.rmtree(os.path.join(dst, fname))
def run(args, verbose=False, exception=False, cwd=None, env=None): def run(args, verbose=False, exception=False, **kwargs):
if verbose: if verbose:
print("running: " + ' '.join(args)) print("running: " + ' '.join(args))
sys.stdout.flush() sys.stdout.flush()
# Use Popen here instead of call() as it apparently allows powershell on # Use Popen here instead of call() as it apparently allows powershell on
# Windows to not lock up waiting for input presumably. # Windows to not lock up waiting for input presumably.
ret = subprocess.Popen(args, cwd=cwd, env=env) ret = subprocess.Popen(args, **kwargs)
code = ret.wait() code = ret.wait()
if code != 0: if code != 0:
err = "failed to run: " + ' '.join(args) err = "failed to run: " + ' '.join(args)
@@ -141,6 +142,7 @@ def run(args, verbose=False, exception=False, cwd=None, env=None):
raise RuntimeError(err) raise RuntimeError(err)
sys.exit(err) sys.exit(err)
def stage0_data(rust_root): def stage0_data(rust_root):
nightlies = os.path.join(rust_root, "src/stage0.txt") nightlies = os.path.join(rust_root, "src/stage0.txt")
data = {} data = {}
@@ -153,11 +155,13 @@ def stage0_data(rust_root):
data[a] = b data[a] = b
return data return data
def format_build_time(duration): def format_build_time(duration):
return str(datetime.timedelta(seconds=int(duration))) return str(datetime.timedelta(seconds=int(duration)))
class RustBuild(object): class RustBuild(object):
def download_stage0(self): def download_stage0(self):
cache_dst = os.path.join(self.build_dir, "cache") cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, self.stage0_date()) rustc_cache = os.path.join(cache_dst, self.stage0_date())
@@ -172,11 +176,13 @@ class RustBuild(object):
self.print_what_it_means_to_bootstrap() self.print_what_it_means_to_bootstrap()
if os.path.exists(self.bin_root()): if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root()) shutil.rmtree(self.bin_root())
filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build) filename = "rust-std-{}-{}.tar.gz".format(
rustc_channel, self.build)
url = self._download_url + "/dist/" + self.stage0_date() url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename) tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball): if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose) get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), unpack(tarball, self.bin_root(),
match="rust-std-" + self.build, match="rust-std-" + self.build,
verbose=self.verbose) verbose=self.verbose)
@@ -185,20 +191,25 @@ class RustBuild(object):
url = self._download_url + "/dist/" + self.stage0_date() url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename) tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball): if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose) get("{}/{}".format(url, filename),
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose) tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rustc", verbose=self.verbose)
self.fix_executable(self.bin_root() + "/bin/rustc") self.fix_executable(self.bin_root() + "/bin/rustc")
self.fix_executable(self.bin_root() + "/bin/rustdoc") self.fix_executable(self.bin_root() + "/bin/rustdoc")
with open(self.rustc_stamp(), 'w') as f: with open(self.rustc_stamp(), 'w') as f:
f.write(self.stage0_date()) f.write(self.stage0_date())
if "pc-windows-gnu" in self.build: if "pc-windows-gnu" in self.build:
filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build) filename = "rust-mingw-{}-{}.tar.gz".format(
rustc_channel, self.build)
url = self._download_url + "/dist/" + self.stage0_date() url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename) tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball): if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose) get("{}/{}".format(url, filename),
unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose) tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rust-mingw", verbose=self.verbose)
if self.cargo().startswith(self.bin_root()) and \ if self.cargo().startswith(self.bin_root()) and \
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()): (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
@@ -207,8 +218,10 @@ class RustBuild(object):
url = self._download_url + "/dist/" + self.stage0_date() url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename) tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball): if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose) get("{}/{}".format(url, filename),
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose) tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="cargo", verbose=self.verbose)
self.fix_executable(self.bin_root() + "/bin/cargo") self.fix_executable(self.bin_root() + "/bin/cargo")
with open(self.cargo_stamp(), 'w') as f: with open(self.cargo_stamp(), 'w') as f:
f.write(self.stage0_date()) f.write(self.stage0_date())
@@ -218,7 +231,8 @@ class RustBuild(object):
default_encoding = sys.getdefaultencoding() default_encoding = sys.getdefaultencoding()
try: try:
ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding) ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, WindowsError): except (subprocess.CalledProcessError, WindowsError):
return return
@@ -234,7 +248,8 @@ class RustBuild(object):
print("info: you seem to be running NixOS. Attempting to patch " + fname) print("info: you seem to be running NixOS. Attempting to patch " + fname)
try: try:
interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname]) interpreter = subprocess.check_output(
["patchelf", "--print-interpreter", fname])
interpreter = interpreter.strip().decode(default_encoding) interpreter = interpreter.strip().decode(default_encoding)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print("warning: failed to call patchelf: %s" % e) print("warning: failed to call patchelf: %s" % e)
@@ -243,7 +258,8 @@ class RustBuild(object):
loader = interpreter.split("/")[-1] loader = interpreter.split("/")[-1]
try: try:
ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh']) ldd_output = subprocess.check_output(
['ldd', '/run/current-system/sw/bin/sh'])
ldd_output = ldd_output.strip().decode(default_encoding) ldd_output = ldd_output.strip().decode(default_encoding)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print("warning: unable to call ldd: %s" % e) print("warning: unable to call ldd: %s" % e)
@@ -261,7 +277,8 @@ class RustBuild(object):
correct_interpreter = loader_path + loader correct_interpreter = loader_path + loader
try: try:
subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname]) subprocess.check_output(
["patchelf", "--set-interpreter", correct_interpreter, fname])
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print("warning: failed to call patchelf: %s" % e) print("warning: failed to call patchelf: %s" % e)
return return
@@ -371,16 +388,16 @@ class RustBuild(object):
env["CARGO_TARGET_DIR"] = build_dir env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc() env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \ (os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else "" if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \ (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else "" if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \ (os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else "" if "LIBRARY_PATH" in env else ""
env["PATH"] = os.path.join(self.bin_root(), "bin") + \ env["PATH"] = os.path.join(self.bin_root(), "bin") + \
os.pathsep + env["PATH"] os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()): if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `%s`" % self.cargo()) raise Exception("no cargo executable found at `%s`" % self.cargo())
args = [self.cargo(), "build", "--manifest-path", args = [self.cargo(), "build", "--manifest-path",
@@ -395,16 +412,6 @@ class RustBuild(object):
args.append("--frozen") args.append("--frozen")
run(args, env=env, verbose=self.verbose) run(args, env=env, verbose=self.verbose)
def output(self, args, env=None, cwd=None):
default_encoding = sys.getdefaultencoding()
proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
(out, err) = proc.communicate()
ret = proc.wait()
if ret != 0:
print(out)
sys.exit(ret)
return out.decode(default_encoding)
def build_triple(self): def build_triple(self):
default_encoding = sys.getdefaultencoding() default_encoding = sys.getdefaultencoding()
config = self.get_toml('build') config = self.get_toml('build')
@@ -414,8 +421,10 @@ class RustBuild(object):
if config: if config:
return config return config
try: try:
ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding) ostype = subprocess.check_output(
cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding) ['uname', '-s']).strip().decode(default_encoding)
cputype = subprocess.check_output(
['uname', '-m']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, OSError): except (subprocess.CalledProcessError, OSError):
if sys.platform == 'win32': if sys.platform == 'win32':
return 'x86_64-pc-windows-msvc' return 'x86_64-pc-windows-msvc'
@@ -427,7 +436,8 @@ class RustBuild(object):
# The goal here is to come up with the same triple as LLVM would, # The goal here is to come up with the same triple as LLVM would,
# at least for the subset of platforms we're willing to target. # at least for the subset of platforms we're willing to target.
if ostype == 'Linux': if ostype == 'Linux':
os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding) os_from_sp = subprocess.check_output(
['uname', '-o']).strip().decode(default_encoding)
if os_from_sp == 'Android': if os_from_sp == 'Android':
ostype = 'linux-android' ostype = 'linux-android'
else: else:
@@ -451,7 +461,7 @@ class RustBuild(object):
# must be used instead. # must be used instead.
try: try:
cputype = subprocess.check_output(['isainfo', cputype = subprocess.check_output(['isainfo',
'-k']).strip().decode(default_encoding) '-k']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, OSError): except (subprocess.CalledProcessError, OSError):
err = "isainfo not found" err = "isainfo not found"
if self.verbose: if self.verbose:
@@ -544,51 +554,29 @@ class RustBuild(object):
def update_submodules(self): def update_submodules(self):
if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
self.get_toml('submodules') == "false" or \ self.get_toml('submodules') == "false" or \
self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1": self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
return return
print('Updating submodules') print('Updating submodules')
output = self.output(["git", "submodule", "status"], cwd=self.rust_root) default_encoding = sys.getdefaultencoding()
submodules = [] run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
for line in output.splitlines(): submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
# NOTE `git submodule status` output looks like this: ["git", "config", "--file", os.path.join(self.rust_root, ".gitmodules"),
# "--get-regexp", "path"]
# -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc ).decode(default_encoding).splitlines()]
# +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..) submodules = [module for module in submodules
# e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6) if not ((module.endswith("llvm") and
# (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT'))) or
# The first character can be '-', '+' or ' ' and denotes the (module.endswith("jemalloc") and
# `State` of the submodule Right next to this character is the (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT'))))
# SHA-1 of the submodule HEAD And after that comes the path to the ]
# submodule run(["git", "submodule", "update",
path = line[1:].split(' ')[1] "--init"] + submodules, cwd=self.rust_root, verbose=self.verbose)
submodules.append([path, line[0]]) run(["git", "submodule", "-q", "foreach", "git",
"reset", "-q", "--hard"], cwd=self.rust_root, verbose=self.verbose)
run(["git", "submodule", "-q", "foreach", "git",
"clean", "-qdfx"], cwd=self.rust_root, verbose=self.verbose)
run(["git", "submodule", "sync"], cwd=self.rust_root)
for submod in submodules:
path, status = submod
if path.endswith('llvm') and \
(self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
continue
if path.endswith('jemalloc') and \
(self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
continue
submod_path = os.path.join(self.rust_root, path)
if status == ' ':
run(["git", "reset", "--hard"], cwd=submod_path)
run(["git", "clean", "-fdx"], cwd=submod_path)
elif status == '+':
run(["git", "submodule", "update", path], cwd=self.rust_root)
run(["git", "reset", "--hard"], cwd=submod_path)
run(["git", "clean", "-fdx"], cwd=submod_path)
elif status == '-':
run(["git", "submodule", "init", path], cwd=self.rust_root)
run(["git", "submodule", "update", path], cwd=self.rust_root)
else:
raise ValueError('unknown submodule status: ' + status)
def bootstrap(): def bootstrap():
parser = argparse.ArgumentParser(description='Build rust') parser = argparse.ArgumentParser(description='Build rust')
@@ -641,7 +629,7 @@ def bootstrap():
if rb.use_vendored_sources: if rb.use_vendored_sources:
if not os.path.exists('.cargo'): if not os.path.exists('.cargo'):
os.makedirs('.cargo') os.makedirs('.cargo')
with open('.cargo/config','w') as f: with open('.cargo/config', 'w') as f:
f.write(""" f.write("""
[source.crates-io] [source.crates-io]
replace-with = 'vendored-sources' replace-with = 'vendored-sources'
@@ -681,13 +669,16 @@ def bootstrap():
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
run(args, env=env, verbose=rb.verbose) run(args, env=env, verbose=rb.verbose)
def main(): def main():
start_time = time() start_time = time()
help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) help_triggered = (
'-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
try: try:
bootstrap() bootstrap()
if not help_triggered: if not help_triggered:
print("Build completed successfully in %s" % format_build_time(time() - start_time)) print("Build completed successfully in %s" %
format_build_time(time() - start_time))
except (SystemExit, KeyboardInterrupt) as e: except (SystemExit, KeyboardInterrupt) as e:
if hasattr(e, 'code') and isinstance(e.code, int): if hasattr(e, 'code') and isinstance(e.code, int):
exit_code = e.code exit_code = e.code
@@ -695,7 +686,8 @@ def main():
exit_code = 1 exit_code = 1
print(e) print(e)
if not help_triggered: if not help_triggered:
print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time)) print("Build completed unsuccessfully in %s" %
format_build_time(time() - start_time))
sys.exit(exit_code) sys.exit(exit_code)
if __name__ == '__main__': if __name__ == '__main__':