diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 0f85ba81d1268..971064fe9fede 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -40,7 +40,8 @@ def get(url, path, verbose=False): return else: if verbose: - print("ignoring already-download file " + path + " due to failed verification") + print("ignoring already-download file " + + path + " due to failed verification") os.unlink(path) download(temp_path, url, True, verbose) if not verify(temp_path, sha_path, verbose): @@ -100,8 +101,8 @@ def verify(path, sha_path, verbose): verified = found == expected if not verified: print("invalid checksum:\n" - " found: {}\n" - " expected: {}".format(found, expected)) + " found: {}\n" + " expected: {}".format(found, expected)) return verified @@ -127,13 +128,13 @@ def unpack(tarball, dst, verbose=False, match=None): shutil.move(tp, fp) shutil.rmtree(os.path.join(dst, fname)) -def run(args, verbose=False, exception=False, cwd=None, env=None): +def run(args, verbose=False, exception=False, **kwargs): if verbose: print("running: " + ' '.join(args)) sys.stdout.flush() # Use Popen here instead of call() as it apparently allows powershell on # Windows to not lock up waiting for input presumably. - ret = subprocess.Popen(args, cwd=cwd, env=env) + ret = subprocess.Popen(args, **kwargs) code = ret.wait() if code != 0: err = "failed to run: " + ' '.join(args) @@ -141,6 +142,7 @@ def run(args, verbose=False, exception=False, cwd=None, env=None): raise RuntimeError(err) sys.exit(err) + def stage0_data(rust_root): nightlies = os.path.join(rust_root, "src/stage0.txt") data = {} @@ -153,11 +155,13 @@ def stage0_data(rust_root): data[a] = b return data + def format_build_time(duration): return str(datetime.timedelta(seconds=int(duration))) class RustBuild(object): + def download_stage0(self): cache_dst = os.path.join(self.build_dir, "cache") rustc_cache = os.path.join(cache_dst, self.stage0_date()) @@ -172,11 +176,13 @@ def download_stage0(self): self.print_what_it_means_to_bootstrap() if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) - filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build) + filename = "rust-std-{}-{}.tar.gz".format( + rustc_channel, self.build) url = self._download_url + "/dist/" + self.stage0_date() tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): - get("{}/{}".format(url, filename), tarball, verbose=self.verbose) + get("{}/{}".format(url, filename), + tarball, verbose=self.verbose) unpack(tarball, self.bin_root(), match="rust-std-" + self.build, verbose=self.verbose) @@ -185,20 +191,25 @@ def download_stage0(self): url = self._download_url + "/dist/" + self.stage0_date() tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): - get("{}/{}".format(url, filename), tarball, verbose=self.verbose) - unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose) + get("{}/{}".format(url, filename), + tarball, verbose=self.verbose) + unpack(tarball, self.bin_root(), + match="rustc", verbose=self.verbose) self.fix_executable(self.bin_root() + "/bin/rustc") self.fix_executable(self.bin_root() + "/bin/rustdoc") with open(self.rustc_stamp(), 'w') as f: f.write(self.stage0_date()) if "pc-windows-gnu" in self.build: - filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build) + filename = "rust-mingw-{}-{}.tar.gz".format( + rustc_channel, self.build) url = self._download_url + "/dist/" + self.stage0_date() tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): - get("{}/{}".format(url, filename), tarball, verbose=self.verbose) - unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose) + get("{}/{}".format(url, filename), + tarball, verbose=self.verbose) + unpack(tarball, self.bin_root(), + match="rust-mingw", verbose=self.verbose) if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.cargo_out_of_date()): @@ -207,8 +218,10 @@ def download_stage0(self): url = self._download_url + "/dist/" + self.stage0_date() tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): - get("{}/{}".format(url, filename), tarball, verbose=self.verbose) - unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose) + get("{}/{}".format(url, filename), + tarball, verbose=self.verbose) + unpack(tarball, self.bin_root(), + match="cargo", verbose=self.verbose) self.fix_executable(self.bin_root() + "/bin/cargo") with open(self.cargo_stamp(), 'w') as f: f.write(self.stage0_date()) @@ -218,7 +231,8 @@ def fix_executable(self, fname): default_encoding = sys.getdefaultencoding() try: - ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding) + ostype = subprocess.check_output( + ['uname', '-s']).strip().decode(default_encoding) except (subprocess.CalledProcessError, WindowsError): return @@ -234,7 +248,8 @@ def fix_executable(self, fname): print("info: you seem to be running NixOS. Attempting to patch " + fname) try: - interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname]) + interpreter = subprocess.check_output( + ["patchelf", "--print-interpreter", fname]) interpreter = interpreter.strip().decode(default_encoding) except subprocess.CalledProcessError as e: print("warning: failed to call patchelf: %s" % e) @@ -243,7 +258,8 @@ def fix_executable(self, fname): loader = interpreter.split("/")[-1] try: - ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh']) + ldd_output = subprocess.check_output( + ['ldd', '/run/current-system/sw/bin/sh']) ldd_output = ldd_output.strip().decode(default_encoding) except subprocess.CalledProcessError as e: print("warning: unable to call ldd: %s" % e) @@ -261,7 +277,8 @@ def fix_executable(self, fname): correct_interpreter = loader_path + loader try: - subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname]) + subprocess.check_output( + ["patchelf", "--set-interpreter", correct_interpreter, fname]) except subprocess.CalledProcessError as e: print("warning: failed to call patchelf: %s" % e) return @@ -371,16 +388,16 @@ def build_bootstrap(self): env["CARGO_TARGET_DIR"] = build_dir env["RUSTC"] = self.rustc() env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LD_LIBRARY_PATH"]) \ - if "LD_LIBRARY_PATH" in env else "" + (os.pathsep + env["LD_LIBRARY_PATH"]) \ + if "LD_LIBRARY_PATH" in env else "" env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ - if "DYLD_LIBRARY_PATH" in env else "" + (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ + if "DYLD_LIBRARY_PATH" in env else "" env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ - (os.pathsep + env["LIBRARY_PATH"]) \ - if "LIBRARY_PATH" in env else "" + (os.pathsep + env["LIBRARY_PATH"]) \ + if "LIBRARY_PATH" in env else "" env["PATH"] = os.path.join(self.bin_root(), "bin") + \ - os.pathsep + env["PATH"] + os.pathsep + env["PATH"] if not os.path.isfile(self.cargo()): raise Exception("no cargo executable found at `%s`" % self.cargo()) args = [self.cargo(), "build", "--manifest-path", @@ -395,16 +412,6 @@ def build_bootstrap(self): args.append("--frozen") run(args, env=env, verbose=self.verbose) - def output(self, args, env=None, cwd=None): - default_encoding = sys.getdefaultencoding() - proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd) - (out, err) = proc.communicate() - ret = proc.wait() - if ret != 0: - print(out) - sys.exit(ret) - return out.decode(default_encoding) - def build_triple(self): default_encoding = sys.getdefaultencoding() config = self.get_toml('build') @@ -414,8 +421,10 @@ def build_triple(self): if config: return config try: - ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding) - cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding) + ostype = subprocess.check_output( + ['uname', '-s']).strip().decode(default_encoding) + cputype = subprocess.check_output( + ['uname', '-m']).strip().decode(default_encoding) except (subprocess.CalledProcessError, OSError): if sys.platform == 'win32': return 'x86_64-pc-windows-msvc' @@ -427,7 +436,8 @@ def build_triple(self): # The goal here is to come up with the same triple as LLVM would, # at least for the subset of platforms we're willing to target. if ostype == 'Linux': - os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding) + os_from_sp = subprocess.check_output( + ['uname', '-o']).strip().decode(default_encoding) if os_from_sp == 'Android': ostype = 'linux-android' else: @@ -451,7 +461,7 @@ def build_triple(self): # must be used instead. try: cputype = subprocess.check_output(['isainfo', - '-k']).strip().decode(default_encoding) + '-k']).strip().decode(default_encoding) except (subprocess.CalledProcessError, OSError): err = "isainfo not found" if self.verbose: @@ -544,51 +554,29 @@ def build_triple(self): def update_submodules(self): if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ - self.get_toml('submodules') == "false" or \ - self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1": + self.get_toml('submodules') == "false" or \ + self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1": return - print('Updating submodules') - output = self.output(["git", "submodule", "status"], cwd=self.rust_root) - submodules = [] - for line in output.splitlines(): - # NOTE `git submodule status` output looks like this: - # - # -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc - # +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..) - # e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6) - # - # The first character can be '-', '+' or ' ' and denotes the - # `State` of the submodule Right next to this character is the - # SHA-1 of the submodule HEAD And after that comes the path to the - # submodule - path = line[1:].split(' ')[1] - submodules.append([path, line[0]]) - - run(["git", "submodule", "sync"], cwd=self.rust_root) - - for submod in submodules: - path, status = submod - if path.endswith('llvm') and \ - (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')): - continue - if path.endswith('jemalloc') and \ - (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')): - continue - submod_path = os.path.join(self.rust_root, path) - - if status == ' ': - run(["git", "reset", "--hard"], cwd=submod_path) - run(["git", "clean", "-fdx"], cwd=submod_path) - elif status == '+': - run(["git", "submodule", "update", path], cwd=self.rust_root) - run(["git", "reset", "--hard"], cwd=submod_path) - run(["git", "clean", "-fdx"], cwd=submod_path) - elif status == '-': - run(["git", "submodule", "init", path], cwd=self.rust_root) - run(["git", "submodule", "update", path], cwd=self.rust_root) - else: - raise ValueError('unknown submodule status: ' + status) + default_encoding = sys.getdefaultencoding() + run(["git", "submodule", "-q", "sync"], cwd=self.rust_root) + submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( + ["git", "config", "--file", os.path.join(self.rust_root, ".gitmodules"), + "--get-regexp", "path"] + ).decode(default_encoding).splitlines()] + submodules = [module for module in submodules + if not ((module.endswith("llvm") and + (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT'))) or + (module.endswith("jemalloc") and + (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')))) + ] + run(["git", "submodule", "update", + "--init"] + submodules, cwd=self.rust_root, verbose=self.verbose) + run(["git", "submodule", "-q", "foreach", "git", + "reset", "-q", "--hard"], cwd=self.rust_root, verbose=self.verbose) + run(["git", "submodule", "-q", "foreach", "git", + "clean", "-qdfx"], cwd=self.rust_root, verbose=self.verbose) + def bootstrap(): parser = argparse.ArgumentParser(description='Build rust') @@ -641,7 +629,7 @@ def bootstrap(): if rb.use_vendored_sources: if not os.path.exists('.cargo'): os.makedirs('.cargo') - with open('.cargo/config','w') as f: + with open('.cargo/config', 'w') as f: f.write(""" [source.crates-io] replace-with = 'vendored-sources' @@ -681,13 +669,16 @@ def bootstrap(): env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) run(args, env=env, verbose=rb.verbose) + def main(): start_time = time() - help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) + help_triggered = ( + '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) try: bootstrap() if not help_triggered: - print("Build completed successfully in %s" % format_build_time(time() - start_time)) + print("Build completed successfully in %s" % + format_build_time(time() - start_time)) except (SystemExit, KeyboardInterrupt) as e: if hasattr(e, 'code') and isinstance(e.code, int): exit_code = e.code @@ -695,7 +686,8 @@ def main(): exit_code = 1 print(e) if not help_triggered: - print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time)) + print("Build completed unsuccessfully in %s" % + format_build_time(time() - start_time)) sys.exit(exit_code) if __name__ == '__main__':