bootstrap/core/
download.rs

1use std::env;
2use std::ffi::OsString;
3use std::fs::{self, File};
4use std::io::{BufRead, BufReader, BufWriter, ErrorKind, Write};
5use std::path::{Path, PathBuf};
6use std::sync::OnceLock;
7
8use xz2::bufread::XzDecoder;
9
10use crate::core::config::BUILDER_CONFIG_FILENAME;
11use crate::utils::build_stamp::BuildStamp;
12use crate::utils::exec::command;
13use crate::utils::helpers::{exe, hex_encode, move_file};
14use crate::{Config, t};
15
16static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock<bool> = OnceLock::new();
17
18fn extract_curl_version(out: String) -> semver::Version {
19    // The output should look like this: "curl <major>.<minor>.<patch> ..."
20    out.lines()
21        .next()
22        .and_then(|line| line.split(" ").nth(1))
23        .and_then(|version| semver::Version::parse(version).ok())
24        .unwrap_or(semver::Version::new(1, 0, 0))
25}
26
27fn curl_version(config: &Config) -> semver::Version {
28    let mut curl = command("curl");
29    curl.arg("-V");
30    let curl = curl.run_capture_stdout(config);
31    if curl.is_failure() {
32        return semver::Version::new(1, 0, 0);
33    }
34    let output = curl.stdout();
35    extract_curl_version(output)
36}
37
38/// Generic helpers that are useful anywhere in bootstrap.
39impl Config {
40    pub fn is_verbose(&self) -> bool {
41        self.exec_ctx.is_verbose()
42    }
43
44    pub(crate) fn create<P: AsRef<Path>>(&self, path: P, s: &str) {
45        if self.dry_run() {
46            return;
47        }
48        t!(fs::write(path, s));
49    }
50
51    pub(crate) fn remove(&self, f: &Path) {
52        if self.dry_run() {
53            return;
54        }
55        fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {f:?}"));
56    }
57
58    /// Create a temporary directory in `out` and return its path.
59    ///
60    /// NOTE: this temporary directory is shared between all steps;
61    /// if you need an empty directory, create a new subdirectory inside it.
62    pub(crate) fn tempdir(&self) -> PathBuf {
63        let tmp = self.out.join("tmp");
64        t!(fs::create_dir_all(&tmp));
65        tmp
66    }
67
68    /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true
69    /// on NixOS
70    fn should_fix_bins_and_dylibs(&self) -> bool {
71        let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| {
72            let uname = command("uname").allow_failure().arg("-s").run_capture_stdout(self);
73            if uname.is_failure() {
74                return false;
75            }
76            let output = uname.stdout();
77            if !output.starts_with("Linux") {
78                return false;
79            }
80            // If the user has asked binaries to be patched for Nix, then
81            // don't check for NixOS or `/lib`.
82            // NOTE: this intentionally comes after the Linux check:
83            // - patchelf only works with ELF files, so no need to run it on Mac or Windows
84            // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
85            if let Some(explicit_value) = self.patch_binaries_for_nix {
86                return explicit_value;
87            }
88
89            // Use `/etc/os-release` instead of `/etc/NIXOS`.
90            // The latter one does not exist on NixOS when using tmpfs as root.
91            let is_nixos = match File::open("/etc/os-release") {
92                Err(e) if e.kind() == ErrorKind::NotFound => false,
93                Err(e) => panic!("failed to access /etc/os-release: {e}"),
94                Ok(os_release) => BufReader::new(os_release).lines().any(|l| {
95                    let l = l.expect("reading /etc/os-release");
96                    matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
97                }),
98            };
99            if !is_nixos {
100                let in_nix_shell = env::var("IN_NIX_SHELL");
101                if let Ok(in_nix_shell) = in_nix_shell {
102                    eprintln!(
103                        "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \
104                         you may need to set `patch-binaries-for-nix=true` in bootstrap.toml"
105                    );
106                }
107            }
108            is_nixos
109        });
110        if val {
111            eprintln!("INFO: You seem to be using Nix.");
112        }
113        val
114    }
115
116    /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
117    /// or the RPATH section, to fix the dynamic library search path
118    ///
119    /// This is only required on NixOS and uses the PatchELF utility to
120    /// change the interpreter/RPATH of ELF executables.
121    ///
122    /// Please see <https://nixos.org/patchelf.html> for more information
123    fn fix_bin_or_dylib(&self, fname: &Path) {
124        assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true));
125        println!("attempting to patch {}", fname.display());
126
127        // Only build `.nix-deps` once.
128        static NIX_DEPS_DIR: OnceLock<PathBuf> = OnceLock::new();
129        let mut nix_build_succeeded = true;
130        let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
131            // Run `nix-build` to "build" each dependency (which will likely reuse
132            // the existing `/nix/store` copy, or at most download a pre-built copy).
133            //
134            // Importantly, we create a gc-root called `.nix-deps` in the `build/`
135            // directory, but still reference the actual `/nix/store` path in the rpath
136            // as it makes it significantly more robust against changes to the location of
137            // the `.nix-deps` location.
138            //
139            // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
140            // zlib: Needed as a system dependency of `libLLVM-*.so`.
141            // patchelf: Needed for patching ELF binaries (see doc comment above).
142            let nix_deps_dir = self.out.join(".nix-deps");
143            const NIX_EXPR: &str = "
144            with (import <nixpkgs> {});
145            symlinkJoin {
146                name = \"rust-stage0-dependencies\";
147                paths = [
148                    zlib
149                    patchelf
150                    stdenv.cc.bintools
151                ];
152            }
153            ";
154            nix_build_succeeded = command("nix-build")
155                .allow_failure()
156                .args([Path::new("-E"), Path::new(NIX_EXPR), Path::new("-o"), &nix_deps_dir])
157                .run_capture_stdout(self)
158                .is_success();
159            nix_deps_dir
160        });
161        if !nix_build_succeeded {
162            return;
163        }
164
165        let mut patchelf = command(nix_deps_dir.join("bin/patchelf"));
166        patchelf.args(&[
167            OsString::from("--add-rpath"),
168            OsString::from(t!(fs::canonicalize(nix_deps_dir)).join("lib")),
169        ]);
170        if !path_is_dylib(fname) {
171            // Finally, set the correct .interp for binaries
172            let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
173            let dynamic_linker = t!(fs::read_to_string(dynamic_linker_path));
174            patchelf.args(["--set-interpreter", dynamic_linker.trim_end()]);
175        }
176        patchelf.arg(fname);
177        let _ = patchelf.allow_failure().run_capture_stdout(self);
178    }
179
180    fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) {
181        self.verbose(|| println!("download {url}"));
182        // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
183        let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
184        // While bootstrap itself only supports http and https downloads, downstream forks might
185        // need to download components from other protocols. The match allows them adding more
186        // protocols without worrying about merge conflicts if we change the HTTP implementation.
187        match url.split_once("://").map(|(proto, _)| proto) {
188            Some("http") | Some("https") => {
189                self.download_http_with_retries(&tempfile, url, help_on_error)
190            }
191            Some(other) => panic!("unsupported protocol {other} in {url}"),
192            None => panic!("no protocol in {url}"),
193        }
194        t!(
195            move_file(&tempfile, dest_path),
196            format!("failed to rename {tempfile:?} to {dest_path:?}")
197        );
198    }
199
200    fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
201        println!("downloading {url}");
202        // Try curl. If that fails and we are on windows, fallback to PowerShell.
203        // options should be kept in sync with
204        // src/bootstrap/src/core/download.rs
205        // for consistency
206        let mut curl = command("curl").allow_failure();
207        curl.args([
208            // follow redirect
209            "--location",
210            // timeout if speed is < 10 bytes/sec for > 30 seconds
211            "--speed-time",
212            "30",
213            "--speed-limit",
214            "10",
215            // timeout if cannot connect within 30 seconds
216            "--connect-timeout",
217            "30",
218            // output file
219            "--output",
220            tempfile.to_str().unwrap(),
221            // if there is an error, don't restart the download,
222            // instead continue where it left off.
223            "--continue-at",
224            "-",
225            // retry up to 3 times.  note that this means a maximum of 4
226            // attempts will be made, since the first attempt isn't a *re*try.
227            "--retry",
228            "3",
229            // show errors, even if --silent is specified
230            "--show-error",
231            // set timestamp of downloaded file to that of the server
232            "--remote-time",
233            // fail on non-ok http status
234            "--fail",
235        ]);
236        // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful.
237        if self.is_running_on_ci {
238            curl.arg("--silent");
239        } else {
240            curl.arg("--progress-bar");
241        }
242        // --retry-all-errors was added in 7.71.0, don't use it if curl is old.
243        if curl_version(self) >= semver::Version::new(7, 71, 0) {
244            curl.arg("--retry-all-errors");
245        }
246        curl.arg(url);
247        if !curl.run(self) {
248            if self.host_target.contains("windows-msvc") {
249                eprintln!("Fallback to PowerShell");
250                for _ in 0..3 {
251                    let powershell = command("PowerShell.exe").allow_failure().args([
252                        "/nologo",
253                        "-Command",
254                        "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
255                        &format!(
256                            "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
257                            url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
258                        ),
259                    ]).run_capture_stdout(self);
260
261                    if powershell.is_failure() {
262                        return;
263                    }
264
265                    eprintln!("\nspurious failure, trying again");
266                }
267            }
268            if !help_on_error.is_empty() {
269                eprintln!("{help_on_error}");
270            }
271            crate::exit!(1);
272        }
273    }
274
275    fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
276        eprintln!("extracting {} to {}", tarball.display(), dst.display());
277        if !dst.exists() {
278            t!(fs::create_dir_all(dst));
279        }
280
281        // `tarball` ends with `.tar.xz`; strip that suffix
282        // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
283        let uncompressed_filename =
284            Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
285        let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
286
287        // decompress the file
288        let data = t!(File::open(tarball), format!("file {} not found", tarball.display()));
289        let decompressor = XzDecoder::new(BufReader::new(data));
290
291        let mut tar = tar::Archive::new(decompressor);
292
293        let is_ci_rustc = dst.ends_with("ci-rustc");
294        let is_ci_llvm = dst.ends_with("ci-llvm");
295
296        // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding
297        // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow.
298        // Cache the entries when we extract it so we only have to read it once.
299        let mut recorded_entries = if is_ci_rustc { recorded_entries(dst, pattern) } else { None };
300
301        for member in t!(tar.entries()) {
302            let mut member = t!(member);
303            let original_path = t!(member.path()).into_owned();
304            // skip the top-level directory
305            if original_path == directory_prefix {
306                continue;
307            }
308            let mut short_path = t!(original_path.strip_prefix(directory_prefix));
309            let is_builder_config = short_path.to_str() == Some(BUILDER_CONFIG_FILENAME);
310
311            if !(short_path.starts_with(pattern)
312                || ((is_ci_rustc || is_ci_llvm) && is_builder_config))
313            {
314                continue;
315            }
316            short_path = short_path.strip_prefix(pattern).unwrap_or(short_path);
317            let dst_path = dst.join(short_path);
318            self.verbose(|| {
319                println!("extracting {} to {}", original_path.display(), dst.display())
320            });
321            if !t!(member.unpack_in(dst)) {
322                panic!("path traversal attack ??");
323            }
324            if let Some(record) = &mut recorded_entries {
325                t!(writeln!(record, "{}", short_path.to_str().unwrap()));
326            }
327            let src_path = dst.join(original_path);
328            if src_path.is_dir() && dst_path.exists() {
329                continue;
330            }
331            t!(move_file(src_path, dst_path));
332        }
333        let dst_dir = dst.join(directory_prefix);
334        if dst_dir.exists() {
335            t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display()));
336        }
337    }
338
339    /// Returns whether the SHA256 checksum of `path` matches `expected`.
340    pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
341        use sha2::Digest;
342
343        self.verbose(|| println!("verifying {}", path.display()));
344
345        if self.dry_run() {
346            return false;
347        }
348
349        let mut hasher = sha2::Sha256::new();
350
351        let file = t!(File::open(path));
352        let mut reader = BufReader::new(file);
353
354        loop {
355            let buffer = t!(reader.fill_buf());
356            let l = buffer.len();
357            // break if EOF
358            if l == 0 {
359                break;
360            }
361            hasher.update(buffer);
362            reader.consume(l);
363        }
364
365        let checksum = hex_encode(hasher.finalize().as_slice());
366        let verified = checksum == expected;
367
368        if !verified {
369            println!(
370                "invalid checksum: \n\
371                found:    {checksum}\n\
372                expected: {expected}",
373            );
374        }
375
376        verified
377    }
378}
379
380fn recorded_entries(dst: &Path, pattern: &str) -> Option<BufWriter<File>> {
381    let name = if pattern == "rustc-dev" {
382        ".rustc-dev-contents"
383    } else if pattern.starts_with("rust-std") {
384        ".rust-std-contents"
385    } else {
386        return None;
387    };
388    Some(BufWriter::new(t!(File::create(dst.join(name)))))
389}
390
391enum DownloadSource {
392    CI,
393    Dist,
394}
395
396/// Functions that are only ever called once, but named for clarity and to avoid thousand-line functions.
397impl Config {
398    pub(crate) fn download_clippy(&self) -> PathBuf {
399        self.verbose(|| println!("downloading stage0 clippy artifacts"));
400
401        let date = &self.stage0_metadata.compiler.date;
402        let version = &self.stage0_metadata.compiler.version;
403        let host = self.host_target;
404
405        let clippy_stamp =
406            BuildStamp::new(&self.initial_sysroot).with_prefix("clippy").add_stamp(date);
407        let cargo_clippy = self.initial_sysroot.join("bin").join(exe("cargo-clippy", host));
408        if cargo_clippy.exists() && clippy_stamp.is_up_to_date() {
409            return cargo_clippy;
410        }
411
412        let filename = format!("clippy-{version}-{host}.tar.xz");
413        self.download_component(DownloadSource::Dist, filename, "clippy-preview", date, "stage0");
414        if self.should_fix_bins_and_dylibs() {
415            self.fix_bin_or_dylib(&cargo_clippy);
416            self.fix_bin_or_dylib(&cargo_clippy.with_file_name(exe("clippy-driver", host)));
417        }
418
419        t!(clippy_stamp.write());
420        cargo_clippy
421    }
422
423    #[cfg(test)]
424    pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
425        Some(PathBuf::new())
426    }
427
428    /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't
429    /// reuse target directories or artifacts
430    #[cfg(not(test))]
431    pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
432        use build_helper::stage0_parser::VersionMetadata;
433
434        if self.dry_run() {
435            return Some(PathBuf::new());
436        }
437
438        let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
439        let channel = format!("{version}-{date}");
440
441        let host = self.host_target;
442        let bin_root = self.out.join(host).join("rustfmt");
443        let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host));
444        let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").add_stamp(channel);
445        if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() {
446            return Some(rustfmt_path);
447        }
448
449        self.download_component(
450            DownloadSource::Dist,
451            format!("rustfmt-{version}-{build}.tar.xz", build = host.triple),
452            "rustfmt-preview",
453            date,
454            "rustfmt",
455        );
456        self.download_component(
457            DownloadSource::Dist,
458            format!("rustc-{version}-{build}.tar.xz", build = host.triple),
459            "rustc",
460            date,
461            "rustfmt",
462        );
463
464        if self.should_fix_bins_and_dylibs() {
465            self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
466            self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
467            let lib_dir = bin_root.join("lib");
468            for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
469                let lib = t!(lib);
470                if path_is_dylib(&lib.path()) {
471                    self.fix_bin_or_dylib(&lib.path());
472                }
473            }
474        }
475
476        t!(rustfmt_stamp.write());
477        Some(rustfmt_path)
478    }
479
480    pub(crate) fn ci_rust_std_contents(&self) -> Vec<String> {
481        self.ci_component_contents(".rust-std-contents")
482    }
483
484    pub(crate) fn ci_rustc_dev_contents(&self) -> Vec<String> {
485        self.ci_component_contents(".rustc-dev-contents")
486    }
487
488    fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> {
489        assert!(self.download_rustc());
490        if self.dry_run() {
491            return vec![];
492        }
493
494        let ci_rustc_dir = self.ci_rustc_dir();
495        let stamp_file = ci_rustc_dir.join(stamp_file);
496        let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string());
497        t!(BufReader::new(contents_file).lines().collect())
498    }
499
500    pub(crate) fn download_ci_rustc(&self, commit: &str) {
501        self.verbose(|| println!("using downloaded stage2 artifacts from CI (commit {commit})"));
502
503        let version = self.artifact_version_part(commit);
504        // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the
505        // `rustc_private` crates for tools.
506        let extra_components = ["rustc-dev"];
507
508        self.download_toolchain(
509            &version,
510            "ci-rustc",
511            &format!("{commit}-{}", self.llvm_assertions),
512            &extra_components,
513            Self::download_ci_component,
514        );
515    }
516
517    #[cfg(test)]
518    pub(crate) fn download_beta_toolchain(&self) {}
519
520    #[cfg(not(test))]
521    pub(crate) fn download_beta_toolchain(&self) {
522        self.verbose(|| println!("downloading stage0 beta artifacts"));
523
524        let date = &self.stage0_metadata.compiler.date;
525        let version = &self.stage0_metadata.compiler.version;
526        let extra_components = ["cargo"];
527
528        let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| {
529            config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0")
530        };
531
532        self.download_toolchain(
533            version,
534            "stage0",
535            date,
536            &extra_components,
537            download_beta_component,
538        );
539    }
540
541    fn download_toolchain(
542        &self,
543        version: &str,
544        sysroot: &str,
545        stamp_key: &str,
546        extra_components: &[&str],
547        download_component: fn(&Config, String, &str, &str),
548    ) {
549        let host = self.host_target.triple;
550        let bin_root = self.out.join(host).join(sysroot);
551        let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").add_stamp(stamp_key);
552
553        if !bin_root.join("bin").join(exe("rustc", self.host_target)).exists()
554            || !rustc_stamp.is_up_to_date()
555        {
556            if bin_root.exists() {
557                t!(fs::remove_dir_all(&bin_root));
558            }
559            let filename = format!("rust-std-{version}-{host}.tar.xz");
560            let pattern = format!("rust-std-{host}");
561            download_component(self, filename, &pattern, stamp_key);
562            let filename = format!("rustc-{version}-{host}.tar.xz");
563            download_component(self, filename, "rustc", stamp_key);
564
565            for component in extra_components {
566                let filename = format!("{component}-{version}-{host}.tar.xz");
567                download_component(self, filename, component, stamp_key);
568            }
569
570            if self.should_fix_bins_and_dylibs() {
571                self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
572                self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
573                self.fix_bin_or_dylib(
574                    &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
575                );
576                let lib_dir = bin_root.join("lib");
577                for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
578                    let lib = t!(lib);
579                    if path_is_dylib(&lib.path()) {
580                        self.fix_bin_or_dylib(&lib.path());
581                    }
582                }
583            }
584
585            t!(rustc_stamp.write());
586        }
587    }
588
589    /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
590    // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
591    fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) {
592        Self::download_component(
593            self,
594            DownloadSource::CI,
595            filename,
596            prefix,
597            commit_with_assertions,
598            "ci-rustc",
599        )
600    }
601
602    fn download_component(
603        &self,
604        mode: DownloadSource,
605        filename: String,
606        prefix: &str,
607        key: &str,
608        destination: &str,
609    ) {
610        if self.dry_run() {
611            return;
612        }
613
614        let cache_dst =
615            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
616
617        let cache_dir = cache_dst.join(key);
618        if !cache_dir.exists() {
619            t!(fs::create_dir_all(&cache_dir));
620        }
621
622        let bin_root = self.out.join(self.host_target).join(destination);
623        let tarball = cache_dir.join(&filename);
624        let (base_url, url, should_verify) = match mode {
625            DownloadSource::CI => {
626                let dist_server = if self.llvm_assertions {
627                    self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone()
628                } else {
629                    self.stage0_metadata.config.artifacts_server.clone()
630                };
631                let url = format!(
632                    "{}/{filename}",
633                    key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap()
634                );
635                (dist_server, url, false)
636            }
637            DownloadSource::Dist => {
638                let dist_server = env::var("RUSTUP_DIST_SERVER")
639                    .unwrap_or(self.stage0_metadata.config.dist_server.to_string());
640                // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0
641                (dist_server, format!("dist/{key}/{filename}"), true)
642            }
643        };
644
645        // For the stage0 compiler, put special effort into ensuring the checksums are valid.
646        let checksum = if should_verify {
647            let error = format!(
648                "src/stage0 doesn't contain a checksum for {url}. \
649                Pre-built artifacts might not be available for this \
650                target at this time, see https://doc.rust-lang.org/nightly\
651                /rustc/platform-support.html for more information."
652            );
653            let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error);
654            if tarball.exists() {
655                if self.verify(&tarball, sha256) {
656                    self.unpack(&tarball, &bin_root, prefix);
657                    return;
658                } else {
659                    self.verbose(|| {
660                        println!(
661                            "ignoring cached file {} due to failed verification",
662                            tarball.display()
663                        )
664                    });
665                    self.remove(&tarball);
666                }
667            }
668            Some(sha256)
669        } else if tarball.exists() {
670            self.unpack(&tarball, &bin_root, prefix);
671            return;
672        } else {
673            None
674        };
675
676        let mut help_on_error = "";
677        if destination == "ci-rustc" {
678            help_on_error = "ERROR: failed to download pre-built rustc from CI
679
680NOTE: old builds get deleted after a certain time
681HELP: if trying to compile an old commit of rustc, disable `download-rustc` in bootstrap.toml:
682
683[rust]
684download-rustc = false
685";
686        }
687        self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error);
688        if let Some(sha256) = checksum
689            && !self.verify(&tarball, sha256)
690        {
691            panic!("failed to verify {}", tarball.display());
692        }
693
694        self.unpack(&tarball, &bin_root, prefix);
695    }
696
697    #[cfg(test)]
698    pub(crate) fn maybe_download_ci_llvm(&self) {}
699
700    #[cfg(not(test))]
701    pub(crate) fn maybe_download_ci_llvm(&self) {
702        use build_helper::exit;
703        use build_helper::git::PathFreshness;
704
705        use crate::core::build_steps::llvm::detect_llvm_freshness;
706        use crate::core::config::toml::llvm::check_incompatible_options_for_ci_llvm;
707
708        if !self.llvm_from_ci {
709            return;
710        }
711
712        let llvm_root = self.ci_llvm_root();
713        let llvm_freshness =
714            detect_llvm_freshness(self, self.rust_info.is_managed_git_subrepository());
715        self.verbose(|| {
716            eprintln!("LLVM freshness: {llvm_freshness:?}");
717        });
718        let llvm_sha = match llvm_freshness {
719            PathFreshness::LastModifiedUpstream { upstream } => upstream,
720            PathFreshness::HasLocalModifications { upstream } => upstream,
721            PathFreshness::MissingUpstream => {
722                eprintln!("error: could not find commit hash for downloading LLVM");
723                eprintln!("HELP: maybe your repository history is too shallow?");
724                eprintln!("HELP: consider disabling `download-ci-llvm`");
725                eprintln!("HELP: or fetch enough history to include one upstream commit");
726                crate::exit!(1);
727            }
728        };
729        let stamp_key = format!("{}{}", llvm_sha, self.llvm_assertions);
730        let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm").add_stamp(stamp_key);
731        if !llvm_stamp.is_up_to_date() && !self.dry_run() {
732            self.download_ci_llvm(&llvm_sha);
733
734            if self.should_fix_bins_and_dylibs() {
735                for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
736                    self.fix_bin_or_dylib(&t!(entry).path());
737                }
738            }
739
740            // Update the timestamp of llvm-config to force rustc_llvm to be
741            // rebuilt. This is a hacky workaround for a deficiency in Cargo where
742            // the rerun-if-changed directive doesn't handle changes very well.
743            // https://github.com/rust-lang/cargo/issues/10791
744            // Cargo only compares the timestamp of the file relative to the last
745            // time `rustc_llvm` build script ran. However, the timestamps of the
746            // files in the tarball are in the past, so it doesn't trigger a
747            // rebuild.
748            let now = std::time::SystemTime::now();
749            let file_times = fs::FileTimes::new().set_accessed(now).set_modified(now);
750
751            let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.host_target));
752            t!(crate::utils::helpers::set_file_times(llvm_config, file_times));
753
754            if self.should_fix_bins_and_dylibs() {
755                let llvm_lib = llvm_root.join("lib");
756                for entry in t!(fs::read_dir(llvm_lib)) {
757                    let lib = t!(entry).path();
758                    if path_is_dylib(&lib) {
759                        self.fix_bin_or_dylib(&lib);
760                    }
761                }
762            }
763
764            t!(llvm_stamp.write());
765        }
766
767        if let Some(config_path) = &self.config {
768            let current_config_toml = Self::get_toml(config_path).unwrap();
769
770            match self.get_builder_toml("ci-llvm") {
771                Ok(ci_config_toml) => {
772                    t!(check_incompatible_options_for_ci_llvm(current_config_toml, ci_config_toml));
773                }
774                Err(e) if e.to_string().contains("unknown field") => {
775                    println!(
776                        "WARNING: CI LLVM has some fields that are no longer supported in bootstrap; download-ci-llvm will be disabled."
777                    );
778                    println!("HELP: Consider rebasing to a newer commit if available.");
779                }
780                Err(e) => {
781                    eprintln!("ERROR: Failed to parse CI LLVM bootstrap.toml: {e}");
782                    exit!(2);
783                }
784            };
785        };
786    }
787
788    #[cfg(not(test))]
789    fn download_ci_llvm(&self, llvm_sha: &str) {
790        let llvm_assertions = self.llvm_assertions;
791
792        let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}");
793        let cache_dst =
794            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
795
796        let rustc_cache = cache_dst.join(cache_prefix);
797        if !rustc_cache.exists() {
798            t!(fs::create_dir_all(&rustc_cache));
799        }
800        let base = if llvm_assertions {
801            &self.stage0_metadata.config.artifacts_with_llvm_assertions_server
802        } else {
803            &self.stage0_metadata.config.artifacts_server
804        };
805        let version = self.artifact_version_part(llvm_sha);
806        let filename = format!("rust-dev-{}-{}.tar.xz", version, self.host_target.triple);
807        let tarball = rustc_cache.join(&filename);
808        if !tarball.exists() {
809            let help_on_error = "ERROR: failed to download llvm from ci
810
811    HELP: There could be two reasons behind this:
812        1) The host triple is not supported for `download-ci-llvm`.
813        2) Old builds get deleted after a certain time.
814    HELP: In either case, disable `download-ci-llvm` in your bootstrap.toml:
815
816    [llvm]
817    download-ci-llvm = false
818    ";
819            self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error);
820        }
821        let llvm_root = self.ci_llvm_root();
822        self.unpack(&tarball, &llvm_root, "rust-dev");
823    }
824
825    pub fn download_ci_gcc(&self, gcc_sha: &str, root_dir: &Path) {
826        let cache_prefix = format!("gcc-{gcc_sha}");
827        let cache_dst =
828            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
829
830        let gcc_cache = cache_dst.join(cache_prefix);
831        if !gcc_cache.exists() {
832            t!(fs::create_dir_all(&gcc_cache));
833        }
834        let base = &self.stage0_metadata.config.artifacts_server;
835        let version = self.artifact_version_part(gcc_sha);
836        let filename = format!("gcc-{version}-{}.tar.xz", self.host_target.triple);
837        let tarball = gcc_cache.join(&filename);
838        if !tarball.exists() {
839            let help_on_error = "ERROR: failed to download gcc from ci
840
841    HELP: There could be two reasons behind this:
842        1) The host triple is not supported for `download-ci-gcc`.
843        2) Old builds get deleted after a certain time.
844    HELP: In either case, disable `download-ci-gcc` in your bootstrap.toml:
845
846    [gcc]
847    download-ci-gcc = false
848    ";
849            self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error);
850        }
851        self.unpack(&tarball, root_dir, "gcc");
852    }
853}
854
855fn path_is_dylib(path: &Path) -> bool {
856    // The .so is not necessarily the extension, it might be libLLVM.so.18.1
857    path.to_str().is_some_and(|path| path.contains(".so"))
858}
859
860/// Checks whether the CI rustc is available for the given target triple.
861pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
862    // All tier 1 targets and tier 2 targets with host tools.
863    const SUPPORTED_PLATFORMS: &[&str] = &[
864        "aarch64-apple-darwin",
865        "aarch64-pc-windows-msvc",
866        "aarch64-unknown-linux-gnu",
867        "aarch64-unknown-linux-musl",
868        "arm-unknown-linux-gnueabi",
869        "arm-unknown-linux-gnueabihf",
870        "armv7-unknown-linux-gnueabihf",
871        "i686-pc-windows-gnu",
872        "i686-pc-windows-msvc",
873        "i686-unknown-linux-gnu",
874        "loongarch64-unknown-linux-gnu",
875        "powerpc-unknown-linux-gnu",
876        "powerpc64-unknown-linux-gnu",
877        "powerpc64le-unknown-linux-gnu",
878        "riscv64gc-unknown-linux-gnu",
879        "s390x-unknown-linux-gnu",
880        "x86_64-apple-darwin",
881        "x86_64-pc-windows-gnu",
882        "x86_64-pc-windows-msvc",
883        "x86_64-unknown-freebsd",
884        "x86_64-unknown-illumos",
885        "x86_64-unknown-linux-gnu",
886        "x86_64-unknown-linux-musl",
887        "x86_64-unknown-netbsd",
888    ];
889
890    const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
891        &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];
892
893    if llvm_assertions {
894        SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
895    } else {
896        SUPPORTED_PLATFORMS.contains(&target_triple)
897    }
898}