bootstrap/utils/
helpers.rs

1//! Various utility functions used throughout bootstrap.
2//!
3//! Simple things like testing the various filesystem operations here and there,
4//! not a lot of interesting happenings here unfortunately.
5
6use std::ffi::OsStr;
7use std::path::{Path, PathBuf};
8use std::process::{Command, Stdio};
9use std::sync::OnceLock;
10use std::time::{Instant, SystemTime, UNIX_EPOCH};
11use std::{env, fs, io, str};
12
13use build_helper::util::fail;
14use object::read::archive::ArchiveFile;
15
16use crate::LldMode;
17use crate::core::builder::Builder;
18use crate::core::config::{Config, TargetSelection};
19use crate::utils::exec::{BootstrapCommand, command};
20pub use crate::utils::shared_helpers::{dylib_path, dylib_path_var};
21
22#[cfg(test)]
23mod tests;
24
25/// A helper macro to `unwrap` a result except also print out details like:
26///
27/// * The file/line of the panic
28/// * The expression that failed
29/// * The error itself
30///
31/// This is currently used judiciously throughout the build system rather than
32/// using a `Result` with `try!`, but this may change one day...
33#[macro_export]
34macro_rules! t {
35    ($e:expr) => {
36        match $e {
37            Ok(e) => e,
38            Err(e) => panic!("{} failed with {}", stringify!($e), e),
39        }
40    };
41    // it can show extra info in the second parameter
42    ($e:expr, $extra:expr) => {
43        match $e {
44            Ok(e) => e,
45            Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra),
46        }
47    };
48}
49
50pub use t;
51pub fn exe(name: &str, target: TargetSelection) -> String {
52    crate::utils::shared_helpers::exe(name, &target.triple)
53}
54
55/// Returns the path to the split debug info for the specified file if it exists.
56pub fn split_debuginfo(name: impl Into<PathBuf>) -> Option<PathBuf> {
57    // FIXME: only msvc is currently supported
58
59    let path = name.into();
60    let pdb = path.with_extension("pdb");
61    if pdb.exists() {
62        return Some(pdb);
63    }
64
65    // pdbs get named with '-' replaced by '_'
66    let file_name = pdb.file_name()?.to_str()?.replace("-", "_");
67
68    let pdb: PathBuf = [path.parent()?, Path::new(&file_name)].into_iter().collect();
69    pdb.exists().then_some(pdb)
70}
71
72/// Returns `true` if the file name given looks like a dynamic library.
73pub fn is_dylib(path: &Path) -> bool {
74    path.extension().and_then(|ext| ext.to_str()).is_some_and(|ext| {
75        ext == "dylib" || ext == "so" || ext == "dll" || (ext == "a" && is_aix_shared_archive(path))
76    })
77}
78
79/// Return the path to the containing submodule if available.
80pub fn submodule_path_of(builder: &Builder<'_>, path: &str) -> Option<String> {
81    let submodule_paths = build_helper::util::parse_gitmodules(&builder.src);
82    submodule_paths.iter().find_map(|submodule_path| {
83        if path.starts_with(submodule_path) { Some(submodule_path.to_string()) } else { None }
84    })
85}
86
87fn is_aix_shared_archive(path: &Path) -> bool {
88    let file = match fs::File::open(path) {
89        Ok(file) => file,
90        Err(_) => return false,
91    };
92    let reader = object::ReadCache::new(file);
93    let archive = match ArchiveFile::parse(&reader) {
94        Ok(result) => result,
95        Err(_) => return false,
96    };
97
98    archive
99        .members()
100        .filter_map(Result::ok)
101        .any(|entry| String::from_utf8_lossy(entry.name()).contains(".so"))
102}
103
104/// Returns `true` if the file name given looks like a debug info file
105pub fn is_debug_info(name: &str) -> bool {
106    // FIXME: consider split debug info on other platforms (e.g., Linux, macOS)
107    name.ends_with(".pdb")
108}
109
110/// Returns the corresponding relative library directory that the compiler's
111/// dylibs will be found in.
112pub fn libdir(target: TargetSelection) -> &'static str {
113    if target.is_windows() { "bin" } else { "lib" }
114}
115
116/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path.
117/// If the dylib_path_var is already set for this cmd, the old value will be overwritten!
118pub fn add_dylib_path(path: Vec<PathBuf>, cmd: &mut BootstrapCommand) {
119    let mut list = dylib_path();
120    for path in path {
121        list.insert(0, path);
122    }
123    cmd.env(dylib_path_var(), t!(env::join_paths(list)));
124}
125
126pub struct TimeIt(bool, Instant);
127
128/// Returns an RAII structure that prints out how long it took to drop.
129pub fn timeit(builder: &Builder<'_>) -> TimeIt {
130    TimeIt(builder.config.dry_run(), Instant::now())
131}
132
133impl Drop for TimeIt {
134    fn drop(&mut self) {
135        let time = self.1.elapsed();
136        if !self.0 {
137            println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis());
138        }
139    }
140}
141
142/// Symlinks two directories, using junctions on Windows and normal symlinks on
143/// Unix.
144pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> {
145    if config.dry_run() {
146        return Ok(());
147    }
148    let _ = fs::remove_dir_all(link);
149    return symlink_dir_inner(original, link);
150
151    #[cfg(not(windows))]
152    fn symlink_dir_inner(original: &Path, link: &Path) -> io::Result<()> {
153        use std::os::unix::fs;
154        fs::symlink(original, link)
155    }
156
157    #[cfg(windows)]
158    fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> {
159        junction::create(target, junction)
160    }
161}
162
163/// Rename a file if from and to are in the same filesystem or
164/// copy and remove the file otherwise
165pub fn move_file<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> io::Result<()> {
166    match fs::rename(&from, &to) {
167        Err(e) if e.kind() == io::ErrorKind::CrossesDevices => {
168            std::fs::copy(&from, &to)?;
169            std::fs::remove_file(&from)
170        }
171        r => r,
172    }
173}
174
175pub fn forcing_clang_based_tests() -> bool {
176    if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") {
177        match &var.to_string_lossy().to_lowercase()[..] {
178            "1" | "yes" | "on" => true,
179            "0" | "no" | "off" => false,
180            other => {
181                // Let's make sure typos don't go unnoticed
182                panic!(
183                    "Unrecognized option '{other}' set in \
184                        RUSTBUILD_FORCE_CLANG_BASED_TESTS"
185                )
186            }
187        }
188    } else {
189        false
190    }
191}
192
193pub fn use_host_linker(target: TargetSelection) -> bool {
194    // FIXME: this information should be gotten by checking the linker flavor
195    // of the rustc target
196    !(target.contains("emscripten")
197        || target.contains("wasm32")
198        || target.contains("nvptx")
199        || target.contains("fortanix")
200        || target.contains("fuchsia")
201        || target.contains("bpf")
202        || target.contains("switch"))
203}
204
205pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool {
206    if target.contains("linux") {
207        target.contains("x86_64")
208            || target.contains("aarch64")
209            || target.contains("s390x")
210            || target.contains("riscv64gc")
211    } else if target.contains("darwin") {
212        target.contains("x86_64") || target.contains("aarch64")
213    } else if target.is_windows() {
214        target.contains("x86_64")
215    } else {
216        false
217    }
218}
219
220pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>(
221    path: &'a Path,
222    suite_path: P,
223    builder: &Builder<'_>,
224) -> Option<&'a str> {
225    let suite_path = suite_path.as_ref();
226    let path = match path.strip_prefix(".") {
227        Ok(p) => p,
228        Err(_) => path,
229    };
230    if !path.starts_with(suite_path) {
231        return None;
232    }
233    let abs_path = builder.src.join(path);
234    let exists = abs_path.is_dir() || abs_path.is_file();
235    if !exists {
236        panic!(
237            "Invalid test suite filter \"{}\": file or directory does not exist",
238            abs_path.display()
239        );
240    }
241    // Since test suite paths are themselves directories, if we don't
242    // specify a directory or file, we'll get an empty string here
243    // (the result of the test suite directory without its suite prefix).
244    // Therefore, we need to filter these out, as only the first --test-args
245    // flag is respected, so providing an empty --test-args conflicts with
246    // any following it.
247    match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) {
248        Some(s) if !s.is_empty() => Some(s),
249        _ => None,
250    }
251}
252
253// FIXME: get rid of this function
254pub fn check_run(cmd: &mut BootstrapCommand, print_cmd_on_fail: bool) -> bool {
255    let status = match cmd.as_command_mut().status() {
256        Ok(status) => status,
257        Err(e) => {
258            println!("failed to execute command: {cmd:?}\nERROR: {e}");
259            return false;
260        }
261    };
262    if !status.success() && print_cmd_on_fail {
263        println!(
264            "\n\ncommand did not execute successfully: {cmd:?}\n\
265             expected success, got: {status}\n\n"
266        );
267    }
268    status.success()
269}
270
271pub fn make(host: &str) -> PathBuf {
272    if host.contains("dragonfly")
273        || host.contains("freebsd")
274        || host.contains("netbsd")
275        || host.contains("openbsd")
276    {
277        PathBuf::from("gmake")
278    } else {
279        PathBuf::from("make")
280    }
281}
282
283#[track_caller]
284pub fn output(cmd: &mut Command) -> String {
285    #[cfg(feature = "tracing")]
286    let _run_span = crate::trace_cmd!(cmd);
287
288    let output = match cmd.stderr(Stdio::inherit()).output() {
289        Ok(status) => status,
290        Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")),
291    };
292    if !output.status.success() {
293        panic!(
294            "command did not execute successfully: {:?}\n\
295             expected success, got: {}",
296            cmd, output.status
297        );
298    }
299    String::from_utf8(output.stdout).unwrap()
300}
301
302/// Spawn a process and return a closure that will wait for the process
303/// to finish and then return its output. This allows the spawned process
304/// to do work without immediately blocking bootstrap.
305#[track_caller]
306pub fn start_process(cmd: &mut Command) -> impl FnOnce() -> String + use<> {
307    let child = match cmd.stderr(Stdio::inherit()).stdout(Stdio::piped()).spawn() {
308        Ok(child) => child,
309        Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")),
310    };
311
312    let command = format!("{:?}", cmd);
313
314    move || {
315        let output = child.wait_with_output().unwrap();
316
317        if !output.status.success() {
318            panic!(
319                "command did not execute successfully: {}\n\
320                 expected success, got: {}",
321                command, output.status
322            );
323        }
324
325        String::from_utf8(output.stdout).unwrap()
326    }
327}
328
329/// Returns the last-modified time for `path`, or zero if it doesn't exist.
330pub fn mtime(path: &Path) -> SystemTime {
331    fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH)
332}
333
334/// Returns `true` if `dst` is up to date given that the file or files in `src`
335/// are used to generate it.
336///
337/// Uses last-modified time checks to verify this.
338pub fn up_to_date(src: &Path, dst: &Path) -> bool {
339    if !dst.exists() {
340        return false;
341    }
342    let threshold = mtime(dst);
343    let meta = match fs::metadata(src) {
344        Ok(meta) => meta,
345        Err(e) => panic!("source {src:?} failed to get metadata: {e}"),
346    };
347    if meta.is_dir() {
348        dir_up_to_date(src, threshold)
349    } else {
350        meta.modified().unwrap_or(UNIX_EPOCH) <= threshold
351    }
352}
353
354/// Returns the filename without the hash prefix added by the cc crate.
355///
356/// Since v1.0.78 of the cc crate, object files are prefixed with a 16-character hash
357/// to avoid filename collisions.
358pub fn unhashed_basename(obj: &Path) -> &str {
359    let basename = obj.file_stem().unwrap().to_str().expect("UTF-8 file name");
360    basename.split_once('-').unwrap().1
361}
362
363fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool {
364    t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
365        let meta = t!(e.metadata());
366        if meta.is_dir() {
367            dir_up_to_date(&e.path(), threshold)
368        } else {
369            meta.modified().unwrap_or(UNIX_EPOCH) < threshold
370        }
371    })
372}
373
374/// Adapted from <https://github.com/llvm/llvm-project/blob/782e91224601e461c019e0a4573bbccc6094fbcd/llvm/cmake/modules/HandleLLVMOptions.cmake#L1058-L1079>
375///
376/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource
377/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime
378/// missing. This function returns the path to that directory.
379pub fn get_clang_cl_resource_dir(builder: &Builder<'_>, clang_cl_path: &str) -> PathBuf {
380    // Similar to how LLVM does it, to find clang's library runtime directory:
381    // - we ask `clang-cl` to locate the `clang_rt.builtins` lib.
382    let mut builtins_locator = command(clang_cl_path);
383    builtins_locator.args(["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]);
384
385    let clang_rt_builtins = builtins_locator.run_capture_stdout(builder).stdout();
386    let clang_rt_builtins = Path::new(clang_rt_builtins.trim());
387    assert!(
388        clang_rt_builtins.exists(),
389        "`clang-cl` must correctly locate the library runtime directory"
390    );
391
392    // - the profiler runtime will be located in the same directory as the builtins lib, like
393    // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`.
394    let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist");
395    clang_rt_dir.to_path_buf()
396}
397
398/// Returns a flag that configures LLD to use only a single thread.
399/// If we use an external LLD, we need to find out which version is it to know which flag should we
400/// pass to it (LLD older than version 10 had a different flag).
401fn lld_flag_no_threads(builder: &Builder<'_>, lld_mode: LldMode, is_windows: bool) -> &'static str {
402    static LLD_NO_THREADS: OnceLock<(&'static str, &'static str)> = OnceLock::new();
403
404    let new_flags = ("/threads:1", "--threads=1");
405    let old_flags = ("/no-threads", "--no-threads");
406
407    let (windows_flag, other_flag) = LLD_NO_THREADS.get_or_init(|| {
408        let newer_version = match lld_mode {
409            LldMode::External => {
410                let mut cmd = command("lld");
411                cmd.arg("-flavor").arg("ld").arg("--version");
412                let out = cmd.run_capture_stdout(builder).stdout();
413                match (out.find(char::is_numeric), out.find('.')) {
414                    (Some(b), Some(e)) => out.as_str()[b..e].parse::<i32>().ok().unwrap_or(14) > 10,
415                    _ => true,
416                }
417            }
418            _ => true,
419        };
420        if newer_version { new_flags } else { old_flags }
421    });
422    if is_windows { windows_flag } else { other_flag }
423}
424
425pub fn dir_is_empty(dir: &Path) -> bool {
426    t!(std::fs::read_dir(dir), dir).next().is_none()
427}
428
429/// Extract the beta revision from the full version string.
430///
431/// The full version string looks like "a.b.c-beta.y". And we need to extract
432/// the "y" part from the string.
433pub fn extract_beta_rev(version: &str) -> Option<String> {
434    let parts = version.splitn(2, "-beta.").collect::<Vec<_>>();
435    let count = parts.get(1).and_then(|s| s.find(' ').map(|p| s[..p].to_string()));
436
437    count
438}
439
440pub enum LldThreads {
441    Yes,
442    No,
443}
444
445/// Returns the linker arguments for rustc/rustdoc for the given builder and target.
446pub fn linker_args(
447    builder: &Builder<'_>,
448    target: TargetSelection,
449    lld_threads: LldThreads,
450    stage: u32,
451) -> Vec<String> {
452    let mut args = linker_flags(builder, target, lld_threads, stage);
453
454    if let Some(linker) = builder.linker(target) {
455        args.push(format!("-Clinker={}", linker.display()));
456    }
457
458    args
459}
460
461/// Returns the linker arguments for rustc/rustdoc for the given builder and target, without the
462/// -Clinker flag.
463pub fn linker_flags(
464    builder: &Builder<'_>,
465    target: TargetSelection,
466    lld_threads: LldThreads,
467    stage: u32,
468) -> Vec<String> {
469    let mut args = vec![];
470    if !builder.is_lld_direct_linker(target) && builder.config.lld_mode.is_used() {
471        match builder.config.lld_mode {
472            LldMode::External => {
473                // cfg(bootstrap) - remove after updating bootstrap compiler (#137498)
474                if stage == 0 && target.is_windows() {
475                    args.push("-Clink-arg=-fuse-ld=lld".to_string());
476                } else {
477                    args.push("-Clinker-flavor=gnu-lld-cc".to_string());
478                }
479                // FIXME(kobzol): remove this flag once MCP510 gets stabilized
480                args.push("-Zunstable-options".to_string());
481            }
482            LldMode::SelfContained => {
483                args.push("-Clinker-flavor=gnu-lld-cc".to_string());
484                args.push("-Clink-self-contained=+linker".to_string());
485                // FIXME(kobzol): remove this flag once MCP510 gets stabilized
486                args.push("-Zunstable-options".to_string());
487            }
488            LldMode::Unused => unreachable!(),
489        };
490
491        if matches!(lld_threads, LldThreads::No) {
492            args.push(format!(
493                "-Clink-arg=-Wl,{}",
494                lld_flag_no_threads(builder, builder.config.lld_mode, target.is_windows())
495            ));
496        }
497    }
498    args
499}
500
501pub fn add_rustdoc_cargo_linker_args(
502    cmd: &mut BootstrapCommand,
503    builder: &Builder<'_>,
504    target: TargetSelection,
505    lld_threads: LldThreads,
506    stage: u32,
507) {
508    let args = linker_args(builder, target, lld_threads, stage);
509    let mut flags = cmd
510        .get_envs()
511        .find_map(|(k, v)| if k == OsStr::new("RUSTDOCFLAGS") { v } else { None })
512        .unwrap_or_default()
513        .to_os_string();
514    for arg in args {
515        if !flags.is_empty() {
516            flags.push(" ");
517        }
518        flags.push(arg);
519    }
520    if !flags.is_empty() {
521        cmd.env("RUSTDOCFLAGS", flags);
522    }
523}
524
525/// Converts `T` into a hexadecimal `String`.
526pub fn hex_encode<T>(input: T) -> String
527where
528    T: AsRef<[u8]>,
529{
530    use std::fmt::Write;
531
532    input.as_ref().iter().fold(String::with_capacity(input.as_ref().len() * 2), |mut acc, &byte| {
533        write!(&mut acc, "{:02x}", byte).expect("Failed to write byte to the hex String.");
534        acc
535    })
536}
537
538/// Create a `--check-cfg` argument invocation for a given name
539/// and it's values.
540pub fn check_cfg_arg(name: &str, values: Option<&[&str]>) -> String {
541    // Creating a string of the values by concatenating each value:
542    // ',values("tvos","watchos")' or '' (nothing) when there are no values.
543    let next = match values {
544        Some(values) => {
545            let mut tmp = values.iter().flat_map(|val| [",", "\"", val, "\""]).collect::<String>();
546
547            tmp.insert_str(1, "values(");
548            tmp.push(')');
549            tmp
550        }
551        None => "".to_string(),
552    };
553    format!("--check-cfg=cfg({name}{next})")
554}
555
556/// Prepares `BootstrapCommand` that runs git inside the source directory if given.
557///
558/// Whenever a git invocation is needed, this function should be preferred over
559/// manually building a git `BootstrapCommand`. This approach allows us to manage
560/// bootstrap-specific needs/hacks from a single source, rather than applying them on next to every
561/// git command creation, which is painful to ensure that the required change is applied
562/// on each one of them correctly.
563#[track_caller]
564pub fn git(source_dir: Option<&Path>) -> BootstrapCommand {
565    let mut git = command("git");
566
567    if let Some(source_dir) = source_dir {
568        git.current_dir(source_dir);
569        // If we are running inside git (e.g. via a hook), `GIT_DIR` is set and takes precedence
570        // over the current dir. Un-set it to make the current dir matter.
571        git.env_remove("GIT_DIR");
572        // Also un-set some other variables, to be on the safe side (based on cargo's
573        // `fetch_with_cli`). In particular un-setting `GIT_INDEX_FILE` is required to fix some odd
574        // misbehavior.
575        git.env_remove("GIT_WORK_TREE")
576            .env_remove("GIT_INDEX_FILE")
577            .env_remove("GIT_OBJECT_DIRECTORY")
578            .env_remove("GIT_ALTERNATE_OBJECT_DIRECTORIES");
579    }
580
581    git
582}
583
584/// Sets the file times for a given file at `path`.
585pub fn set_file_times<P: AsRef<Path>>(path: P, times: fs::FileTimes) -> io::Result<()> {
586    // Windows requires file to be writable to modify file times. But on Linux CI the file does not
587    // need to be writable to modify file times and might be read-only.
588    let f = if cfg!(windows) {
589        fs::File::options().write(true).open(path)?
590    } else {
591        fs::File::open(path)?
592    };
593    f.set_times(times)
594}
bootstrap/utils/helpers.rs

bootstrap/utils/
helpers.rs
