compiletest/
lib.rs

1#![crate_name = "compiletest"]
2// Needed by the "new" test executor that does not depend on libtest.
3// FIXME(Zalathar): We should be able to get rid of `internal_output_capture`,
4// by having `runtest` manually capture all of its println-like output instead.
5// That would result in compiletest being written entirely in stable Rust!
6#![feature(internal_output_capture)]
7
8#[cfg(test)]
9mod tests;
10
11pub mod common;
12pub mod compute_diff;
13mod debuggers;
14pub mod diagnostics;
15pub mod directives;
16pub mod errors;
17mod executor;
18mod json;
19mod raise_fd_limit;
20mod read2;
21pub mod runtest;
22pub mod util;
23
24use core::panic;
25use std::collections::HashSet;
26use std::fmt::Write;
27use std::io::{self, ErrorKind};
28use std::process::{Command, Stdio};
29use std::sync::{Arc, OnceLock};
30use std::time::SystemTime;
31use std::{env, fs, vec};
32
33use build_helper::git::{get_git_modified_files, get_git_untracked_files};
34use camino::{Utf8Path, Utf8PathBuf};
35use getopts::Options;
36use rayon::iter::{ParallelBridge, ParallelIterator};
37use tracing::debug;
38use walkdir::WalkDir;
39
40use self::directives::{EarlyProps, make_test_description};
41use crate::common::{
42    CompareMode, Config, Debugger, Mode, PassMode, TestPaths, UI_EXTENSIONS, expected_output_path,
43    output_base_dir, output_relative_path,
44};
45use crate::directives::DirectivesCache;
46use crate::executor::{CollectedTest, ColorConfig, OutputFormat};
47use crate::util::logv;
48
49/// Creates the `Config` instance for this invocation of compiletest.
50///
51/// The config mostly reflects command-line arguments, but there might also be
52/// some code here that inspects environment variables or even runs executables
53/// (e.g. when discovering debugger versions).
54pub fn parse_config(args: Vec<String>) -> Config {
55    let mut opts = Options::new();
56    opts.reqopt("", "compile-lib-path", "path to host shared libraries", "PATH")
57        .reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
58        .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
59        .optopt("", "cargo-path", "path to cargo to use for compiling", "PATH")
60        .optopt(
61            "",
62            "stage0-rustc-path",
63            "path to rustc to use for compiling run-make recipes",
64            "PATH",
65        )
66        .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
67        .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
68        .reqopt("", "python", "path to python to use for doc tests", "PATH")
69        .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
70        .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
71        .optopt("", "run-clang-based-tests-with", "path to Clang executable", "PATH")
72        .optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR")
73        .reqopt("", "src-root", "directory containing sources", "PATH")
74        .reqopt("", "src-test-suite-root", "directory containing test suite sources", "PATH")
75        .reqopt("", "build-root", "path to root build directory", "PATH")
76        .reqopt("", "build-test-suite-root", "path to test suite specific build directory", "PATH")
77        .reqopt("", "sysroot-base", "directory containing the compiler sysroot", "PATH")
78        .reqopt("", "stage", "stage number under test", "N")
79        .reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET")
80        .reqopt(
81            "",
82            "mode",
83            "which sort of compile tests to run",
84            "pretty | debug-info | codegen | rustdoc \
85            | rustdoc-json | codegen-units | incremental | run-make | ui \
86            | rustdoc-js | mir-opt | assembly | crashes",
87        )
88        .reqopt(
89            "",
90            "suite",
91            "which suite of compile tests to run. used for nicer error reporting.",
92            "SUITE",
93        )
94        .optopt(
95            "",
96            "pass",
97            "force {check,build,run}-pass tests to this mode.",
98            "check | build | run",
99        )
100        .optopt("", "run", "whether to execute run-* tests", "auto | always | never")
101        .optflag("", "ignored", "run tests marked as ignored")
102        .optflag("", "has-enzyme", "run tests that require enzyme")
103        .optflag("", "with-rustc-debug-assertions", "whether rustc was built with debug assertions")
104        .optflag("", "with-std-debug-assertions", "whether std was built with debug assertions")
105        .optmulti(
106            "",
107            "skip",
108            "skip tests matching SUBSTRING. Can be passed multiple times",
109            "SUBSTRING",
110        )
111        .optflag("", "exact", "filters match exactly")
112        .optopt(
113            "",
114            "runner",
115            "supervisor program to run tests under \
116             (eg. emulator, valgrind)",
117            "PROGRAM",
118        )
119        .optmulti("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS")
120        .optmulti("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS")
121        .optflag(
122            "",
123            "rust-randomized-layout",
124            "set this when rustc/stdlib were compiled with randomized layouts",
125        )
126        .optflag("", "optimize-tests", "run tests with optimizations enabled")
127        .optflag("", "verbose", "run tests verbosely, showing all output")
128        .optflag(
129            "",
130            "bless",
131            "overwrite stderr/stdout files instead of complaining about a mismatch",
132        )
133        .optflag("", "fail-fast", "stop as soon as possible after any test fails")
134        .optflag("", "quiet", "print one character per test instead of one line")
135        .optopt("", "color", "coloring: auto, always, never", "WHEN")
136        .optflag("", "json", "emit json output instead of plaintext output")
137        .optopt("", "target", "the target to build for", "TARGET")
138        .optopt("", "host", "the host to build for", "HOST")
139        .optopt("", "cdb", "path to CDB to use for CDB debuginfo tests", "PATH")
140        .optopt("", "gdb", "path to GDB to use for GDB debuginfo tests", "PATH")
141        .optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING")
142        .optopt("", "llvm-version", "the version of LLVM used", "VERSION STRING")
143        .optflag("", "system-llvm", "is LLVM the system LLVM")
144        .optopt("", "android-cross-path", "Android NDK standalone path", "PATH")
145        .optopt("", "adb-path", "path to the android debugger", "PATH")
146        .optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH")
147        .optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH")
148        .reqopt("", "cc", "path to a C compiler", "PATH")
149        .reqopt("", "cxx", "path to a C++ compiler", "PATH")
150        .reqopt("", "cflags", "flags for the C compiler", "FLAGS")
151        .reqopt("", "cxxflags", "flags for the CXX compiler", "FLAGS")
152        .optopt("", "ar", "path to an archiver", "PATH")
153        .optopt("", "target-linker", "path to a linker for the target", "PATH")
154        .optopt("", "host-linker", "path to a linker for the host", "PATH")
155        .reqopt("", "llvm-components", "list of LLVM components built in", "LIST")
156        .optopt("", "llvm-bin-dir", "Path to LLVM's `bin` directory", "PATH")
157        .optopt("", "nodejs", "the name of nodejs", "PATH")
158        .optopt("", "npm", "the name of npm", "PATH")
159        .optopt("", "remote-test-client", "path to the remote test client", "PATH")
160        .optopt(
161            "",
162            "compare-mode",
163            "mode describing what file the actual ui output will be compared to",
164            "COMPARE MODE",
165        )
166        .optflag(
167            "",
168            "rustfix-coverage",
169            "enable this to generate a Rustfix coverage file, which is saved in \
170            `./<build_test_suite_root>/rustfix_missing_coverage.txt`",
171        )
172        .optflag("", "force-rerun", "rerun tests even if the inputs are unchanged")
173        .optflag("", "only-modified", "only run tests that result been modified")
174        // FIXME: Temporarily retained so we can point users to `--no-capture`
175        .optflag("", "nocapture", "")
176        .optflag("", "no-capture", "don't capture stdout/stderr of tests")
177        .optflag("", "profiler-runtime", "is the profiler runtime enabled for this target")
178        .optflag("h", "help", "show this message")
179        .reqopt("", "channel", "current Rust channel", "CHANNEL")
180        .optflag(
181            "",
182            "git-hash",
183            "run tests which rely on commit version being compiled into the binaries",
184        )
185        .optopt("", "edition", "default Rust edition", "EDITION")
186        .reqopt("", "nightly-branch", "name of the git branch for nightly", "BRANCH")
187        .reqopt(
188            "",
189            "git-merge-commit-email",
190            "email address used for finding merge commits",
191            "EMAIL",
192        )
193        .optopt(
194            "",
195            "compiletest-diff-tool",
196            "What custom diff tool to use for displaying compiletest tests.",
197            "COMMAND",
198        )
199        .reqopt("", "minicore-path", "path to minicore aux library", "PATH")
200        .optflag("N", "no-new-executor", "disables the new test executor, and uses libtest instead")
201        .optopt(
202            "",
203            "debugger",
204            "only test a specific debugger in debuginfo tests",
205            "gdb | lldb | cdb",
206        );
207
208    let (argv0, args_) = args.split_first().unwrap();
209    if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
210        let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
211        println!("{}", opts.usage(&message));
212        println!();
213        panic!()
214    }
215
216    let matches = &match opts.parse(args_) {
217        Ok(m) => m,
218        Err(f) => panic!("{:?}", f),
219    };
220
221    if matches.opt_present("h") || matches.opt_present("help") {
222        let message = format!("Usage: {} [OPTIONS]  [TESTNAME...]", argv0);
223        println!("{}", opts.usage(&message));
224        println!();
225        panic!()
226    }
227
228    fn make_absolute(path: Utf8PathBuf) -> Utf8PathBuf {
229        if path.is_relative() {
230            Utf8PathBuf::try_from(env::current_dir().unwrap()).unwrap().join(path)
231        } else {
232            path
233        }
234    }
235
236    fn opt_path(m: &getopts::Matches, nm: &str) -> Utf8PathBuf {
237        match m.opt_str(nm) {
238            Some(s) => Utf8PathBuf::from(&s),
239            None => panic!("no option (=path) found for {}", nm),
240        }
241    }
242
243    let target = opt_str2(matches.opt_str("target"));
244    let android_cross_path = opt_path(matches, "android-cross-path");
245    // FIXME: `cdb_version` is *derived* from cdb, but it's *not* technically a config!
246    let (cdb, cdb_version) = debuggers::analyze_cdb(matches.opt_str("cdb"), &target);
247    // FIXME: `gdb_version` is *derived* from gdb, but it's *not* technically a config!
248    let (gdb, gdb_version) =
249        debuggers::analyze_gdb(matches.opt_str("gdb"), &target, &android_cross_path);
250    // FIXME: `lldb_version` is *derived* from lldb, but it's *not* technically a config!
251    let lldb_version =
252        matches.opt_str("lldb-version").as_deref().and_then(debuggers::extract_lldb_version);
253    let color = match matches.opt_str("color").as_deref() {
254        Some("auto") | None => ColorConfig::AutoColor,
255        Some("always") => ColorConfig::AlwaysColor,
256        Some("never") => ColorConfig::NeverColor,
257        Some(x) => panic!("argument for --color must be auto, always, or never, but found `{}`", x),
258    };
259    // FIXME: this is very questionable, we really should be obtaining LLVM version info from
260    // `bootstrap`, and not trying to be figuring out that in `compiletest` by running the
261    // `FileCheck` binary.
262    let llvm_version =
263        matches.opt_str("llvm-version").as_deref().map(directives::extract_llvm_version).or_else(
264            || directives::extract_llvm_version_from_binary(&matches.opt_str("llvm-filecheck")?),
265        );
266
267    let run_ignored = matches.opt_present("ignored");
268    let with_rustc_debug_assertions = matches.opt_present("with-rustc-debug-assertions");
269    let with_std_debug_assertions = matches.opt_present("with-std-debug-assertions");
270    let mode = matches.opt_str("mode").unwrap().parse().expect("invalid mode");
271    let has_html_tidy = if mode == Mode::Rustdoc {
272        Command::new("tidy")
273            .arg("--version")
274            .stdout(Stdio::null())
275            .status()
276            .map_or(false, |status| status.success())
277    } else {
278        // Avoid spawning an external command when we know html-tidy won't be used.
279        false
280    };
281    let has_enzyme = matches.opt_present("has-enzyme");
282    let filters = if mode == Mode::RunMake {
283        matches
284            .free
285            .iter()
286            .map(|f| {
287                let path = Utf8Path::new(f);
288                let mut iter = path.iter().skip(1);
289
290                // We skip the test folder and check if the user passed `rmake.rs`.
291                if iter.next().is_some_and(|s| s == "rmake.rs") && iter.next().is_none() {
292                    path.parent().unwrap().to_string()
293                } else {
294                    f.to_string()
295                }
296            })
297            .collect::<Vec<_>>()
298    } else {
299        matches.free.clone()
300    };
301    let compare_mode = matches.opt_str("compare-mode").map(|s| {
302        s.parse().unwrap_or_else(|_| {
303            let variants: Vec<_> = CompareMode::STR_VARIANTS.iter().copied().collect();
304            panic!(
305                "`{s}` is not a valid value for `--compare-mode`, it should be one of: {}",
306                variants.join(", ")
307            );
308        })
309    });
310    if matches.opt_present("nocapture") {
311        panic!("`--nocapture` is deprecated; please use `--no-capture`");
312    }
313
314    let stage = match matches.opt_str("stage") {
315        Some(stage) => stage.parse::<u32>().expect("expected `--stage` to be an unsigned integer"),
316        None => panic!("`--stage` is required"),
317    };
318
319    let src_root = opt_path(matches, "src-root");
320    let src_test_suite_root = opt_path(matches, "src-test-suite-root");
321    assert!(
322        src_test_suite_root.starts_with(&src_root),
323        "`src-root` must be a parent of `src-test-suite-root`: `src-root`=`{}`, `src-test-suite-root` = `{}`",
324        src_root,
325        src_test_suite_root
326    );
327
328    let build_root = opt_path(matches, "build-root");
329    let build_test_suite_root = opt_path(matches, "build-test-suite-root");
330    assert!(build_test_suite_root.starts_with(&build_root));
331
332    Config {
333        bless: matches.opt_present("bless"),
334        fail_fast: matches.opt_present("fail-fast")
335            || env::var_os("RUSTC_TEST_FAIL_FAST").is_some(),
336
337        compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
338        run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
339        rustc_path: opt_path(matches, "rustc-path"),
340        cargo_path: matches.opt_str("cargo-path").map(Utf8PathBuf::from),
341        stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(Utf8PathBuf::from),
342        rustdoc_path: matches.opt_str("rustdoc-path").map(Utf8PathBuf::from),
343        coverage_dump_path: matches.opt_str("coverage-dump-path").map(Utf8PathBuf::from),
344        python: matches.opt_str("python").unwrap(),
345        jsondocck_path: matches.opt_str("jsondocck-path"),
346        jsondoclint_path: matches.opt_str("jsondoclint-path"),
347        run_clang_based_tests_with: matches.opt_str("run-clang-based-tests-with"),
348        llvm_filecheck: matches.opt_str("llvm-filecheck").map(Utf8PathBuf::from),
349        llvm_bin_dir: matches.opt_str("llvm-bin-dir").map(Utf8PathBuf::from),
350
351        src_root,
352        src_test_suite_root,
353
354        build_root,
355        build_test_suite_root,
356
357        sysroot_base: opt_path(matches, "sysroot-base"),
358
359        stage,
360        stage_id: matches.opt_str("stage-id").unwrap(),
361
362        mode,
363        suite: matches.opt_str("suite").unwrap(),
364        debugger: matches.opt_str("debugger").map(|debugger| {
365            debugger
366                .parse::<Debugger>()
367                .unwrap_or_else(|_| panic!("unknown `--debugger` option `{debugger}` given"))
368        }),
369        run_ignored,
370        with_rustc_debug_assertions,
371        with_std_debug_assertions,
372        filters,
373        skip: matches.opt_strs("skip"),
374        filter_exact: matches.opt_present("exact"),
375        force_pass_mode: matches.opt_str("pass").map(|mode| {
376            mode.parse::<PassMode>()
377                .unwrap_or_else(|_| panic!("unknown `--pass` option `{}` given", mode))
378        }),
379        // FIXME: this run scheme is... confusing.
380        run: matches.opt_str("run").and_then(|mode| match mode.as_str() {
381            "auto" => None,
382            "always" => Some(true),
383            "never" => Some(false),
384            _ => panic!("unknown `--run` option `{}` given", mode),
385        }),
386        runner: matches.opt_str("runner"),
387        host_rustcflags: matches.opt_strs("host-rustcflags"),
388        target_rustcflags: matches.opt_strs("target-rustcflags"),
389        optimize_tests: matches.opt_present("optimize-tests"),
390        rust_randomized_layout: matches.opt_present("rust-randomized-layout"),
391        target,
392        host: opt_str2(matches.opt_str("host")),
393        cdb,
394        cdb_version,
395        gdb,
396        gdb_version,
397        lldb_version,
398        llvm_version,
399        system_llvm: matches.opt_present("system-llvm"),
400        android_cross_path,
401        adb_path: opt_str2(matches.opt_str("adb-path")),
402        adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")),
403        adb_device_status: opt_str2(matches.opt_str("target")).contains("android")
404            && "(none)" != opt_str2(matches.opt_str("adb-test-dir"))
405            && !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
406        lldb_python_dir: matches.opt_str("lldb-python-dir"),
407        verbose: matches.opt_present("verbose"),
408        format: match (matches.opt_present("quiet"), matches.opt_present("json")) {
409            (true, true) => panic!("--quiet and --json are incompatible"),
410            (true, false) => OutputFormat::Terse,
411            (false, true) => OutputFormat::Json,
412            (false, false) => OutputFormat::Pretty,
413        },
414        only_modified: matches.opt_present("only-modified"),
415        color,
416        remote_test_client: matches.opt_str("remote-test-client").map(Utf8PathBuf::from),
417        compare_mode,
418        rustfix_coverage: matches.opt_present("rustfix-coverage"),
419        has_html_tidy,
420        has_enzyme,
421        channel: matches.opt_str("channel").unwrap(),
422        git_hash: matches.opt_present("git-hash"),
423        edition: matches.opt_str("edition"),
424
425        cc: matches.opt_str("cc").unwrap(),
426        cxx: matches.opt_str("cxx").unwrap(),
427        cflags: matches.opt_str("cflags").unwrap(),
428        cxxflags: matches.opt_str("cxxflags").unwrap(),
429        ar: matches.opt_str("ar").unwrap_or_else(|| String::from("ar")),
430        target_linker: matches.opt_str("target-linker"),
431        host_linker: matches.opt_str("host-linker"),
432        llvm_components: matches.opt_str("llvm-components").unwrap(),
433        nodejs: matches.opt_str("nodejs"),
434        npm: matches.opt_str("npm"),
435
436        force_rerun: matches.opt_present("force-rerun"),
437
438        target_cfgs: OnceLock::new(),
439        builtin_cfg_names: OnceLock::new(),
440        supported_crate_types: OnceLock::new(),
441
442        nocapture: matches.opt_present("no-capture"),
443
444        nightly_branch: matches.opt_str("nightly-branch").unwrap(),
445        git_merge_commit_email: matches.opt_str("git-merge-commit-email").unwrap(),
446
447        profiler_runtime: matches.opt_present("profiler-runtime"),
448
449        diff_command: matches.opt_str("compiletest-diff-tool"),
450
451        minicore_path: opt_path(matches, "minicore-path"),
452    }
453}
454
455pub fn log_config(config: &Config) {
456    let c = config;
457    logv(c, "configuration:".to_string());
458    logv(c, format!("compile_lib_path: {}", config.compile_lib_path));
459    logv(c, format!("run_lib_path: {}", config.run_lib_path));
460    logv(c, format!("rustc_path: {}", config.rustc_path));
461    logv(c, format!("cargo_path: {:?}", config.cargo_path));
462    logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path));
463
464    logv(c, format!("src_root: {}", config.src_root));
465    logv(c, format!("src_test_suite_root: {}", config.src_test_suite_root));
466
467    logv(c, format!("build_root: {}", config.build_root));
468    logv(c, format!("build_test_suite_root: {}", config.build_test_suite_root));
469
470    logv(c, format!("sysroot_base: {}", config.sysroot_base));
471
472    logv(c, format!("stage: {}", config.stage));
473    logv(c, format!("stage_id: {}", config.stage_id));
474    logv(c, format!("mode: {}", config.mode));
475    logv(c, format!("run_ignored: {}", config.run_ignored));
476    logv(c, format!("filters: {:?}", config.filters));
477    logv(c, format!("skip: {:?}", config.skip));
478    logv(c, format!("filter_exact: {}", config.filter_exact));
479    logv(
480        c,
481        format!("force_pass_mode: {}", opt_str(&config.force_pass_mode.map(|m| format!("{}", m))),),
482    );
483    logv(c, format!("runner: {}", opt_str(&config.runner)));
484    logv(c, format!("host-rustcflags: {:?}", config.host_rustcflags));
485    logv(c, format!("target-rustcflags: {:?}", config.target_rustcflags));
486    logv(c, format!("target: {}", config.target));
487    logv(c, format!("host: {}", config.host));
488    logv(c, format!("android-cross-path: {}", config.android_cross_path));
489    logv(c, format!("adb_path: {}", config.adb_path));
490    logv(c, format!("adb_test_dir: {}", config.adb_test_dir));
491    logv(c, format!("adb_device_status: {}", config.adb_device_status));
492    logv(c, format!("ar: {}", config.ar));
493    logv(c, format!("target-linker: {:?}", config.target_linker));
494    logv(c, format!("host-linker: {:?}", config.host_linker));
495    logv(c, format!("verbose: {}", config.verbose));
496    logv(c, format!("format: {:?}", config.format));
497    logv(c, format!("minicore_path: {}", config.minicore_path));
498    logv(c, "\n".to_string());
499}
500
501pub fn opt_str(maybestr: &Option<String>) -> &str {
502    match *maybestr {
503        None => "(none)",
504        Some(ref s) => s,
505    }
506}
507
508pub fn opt_str2(maybestr: Option<String>) -> String {
509    match maybestr {
510        None => "(none)".to_owned(),
511        Some(s) => s,
512    }
513}
514
515/// Called by `main` after the config has been parsed.
516pub fn run_tests(config: Arc<Config>) {
517    // If we want to collect rustfix coverage information,
518    // we first make sure that the coverage file does not exist.
519    // It will be created later on.
520    if config.rustfix_coverage {
521        let mut coverage_file_path = config.build_test_suite_root.clone();
522        coverage_file_path.push("rustfix_missing_coverage.txt");
523        if coverage_file_path.exists() {
524            if let Err(e) = fs::remove_file(&coverage_file_path) {
525                panic!("Could not delete {} due to {}", coverage_file_path, e)
526            }
527        }
528    }
529
530    // sadly osx needs some file descriptor limits raised for running tests in
531    // parallel (especially when we have lots and lots of child processes).
532    // For context, see #8904
533    unsafe {
534        raise_fd_limit::raise_fd_limit();
535    }
536    // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
537    // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
538    //
539    // SAFETY: at this point we're still single-threaded.
540    unsafe { env::set_var("__COMPAT_LAYER", "RunAsInvoker") };
541
542    // Let tests know which target they're running as.
543    //
544    // SAFETY: at this point we're still single-threaded.
545    unsafe { env::set_var("TARGET", &config.target) };
546
547    let mut configs = Vec::new();
548    if let Mode::DebugInfo = config.mode {
549        // Debugging emscripten code doesn't make sense today
550        if !config.target.contains("emscripten") {
551            match config.debugger {
552                Some(Debugger::Cdb) => configs.extend(debuggers::configure_cdb(&config)),
553                Some(Debugger::Gdb) => configs.extend(debuggers::configure_gdb(&config)),
554                Some(Debugger::Lldb) => configs.extend(debuggers::configure_lldb(&config)),
555                // FIXME: the *implicit* debugger discovery makes it really difficult to control
556                // which {`cdb`, `gdb`, `lldb`} are used. These should **not** be implicitly
557                // discovered by `compiletest`; these should be explicit `bootstrap` configuration
558                // options that are passed to `compiletest`!
559                None => {
560                    configs.extend(debuggers::configure_cdb(&config));
561                    configs.extend(debuggers::configure_gdb(&config));
562                    configs.extend(debuggers::configure_lldb(&config));
563                }
564            }
565        }
566    } else {
567        configs.push(config.clone());
568    };
569
570    // Discover all of the tests in the test suite directory, and build a libtest
571    // structure for each test (or each revision of a multi-revision test).
572    let mut tests = Vec::new();
573    for c in configs {
574        tests.extend(collect_and_make_tests(c));
575    }
576
577    tests.sort_by(|a, b| Ord::cmp(&a.desc.name, &b.desc.name));
578
579    // Delegate to the executor to filter and run the big list of test structures
580    // created during test discovery. When the executor decides to run a test,
581    // it will return control to the rest of compiletest by calling `runtest::run`.
582    // FIXME(Zalathar): Once we're confident that we won't need to revert the
583    // removal of the libtest-based executor, remove this Result and other
584    // remnants of the old executor.
585    let res: io::Result<bool> = Ok(executor::run_tests(&config, tests));
586
587    // Check the outcome reported by libtest.
588    match res {
589        Ok(true) => {}
590        Ok(false) => {
591            // We want to report that the tests failed, but we also want to give
592            // some indication of just what tests we were running. Especially on
593            // CI, where there can be cross-compiled tests for a lot of
594            // architectures, without this critical information it can be quite
595            // easy to miss which tests failed, and as such fail to reproduce
596            // the failure locally.
597
598            let mut msg = String::from("Some tests failed in compiletest");
599            write!(msg, " suite={}", config.suite).unwrap();
600
601            if let Some(compare_mode) = config.compare_mode.as_ref() {
602                write!(msg, " compare_mode={}", compare_mode).unwrap();
603            }
604
605            if let Some(pass_mode) = config.force_pass_mode.as_ref() {
606                write!(msg, " pass_mode={}", pass_mode).unwrap();
607            }
608
609            write!(msg, " mode={}", config.mode).unwrap();
610            write!(msg, " host={}", config.host).unwrap();
611            write!(msg, " target={}", config.target).unwrap();
612
613            println!("{msg}");
614
615            std::process::exit(1);
616        }
617        Err(e) => {
618            // We don't know if tests passed or not, but if there was an error
619            // during testing we don't want to just succeed (we may not have
620            // tested something), so fail.
621            //
622            // This should realistically "never" happen, so don't try to make
623            // this a pretty error message.
624            panic!("I/O failure during tests: {:?}", e);
625        }
626    }
627}
628
629/// Read-only context data used during test collection.
630struct TestCollectorCx {
631    config: Arc<Config>,
632    cache: DirectivesCache,
633    common_inputs_stamp: Stamp,
634    modified_tests: Vec<Utf8PathBuf>,
635}
636
637/// Mutable state used during test collection.
638struct TestCollector {
639    tests: Vec<CollectedTest>,
640    found_path_stems: HashSet<Utf8PathBuf>,
641    poisoned: bool,
642}
643
644impl TestCollector {
645    fn new() -> Self {
646        TestCollector { tests: vec![], found_path_stems: HashSet::new(), poisoned: false }
647    }
648
649    fn merge(&mut self, mut other: Self) {
650        self.tests.append(&mut other.tests);
651        self.found_path_stems.extend(other.found_path_stems);
652        self.poisoned |= other.poisoned;
653    }
654}
655
656/// Creates test structures for every test/revision in the test suite directory.
657///
658/// This always inspects _all_ test files in the suite (e.g. all 17k+ ui tests),
659/// regardless of whether any filters/tests were specified on the command-line,
660/// because filtering is handled later by libtest.
661pub(crate) fn collect_and_make_tests(config: Arc<Config>) -> Vec<CollectedTest> {
662    debug!("making tests from {}", config.src_test_suite_root);
663    let common_inputs_stamp = common_inputs_stamp(&config);
664    let modified_tests =
665        modified_tests(&config, &config.src_test_suite_root).unwrap_or_else(|err| {
666            fatal!("modified_tests: {}: {err}", config.src_test_suite_root);
667        });
668    let cache = DirectivesCache::load(&config);
669
670    let cx = TestCollectorCx { config, cache, common_inputs_stamp, modified_tests };
671    let collector = collect_tests_from_dir(&cx, &cx.config.src_test_suite_root, Utf8Path::new(""))
672        .unwrap_or_else(|reason| {
673            panic!("Could not read tests from {}: {reason}", cx.config.src_test_suite_root)
674        });
675
676    let TestCollector { tests, found_path_stems, poisoned } = collector;
677
678    if poisoned {
679        eprintln!();
680        panic!("there are errors in tests");
681    }
682
683    check_for_overlapping_test_paths(&found_path_stems);
684
685    tests
686}
687
688/// Returns the most recent last-modified timestamp from among the input files
689/// that are considered relevant to all tests (e.g. the compiler, std, and
690/// compiletest itself).
691///
692/// (Some of these inputs aren't actually relevant to _all_ tests, but they are
693/// common to some subset of tests, and are hopefully unlikely to be modified
694/// while working on other tests.)
695fn common_inputs_stamp(config: &Config) -> Stamp {
696    let src_root = &config.src_root;
697
698    let mut stamp = Stamp::from_path(&config.rustc_path);
699
700    // Relevant pretty printer files
701    let pretty_printer_files = [
702        "src/etc/rust_types.py",
703        "src/etc/gdb_load_rust_pretty_printers.py",
704        "src/etc/gdb_lookup.py",
705        "src/etc/gdb_providers.py",
706        "src/etc/lldb_batchmode.py",
707        "src/etc/lldb_lookup.py",
708        "src/etc/lldb_providers.py",
709    ];
710    for file in &pretty_printer_files {
711        let path = src_root.join(file);
712        stamp.add_path(&path);
713    }
714
715    stamp.add_dir(&src_root.join("src/etc/natvis"));
716
717    stamp.add_dir(&config.run_lib_path);
718
719    if let Some(ref rustdoc_path) = config.rustdoc_path {
720        stamp.add_path(&rustdoc_path);
721        stamp.add_path(&src_root.join("src/etc/htmldocck.py"));
722    }
723
724    // Re-run coverage tests if the `coverage-dump` tool was modified,
725    // because its output format might have changed.
726    if let Some(coverage_dump_path) = &config.coverage_dump_path {
727        stamp.add_path(coverage_dump_path)
728    }
729
730    stamp.add_dir(&src_root.join("src/tools/run-make-support"));
731
732    // Compiletest itself.
733    stamp.add_dir(&src_root.join("src/tools/compiletest"));
734
735    stamp
736}
737
738/// Returns a list of modified/untracked test files that should be run when
739/// the `--only-modified` flag is in use.
740///
741/// (Might be inaccurate in some cases.)
742fn modified_tests(config: &Config, dir: &Utf8Path) -> Result<Vec<Utf8PathBuf>, String> {
743    // If `--only-modified` wasn't passed, the list of modified tests won't be
744    // used for anything, so avoid some work and just return an empty list.
745    if !config.only_modified {
746        return Ok(vec![]);
747    }
748
749    let files = get_git_modified_files(
750        &config.git_config(),
751        Some(dir.as_std_path()),
752        &vec!["rs", "stderr", "fixed"],
753    )?;
754    // Add new test cases to the list, it will be convenient in daily development.
755    let untracked_files = get_git_untracked_files(Some(dir.as_std_path()))?.unwrap_or(vec![]);
756
757    let all_paths = [&files[..], &untracked_files[..]].concat();
758    let full_paths = {
759        let mut full_paths: Vec<Utf8PathBuf> = all_paths
760            .into_iter()
761            .map(|f| Utf8PathBuf::from(f).with_extension("").with_extension("rs"))
762            .filter_map(
763                |f| if Utf8Path::new(&f).exists() { f.canonicalize_utf8().ok() } else { None },
764            )
765            .collect();
766        full_paths.dedup();
767        full_paths.sort_unstable();
768        full_paths
769    };
770    Ok(full_paths)
771}
772
773/// Recursively scans a directory to find test files and create test structures
774/// that will be handed over to libtest.
775fn collect_tests_from_dir(
776    cx: &TestCollectorCx,
777    dir: &Utf8Path,
778    relative_dir_path: &Utf8Path,
779) -> io::Result<TestCollector> {
780    // Ignore directories that contain a file named `compiletest-ignore-dir`.
781    if dir.join("compiletest-ignore-dir").exists() {
782        return Ok(TestCollector::new());
783    }
784
785    // For run-make tests, a "test file" is actually a directory that contains an `rmake.rs`.
786    if cx.config.mode == Mode::RunMake {
787        let mut collector = TestCollector::new();
788        if dir.join("rmake.rs").exists() {
789            let paths = TestPaths {
790                file: dir.to_path_buf(),
791                relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
792            };
793            make_test(cx, &mut collector, &paths);
794            // This directory is a test, so don't try to find other tests inside it.
795            return Ok(collector);
796        }
797    }
798
799    // If we find a test foo/bar.rs, we have to build the
800    // output directory `$build/foo` so we can write
801    // `$build/foo/bar` into it. We do this *now* in this
802    // sequential loop because otherwise, if we do it in the
803    // tests themselves, they race for the privilege of
804    // creating the directories and sometimes fail randomly.
805    let build_dir = output_relative_path(&cx.config, relative_dir_path);
806    fs::create_dir_all(&build_dir).unwrap();
807
808    // Add each `.rs` file as a test, and recurse further on any
809    // subdirectories we find, except for `auxiliary` directories.
810    // FIXME: this walks full tests tree, even if we have something to ignore
811    // use walkdir/ignore like in tidy?
812    fs::read_dir(dir.as_std_path())?
813        .par_bridge()
814        .map(|file| {
815            let mut collector = TestCollector::new();
816            let file = file?;
817            let file_path = Utf8PathBuf::try_from(file.path()).unwrap();
818            let file_name = file_path.file_name().unwrap();
819
820            if is_test(file_name)
821                && (!cx.config.only_modified || cx.modified_tests.contains(&file_path))
822            {
823                // We found a test file, so create the corresponding libtest structures.
824                debug!(%file_path, "found test file");
825
826                // Record the stem of the test file, to check for overlaps later.
827                let rel_test_path = relative_dir_path.join(file_path.file_stem().unwrap());
828                collector.found_path_stems.insert(rel_test_path);
829
830                let paths =
831                    TestPaths { file: file_path, relative_dir: relative_dir_path.to_path_buf() };
832                make_test(cx, &mut collector, &paths);
833            } else if file_path.is_dir() {
834                // Recurse to find more tests in a subdirectory.
835                let relative_file_path = relative_dir_path.join(file_name);
836                if file_name != "auxiliary" {
837                    debug!(%file_path, "found directory");
838                    collector.merge(collect_tests_from_dir(cx, &file_path, &relative_file_path)?);
839                }
840            } else {
841                debug!(%file_path, "found other file/directory");
842            }
843            Ok(collector)
844        })
845        .reduce(
846            || Ok(TestCollector::new()),
847            |a, b| {
848                let mut a = a?;
849                a.merge(b?);
850                Ok(a)
851            },
852        )
853}
854
855/// Returns true if `file_name` looks like a proper test file name.
856pub fn is_test(file_name: &str) -> bool {
857    if !file_name.ends_with(".rs") {
858        return false;
859    }
860
861    // `.`, `#`, and `~` are common temp-file prefixes.
862    let invalid_prefixes = &[".", "#", "~"];
863    !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
864}
865
866/// For a single test file, creates one or more test structures (one per revision) that can be
867/// handed over to libtest to run, possibly in parallel.
868fn make_test(cx: &TestCollectorCx, collector: &mut TestCollector, testpaths: &TestPaths) {
869    // For run-make tests, each "test file" is actually a _directory_ containing an `rmake.rs`. But
870    // for the purposes of directive parsing, we want to look at that recipe file, not the directory
871    // itself.
872    let test_path = if cx.config.mode == Mode::RunMake {
873        testpaths.file.join("rmake.rs")
874    } else {
875        testpaths.file.clone()
876    };
877
878    // Scan the test file to discover its revisions, if any.
879    let early_props = EarlyProps::from_file(&cx.config, &test_path);
880
881    // Normally we create one libtest structure per revision, with two exceptions:
882    // - If a test doesn't use revisions, create a dummy revision (None) so that
883    //   the test can still run.
884    // - Incremental tests inherently can't run their revisions in parallel, so
885    //   we treat them like non-revisioned tests here. Incremental revisions are
886    //   handled internally by `runtest::run` instead.
887    let revisions = if early_props.revisions.is_empty() || cx.config.mode == Mode::Incremental {
888        vec![None]
889    } else {
890        early_props.revisions.iter().map(|r| Some(r.as_str())).collect()
891    };
892
893    // For each revision (or the sole dummy revision), create and append a
894    // `CollectedTest` that can be handed over to the test executor.
895    collector.tests.extend(revisions.into_iter().map(|revision| {
896        // Create a test name and description to hand over to libtest.
897        let src_file = fs::File::open(&test_path).expect("open test file to parse ignores");
898        let test_name = make_test_name(&cx.config, testpaths, revision);
899        // Create a libtest description for the test/revision.
900        // This is where `ignore-*`/`only-*`/`needs-*` directives are handled,
901        // because they need to set the libtest ignored flag.
902        let mut desc = make_test_description(
903            &cx.config,
904            &cx.cache,
905            test_name,
906            &test_path,
907            src_file,
908            revision,
909            &mut collector.poisoned,
910        );
911
912        // If a test's inputs haven't changed since the last time it ran,
913        // mark it as ignored so that libtest will skip it.
914        if !cx.config.force_rerun && is_up_to_date(cx, testpaths, &early_props, revision) {
915            desc.ignore = true;
916            // Keep this in sync with the "up-to-date" message detected by bootstrap.
917            desc.ignore_message = Some("up-to-date".into());
918        }
919
920        let config = Arc::clone(&cx.config);
921        let testpaths = testpaths.clone();
922        let revision = revision.map(str::to_owned);
923
924        CollectedTest { desc, config, testpaths, revision }
925    }));
926}
927
928/// The path of the `stamp` file that gets created or updated whenever a
929/// particular test completes successfully.
930fn stamp_file_path(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> Utf8PathBuf {
931    output_base_dir(config, testpaths, revision).join("stamp")
932}
933
934/// Returns a list of files that, if modified, would cause this test to no
935/// longer be up-to-date.
936///
937/// (Might be inaccurate in some cases.)
938fn files_related_to_test(
939    config: &Config,
940    testpaths: &TestPaths,
941    props: &EarlyProps,
942    revision: Option<&str>,
943) -> Vec<Utf8PathBuf> {
944    let mut related = vec![];
945
946    if testpaths.file.is_dir() {
947        // run-make tests use their individual directory
948        for entry in WalkDir::new(&testpaths.file) {
949            let path = entry.unwrap().into_path();
950            if path.is_file() {
951                related.push(Utf8PathBuf::try_from(path).unwrap());
952            }
953        }
954    } else {
955        related.push(testpaths.file.clone());
956    }
957
958    for aux in props.aux.all_aux_path_strings() {
959        // FIXME(Zalathar): Perform all `auxiliary` path resolution in one place.
960        let path = testpaths.file.parent().unwrap().join("auxiliary").join(aux);
961        related.push(path);
962    }
963
964    // UI test files.
965    for extension in UI_EXTENSIONS {
966        let path = expected_output_path(testpaths, revision, &config.compare_mode, extension);
967        related.push(path);
968    }
969
970    // `minicore.rs` test auxiliary: we need to make sure tests get rerun if this changes.
971    related.push(config.src_root.join("tests").join("auxiliary").join("minicore.rs"));
972
973    related
974}
975
976/// Checks whether a particular test/revision is "up-to-date", meaning that no
977/// relevant files/settings have changed since the last time the test succeeded.
978///
979/// (This is not very reliable in some circumstances, so the `--force-rerun`
980/// flag can be used to ignore up-to-date checking and always re-run tests.)
981fn is_up_to_date(
982    cx: &TestCollectorCx,
983    testpaths: &TestPaths,
984    props: &EarlyProps,
985    revision: Option<&str>,
986) -> bool {
987    let stamp_file_path = stamp_file_path(&cx.config, testpaths, revision);
988    // Check the config hash inside the stamp file.
989    let contents = match fs::read_to_string(&stamp_file_path) {
990        Ok(f) => f,
991        Err(ref e) if e.kind() == ErrorKind::InvalidData => panic!("Can't read stamp contents"),
992        // The test hasn't succeeded yet, so it is not up-to-date.
993        Err(_) => return false,
994    };
995    let expected_hash = runtest::compute_stamp_hash(&cx.config);
996    if contents != expected_hash {
997        // Some part of compiletest configuration has changed since the test
998        // last succeeded, so it is not up-to-date.
999        return false;
1000    }
1001
1002    // Check the timestamp of the stamp file against the last modified time
1003    // of all files known to be relevant to the test.
1004    let mut inputs_stamp = cx.common_inputs_stamp.clone();
1005    for path in files_related_to_test(&cx.config, testpaths, props, revision) {
1006        inputs_stamp.add_path(&path);
1007    }
1008
1009    // If no relevant files have been modified since the stamp file was last
1010    // written, the test is up-to-date.
1011    inputs_stamp < Stamp::from_path(&stamp_file_path)
1012}
1013
1014/// The maximum of a set of file-modified timestamps.
1015#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
1016struct Stamp {
1017    time: SystemTime,
1018}
1019
1020impl Stamp {
1021    /// Creates a timestamp holding the last-modified time of the specified file.
1022    fn from_path(path: &Utf8Path) -> Self {
1023        let mut stamp = Stamp { time: SystemTime::UNIX_EPOCH };
1024        stamp.add_path(path);
1025        stamp
1026    }
1027
1028    /// Updates this timestamp to the last-modified time of the specified file,
1029    /// if it is later than the currently-stored timestamp.
1030    fn add_path(&mut self, path: &Utf8Path) {
1031        let modified = fs::metadata(path.as_std_path())
1032            .and_then(|metadata| metadata.modified())
1033            .unwrap_or(SystemTime::UNIX_EPOCH);
1034        self.time = self.time.max(modified);
1035    }
1036
1037    /// Updates this timestamp to the most recent last-modified time of all files
1038    /// recursively contained in the given directory, if it is later than the
1039    /// currently-stored timestamp.
1040    fn add_dir(&mut self, path: &Utf8Path) {
1041        let path = path.as_std_path();
1042        for entry in WalkDir::new(path) {
1043            let entry = entry.unwrap();
1044            if entry.file_type().is_file() {
1045                let modified = entry
1046                    .metadata()
1047                    .ok()
1048                    .and_then(|metadata| metadata.modified().ok())
1049                    .unwrap_or(SystemTime::UNIX_EPOCH);
1050                self.time = self.time.max(modified);
1051            }
1052        }
1053    }
1054}
1055
1056/// Creates a name for this test/revision that can be handed over to libtest.
1057fn make_test_name(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> String {
1058    // Print the name of the file, relative to the sources root.
1059    let path = testpaths.file.strip_prefix(&config.src_root).unwrap();
1060    let debugger = match config.debugger {
1061        Some(d) => format!("-{}", d),
1062        None => String::new(),
1063    };
1064    let mode_suffix = match config.compare_mode {
1065        Some(ref mode) => format!(" ({})", mode.to_str()),
1066        None => String::new(),
1067    };
1068
1069    format!(
1070        "[{}{}{}] {}{}",
1071        config.mode,
1072        debugger,
1073        mode_suffix,
1074        path,
1075        revision.map_or("".to_string(), |rev| format!("#{}", rev))
1076    )
1077}
1078
1079/// Checks that test discovery didn't find any tests whose name stem is a prefix
1080/// of some other tests's name.
1081///
1082/// For example, suppose the test suite contains these two test files:
1083/// - `tests/rustdoc/primitive.rs`
1084/// - `tests/rustdoc/primitive/no_std.rs`
1085///
1086/// The test runner might put the output from those tests in these directories:
1087/// - `$build/test/rustdoc/primitive/`
1088/// - `$build/test/rustdoc/primitive/no_std/`
1089///
1090/// Because one output path is a subdirectory of the other, the two tests might
1091/// interfere with each other in unwanted ways, especially if the test runner
1092/// decides to delete test output directories to clean them between runs.
1093/// To avoid problems, we forbid test names from overlapping in this way.
1094///
1095/// See <https://github.com/rust-lang/rust/pull/109509> for more context.
1096fn check_for_overlapping_test_paths(found_path_stems: &HashSet<Utf8PathBuf>) {
1097    let mut collisions = Vec::new();
1098    for path in found_path_stems {
1099        for ancestor in path.ancestors().skip(1) {
1100            if found_path_stems.contains(ancestor) {
1101                collisions.push((path, ancestor));
1102            }
1103        }
1104    }
1105    if !collisions.is_empty() {
1106        collisions.sort();
1107        let collisions: String = collisions
1108            .into_iter()
1109            .map(|(path, check_parent)| format!("test {path} clashes with {check_parent}\n"))
1110            .collect();
1111        panic!(
1112            "{collisions}\n\
1113            Tests cannot have overlapping names. Make sure they use unique prefixes."
1114        );
1115    }
1116}
1117
1118pub fn early_config_check(config: &Config) {
1119    if !config.has_html_tidy && config.mode == Mode::Rustdoc {
1120        warning!("`tidy` (html-tidy.org) is not installed; diffs will not be generated");
1121    }
1122
1123    if !config.profiler_runtime && config.mode == Mode::CoverageRun {
1124        let actioned = if config.bless { "blessed" } else { "checked" };
1125        warning!("profiler runtime is not available, so `.coverage` files won't be {actioned}");
1126        help!("try setting `profiler = true` in the `[build]` section of `bootstrap.toml`");
1127    }
1128
1129    // `RUST_TEST_NOCAPTURE` is a libtest env var, but we don't callout to libtest.
1130    if env::var("RUST_TEST_NOCAPTURE").is_ok() {
1131        warning!("`RUST_TEST_NOCAPTURE` is not supported; use the `--no-capture` flag instead");
1132    }
1133}