compiletest/
lib.rs

1#![crate_name = "compiletest"]
2// Needed by the "new" test executor that does not depend on libtest.
3// FIXME(Zalathar): We should be able to get rid of `internal_output_capture`,
4// by having `runtest` manually capture all of its println-like output instead.
5// That would result in compiletest being written entirely in stable Rust!
6#![feature(internal_output_capture)]
7
8#[cfg(test)]
9mod tests;
10
11pub mod common;
12pub mod compute_diff;
13mod debuggers;
14pub mod errors;
15mod executor;
16pub mod header;
17mod json;
18mod raise_fd_limit;
19mod read2;
20pub mod runtest;
21pub mod util;
22
23use core::panic;
24use std::collections::HashSet;
25use std::fmt::Write;
26use std::io::{self, ErrorKind};
27use std::process::{Command, Stdio};
28use std::sync::{Arc, OnceLock};
29use std::time::SystemTime;
30use std::{env, fs, vec};
31
32use build_helper::git::{get_git_modified_files, get_git_untracked_files};
33use camino::{Utf8Path, Utf8PathBuf};
34use getopts::Options;
35use rayon::iter::{ParallelBridge, ParallelIterator};
36use tracing::*;
37use walkdir::WalkDir;
38
39use self::header::{EarlyProps, make_test_description};
40use crate::common::{
41    CompareMode, Config, Debugger, Mode, PassMode, TestPaths, UI_EXTENSIONS, expected_output_path,
42    output_base_dir, output_relative_path,
43};
44use crate::executor::{CollectedTest, ColorConfig, OutputFormat};
45use crate::header::HeadersCache;
46use crate::util::logv;
47
48/// Creates the `Config` instance for this invocation of compiletest.
49///
50/// The config mostly reflects command-line arguments, but there might also be
51/// some code here that inspects environment variables or even runs executables
52/// (e.g. when discovering debugger versions).
53pub fn parse_config(args: Vec<String>) -> Config {
54    if env::var("RUST_TEST_NOCAPTURE").is_ok() {
55        eprintln!(
56            "WARNING: RUST_TEST_NOCAPTURE is not supported. Use the `--no-capture` flag instead."
57        );
58    }
59
60    let mut opts = Options::new();
61    opts.reqopt("", "compile-lib-path", "path to host shared libraries", "PATH")
62        .reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
63        .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
64        .optopt("", "cargo-path", "path to cargo to use for compiling", "PATH")
65        .optopt(
66            "",
67            "stage0-rustc-path",
68            "path to rustc to use for compiling run-make recipes",
69            "PATH",
70        )
71        .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
72        .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
73        .reqopt("", "python", "path to python to use for doc tests", "PATH")
74        .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
75        .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
76        .optopt("", "run-clang-based-tests-with", "path to Clang executable", "PATH")
77        .optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR")
78        .reqopt("", "src-root", "directory containing sources", "PATH")
79        .reqopt("", "src-test-suite-root", "directory containing test suite sources", "PATH")
80        .reqopt("", "build-root", "path to root build directory", "PATH")
81        .reqopt("", "build-test-suite-root", "path to test suite specific build directory", "PATH")
82        .reqopt("", "sysroot-base", "directory containing the compiler sysroot", "PATH")
83        .reqopt("", "stage", "stage number under test", "N")
84        .reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET")
85        .reqopt(
86            "",
87            "mode",
88            "which sort of compile tests to run",
89            "pretty | debug-info | codegen | rustdoc \
90            | rustdoc-json | codegen-units | incremental | run-make | ui \
91            | rustdoc-js | mir-opt | assembly | crashes",
92        )
93        .reqopt(
94            "",
95            "suite",
96            "which suite of compile tests to run. used for nicer error reporting.",
97            "SUITE",
98        )
99        .optopt(
100            "",
101            "pass",
102            "force {check,build,run}-pass tests to this mode.",
103            "check | build | run",
104        )
105        .optopt("", "run", "whether to execute run-* tests", "auto | always | never")
106        .optflag("", "ignored", "run tests marked as ignored")
107        .optflag("", "has-enzyme", "run tests that require enzyme")
108        .optflag("", "with-rustc-debug-assertions", "whether rustc was built with debug assertions")
109        .optflag("", "with-std-debug-assertions", "whether std was built with debug assertions")
110        .optmulti(
111            "",
112            "skip",
113            "skip tests matching SUBSTRING. Can be passed multiple times",
114            "SUBSTRING",
115        )
116        .optflag("", "exact", "filters match exactly")
117        .optopt(
118            "",
119            "runner",
120            "supervisor program to run tests under \
121             (eg. emulator, valgrind)",
122            "PROGRAM",
123        )
124        .optmulti("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS")
125        .optmulti("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS")
126        .optflag(
127            "",
128            "rust-randomized-layout",
129            "set this when rustc/stdlib were compiled with randomized layouts",
130        )
131        .optflag("", "optimize-tests", "run tests with optimizations enabled")
132        .optflag("", "verbose", "run tests verbosely, showing all output")
133        .optflag(
134            "",
135            "bless",
136            "overwrite stderr/stdout files instead of complaining about a mismatch",
137        )
138        .optflag("", "fail-fast", "stop as soon as possible after any test fails")
139        .optflag("", "quiet", "print one character per test instead of one line")
140        .optopt("", "color", "coloring: auto, always, never", "WHEN")
141        .optflag("", "json", "emit json output instead of plaintext output")
142        .optopt("", "target", "the target to build for", "TARGET")
143        .optopt("", "host", "the host to build for", "HOST")
144        .optopt("", "cdb", "path to CDB to use for CDB debuginfo tests", "PATH")
145        .optopt("", "gdb", "path to GDB to use for GDB debuginfo tests", "PATH")
146        .optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING")
147        .optopt("", "llvm-version", "the version of LLVM used", "VERSION STRING")
148        .optflag("", "system-llvm", "is LLVM the system LLVM")
149        .optopt("", "android-cross-path", "Android NDK standalone path", "PATH")
150        .optopt("", "adb-path", "path to the android debugger", "PATH")
151        .optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH")
152        .optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH")
153        .reqopt("", "cc", "path to a C compiler", "PATH")
154        .reqopt("", "cxx", "path to a C++ compiler", "PATH")
155        .reqopt("", "cflags", "flags for the C compiler", "FLAGS")
156        .reqopt("", "cxxflags", "flags for the CXX compiler", "FLAGS")
157        .optopt("", "ar", "path to an archiver", "PATH")
158        .optopt("", "target-linker", "path to a linker for the target", "PATH")
159        .optopt("", "host-linker", "path to a linker for the host", "PATH")
160        .reqopt("", "llvm-components", "list of LLVM components built in", "LIST")
161        .optopt("", "llvm-bin-dir", "Path to LLVM's `bin` directory", "PATH")
162        .optopt("", "nodejs", "the name of nodejs", "PATH")
163        .optopt("", "npm", "the name of npm", "PATH")
164        .optopt("", "remote-test-client", "path to the remote test client", "PATH")
165        .optopt(
166            "",
167            "compare-mode",
168            "mode describing what file the actual ui output will be compared to",
169            "COMPARE MODE",
170        )
171        .optflag(
172            "",
173            "rustfix-coverage",
174            "enable this to generate a Rustfix coverage file, which is saved in \
175            `./<build_test_suite_root>/rustfix_missing_coverage.txt`",
176        )
177        .optflag("", "force-rerun", "rerun tests even if the inputs are unchanged")
178        .optflag("", "only-modified", "only run tests that result been modified")
179        // FIXME: Temporarily retained so we can point users to `--no-capture`
180        .optflag("", "nocapture", "")
181        .optflag("", "no-capture", "don't capture stdout/stderr of tests")
182        .optflag("", "profiler-runtime", "is the profiler runtime enabled for this target")
183        .optflag("h", "help", "show this message")
184        .reqopt("", "channel", "current Rust channel", "CHANNEL")
185        .optflag(
186            "",
187            "git-hash",
188            "run tests which rely on commit version being compiled into the binaries",
189        )
190        .optopt("", "edition", "default Rust edition", "EDITION")
191        .reqopt("", "nightly-branch", "name of the git branch for nightly", "BRANCH")
192        .reqopt(
193            "",
194            "git-merge-commit-email",
195            "email address used for finding merge commits",
196            "EMAIL",
197        )
198        .optopt(
199            "",
200            "compiletest-diff-tool",
201            "What custom diff tool to use for displaying compiletest tests.",
202            "COMMAND",
203        )
204        .reqopt("", "minicore-path", "path to minicore aux library", "PATH")
205        .optflag("N", "no-new-executor", "disables the new test executor, and uses libtest instead")
206        .optopt(
207            "",
208            "debugger",
209            "only test a specific debugger in debuginfo tests",
210            "gdb | lldb | cdb",
211        );
212
213    let (argv0, args_) = args.split_first().unwrap();
214    if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
215        let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
216        println!("{}", opts.usage(&message));
217        println!();
218        panic!()
219    }
220
221    let matches = &match opts.parse(args_) {
222        Ok(m) => m,
223        Err(f) => panic!("{:?}", f),
224    };
225
226    if matches.opt_present("h") || matches.opt_present("help") {
227        let message = format!("Usage: {} [OPTIONS]  [TESTNAME...]", argv0);
228        println!("{}", opts.usage(&message));
229        println!();
230        panic!()
231    }
232
233    fn make_absolute(path: Utf8PathBuf) -> Utf8PathBuf {
234        if path.is_relative() {
235            Utf8PathBuf::try_from(env::current_dir().unwrap()).unwrap().join(path)
236        } else {
237            path
238        }
239    }
240
241    fn opt_path(m: &getopts::Matches, nm: &str) -> Utf8PathBuf {
242        match m.opt_str(nm) {
243            Some(s) => Utf8PathBuf::from(&s),
244            None => panic!("no option (=path) found for {}", nm),
245        }
246    }
247
248    let target = opt_str2(matches.opt_str("target"));
249    let android_cross_path = opt_path(matches, "android-cross-path");
250    let (cdb, cdb_version) = debuggers::analyze_cdb(matches.opt_str("cdb"), &target);
251    let (gdb, gdb_version) =
252        debuggers::analyze_gdb(matches.opt_str("gdb"), &target, &android_cross_path);
253    let lldb_version =
254        matches.opt_str("lldb-version").as_deref().and_then(debuggers::extract_lldb_version);
255    let color = match matches.opt_str("color").as_deref() {
256        Some("auto") | None => ColorConfig::AutoColor,
257        Some("always") => ColorConfig::AlwaysColor,
258        Some("never") => ColorConfig::NeverColor,
259        Some(x) => panic!("argument for --color must be auto, always, or never, but found `{}`", x),
260    };
261    let llvm_version =
262        matches.opt_str("llvm-version").as_deref().map(header::extract_llvm_version).or_else(
263            || header::extract_llvm_version_from_binary(&matches.opt_str("llvm-filecheck")?),
264        );
265
266    let run_ignored = matches.opt_present("ignored");
267    let with_rustc_debug_assertions = matches.opt_present("with-rustc-debug-assertions");
268    let with_std_debug_assertions = matches.opt_present("with-std-debug-assertions");
269    let mode = matches.opt_str("mode").unwrap().parse().expect("invalid mode");
270    let has_html_tidy = if mode == Mode::Rustdoc {
271        Command::new("tidy")
272            .arg("--version")
273            .stdout(Stdio::null())
274            .status()
275            .map_or(false, |status| status.success())
276    } else {
277        // Avoid spawning an external command when we know html-tidy won't be used.
278        false
279    };
280    let has_enzyme = matches.opt_present("has-enzyme");
281    let filters = if mode == Mode::RunMake {
282        matches
283            .free
284            .iter()
285            .map(|f| {
286                let path = Utf8Path::new(f);
287                let mut iter = path.iter().skip(1);
288
289                // We skip the test folder and check if the user passed `rmake.rs`.
290                if iter.next().is_some_and(|s| s == "rmake.rs") && iter.next().is_none() {
291                    path.parent().unwrap().to_string()
292                } else {
293                    f.to_string()
294                }
295            })
296            .collect::<Vec<_>>()
297    } else {
298        matches.free.clone()
299    };
300    let compare_mode = matches.opt_str("compare-mode").map(|s| {
301        s.parse().unwrap_or_else(|_| {
302            let variants: Vec<_> = CompareMode::STR_VARIANTS.iter().copied().collect();
303            panic!(
304                "`{s}` is not a valid value for `--compare-mode`, it should be one of: {}",
305                variants.join(", ")
306            );
307        })
308    });
309    if matches.opt_present("nocapture") {
310        panic!("`--nocapture` is deprecated; please use `--no-capture`");
311    }
312
313    let stage = match matches.opt_str("stage") {
314        Some(stage) => stage.parse::<u32>().expect("expected `--stage` to be an unsigned integer"),
315        None => panic!("`--stage` is required"),
316    };
317
318    let src_root = opt_path(matches, "src-root");
319    let src_test_suite_root = opt_path(matches, "src-test-suite-root");
320    assert!(
321        src_test_suite_root.starts_with(&src_root),
322        "`src-root` must be a parent of `src-test-suite-root`: `src-root`=`{}`, `src-test-suite-root` = `{}`",
323        src_root,
324        src_test_suite_root
325    );
326
327    let build_root = opt_path(matches, "build-root");
328    let build_test_suite_root = opt_path(matches, "build-test-suite-root");
329    assert!(build_test_suite_root.starts_with(&build_root));
330
331    Config {
332        bless: matches.opt_present("bless"),
333        fail_fast: matches.opt_present("fail-fast")
334            || env::var_os("RUSTC_TEST_FAIL_FAST").is_some(),
335
336        compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
337        run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
338        rustc_path: opt_path(matches, "rustc-path"),
339        cargo_path: matches.opt_str("cargo-path").map(Utf8PathBuf::from),
340        stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(Utf8PathBuf::from),
341        rustdoc_path: matches.opt_str("rustdoc-path").map(Utf8PathBuf::from),
342        coverage_dump_path: matches.opt_str("coverage-dump-path").map(Utf8PathBuf::from),
343        python: matches.opt_str("python").unwrap(),
344        jsondocck_path: matches.opt_str("jsondocck-path"),
345        jsondoclint_path: matches.opt_str("jsondoclint-path"),
346        run_clang_based_tests_with: matches.opt_str("run-clang-based-tests-with"),
347        llvm_filecheck: matches.opt_str("llvm-filecheck").map(Utf8PathBuf::from),
348        llvm_bin_dir: matches.opt_str("llvm-bin-dir").map(Utf8PathBuf::from),
349
350        src_root,
351        src_test_suite_root,
352
353        build_root,
354        build_test_suite_root,
355
356        sysroot_base: opt_path(matches, "sysroot-base"),
357
358        stage,
359        stage_id: matches.opt_str("stage-id").unwrap(),
360
361        mode,
362        suite: matches.opt_str("suite").unwrap(),
363        debugger: matches.opt_str("debugger").map(|debugger| {
364            debugger
365                .parse::<Debugger>()
366                .unwrap_or_else(|_| panic!("unknown `--debugger` option `{debugger}` given"))
367        }),
368        run_ignored,
369        with_rustc_debug_assertions,
370        with_std_debug_assertions,
371        filters,
372        skip: matches.opt_strs("skip"),
373        filter_exact: matches.opt_present("exact"),
374        force_pass_mode: matches.opt_str("pass").map(|mode| {
375            mode.parse::<PassMode>()
376                .unwrap_or_else(|_| panic!("unknown `--pass` option `{}` given", mode))
377        }),
378        run: matches.opt_str("run").and_then(|mode| match mode.as_str() {
379            "auto" => None,
380            "always" => Some(true),
381            "never" => Some(false),
382            _ => panic!("unknown `--run` option `{}` given", mode),
383        }),
384        runner: matches.opt_str("runner"),
385        host_rustcflags: matches.opt_strs("host-rustcflags"),
386        target_rustcflags: matches.opt_strs("target-rustcflags"),
387        optimize_tests: matches.opt_present("optimize-tests"),
388        rust_randomized_layout: matches.opt_present("rust-randomized-layout"),
389        target,
390        host: opt_str2(matches.opt_str("host")),
391        cdb,
392        cdb_version,
393        gdb,
394        gdb_version,
395        lldb_version,
396        llvm_version,
397        system_llvm: matches.opt_present("system-llvm"),
398        android_cross_path,
399        adb_path: opt_str2(matches.opt_str("adb-path")),
400        adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")),
401        adb_device_status: opt_str2(matches.opt_str("target")).contains("android")
402            && "(none)" != opt_str2(matches.opt_str("adb-test-dir"))
403            && !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
404        lldb_python_dir: matches.opt_str("lldb-python-dir"),
405        verbose: matches.opt_present("verbose"),
406        format: match (matches.opt_present("quiet"), matches.opt_present("json")) {
407            (true, true) => panic!("--quiet and --json are incompatible"),
408            (true, false) => OutputFormat::Terse,
409            (false, true) => OutputFormat::Json,
410            (false, false) => OutputFormat::Pretty,
411        },
412        only_modified: matches.opt_present("only-modified"),
413        color,
414        remote_test_client: matches.opt_str("remote-test-client").map(Utf8PathBuf::from),
415        compare_mode,
416        rustfix_coverage: matches.opt_present("rustfix-coverage"),
417        has_html_tidy,
418        has_enzyme,
419        channel: matches.opt_str("channel").unwrap(),
420        git_hash: matches.opt_present("git-hash"),
421        edition: matches.opt_str("edition"),
422
423        cc: matches.opt_str("cc").unwrap(),
424        cxx: matches.opt_str("cxx").unwrap(),
425        cflags: matches.opt_str("cflags").unwrap(),
426        cxxflags: matches.opt_str("cxxflags").unwrap(),
427        ar: matches.opt_str("ar").unwrap_or_else(|| String::from("ar")),
428        target_linker: matches.opt_str("target-linker"),
429        host_linker: matches.opt_str("host-linker"),
430        llvm_components: matches.opt_str("llvm-components").unwrap(),
431        nodejs: matches.opt_str("nodejs"),
432        npm: matches.opt_str("npm"),
433
434        force_rerun: matches.opt_present("force-rerun"),
435
436        target_cfgs: OnceLock::new(),
437        builtin_cfg_names: OnceLock::new(),
438        supported_crate_types: OnceLock::new(),
439
440        nocapture: matches.opt_present("no-capture"),
441
442        nightly_branch: matches.opt_str("nightly-branch").unwrap(),
443        git_merge_commit_email: matches.opt_str("git-merge-commit-email").unwrap(),
444
445        profiler_runtime: matches.opt_present("profiler-runtime"),
446
447        diff_command: matches.opt_str("compiletest-diff-tool"),
448
449        minicore_path: opt_path(matches, "minicore-path"),
450    }
451}
452
453pub fn log_config(config: &Config) {
454    let c = config;
455    logv(c, "configuration:".to_string());
456    logv(c, format!("compile_lib_path: {}", config.compile_lib_path));
457    logv(c, format!("run_lib_path: {}", config.run_lib_path));
458    logv(c, format!("rustc_path: {}", config.rustc_path));
459    logv(c, format!("cargo_path: {:?}", config.cargo_path));
460    logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path));
461
462    logv(c, format!("src_root: {}", config.src_root));
463    logv(c, format!("src_test_suite_root: {}", config.src_test_suite_root));
464
465    logv(c, format!("build_root: {}", config.build_root));
466    logv(c, format!("build_test_suite_root: {}", config.build_test_suite_root));
467
468    logv(c, format!("sysroot_base: {}", config.sysroot_base));
469
470    logv(c, format!("stage: {}", config.stage));
471    logv(c, format!("stage_id: {}", config.stage_id));
472    logv(c, format!("mode: {}", config.mode));
473    logv(c, format!("run_ignored: {}", config.run_ignored));
474    logv(c, format!("filters: {:?}", config.filters));
475    logv(c, format!("skip: {:?}", config.skip));
476    logv(c, format!("filter_exact: {}", config.filter_exact));
477    logv(
478        c,
479        format!("force_pass_mode: {}", opt_str(&config.force_pass_mode.map(|m| format!("{}", m))),),
480    );
481    logv(c, format!("runner: {}", opt_str(&config.runner)));
482    logv(c, format!("host-rustcflags: {:?}", config.host_rustcflags));
483    logv(c, format!("target-rustcflags: {:?}", config.target_rustcflags));
484    logv(c, format!("target: {}", config.target));
485    logv(c, format!("host: {}", config.host));
486    logv(c, format!("android-cross-path: {}", config.android_cross_path));
487    logv(c, format!("adb_path: {}", config.adb_path));
488    logv(c, format!("adb_test_dir: {}", config.adb_test_dir));
489    logv(c, format!("adb_device_status: {}", config.adb_device_status));
490    logv(c, format!("ar: {}", config.ar));
491    logv(c, format!("target-linker: {:?}", config.target_linker));
492    logv(c, format!("host-linker: {:?}", config.host_linker));
493    logv(c, format!("verbose: {}", config.verbose));
494    logv(c, format!("format: {:?}", config.format));
495    logv(c, format!("minicore_path: {}", config.minicore_path));
496    logv(c, "\n".to_string());
497}
498
499pub fn opt_str(maybestr: &Option<String>) -> &str {
500    match *maybestr {
501        None => "(none)",
502        Some(ref s) => s,
503    }
504}
505
506pub fn opt_str2(maybestr: Option<String>) -> String {
507    match maybestr {
508        None => "(none)".to_owned(),
509        Some(s) => s,
510    }
511}
512
513/// Called by `main` after the config has been parsed.
514pub fn run_tests(config: Arc<Config>) {
515    // If we want to collect rustfix coverage information,
516    // we first make sure that the coverage file does not exist.
517    // It will be created later on.
518    if config.rustfix_coverage {
519        let mut coverage_file_path = config.build_test_suite_root.clone();
520        coverage_file_path.push("rustfix_missing_coverage.txt");
521        if coverage_file_path.exists() {
522            if let Err(e) = fs::remove_file(&coverage_file_path) {
523                panic!("Could not delete {} due to {}", coverage_file_path, e)
524            }
525        }
526    }
527
528    // sadly osx needs some file descriptor limits raised for running tests in
529    // parallel (especially when we have lots and lots of child processes).
530    // For context, see #8904
531    unsafe {
532        raise_fd_limit::raise_fd_limit();
533    }
534    // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
535    // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
536    //
537    // SAFETY: at this point we're still single-threaded.
538    unsafe { env::set_var("__COMPAT_LAYER", "RunAsInvoker") };
539
540    // Let tests know which target they're running as.
541    //
542    // SAFETY: at this point we're still single-threaded.
543    unsafe { env::set_var("TARGET", &config.target) };
544
545    let mut configs = Vec::new();
546    if let Mode::DebugInfo = config.mode {
547        // Debugging emscripten code doesn't make sense today
548        if !config.target.contains("emscripten") {
549            match config.debugger {
550                Some(Debugger::Cdb) => configs.extend(debuggers::configure_cdb(&config)),
551                Some(Debugger::Gdb) => configs.extend(debuggers::configure_gdb(&config)),
552                Some(Debugger::Lldb) => configs.extend(debuggers::configure_lldb(&config)),
553                None => {
554                    configs.extend(debuggers::configure_cdb(&config));
555                    configs.extend(debuggers::configure_gdb(&config));
556                    configs.extend(debuggers::configure_lldb(&config));
557                }
558            }
559        }
560    } else {
561        configs.push(config.clone());
562    };
563
564    // Discover all of the tests in the test suite directory, and build a libtest
565    // structure for each test (or each revision of a multi-revision test).
566    let mut tests = Vec::new();
567    for c in configs {
568        tests.extend(collect_and_make_tests(c));
569    }
570
571    tests.sort_by(|a, b| Ord::cmp(&a.desc.name, &b.desc.name));
572
573    // Delegate to the executor to filter and run the big list of test structures
574    // created during test discovery. When the executor decides to run a test,
575    // it will return control to the rest of compiletest by calling `runtest::run`.
576    // FIXME(Zalathar): Once we're confident that we won't need to revert the
577    // removal of the libtest-based executor, remove this Result and other
578    // remnants of the old executor.
579    let res: io::Result<bool> = Ok(executor::run_tests(&config, tests));
580
581    // Check the outcome reported by libtest.
582    match res {
583        Ok(true) => {}
584        Ok(false) => {
585            // We want to report that the tests failed, but we also want to give
586            // some indication of just what tests we were running. Especially on
587            // CI, where there can be cross-compiled tests for a lot of
588            // architectures, without this critical information it can be quite
589            // easy to miss which tests failed, and as such fail to reproduce
590            // the failure locally.
591
592            let mut msg = String::from("Some tests failed in compiletest");
593            write!(msg, " suite={}", config.suite).unwrap();
594
595            if let Some(compare_mode) = config.compare_mode.as_ref() {
596                write!(msg, " compare_mode={}", compare_mode).unwrap();
597            }
598
599            if let Some(pass_mode) = config.force_pass_mode.as_ref() {
600                write!(msg, " pass_mode={}", pass_mode).unwrap();
601            }
602
603            write!(msg, " mode={}", config.mode).unwrap();
604            write!(msg, " host={}", config.host).unwrap();
605            write!(msg, " target={}", config.target).unwrap();
606
607            println!("{msg}");
608
609            std::process::exit(1);
610        }
611        Err(e) => {
612            // We don't know if tests passed or not, but if there was an error
613            // during testing we don't want to just succeed (we may not have
614            // tested something), so fail.
615            //
616            // This should realistically "never" happen, so don't try to make
617            // this a pretty error message.
618            panic!("I/O failure during tests: {:?}", e);
619        }
620    }
621}
622
623/// Read-only context data used during test collection.
624struct TestCollectorCx {
625    config: Arc<Config>,
626    cache: HeadersCache,
627    common_inputs_stamp: Stamp,
628    modified_tests: Vec<Utf8PathBuf>,
629}
630
631/// Mutable state used during test collection.
632struct TestCollector {
633    tests: Vec<CollectedTest>,
634    found_path_stems: HashSet<Utf8PathBuf>,
635    poisoned: bool,
636}
637
638impl TestCollector {
639    fn new() -> Self {
640        TestCollector { tests: vec![], found_path_stems: HashSet::new(), poisoned: false }
641    }
642
643    fn merge(&mut self, mut other: Self) {
644        self.tests.append(&mut other.tests);
645        self.found_path_stems.extend(other.found_path_stems);
646        self.poisoned |= other.poisoned;
647    }
648}
649
650/// Creates test structures for every test/revision in the test suite directory.
651///
652/// This always inspects _all_ test files in the suite (e.g. all 17k+ ui tests),
653/// regardless of whether any filters/tests were specified on the command-line,
654/// because filtering is handled later by libtest.
655pub(crate) fn collect_and_make_tests(config: Arc<Config>) -> Vec<CollectedTest> {
656    debug!("making tests from {}", config.src_test_suite_root);
657    let common_inputs_stamp = common_inputs_stamp(&config);
658    let modified_tests =
659        modified_tests(&config, &config.src_test_suite_root).unwrap_or_else(|err| {
660            panic!(
661                "modified_tests got error from dir: {}, error: {}",
662                config.src_test_suite_root, err
663            )
664        });
665    let cache = HeadersCache::load(&config);
666
667    let cx = TestCollectorCx { config, cache, common_inputs_stamp, modified_tests };
668    let collector = collect_tests_from_dir(&cx, &cx.config.src_test_suite_root, Utf8Path::new(""))
669        .unwrap_or_else(|reason| {
670            panic!("Could not read tests from {}: {reason}", cx.config.src_test_suite_root)
671        });
672
673    let TestCollector { tests, found_path_stems, poisoned } = collector;
674
675    if poisoned {
676        eprintln!();
677        panic!("there are errors in tests");
678    }
679
680    check_for_overlapping_test_paths(&found_path_stems);
681
682    tests
683}
684
685/// Returns the most recent last-modified timestamp from among the input files
686/// that are considered relevant to all tests (e.g. the compiler, std, and
687/// compiletest itself).
688///
689/// (Some of these inputs aren't actually relevant to _all_ tests, but they are
690/// common to some subset of tests, and are hopefully unlikely to be modified
691/// while working on other tests.)
692fn common_inputs_stamp(config: &Config) -> Stamp {
693    let src_root = &config.src_root;
694
695    let mut stamp = Stamp::from_path(&config.rustc_path);
696
697    // Relevant pretty printer files
698    let pretty_printer_files = [
699        "src/etc/rust_types.py",
700        "src/etc/gdb_load_rust_pretty_printers.py",
701        "src/etc/gdb_lookup.py",
702        "src/etc/gdb_providers.py",
703        "src/etc/lldb_batchmode.py",
704        "src/etc/lldb_lookup.py",
705        "src/etc/lldb_providers.py",
706    ];
707    for file in &pretty_printer_files {
708        let path = src_root.join(file);
709        stamp.add_path(&path);
710    }
711
712    stamp.add_dir(&src_root.join("src/etc/natvis"));
713
714    stamp.add_dir(&config.run_lib_path);
715
716    if let Some(ref rustdoc_path) = config.rustdoc_path {
717        stamp.add_path(&rustdoc_path);
718        stamp.add_path(&src_root.join("src/etc/htmldocck.py"));
719    }
720
721    // Re-run coverage tests if the `coverage-dump` tool was modified,
722    // because its output format might have changed.
723    if let Some(coverage_dump_path) = &config.coverage_dump_path {
724        stamp.add_path(coverage_dump_path)
725    }
726
727    stamp.add_dir(&src_root.join("src/tools/run-make-support"));
728
729    // Compiletest itself.
730    stamp.add_dir(&src_root.join("src/tools/compiletest"));
731
732    stamp
733}
734
735/// Returns a list of modified/untracked test files that should be run when
736/// the `--only-modified` flag is in use.
737///
738/// (Might be inaccurate in some cases.)
739fn modified_tests(config: &Config, dir: &Utf8Path) -> Result<Vec<Utf8PathBuf>, String> {
740    // If `--only-modified` wasn't passed, the list of modified tests won't be
741    // used for anything, so avoid some work and just return an empty list.
742    if !config.only_modified {
743        return Ok(vec![]);
744    }
745
746    let files = get_git_modified_files(
747        &config.git_config(),
748        Some(dir.as_std_path()),
749        &vec!["rs", "stderr", "fixed"],
750    )?;
751    // Add new test cases to the list, it will be convenient in daily development.
752    let untracked_files = get_git_untracked_files(Some(dir.as_std_path()))?.unwrap_or(vec![]);
753
754    let all_paths = [&files[..], &untracked_files[..]].concat();
755    let full_paths = {
756        let mut full_paths: Vec<Utf8PathBuf> = all_paths
757            .into_iter()
758            .map(|f| Utf8PathBuf::from(f).with_extension("").with_extension("rs"))
759            .filter_map(
760                |f| if Utf8Path::new(&f).exists() { f.canonicalize_utf8().ok() } else { None },
761            )
762            .collect();
763        full_paths.dedup();
764        full_paths.sort_unstable();
765        full_paths
766    };
767    Ok(full_paths)
768}
769
770/// Recursively scans a directory to find test files and create test structures
771/// that will be handed over to libtest.
772fn collect_tests_from_dir(
773    cx: &TestCollectorCx,
774    dir: &Utf8Path,
775    relative_dir_path: &Utf8Path,
776) -> io::Result<TestCollector> {
777    // Ignore directories that contain a file named `compiletest-ignore-dir`.
778    if dir.join("compiletest-ignore-dir").exists() {
779        return Ok(TestCollector::new());
780    }
781
782    // For run-make tests, a "test file" is actually a directory that contains an `rmake.rs`.
783    if cx.config.mode == Mode::RunMake {
784        let mut collector = TestCollector::new();
785        if dir.join("rmake.rs").exists() {
786            let paths = TestPaths {
787                file: dir.to_path_buf(),
788                relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
789            };
790            make_test(cx, &mut collector, &paths);
791            // This directory is a test, so don't try to find other tests inside it.
792            return Ok(collector);
793        }
794    }
795
796    // If we find a test foo/bar.rs, we have to build the
797    // output directory `$build/foo` so we can write
798    // `$build/foo/bar` into it. We do this *now* in this
799    // sequential loop because otherwise, if we do it in the
800    // tests themselves, they race for the privilege of
801    // creating the directories and sometimes fail randomly.
802    let build_dir = output_relative_path(&cx.config, relative_dir_path);
803    fs::create_dir_all(&build_dir).unwrap();
804
805    // Add each `.rs` file as a test, and recurse further on any
806    // subdirectories we find, except for `auxiliary` directories.
807    // FIXME: this walks full tests tree, even if we have something to ignore
808    // use walkdir/ignore like in tidy?
809    fs::read_dir(dir.as_std_path())?
810        .par_bridge()
811        .map(|file| {
812            let mut collector = TestCollector::new();
813            let file = file?;
814            let file_path = Utf8PathBuf::try_from(file.path()).unwrap();
815            let file_name = file_path.file_name().unwrap();
816
817            if is_test(file_name)
818                && (!cx.config.only_modified || cx.modified_tests.contains(&file_path))
819            {
820                // We found a test file, so create the corresponding libtest structures.
821                debug!(%file_path, "found test file");
822
823                // Record the stem of the test file, to check for overlaps later.
824                let rel_test_path = relative_dir_path.join(file_path.file_stem().unwrap());
825                collector.found_path_stems.insert(rel_test_path);
826
827                let paths =
828                    TestPaths { file: file_path, relative_dir: relative_dir_path.to_path_buf() };
829                make_test(cx, &mut collector, &paths);
830            } else if file_path.is_dir() {
831                // Recurse to find more tests in a subdirectory.
832                let relative_file_path = relative_dir_path.join(file_name);
833                if file_name != "auxiliary" {
834                    debug!(%file_path, "found directory");
835                    collector.merge(collect_tests_from_dir(cx, &file_path, &relative_file_path)?);
836                }
837            } else {
838                debug!(%file_path, "found other file/directory");
839            }
840            Ok(collector)
841        })
842        .reduce(
843            || Ok(TestCollector::new()),
844            |a, b| {
845                let mut a = a?;
846                a.merge(b?);
847                Ok(a)
848            },
849        )
850}
851
852/// Returns true if `file_name` looks like a proper test file name.
853pub fn is_test(file_name: &str) -> bool {
854    if !file_name.ends_with(".rs") {
855        return false;
856    }
857
858    // `.`, `#`, and `~` are common temp-file prefixes.
859    let invalid_prefixes = &[".", "#", "~"];
860    !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
861}
862
863/// For a single test file, creates one or more test structures (one per revision) that can be
864/// handed over to libtest to run, possibly in parallel.
865fn make_test(cx: &TestCollectorCx, collector: &mut TestCollector, testpaths: &TestPaths) {
866    // For run-make tests, each "test file" is actually a _directory_ containing an `rmake.rs`. But
867    // for the purposes of directive parsing, we want to look at that recipe file, not the directory
868    // itself.
869    let test_path = if cx.config.mode == Mode::RunMake {
870        testpaths.file.join("rmake.rs")
871    } else {
872        testpaths.file.clone()
873    };
874
875    // Scan the test file to discover its revisions, if any.
876    let early_props = EarlyProps::from_file(&cx.config, &test_path);
877
878    // Normally we create one libtest structure per revision, with two exceptions:
879    // - If a test doesn't use revisions, create a dummy revision (None) so that
880    //   the test can still run.
881    // - Incremental tests inherently can't run their revisions in parallel, so
882    //   we treat them like non-revisioned tests here. Incremental revisions are
883    //   handled internally by `runtest::run` instead.
884    let revisions = if early_props.revisions.is_empty() || cx.config.mode == Mode::Incremental {
885        vec![None]
886    } else {
887        early_props.revisions.iter().map(|r| Some(r.as_str())).collect()
888    };
889
890    // For each revision (or the sole dummy revision), create and append a
891    // `CollectedTest` that can be handed over to the test executor.
892    collector.tests.extend(revisions.into_iter().map(|revision| {
893        // Create a test name and description to hand over to libtest.
894        let src_file = fs::File::open(&test_path).expect("open test file to parse ignores");
895        let test_name = make_test_name(&cx.config, testpaths, revision);
896        // Create a libtest description for the test/revision.
897        // This is where `ignore-*`/`only-*`/`needs-*` directives are handled,
898        // because they need to set the libtest ignored flag.
899        let mut desc = make_test_description(
900            &cx.config,
901            &cx.cache,
902            test_name,
903            &test_path,
904            src_file,
905            revision,
906            &mut collector.poisoned,
907        );
908
909        // If a test's inputs haven't changed since the last time it ran,
910        // mark it as ignored so that libtest will skip it.
911        if !cx.config.force_rerun && is_up_to_date(cx, testpaths, &early_props, revision) {
912            desc.ignore = true;
913            // Keep this in sync with the "up-to-date" message detected by bootstrap.
914            desc.ignore_message = Some("up-to-date".into());
915        }
916
917        let config = Arc::clone(&cx.config);
918        let testpaths = testpaths.clone();
919        let revision = revision.map(str::to_owned);
920
921        CollectedTest { desc, config, testpaths, revision }
922    }));
923}
924
925/// The path of the `stamp` file that gets created or updated whenever a
926/// particular test completes successfully.
927fn stamp_file_path(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> Utf8PathBuf {
928    output_base_dir(config, testpaths, revision).join("stamp")
929}
930
931/// Returns a list of files that, if modified, would cause this test to no
932/// longer be up-to-date.
933///
934/// (Might be inaccurate in some cases.)
935fn files_related_to_test(
936    config: &Config,
937    testpaths: &TestPaths,
938    props: &EarlyProps,
939    revision: Option<&str>,
940) -> Vec<Utf8PathBuf> {
941    let mut related = vec![];
942
943    if testpaths.file.is_dir() {
944        // run-make tests use their individual directory
945        for entry in WalkDir::new(&testpaths.file) {
946            let path = entry.unwrap().into_path();
947            if path.is_file() {
948                related.push(Utf8PathBuf::try_from(path).unwrap());
949            }
950        }
951    } else {
952        related.push(testpaths.file.clone());
953    }
954
955    for aux in props.aux.all_aux_path_strings() {
956        // FIXME(Zalathar): Perform all `auxiliary` path resolution in one place.
957        let path = testpaths.file.parent().unwrap().join("auxiliary").join(aux);
958        related.push(path);
959    }
960
961    // UI test files.
962    for extension in UI_EXTENSIONS {
963        let path = expected_output_path(testpaths, revision, &config.compare_mode, extension);
964        related.push(path);
965    }
966
967    // `minicore.rs` test auxiliary: we need to make sure tests get rerun if this changes.
968    related.push(config.src_root.join("tests").join("auxiliary").join("minicore.rs"));
969
970    related
971}
972
973/// Checks whether a particular test/revision is "up-to-date", meaning that no
974/// relevant files/settings have changed since the last time the test succeeded.
975///
976/// (This is not very reliable in some circumstances, so the `--force-rerun`
977/// flag can be used to ignore up-to-date checking and always re-run tests.)
978fn is_up_to_date(
979    cx: &TestCollectorCx,
980    testpaths: &TestPaths,
981    props: &EarlyProps,
982    revision: Option<&str>,
983) -> bool {
984    let stamp_file_path = stamp_file_path(&cx.config, testpaths, revision);
985    // Check the config hash inside the stamp file.
986    let contents = match fs::read_to_string(&stamp_file_path) {
987        Ok(f) => f,
988        Err(ref e) if e.kind() == ErrorKind::InvalidData => panic!("Can't read stamp contents"),
989        // The test hasn't succeeded yet, so it is not up-to-date.
990        Err(_) => return false,
991    };
992    let expected_hash = runtest::compute_stamp_hash(&cx.config);
993    if contents != expected_hash {
994        // Some part of compiletest configuration has changed since the test
995        // last succeeded, so it is not up-to-date.
996        return false;
997    }
998
999    // Check the timestamp of the stamp file against the last modified time
1000    // of all files known to be relevant to the test.
1001    let mut inputs_stamp = cx.common_inputs_stamp.clone();
1002    for path in files_related_to_test(&cx.config, testpaths, props, revision) {
1003        inputs_stamp.add_path(&path);
1004    }
1005
1006    // If no relevant files have been modified since the stamp file was last
1007    // written, the test is up-to-date.
1008    inputs_stamp < Stamp::from_path(&stamp_file_path)
1009}
1010
1011/// The maximum of a set of file-modified timestamps.
1012#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
1013struct Stamp {
1014    time: SystemTime,
1015}
1016
1017impl Stamp {
1018    /// Creates a timestamp holding the last-modified time of the specified file.
1019    fn from_path(path: &Utf8Path) -> Self {
1020        let mut stamp = Stamp { time: SystemTime::UNIX_EPOCH };
1021        stamp.add_path(path);
1022        stamp
1023    }
1024
1025    /// Updates this timestamp to the last-modified time of the specified file,
1026    /// if it is later than the currently-stored timestamp.
1027    fn add_path(&mut self, path: &Utf8Path) {
1028        let modified = fs::metadata(path.as_std_path())
1029            .and_then(|metadata| metadata.modified())
1030            .unwrap_or(SystemTime::UNIX_EPOCH);
1031        self.time = self.time.max(modified);
1032    }
1033
1034    /// Updates this timestamp to the most recent last-modified time of all files
1035    /// recursively contained in the given directory, if it is later than the
1036    /// currently-stored timestamp.
1037    fn add_dir(&mut self, path: &Utf8Path) {
1038        let path = path.as_std_path();
1039        for entry in WalkDir::new(path) {
1040            let entry = entry.unwrap();
1041            if entry.file_type().is_file() {
1042                let modified = entry
1043                    .metadata()
1044                    .ok()
1045                    .and_then(|metadata| metadata.modified().ok())
1046                    .unwrap_or(SystemTime::UNIX_EPOCH);
1047                self.time = self.time.max(modified);
1048            }
1049        }
1050    }
1051}
1052
1053/// Creates a name for this test/revision that can be handed over to libtest.
1054fn make_test_name(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> String {
1055    // Print the name of the file, relative to the sources root.
1056    let path = testpaths.file.strip_prefix(&config.src_root).unwrap();
1057    let debugger = match config.debugger {
1058        Some(d) => format!("-{}", d),
1059        None => String::new(),
1060    };
1061    let mode_suffix = match config.compare_mode {
1062        Some(ref mode) => format!(" ({})", mode.to_str()),
1063        None => String::new(),
1064    };
1065
1066    format!(
1067        "[{}{}{}] {}{}",
1068        config.mode,
1069        debugger,
1070        mode_suffix,
1071        path,
1072        revision.map_or("".to_string(), |rev| format!("#{}", rev))
1073    )
1074}
1075
1076/// Checks that test discovery didn't find any tests whose name stem is a prefix
1077/// of some other tests's name.
1078///
1079/// For example, suppose the test suite contains these two test files:
1080/// - `tests/rustdoc/primitive.rs`
1081/// - `tests/rustdoc/primitive/no_std.rs`
1082///
1083/// The test runner might put the output from those tests in these directories:
1084/// - `$build/test/rustdoc/primitive/`
1085/// - `$build/test/rustdoc/primitive/no_std/`
1086///
1087/// Because one output path is a subdirectory of the other, the two tests might
1088/// interfere with each other in unwanted ways, especially if the test runner
1089/// decides to delete test output directories to clean them between runs.
1090/// To avoid problems, we forbid test names from overlapping in this way.
1091///
1092/// See <https://github.com/rust-lang/rust/pull/109509> for more context.
1093fn check_for_overlapping_test_paths(found_path_stems: &HashSet<Utf8PathBuf>) {
1094    let mut collisions = Vec::new();
1095    for path in found_path_stems {
1096        for ancestor in path.ancestors().skip(1) {
1097            if found_path_stems.contains(ancestor) {
1098                collisions.push((path, ancestor));
1099            }
1100        }
1101    }
1102    if !collisions.is_empty() {
1103        collisions.sort();
1104        let collisions: String = collisions
1105            .into_iter()
1106            .map(|(path, check_parent)| format!("test {path} clashes with {check_parent}\n"))
1107            .collect();
1108        panic!(
1109            "{collisions}\n\
1110            Tests cannot have overlapping names. Make sure they use unique prefixes."
1111        );
1112    }
1113}