compiletest/
lib.rs

1#![crate_name = "compiletest"]
2// The `test` crate is the only unstable feature
3// allowed here, just to share similar code.
4#![feature(test)]
5
6extern crate test;
7
8#[cfg(test)]
9mod tests;
10
11pub mod common;
12pub mod compute_diff;
13mod debuggers;
14pub mod errors;
15pub mod header;
16mod json;
17mod raise_fd_limit;
18mod read2;
19pub mod runtest;
20pub mod util;
21
22use core::panic;
23use std::collections::HashSet;
24use std::ffi::OsString;
25use std::fmt::Write;
26use std::io::{self, ErrorKind};
27use std::path::{Path, PathBuf};
28use std::process::{Command, Stdio};
29use std::sync::{Arc, OnceLock};
30use std::time::SystemTime;
31use std::{env, fs, vec};
32
33use build_helper::git::{get_git_modified_files, get_git_untracked_files};
34use getopts::Options;
35use test::ColorConfig;
36use tracing::*;
37use walkdir::WalkDir;
38
39use self::header::{EarlyProps, make_test_description};
40use crate::common::{
41    CompareMode, Config, Debugger, Mode, PassMode, TestPaths, UI_EXTENSIONS, expected_output_path,
42    output_base_dir, output_relative_path,
43};
44use crate::header::HeadersCache;
45use crate::util::logv;
46
47/// Creates the `Config` instance for this invocation of compiletest.
48///
49/// The config mostly reflects command-line arguments, but there might also be
50/// some code here that inspects environment variables or even runs executables
51/// (e.g. when discovering debugger versions).
52pub fn parse_config(args: Vec<String>) -> Config {
53    let mut opts = Options::new();
54    opts.reqopt("", "compile-lib-path", "path to host shared libraries", "PATH")
55        .reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
56        .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
57        .optopt("", "cargo-path", "path to cargo to use for compiling", "PATH")
58        .optopt(
59            "",
60            "stage0-rustc-path",
61            "path to rustc to use for compiling run-make recipes",
62            "PATH",
63        )
64        .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
65        .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
66        .reqopt("", "python", "path to python to use for doc tests", "PATH")
67        .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
68        .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
69        .optopt("", "run-clang-based-tests-with", "path to Clang executable", "PATH")
70        .optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR")
71        .reqopt("", "src-root", "directory containing sources", "PATH")
72        .reqopt("", "src-test-suite-root", "directory containing test suite sources", "PATH")
73        .reqopt("", "build-root", "path to root build directory", "PATH")
74        .reqopt("", "build-test-suite-root", "path to test suite specific build directory", "PATH")
75        .reqopt("", "sysroot-base", "directory containing the compiler sysroot", "PATH")
76        .reqopt("", "stage", "stage number under test", "N")
77        .reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET")
78        .reqopt(
79            "",
80            "mode",
81            "which sort of compile tests to run",
82            "pretty | debug-info | codegen | rustdoc \
83            | rustdoc-json | codegen-units | incremental | run-make | ui \
84            | rustdoc-js | mir-opt | assembly | crashes",
85        )
86        .reqopt(
87            "",
88            "suite",
89            "which suite of compile tests to run. used for nicer error reporting.",
90            "SUITE",
91        )
92        .optopt(
93            "",
94            "pass",
95            "force {check,build,run}-pass tests to this mode.",
96            "check | build | run",
97        )
98        .optopt("", "run", "whether to execute run-* tests", "auto | always | never")
99        .optflag("", "ignored", "run tests marked as ignored")
100        .optflag("", "has-enzyme", "run tests that require enzyme")
101        .optflag("", "with-rustc-debug-assertions", "whether rustc was built with debug assertions")
102        .optflag("", "with-std-debug-assertions", "whether std was built with debug assertions")
103        .optmulti(
104            "",
105            "skip",
106            "skip tests matching SUBSTRING. Can be passed multiple times",
107            "SUBSTRING",
108        )
109        .optflag("", "exact", "filters match exactly")
110        .optopt(
111            "",
112            "runner",
113            "supervisor program to run tests under \
114             (eg. emulator, valgrind)",
115            "PROGRAM",
116        )
117        .optmulti("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS")
118        .optmulti("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS")
119        .optflag(
120            "",
121            "rust-randomized-layout",
122            "set this when rustc/stdlib were compiled with randomized layouts",
123        )
124        .optflag("", "optimize-tests", "run tests with optimizations enabled")
125        .optflag("", "verbose", "run tests verbosely, showing all output")
126        .optflag(
127            "",
128            "bless",
129            "overwrite stderr/stdout files instead of complaining about a mismatch",
130        )
131        .optflag("", "quiet", "print one character per test instead of one line")
132        .optopt("", "color", "coloring: auto, always, never", "WHEN")
133        .optflag("", "json", "emit json output instead of plaintext output")
134        .optopt("", "logfile", "file to log test execution to", "FILE")
135        .optopt("", "target", "the target to build for", "TARGET")
136        .optopt("", "host", "the host to build for", "HOST")
137        .optopt("", "cdb", "path to CDB to use for CDB debuginfo tests", "PATH")
138        .optopt("", "gdb", "path to GDB to use for GDB debuginfo tests", "PATH")
139        .optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING")
140        .optopt("", "llvm-version", "the version of LLVM used", "VERSION STRING")
141        .optflag("", "system-llvm", "is LLVM the system LLVM")
142        .optopt("", "android-cross-path", "Android NDK standalone path", "PATH")
143        .optopt("", "adb-path", "path to the android debugger", "PATH")
144        .optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH")
145        .optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH")
146        .reqopt("", "cc", "path to a C compiler", "PATH")
147        .reqopt("", "cxx", "path to a C++ compiler", "PATH")
148        .reqopt("", "cflags", "flags for the C compiler", "FLAGS")
149        .reqopt("", "cxxflags", "flags for the CXX compiler", "FLAGS")
150        .optopt("", "ar", "path to an archiver", "PATH")
151        .optopt("", "target-linker", "path to a linker for the target", "PATH")
152        .optopt("", "host-linker", "path to a linker for the host", "PATH")
153        .reqopt("", "llvm-components", "list of LLVM components built in", "LIST")
154        .optopt("", "llvm-bin-dir", "Path to LLVM's `bin` directory", "PATH")
155        .optopt("", "nodejs", "the name of nodejs", "PATH")
156        .optopt("", "npm", "the name of npm", "PATH")
157        .optopt("", "remote-test-client", "path to the remote test client", "PATH")
158        .optopt(
159            "",
160            "compare-mode",
161            "mode describing what file the actual ui output will be compared to",
162            "COMPARE MODE",
163        )
164        .optflag(
165            "",
166            "rustfix-coverage",
167            "enable this to generate a Rustfix coverage file, which is saved in \
168            `./<build_test_suite_root>/rustfix_missing_coverage.txt`",
169        )
170        .optflag("", "force-rerun", "rerun tests even if the inputs are unchanged")
171        .optflag("", "only-modified", "only run tests that result been modified")
172        // FIXME: Temporarily retained so we can point users to `--no-capture`
173        .optflag("", "nocapture", "")
174        .optflag("", "no-capture", "don't capture stdout/stderr of tests")
175        .optflag("", "profiler-runtime", "is the profiler runtime enabled for this target")
176        .optflag("h", "help", "show this message")
177        .reqopt("", "channel", "current Rust channel", "CHANNEL")
178        .optflag(
179            "",
180            "git-hash",
181            "run tests which rely on commit version being compiled into the binaries",
182        )
183        .optopt("", "edition", "default Rust edition", "EDITION")
184        .reqopt("", "git-repository", "name of the git repository", "ORG/REPO")
185        .reqopt("", "nightly-branch", "name of the git branch for nightly", "BRANCH")
186        .reqopt(
187            "",
188            "git-merge-commit-email",
189            "email address used for finding merge commits",
190            "EMAIL",
191        )
192        .optopt(
193            "",
194            "compiletest-diff-tool",
195            "What custom diff tool to use for displaying compiletest tests.",
196            "COMMAND",
197        )
198        .reqopt("", "minicore-path", "path to minicore aux library", "PATH")
199        .optopt(
200            "",
201            "debugger",
202            "only test a specific debugger in debuginfo tests",
203            "gdb | lldb | cdb",
204        );
205
206    let (argv0, args_) = args.split_first().unwrap();
207    if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
208        let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
209        println!("{}", opts.usage(&message));
210        println!();
211        panic!()
212    }
213
214    let matches = &match opts.parse(args_) {
215        Ok(m) => m,
216        Err(f) => panic!("{:?}", f),
217    };
218
219    if matches.opt_present("h") || matches.opt_present("help") {
220        let message = format!("Usage: {} [OPTIONS]  [TESTNAME...]", argv0);
221        println!("{}", opts.usage(&message));
222        println!();
223        panic!()
224    }
225
226    fn opt_path(m: &getopts::Matches, nm: &str) -> PathBuf {
227        match m.opt_str(nm) {
228            Some(s) => PathBuf::from(&s),
229            None => panic!("no option (=path) found for {}", nm),
230        }
231    }
232
233    fn make_absolute(path: PathBuf) -> PathBuf {
234        if path.is_relative() { env::current_dir().unwrap().join(path) } else { path }
235    }
236
237    let target = opt_str2(matches.opt_str("target"));
238    let android_cross_path = opt_path(matches, "android-cross-path");
239    let (cdb, cdb_version) = debuggers::analyze_cdb(matches.opt_str("cdb"), &target);
240    let (gdb, gdb_version) =
241        debuggers::analyze_gdb(matches.opt_str("gdb"), &target, &android_cross_path);
242    let lldb_version =
243        matches.opt_str("lldb-version").as_deref().and_then(debuggers::extract_lldb_version);
244    let color = match matches.opt_str("color").as_deref() {
245        Some("auto") | None => ColorConfig::AutoColor,
246        Some("always") => ColorConfig::AlwaysColor,
247        Some("never") => ColorConfig::NeverColor,
248        Some(x) => panic!("argument for --color must be auto, always, or never, but found `{}`", x),
249    };
250    let llvm_version =
251        matches.opt_str("llvm-version").as_deref().map(header::extract_llvm_version).or_else(
252            || header::extract_llvm_version_from_binary(&matches.opt_str("llvm-filecheck")?),
253        );
254
255    let run_ignored = matches.opt_present("ignored");
256    let with_rustc_debug_assertions = matches.opt_present("with-rustc-debug-assertions");
257    let with_std_debug_assertions = matches.opt_present("with-std-debug-assertions");
258    let mode = matches.opt_str("mode").unwrap().parse().expect("invalid mode");
259    let has_html_tidy = if mode == Mode::Rustdoc {
260        Command::new("tidy")
261            .arg("--version")
262            .stdout(Stdio::null())
263            .status()
264            .map_or(false, |status| status.success())
265    } else {
266        // Avoid spawning an external command when we know html-tidy won't be used.
267        false
268    };
269    let has_enzyme = matches.opt_present("has-enzyme");
270    let filters = if mode == Mode::RunMake {
271        matches
272            .free
273            .iter()
274            .map(|f| {
275                let path = Path::new(f);
276                let mut iter = path.iter().skip(1);
277
278                // We skip the test folder and check if the user passed `rmake.rs`.
279                if iter.next().is_some_and(|s| s == "rmake.rs") && iter.next().is_none() {
280                    path.parent().unwrap().to_str().unwrap().to_string()
281                } else {
282                    f.to_string()
283                }
284            })
285            .collect::<Vec<_>>()
286    } else {
287        matches.free.clone()
288    };
289    let compare_mode = matches.opt_str("compare-mode").map(|s| {
290        s.parse().unwrap_or_else(|_| {
291            let variants: Vec<_> = CompareMode::STR_VARIANTS.iter().copied().collect();
292            panic!(
293                "`{s}` is not a valid value for `--compare-mode`, it should be one of: {}",
294                variants.join(", ")
295            );
296        })
297    });
298    if matches.opt_present("nocapture") {
299        panic!("`--nocapture` is deprecated; please use `--no-capture`");
300    }
301
302    let stage = match matches.opt_str("stage") {
303        Some(stage) => stage.parse::<u32>().expect("expected `--stage` to be an unsigned integer"),
304        None => panic!("`--stage` is required"),
305    };
306
307    let src_root = opt_path(matches, "src-root");
308    let src_test_suite_root = opt_path(matches, "src-test-suite-root");
309    assert!(
310        src_test_suite_root.starts_with(&src_root),
311        "`src-root` must be a parent of `src-test-suite-root`: `src-root`=`{}`, `src-test-suite-root` = `{}`",
312        src_root.display(),
313        src_test_suite_root.display()
314    );
315
316    let build_root = opt_path(matches, "build-root");
317    let build_test_suite_root = opt_path(matches, "build-test-suite-root");
318    assert!(build_test_suite_root.starts_with(&build_root));
319
320    Config {
321        bless: matches.opt_present("bless"),
322        compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
323        run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
324        rustc_path: opt_path(matches, "rustc-path"),
325        cargo_path: matches.opt_str("cargo-path").map(PathBuf::from),
326        stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(PathBuf::from),
327        rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
328        coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
329        python: matches.opt_str("python").unwrap(),
330        jsondocck_path: matches.opt_str("jsondocck-path"),
331        jsondoclint_path: matches.opt_str("jsondoclint-path"),
332        run_clang_based_tests_with: matches.opt_str("run-clang-based-tests-with"),
333        llvm_filecheck: matches.opt_str("llvm-filecheck").map(PathBuf::from),
334        llvm_bin_dir: matches.opt_str("llvm-bin-dir").map(PathBuf::from),
335
336        src_root,
337        src_test_suite_root,
338
339        build_root,
340        build_test_suite_root,
341
342        sysroot_base: opt_path(matches, "sysroot-base"),
343
344        stage,
345        stage_id: matches.opt_str("stage-id").unwrap(),
346
347        mode,
348        suite: matches.opt_str("suite").unwrap(),
349        debugger: matches.opt_str("debugger").map(|debugger| {
350            debugger
351                .parse::<Debugger>()
352                .unwrap_or_else(|_| panic!("unknown `--debugger` option `{debugger}` given"))
353        }),
354        run_ignored,
355        with_rustc_debug_assertions,
356        with_std_debug_assertions,
357        filters,
358        skip: matches.opt_strs("skip"),
359        filter_exact: matches.opt_present("exact"),
360        force_pass_mode: matches.opt_str("pass").map(|mode| {
361            mode.parse::<PassMode>()
362                .unwrap_or_else(|_| panic!("unknown `--pass` option `{}` given", mode))
363        }),
364        run: matches.opt_str("run").and_then(|mode| match mode.as_str() {
365            "auto" => None,
366            "always" => Some(true),
367            "never" => Some(false),
368            _ => panic!("unknown `--run` option `{}` given", mode),
369        }),
370        logfile: matches.opt_str("logfile").map(|s| PathBuf::from(&s)),
371        runner: matches.opt_str("runner"),
372        host_rustcflags: matches.opt_strs("host-rustcflags"),
373        target_rustcflags: matches.opt_strs("target-rustcflags"),
374        optimize_tests: matches.opt_present("optimize-tests"),
375        rust_randomized_layout: matches.opt_present("rust-randomized-layout"),
376        target,
377        host: opt_str2(matches.opt_str("host")),
378        cdb,
379        cdb_version,
380        gdb,
381        gdb_version,
382        lldb_version,
383        llvm_version,
384        system_llvm: matches.opt_present("system-llvm"),
385        android_cross_path,
386        adb_path: opt_str2(matches.opt_str("adb-path")),
387        adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")),
388        adb_device_status: opt_str2(matches.opt_str("target")).contains("android")
389            && "(none)" != opt_str2(matches.opt_str("adb-test-dir"))
390            && !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
391        lldb_python_dir: matches.opt_str("lldb-python-dir"),
392        verbose: matches.opt_present("verbose"),
393        format: match (matches.opt_present("quiet"), matches.opt_present("json")) {
394            (true, true) => panic!("--quiet and --json are incompatible"),
395            (true, false) => test::OutputFormat::Terse,
396            (false, true) => test::OutputFormat::Json,
397            (false, false) => test::OutputFormat::Pretty,
398        },
399        only_modified: matches.opt_present("only-modified"),
400        color,
401        remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from),
402        compare_mode,
403        rustfix_coverage: matches.opt_present("rustfix-coverage"),
404        has_html_tidy,
405        has_enzyme,
406        channel: matches.opt_str("channel").unwrap(),
407        git_hash: matches.opt_present("git-hash"),
408        edition: matches.opt_str("edition"),
409
410        cc: matches.opt_str("cc").unwrap(),
411        cxx: matches.opt_str("cxx").unwrap(),
412        cflags: matches.opt_str("cflags").unwrap(),
413        cxxflags: matches.opt_str("cxxflags").unwrap(),
414        ar: matches.opt_str("ar").unwrap_or_else(|| String::from("ar")),
415        target_linker: matches.opt_str("target-linker"),
416        host_linker: matches.opt_str("host-linker"),
417        llvm_components: matches.opt_str("llvm-components").unwrap(),
418        nodejs: matches.opt_str("nodejs"),
419        npm: matches.opt_str("npm"),
420
421        force_rerun: matches.opt_present("force-rerun"),
422
423        target_cfgs: OnceLock::new(),
424        builtin_cfg_names: OnceLock::new(),
425
426        nocapture: matches.opt_present("no-capture"),
427
428        git_repository: matches.opt_str("git-repository").unwrap(),
429        nightly_branch: matches.opt_str("nightly-branch").unwrap(),
430        git_merge_commit_email: matches.opt_str("git-merge-commit-email").unwrap(),
431
432        profiler_runtime: matches.opt_present("profiler-runtime"),
433
434        diff_command: matches.opt_str("compiletest-diff-tool"),
435
436        minicore_path: opt_path(matches, "minicore-path"),
437    }
438}
439
440pub fn log_config(config: &Config) {
441    let c = config;
442    logv(c, "configuration:".to_string());
443    logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
444    logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
445    logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
446    logv(c, format!("cargo_path: {:?}", config.cargo_path));
447    logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path));
448
449    logv(c, format!("src_root: {}", config.src_root.display()));
450    logv(c, format!("src_test_suite_root: {}", config.src_test_suite_root.display()));
451
452    logv(c, format!("build_root: {}", config.build_root.display()));
453    logv(c, format!("build_test_suite_root: {}", config.build_test_suite_root.display()));
454
455    logv(c, format!("sysroot_base: {}", config.sysroot_base.display()));
456
457    logv(c, format!("stage: {}", config.stage));
458    logv(c, format!("stage_id: {}", config.stage_id));
459    logv(c, format!("mode: {}", config.mode));
460    logv(c, format!("run_ignored: {}", config.run_ignored));
461    logv(c, format!("filters: {:?}", config.filters));
462    logv(c, format!("skip: {:?}", config.skip));
463    logv(c, format!("filter_exact: {}", config.filter_exact));
464    logv(
465        c,
466        format!("force_pass_mode: {}", opt_str(&config.force_pass_mode.map(|m| format!("{}", m))),),
467    );
468    logv(c, format!("runner: {}", opt_str(&config.runner)));
469    logv(c, format!("host-rustcflags: {:?}", config.host_rustcflags));
470    logv(c, format!("target-rustcflags: {:?}", config.target_rustcflags));
471    logv(c, format!("target: {}", config.target));
472    logv(c, format!("host: {}", config.host));
473    logv(c, format!("android-cross-path: {:?}", config.android_cross_path.display()));
474    logv(c, format!("adb_path: {:?}", config.adb_path));
475    logv(c, format!("adb_test_dir: {:?}", config.adb_test_dir));
476    logv(c, format!("adb_device_status: {}", config.adb_device_status));
477    logv(c, format!("ar: {}", config.ar));
478    logv(c, format!("target-linker: {:?}", config.target_linker));
479    logv(c, format!("host-linker: {:?}", config.host_linker));
480    logv(c, format!("verbose: {}", config.verbose));
481    logv(c, format!("format: {:?}", config.format));
482    logv(c, format!("minicore_path: {:?}", config.minicore_path.display()));
483    logv(c, "\n".to_string());
484}
485
486pub fn opt_str(maybestr: &Option<String>) -> &str {
487    match *maybestr {
488        None => "(none)",
489        Some(ref s) => s,
490    }
491}
492
493pub fn opt_str2(maybestr: Option<String>) -> String {
494    match maybestr {
495        None => "(none)".to_owned(),
496        Some(s) => s,
497    }
498}
499
500/// Called by `main` after the config has been parsed.
501pub fn run_tests(config: Arc<Config>) {
502    // If we want to collect rustfix coverage information,
503    // we first make sure that the coverage file does not exist.
504    // It will be created later on.
505    if config.rustfix_coverage {
506        let mut coverage_file_path = config.build_test_suite_root.clone();
507        coverage_file_path.push("rustfix_missing_coverage.txt");
508        if coverage_file_path.exists() {
509            if let Err(e) = fs::remove_file(&coverage_file_path) {
510                panic!("Could not delete {} due to {}", coverage_file_path.display(), e)
511            }
512        }
513    }
514
515    // sadly osx needs some file descriptor limits raised for running tests in
516    // parallel (especially when we have lots and lots of child processes).
517    // For context, see #8904
518    unsafe {
519        raise_fd_limit::raise_fd_limit();
520    }
521    // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
522    // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
523    env::set_var("__COMPAT_LAYER", "RunAsInvoker");
524
525    // Let tests know which target they're running as
526    env::set_var("TARGET", &config.target);
527
528    let opts = test_opts(&config);
529
530    let mut configs = Vec::new();
531    if let Mode::DebugInfo = config.mode {
532        // Debugging emscripten code doesn't make sense today
533        if !config.target.contains("emscripten") {
534            match config.debugger {
535                Some(Debugger::Cdb) => configs.extend(debuggers::configure_cdb(&config)),
536                Some(Debugger::Gdb) => configs.extend(debuggers::configure_gdb(&config)),
537                Some(Debugger::Lldb) => configs.extend(debuggers::configure_lldb(&config)),
538                None => {
539                    configs.extend(debuggers::configure_cdb(&config));
540                    configs.extend(debuggers::configure_gdb(&config));
541                    configs.extend(debuggers::configure_lldb(&config));
542                }
543            }
544        }
545    } else {
546        configs.push(config.clone());
547    };
548
549    // Discover all of the tests in the test suite directory, and build a libtest
550    // structure for each test (or each revision of a multi-revision test).
551    let mut tests = Vec::new();
552    for c in configs {
553        tests.extend(collect_and_make_tests(c));
554    }
555
556    tests.sort_by(|a, b| a.desc.name.as_slice().cmp(&b.desc.name.as_slice()));
557
558    // Delegate to libtest to filter and run the big list of structures created
559    // during test discovery. When libtest decides to run a test, it will invoke
560    // the corresponding closure created by `make_test_closure`.
561    let res = test::run_tests_console(&opts, tests);
562
563    // Check the outcome reported by libtest.
564    match res {
565        Ok(true) => {}
566        Ok(false) => {
567            // We want to report that the tests failed, but we also want to give
568            // some indication of just what tests we were running. Especially on
569            // CI, where there can be cross-compiled tests for a lot of
570            // architectures, without this critical information it can be quite
571            // easy to miss which tests failed, and as such fail to reproduce
572            // the failure locally.
573
574            let mut msg = String::from("Some tests failed in compiletest");
575            write!(msg, " suite={}", config.suite).unwrap();
576
577            if let Some(compare_mode) = config.compare_mode.as_ref() {
578                write!(msg, " compare_mode={}", compare_mode).unwrap();
579            }
580
581            if let Some(pass_mode) = config.force_pass_mode.as_ref() {
582                write!(msg, " pass_mode={}", pass_mode).unwrap();
583            }
584
585            write!(msg, " mode={}", config.mode).unwrap();
586            write!(msg, " host={}", config.host).unwrap();
587            write!(msg, " target={}", config.target).unwrap();
588
589            println!("{msg}");
590
591            std::process::exit(1);
592        }
593        Err(e) => {
594            // We don't know if tests passed or not, but if there was an error
595            // during testing we don't want to just succeed (we may not have
596            // tested something), so fail.
597            //
598            // This should realistically "never" happen, so don't try to make
599            // this a pretty error message.
600            panic!("I/O failure during tests: {:?}", e);
601        }
602    }
603}
604
605pub fn test_opts(config: &Config) -> test::TestOpts {
606    if env::var("RUST_TEST_NOCAPTURE").is_ok() {
607        eprintln!(
608            "WARNING: RUST_TEST_NOCAPTURE is no longer used. \
609                   Use the `--nocapture` flag instead."
610        );
611    }
612
613    test::TestOpts {
614        exclude_should_panic: false,
615        filters: config.filters.clone(),
616        filter_exact: config.filter_exact,
617        run_ignored: if config.run_ignored { test::RunIgnored::Yes } else { test::RunIgnored::No },
618        format: config.format,
619        logfile: config.logfile.clone(),
620        run_tests: true,
621        bench_benchmarks: true,
622        nocapture: config.nocapture,
623        color: config.color,
624        shuffle: false,
625        shuffle_seed: None,
626        test_threads: None,
627        skip: config.skip.clone(),
628        list: false,
629        options: test::Options::new(),
630        time_options: None,
631        force_run_in_process: false,
632        fail_fast: std::env::var_os("RUSTC_TEST_FAIL_FAST").is_some(),
633    }
634}
635
636/// Read-only context data used during test collection.
637struct TestCollectorCx {
638    config: Arc<Config>,
639    cache: HeadersCache,
640    common_inputs_stamp: Stamp,
641    modified_tests: Vec<PathBuf>,
642}
643
644/// Mutable state used during test collection.
645struct TestCollector {
646    tests: Vec<test::TestDescAndFn>,
647    found_path_stems: HashSet<PathBuf>,
648    poisoned: bool,
649}
650
651/// Creates libtest structures for every test/revision in the test suite directory.
652///
653/// This always inspects _all_ test files in the suite (e.g. all 17k+ ui tests),
654/// regardless of whether any filters/tests were specified on the command-line,
655/// because filtering is handled later by libtest.
656pub fn collect_and_make_tests(config: Arc<Config>) -> Vec<test::TestDescAndFn> {
657    debug!("making tests from {}", config.src_test_suite_root.display());
658    let common_inputs_stamp = common_inputs_stamp(&config);
659    let modified_tests =
660        modified_tests(&config, &config.src_test_suite_root).unwrap_or_else(|err| {
661            panic!(
662                "modified_tests got error from dir: {}, error: {}",
663                config.src_test_suite_root.display(),
664                err
665            )
666        });
667    let cache = HeadersCache::load(&config);
668
669    let cx = TestCollectorCx { config, cache, common_inputs_stamp, modified_tests };
670    let mut collector =
671        TestCollector { tests: vec![], found_path_stems: HashSet::new(), poisoned: false };
672
673    collect_tests_from_dir(&cx, &mut collector, &cx.config.src_test_suite_root, Path::new(""))
674        .unwrap_or_else(|reason| {
675            panic!(
676                "Could not read tests from {}: {reason}",
677                cx.config.src_test_suite_root.display()
678            )
679        });
680
681    let TestCollector { tests, found_path_stems, poisoned } = collector;
682
683    if poisoned {
684        eprintln!();
685        panic!("there are errors in tests");
686    }
687
688    check_for_overlapping_test_paths(&found_path_stems);
689
690    tests
691}
692
693/// Returns the most recent last-modified timestamp from among the input files
694/// that are considered relevant to all tests (e.g. the compiler, std, and
695/// compiletest itself).
696///
697/// (Some of these inputs aren't actually relevant to _all_ tests, but they are
698/// common to some subset of tests, and are hopefully unlikely to be modified
699/// while working on other tests.)
700fn common_inputs_stamp(config: &Config) -> Stamp {
701    let src_root = &config.src_root;
702
703    let mut stamp = Stamp::from_path(&config.rustc_path);
704
705    // Relevant pretty printer files
706    let pretty_printer_files = [
707        "src/etc/rust_types.py",
708        "src/etc/gdb_load_rust_pretty_printers.py",
709        "src/etc/gdb_lookup.py",
710        "src/etc/gdb_providers.py",
711        "src/etc/lldb_batchmode.py",
712        "src/etc/lldb_lookup.py",
713        "src/etc/lldb_providers.py",
714    ];
715    for file in &pretty_printer_files {
716        let path = src_root.join(file);
717        stamp.add_path(&path);
718    }
719
720    stamp.add_dir(&src_root.join("src/etc/natvis"));
721
722    stamp.add_dir(&config.run_lib_path);
723
724    if let Some(ref rustdoc_path) = config.rustdoc_path {
725        stamp.add_path(&rustdoc_path);
726        stamp.add_path(&src_root.join("src/etc/htmldocck.py"));
727    }
728
729    // Re-run coverage tests if the `coverage-dump` tool was modified,
730    // because its output format might have changed.
731    if let Some(coverage_dump_path) = &config.coverage_dump_path {
732        stamp.add_path(coverage_dump_path)
733    }
734
735    stamp.add_dir(&src_root.join("src/tools/run-make-support"));
736
737    // Compiletest itself.
738    stamp.add_dir(&src_root.join("src/tools/compiletest"));
739
740    stamp
741}
742
743/// Returns a list of modified/untracked test files that should be run when
744/// the `--only-modified` flag is in use.
745///
746/// (Might be inaccurate in some cases.)
747fn modified_tests(config: &Config, dir: &Path) -> Result<Vec<PathBuf>, String> {
748    // If `--only-modified` wasn't passed, the list of modified tests won't be
749    // used for anything, so avoid some work and just return an empty list.
750    if !config.only_modified {
751        return Ok(vec![]);
752    }
753
754    let files =
755        get_git_modified_files(&config.git_config(), Some(dir), &vec!["rs", "stderr", "fixed"])?;
756    // Add new test cases to the list, it will be convenient in daily development.
757    let untracked_files = get_git_untracked_files(&config.git_config(), None)?.unwrap_or(vec![]);
758
759    let all_paths = [&files[..], &untracked_files[..]].concat();
760    let full_paths = {
761        let mut full_paths: Vec<PathBuf> = all_paths
762            .into_iter()
763            .map(|f| PathBuf::from(f).with_extension("").with_extension("rs"))
764            .filter_map(|f| if Path::new(&f).exists() { f.canonicalize().ok() } else { None })
765            .collect();
766        full_paths.dedup();
767        full_paths.sort_unstable();
768        full_paths
769    };
770    Ok(full_paths)
771}
772
773/// Recursively scans a directory to find test files and create test structures
774/// that will be handed over to libtest.
775fn collect_tests_from_dir(
776    cx: &TestCollectorCx,
777    collector: &mut TestCollector,
778    dir: &Path,
779    relative_dir_path: &Path,
780) -> io::Result<()> {
781    // Ignore directories that contain a file named `compiletest-ignore-dir`.
782    if dir.join("compiletest-ignore-dir").exists() {
783        return Ok(());
784    }
785
786    // For run-make tests, a "test file" is actually a directory that contains an `rmake.rs`.
787    if cx.config.mode == Mode::RunMake {
788        if dir.join("rmake.rs").exists() {
789            let paths = TestPaths {
790                file: dir.to_path_buf(),
791                relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
792            };
793            make_test(cx, collector, &paths);
794            // This directory is a test, so don't try to find other tests inside it.
795            return Ok(());
796        }
797    }
798
799    // If we find a test foo/bar.rs, we have to build the
800    // output directory `$build/foo` so we can write
801    // `$build/foo/bar` into it. We do this *now* in this
802    // sequential loop because otherwise, if we do it in the
803    // tests themselves, they race for the privilege of
804    // creating the directories and sometimes fail randomly.
805    let build_dir = output_relative_path(&cx.config, relative_dir_path);
806    fs::create_dir_all(&build_dir).unwrap();
807
808    // Add each `.rs` file as a test, and recurse further on any
809    // subdirectories we find, except for `auxiliary` directories.
810    // FIXME: this walks full tests tree, even if we have something to ignore
811    // use walkdir/ignore like in tidy?
812    for file in fs::read_dir(dir)? {
813        let file = file?;
814        let file_path = file.path();
815        let file_name = file.file_name();
816
817        if is_test(&file_name)
818            && (!cx.config.only_modified || cx.modified_tests.contains(&file_path))
819        {
820            // We found a test file, so create the corresponding libtest structures.
821            debug!("found test file: {:?}", file_path.display());
822
823            // Record the stem of the test file, to check for overlaps later.
824            let rel_test_path = relative_dir_path.join(file_path.file_stem().unwrap());
825            collector.found_path_stems.insert(rel_test_path);
826
827            let paths =
828                TestPaths { file: file_path, relative_dir: relative_dir_path.to_path_buf() };
829            make_test(cx, collector, &paths);
830        } else if file_path.is_dir() {
831            // Recurse to find more tests in a subdirectory.
832            let relative_file_path = relative_dir_path.join(file.file_name());
833            if &file_name != "auxiliary" {
834                debug!("found directory: {:?}", file_path.display());
835                collect_tests_from_dir(cx, collector, &file_path, &relative_file_path)?;
836            }
837        } else {
838            debug!("found other file/directory: {:?}", file_path.display());
839        }
840    }
841    Ok(())
842}
843
844/// Returns true if `file_name` looks like a proper test file name.
845pub fn is_test(file_name: &OsString) -> bool {
846    let file_name = file_name.to_str().unwrap();
847
848    if !file_name.ends_with(".rs") {
849        return false;
850    }
851
852    // `.`, `#`, and `~` are common temp-file prefixes.
853    let invalid_prefixes = &[".", "#", "~"];
854    !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
855}
856
857/// For a single test file, creates one or more test structures (one per revision) that can be
858/// handed over to libtest to run, possibly in parallel.
859fn make_test(cx: &TestCollectorCx, collector: &mut TestCollector, testpaths: &TestPaths) {
860    // For run-make tests, each "test file" is actually a _directory_ containing an `rmake.rs`. But
861    // for the purposes of directive parsing, we want to look at that recipe file, not the directory
862    // itself.
863    let test_path = if cx.config.mode == Mode::RunMake {
864        testpaths.file.join("rmake.rs")
865    } else {
866        PathBuf::from(&testpaths.file)
867    };
868
869    // Scan the test file to discover its revisions, if any.
870    let early_props = EarlyProps::from_file(&cx.config, &test_path);
871
872    // Normally we create one libtest structure per revision, with two exceptions:
873    // - If a test doesn't use revisions, create a dummy revision (None) so that
874    //   the test can still run.
875    // - Incremental tests inherently can't run their revisions in parallel, so
876    //   we treat them like non-revisioned tests here. Incremental revisions are
877    //   handled internally by `runtest::run` instead.
878    let revisions = if early_props.revisions.is_empty() || cx.config.mode == Mode::Incremental {
879        vec![None]
880    } else {
881        early_props.revisions.iter().map(|r| Some(r.as_str())).collect()
882    };
883
884    // For each revision (or the sole dummy revision), create and append a
885    // `test::TestDescAndFn` that can be handed over to libtest.
886    collector.tests.extend(revisions.into_iter().map(|revision| {
887        // Create a test name and description to hand over to libtest.
888        let src_file = fs::File::open(&test_path).expect("open test file to parse ignores");
889        let test_name = make_test_name(&cx.config, testpaths, revision);
890        // Create a libtest description for the test/revision.
891        // This is where `ignore-*`/`only-*`/`needs-*` directives are handled,
892        // because they need to set the libtest ignored flag.
893        let mut desc = make_test_description(
894            &cx.config,
895            &cx.cache,
896            test_name,
897            &test_path,
898            src_file,
899            revision,
900            &mut collector.poisoned,
901        );
902
903        // If a test's inputs haven't changed since the last time it ran,
904        // mark it as ignored so that libtest will skip it.
905        if !cx.config.force_rerun && is_up_to_date(cx, testpaths, &early_props, revision) {
906            desc.ignore = true;
907            // Keep this in sync with the "up-to-date" message detected by bootstrap.
908            desc.ignore_message = Some("up-to-date");
909        }
910
911        // Create the callback that will run this test/revision when libtest calls it.
912        let testfn = make_test_closure(Arc::clone(&cx.config), testpaths, revision);
913
914        test::TestDescAndFn { desc, testfn }
915    }));
916}
917
918/// The path of the `stamp` file that gets created or updated whenever a
919/// particular test completes successfully.
920fn stamp_file_path(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> PathBuf {
921    output_base_dir(config, testpaths, revision).join("stamp")
922}
923
924/// Returns a list of files that, if modified, would cause this test to no
925/// longer be up-to-date.
926///
927/// (Might be inaccurate in some cases.)
928fn files_related_to_test(
929    config: &Config,
930    testpaths: &TestPaths,
931    props: &EarlyProps,
932    revision: Option<&str>,
933) -> Vec<PathBuf> {
934    let mut related = vec![];
935
936    if testpaths.file.is_dir() {
937        // run-make tests use their individual directory
938        for entry in WalkDir::new(&testpaths.file) {
939            let path = entry.unwrap().into_path();
940            if path.is_file() {
941                related.push(path);
942            }
943        }
944    } else {
945        related.push(testpaths.file.clone());
946    }
947
948    for aux in props.aux.all_aux_path_strings() {
949        // FIXME(Zalathar): Perform all `auxiliary` path resolution in one place.
950        let path = testpaths.file.parent().unwrap().join("auxiliary").join(aux);
951        related.push(path);
952    }
953
954    // UI test files.
955    for extension in UI_EXTENSIONS {
956        let path = expected_output_path(testpaths, revision, &config.compare_mode, extension);
957        related.push(path);
958    }
959
960    // `minicore.rs` test auxiliary: we need to make sure tests get rerun if this changes.
961    related.push(config.src_root.join("tests").join("auxiliary").join("minicore.rs"));
962
963    related
964}
965
966/// Checks whether a particular test/revision is "up-to-date", meaning that no
967/// relevant files/settings have changed since the last time the test succeeded.
968///
969/// (This is not very reliable in some circumstances, so the `--force-rerun`
970/// flag can be used to ignore up-to-date checking and always re-run tests.)
971fn is_up_to_date(
972    cx: &TestCollectorCx,
973    testpaths: &TestPaths,
974    props: &EarlyProps,
975    revision: Option<&str>,
976) -> bool {
977    let stamp_file_path = stamp_file_path(&cx.config, testpaths, revision);
978    // Check the config hash inside the stamp file.
979    let contents = match fs::read_to_string(&stamp_file_path) {
980        Ok(f) => f,
981        Err(ref e) if e.kind() == ErrorKind::InvalidData => panic!("Can't read stamp contents"),
982        // The test hasn't succeeded yet, so it is not up-to-date.
983        Err(_) => return false,
984    };
985    let expected_hash = runtest::compute_stamp_hash(&cx.config);
986    if contents != expected_hash {
987        // Some part of compiletest configuration has changed since the test
988        // last succeeded, so it is not up-to-date.
989        return false;
990    }
991
992    // Check the timestamp of the stamp file against the last modified time
993    // of all files known to be relevant to the test.
994    let mut inputs_stamp = cx.common_inputs_stamp.clone();
995    for path in files_related_to_test(&cx.config, testpaths, props, revision) {
996        inputs_stamp.add_path(&path);
997    }
998
999    // If no relevant files have been modified since the stamp file was last
1000    // written, the test is up-to-date.
1001    inputs_stamp < Stamp::from_path(&stamp_file_path)
1002}
1003
1004/// The maximum of a set of file-modified timestamps.
1005#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
1006struct Stamp {
1007    time: SystemTime,
1008}
1009
1010impl Stamp {
1011    /// Creates a timestamp holding the last-modified time of the specified file.
1012    fn from_path(path: &Path) -> Self {
1013        let mut stamp = Stamp { time: SystemTime::UNIX_EPOCH };
1014        stamp.add_path(path);
1015        stamp
1016    }
1017
1018    /// Updates this timestamp to the last-modified time of the specified file,
1019    /// if it is later than the currently-stored timestamp.
1020    fn add_path(&mut self, path: &Path) {
1021        let modified = fs::metadata(path)
1022            .and_then(|metadata| metadata.modified())
1023            .unwrap_or(SystemTime::UNIX_EPOCH);
1024        self.time = self.time.max(modified);
1025    }
1026
1027    /// Updates this timestamp to the most recent last-modified time of all files
1028    /// recursively contained in the given directory, if it is later than the
1029    /// currently-stored timestamp.
1030    fn add_dir(&mut self, path: &Path) {
1031        for entry in WalkDir::new(path) {
1032            let entry = entry.unwrap();
1033            if entry.file_type().is_file() {
1034                let modified = entry
1035                    .metadata()
1036                    .ok()
1037                    .and_then(|metadata| metadata.modified().ok())
1038                    .unwrap_or(SystemTime::UNIX_EPOCH);
1039                self.time = self.time.max(modified);
1040            }
1041        }
1042    }
1043}
1044
1045/// Creates a name for this test/revision that can be handed over to libtest.
1046fn make_test_name(
1047    config: &Config,
1048    testpaths: &TestPaths,
1049    revision: Option<&str>,
1050) -> test::TestName {
1051    // Print the name of the file, relative to the sources root.
1052    let path = testpaths.file.strip_prefix(&config.src_root).unwrap();
1053    let debugger = match config.debugger {
1054        Some(d) => format!("-{}", d),
1055        None => String::new(),
1056    };
1057    let mode_suffix = match config.compare_mode {
1058        Some(ref mode) => format!(" ({})", mode.to_str()),
1059        None => String::new(),
1060    };
1061
1062    test::DynTestName(format!(
1063        "[{}{}{}] {}{}",
1064        config.mode,
1065        debugger,
1066        mode_suffix,
1067        path.display(),
1068        revision.map_or("".to_string(), |rev| format!("#{}", rev))
1069    ))
1070}
1071
1072/// Creates a callback for this test/revision that libtest will call when it
1073/// decides to actually run the underlying test.
1074fn make_test_closure(
1075    config: Arc<Config>,
1076    testpaths: &TestPaths,
1077    revision: Option<&str>,
1078) -> test::TestFn {
1079    let testpaths = testpaths.clone();
1080    let revision = revision.map(str::to_owned);
1081
1082    // This callback is the link between compiletest's test discovery code,
1083    // and the parts of compiletest that know how to run an individual test.
1084    test::DynTestFn(Box::new(move || {
1085        runtest::run(config, &testpaths, revision.as_deref());
1086        Ok(())
1087    }))
1088}
1089
1090/// Checks that test discovery didn't find any tests whose name stem is a prefix
1091/// of some other tests's name.
1092///
1093/// For example, suppose the test suite contains these two test files:
1094/// - `tests/rustdoc/primitive.rs`
1095/// - `tests/rustdoc/primitive/no_std.rs`
1096///
1097/// The test runner might put the output from those tests in these directories:
1098/// - `$build/test/rustdoc/primitive/`
1099/// - `$build/test/rustdoc/primitive/no_std/`
1100///
1101/// Because one output path is a subdirectory of the other, the two tests might
1102/// interfere with each other in unwanted ways, especially if the test runner
1103/// decides to delete test output directories to clean them between runs.
1104/// To avoid problems, we forbid test names from overlapping in this way.
1105///
1106/// See <https://github.com/rust-lang/rust/pull/109509> for more context.
1107fn check_for_overlapping_test_paths(found_path_stems: &HashSet<PathBuf>) {
1108    let mut collisions = Vec::new();
1109    for path in found_path_stems {
1110        for ancestor in path.ancestors().skip(1) {
1111            if found_path_stems.contains(ancestor) {
1112                collisions.push((path, ancestor));
1113            }
1114        }
1115    }
1116    if !collisions.is_empty() {
1117        collisions.sort();
1118        let collisions: String = collisions
1119            .into_iter()
1120            .map(|(path, check_parent)| format!("test {path:?} clashes with {check_parent:?}\n"))
1121            .collect();
1122        panic!(
1123            "{collisions}\n\
1124            Tests cannot have overlapping names. Make sure they use unique prefixes."
1125        );
1126    }
1127}