rustdoc/
doctest.rs

1mod extracted;
2mod make;
3mod markdown;
4mod runner;
5mod rust;
6
7use std::fs::File;
8use std::hash::{Hash, Hasher};
9use std::io::{self, Write};
10use std::path::{Path, PathBuf};
11use std::process::{self, Command, Stdio};
12use std::sync::atomic::{AtomicUsize, Ordering};
13use std::sync::{Arc, Mutex};
14use std::{panic, str};
15
16pub(crate) use make::{BuildDocTestBuilder, DocTestBuilder};
17pub(crate) use markdown::test as test_markdown;
18use rustc_data_structures::fx::{FxHashMap, FxHasher, FxIndexMap, FxIndexSet};
19use rustc_errors::emitter::HumanReadableErrorType;
20use rustc_errors::{ColorConfig, DiagCtxtHandle};
21use rustc_hir as hir;
22use rustc_hir::CRATE_HIR_ID;
23use rustc_hir::def_id::LOCAL_CRATE;
24use rustc_interface::interface;
25use rustc_session::config::{self, CrateType, ErrorOutputType, Input};
26use rustc_session::lint;
27use rustc_span::edition::Edition;
28use rustc_span::symbol::sym;
29use rustc_span::{FileName, Span};
30use rustc_target::spec::{Target, TargetTuple};
31use tempfile::{Builder as TempFileBuilder, TempDir};
32use tracing::debug;
33
34use self::rust::HirCollector;
35use crate::config::{Options as RustdocOptions, OutputFormat};
36use crate::html::markdown::{ErrorCodes, Ignore, LangString, MdRelLine};
37use crate::lint::init_lints;
38
39/// Options that apply to all doctests in a crate or Markdown file (for `rustdoc foo.md`).
40#[derive(Clone)]
41pub(crate) struct GlobalTestOptions {
42    /// Name of the crate (for regular `rustdoc`) or Markdown file (for `rustdoc foo.md`).
43    pub(crate) crate_name: String,
44    /// Whether to disable the default `extern crate my_crate;` when creating doctests.
45    pub(crate) no_crate_inject: bool,
46    /// Whether inserting extra indent spaces in code block,
47    /// default is `false`, only `true` for generating code link of Rust playground
48    pub(crate) insert_indent_space: bool,
49    /// Path to file containing arguments for the invocation of rustc.
50    pub(crate) args_file: PathBuf,
51}
52
53pub(crate) fn generate_args_file(file_path: &Path, options: &RustdocOptions) -> Result<(), String> {
54    let mut file = File::create(file_path)
55        .map_err(|error| format!("failed to create args file: {error:?}"))?;
56
57    // We now put the common arguments into the file we created.
58    let mut content = vec![];
59
60    for cfg in &options.cfgs {
61        content.push(format!("--cfg={cfg}"));
62    }
63    for check_cfg in &options.check_cfgs {
64        content.push(format!("--check-cfg={check_cfg}"));
65    }
66
67    for lib_str in &options.lib_strs {
68        content.push(format!("-L{lib_str}"));
69    }
70    for extern_str in &options.extern_strs {
71        content.push(format!("--extern={extern_str}"));
72    }
73    content.push("-Ccodegen-units=1".to_string());
74    for codegen_options_str in &options.codegen_options_strs {
75        content.push(format!("-C{codegen_options_str}"));
76    }
77    for unstable_option_str in &options.unstable_opts_strs {
78        content.push(format!("-Z{unstable_option_str}"));
79    }
80
81    content.extend(options.doctest_build_args.clone());
82
83    let content = content.join("\n");
84
85    file.write_all(content.as_bytes())
86        .map_err(|error| format!("failed to write arguments to temporary file: {error:?}"))?;
87    Ok(())
88}
89
90fn get_doctest_dir() -> io::Result<TempDir> {
91    TempFileBuilder::new().prefix("rustdoctest").tempdir()
92}
93
94pub(crate) fn run(dcx: DiagCtxtHandle<'_>, input: Input, options: RustdocOptions) {
95    let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
96
97    // See core::create_config for what's going on here.
98    let allowed_lints = vec![
99        invalid_codeblock_attributes_name.to_owned(),
100        lint::builtin::UNKNOWN_LINTS.name.to_owned(),
101        lint::builtin::RENAMED_AND_REMOVED_LINTS.name.to_owned(),
102    ];
103
104    let (lint_opts, lint_caps) = init_lints(allowed_lints, options.lint_opts.clone(), |lint| {
105        if lint.name == invalid_codeblock_attributes_name {
106            None
107        } else {
108            Some((lint.name_lower(), lint::Allow))
109        }
110    });
111
112    debug!(?lint_opts);
113
114    let crate_types =
115        if options.proc_macro_crate { vec![CrateType::ProcMacro] } else { vec![CrateType::Rlib] };
116
117    let sessopts = config::Options {
118        sysroot: options.sysroot.clone(),
119        search_paths: options.libs.clone(),
120        crate_types,
121        lint_opts,
122        lint_cap: Some(options.lint_cap.unwrap_or(lint::Forbid)),
123        cg: options.codegen_options.clone(),
124        externs: options.externs.clone(),
125        unstable_features: options.unstable_features,
126        actually_rustdoc: true,
127        edition: options.edition,
128        target_triple: options.target.clone(),
129        crate_name: options.crate_name.clone(),
130        remap_path_prefix: options.remap_path_prefix.clone(),
131        ..config::Options::default()
132    };
133
134    let mut cfgs = options.cfgs.clone();
135    cfgs.push("doc".to_owned());
136    cfgs.push("doctest".to_owned());
137    let config = interface::Config {
138        opts: sessopts,
139        crate_cfg: cfgs,
140        crate_check_cfg: options.check_cfgs.clone(),
141        input: input.clone(),
142        output_file: None,
143        output_dir: None,
144        file_loader: None,
145        locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
146        lint_caps,
147        psess_created: None,
148        hash_untracked_state: None,
149        register_lints: Some(Box::new(crate::lint::register_lints)),
150        override_queries: None,
151        extra_symbols: Vec::new(),
152        make_codegen_backend: None,
153        registry: rustc_driver::diagnostics_registry(),
154        ice_file: None,
155        using_internal_features: &rustc_driver::USING_INTERNAL_FEATURES,
156        expanded_args: options.expanded_args.clone(),
157    };
158
159    let externs = options.externs.clone();
160    let json_unused_externs = options.json_unused_externs;
161
162    let temp_dir = match get_doctest_dir()
163        .map_err(|error| format!("failed to create temporary directory: {error:?}"))
164    {
165        Ok(temp_dir) => temp_dir,
166        Err(error) => return crate::wrap_return(dcx, Err(error)),
167    };
168    let args_path = temp_dir.path().join("rustdoc-cfgs");
169    crate::wrap_return(dcx, generate_args_file(&args_path, &options));
170
171    let extract_doctests = options.output_format == OutputFormat::Doctest;
172    let result = interface::run_compiler(config, |compiler| {
173        let krate = rustc_interface::passes::parse(&compiler.sess);
174
175        let collector = rustc_interface::create_and_enter_global_ctxt(compiler, krate, |tcx| {
176            let crate_name = tcx.crate_name(LOCAL_CRATE).to_string();
177            let crate_attrs = tcx.hir_attrs(CRATE_HIR_ID);
178            let opts = scrape_test_config(crate_name, crate_attrs, args_path);
179
180            let hir_collector = HirCollector::new(
181                ErrorCodes::from(compiler.sess.opts.unstable_features.is_nightly_build()),
182                tcx,
183            );
184            let tests = hir_collector.collect_crate();
185            if extract_doctests {
186                let mut collector = extracted::ExtractedDocTests::new();
187                tests.into_iter().for_each(|t| collector.add_test(t, &opts, &options));
188
189                let stdout = std::io::stdout();
190                let mut stdout = stdout.lock();
191                if let Err(error) = serde_json::ser::to_writer(&mut stdout, &collector) {
192                    eprintln!();
193                    Err(format!("Failed to generate JSON output for doctests: {error:?}"))
194                } else {
195                    Ok(None)
196                }
197            } else {
198                let mut collector = CreateRunnableDocTests::new(options, opts);
199                tests.into_iter().for_each(|t| collector.add_test(t, Some(compiler.sess.dcx())));
200
201                Ok(Some(collector))
202            }
203        });
204        compiler.sess.dcx().abort_if_errors();
205
206        collector
207    });
208
209    let CreateRunnableDocTests {
210        standalone_tests,
211        mergeable_tests,
212        rustdoc_options,
213        opts,
214        unused_extern_reports,
215        compiling_test_count,
216        ..
217    } = match result {
218        Ok(Some(collector)) => collector,
219        Ok(None) => return,
220        Err(error) => {
221            eprintln!("{error}");
222            // Since some files in the temporary folder are still owned and alive, we need
223            // to manually remove the folder.
224            let _ = std::fs::remove_dir_all(temp_dir.path());
225            std::process::exit(1);
226        }
227    };
228
229    run_tests(
230        opts,
231        &rustdoc_options,
232        &unused_extern_reports,
233        standalone_tests,
234        mergeable_tests,
235        Some(temp_dir),
236    );
237
238    let compiling_test_count = compiling_test_count.load(Ordering::SeqCst);
239
240    // Collect and warn about unused externs, but only if we've gotten
241    // reports for each doctest
242    if json_unused_externs.is_enabled() {
243        let unused_extern_reports: Vec<_> =
244            std::mem::take(&mut unused_extern_reports.lock().unwrap());
245        if unused_extern_reports.len() == compiling_test_count {
246            let extern_names =
247                externs.iter().map(|(name, _)| name).collect::<FxIndexSet<&String>>();
248            let mut unused_extern_names = unused_extern_reports
249                .iter()
250                .map(|uexts| uexts.unused_extern_names.iter().collect::<FxIndexSet<&String>>())
251                .fold(extern_names, |uextsa, uextsb| {
252                    uextsa.intersection(&uextsb).copied().collect::<FxIndexSet<&String>>()
253                })
254                .iter()
255                .map(|v| (*v).clone())
256                .collect::<Vec<String>>();
257            unused_extern_names.sort();
258            // Take the most severe lint level
259            let lint_level = unused_extern_reports
260                .iter()
261                .map(|uexts| uexts.lint_level.as_str())
262                .max_by_key(|v| match *v {
263                    "warn" => 1,
264                    "deny" => 2,
265                    "forbid" => 3,
266                    // The allow lint level is not expected,
267                    // as if allow is specified, no message
268                    // is to be emitted.
269                    v => unreachable!("Invalid lint level '{v}'"),
270                })
271                .unwrap_or("warn")
272                .to_string();
273            let uext = UnusedExterns { lint_level, unused_extern_names };
274            let unused_extern_json = serde_json::to_string(&uext).unwrap();
275            eprintln!("{unused_extern_json}");
276        }
277    }
278}
279
280pub(crate) fn run_tests(
281    opts: GlobalTestOptions,
282    rustdoc_options: &Arc<RustdocOptions>,
283    unused_extern_reports: &Arc<Mutex<Vec<UnusedExterns>>>,
284    mut standalone_tests: Vec<test::TestDescAndFn>,
285    mergeable_tests: FxIndexMap<MergeableTestKey, Vec<(DocTestBuilder, ScrapedDocTest)>>,
286    // We pass this argument so we can drop it manually before using `exit`.
287    mut temp_dir: Option<TempDir>,
288) {
289    let mut test_args = Vec::with_capacity(rustdoc_options.test_args.len() + 1);
290    test_args.insert(0, "rustdoctest".to_string());
291    test_args.extend_from_slice(&rustdoc_options.test_args);
292    if rustdoc_options.nocapture {
293        test_args.push("--nocapture".to_string());
294    }
295
296    let mut nb_errors = 0;
297    let mut ran_edition_tests = 0;
298    let target_str = rustdoc_options.target.to_string();
299
300    for (MergeableTestKey { edition, global_crate_attrs_hash }, mut doctests) in mergeable_tests {
301        if doctests.is_empty() {
302            continue;
303        }
304        doctests.sort_by(|(_, a), (_, b)| a.name.cmp(&b.name));
305
306        let mut tests_runner = runner::DocTestRunner::new();
307
308        let rustdoc_test_options = IndividualTestOptions::new(
309            rustdoc_options,
310            &Some(format!("merged_doctest_{edition}_{global_crate_attrs_hash}")),
311            PathBuf::from(format!("doctest_{edition}_{global_crate_attrs_hash}.rs")),
312        );
313
314        for (doctest, scraped_test) in &doctests {
315            tests_runner.add_test(doctest, scraped_test, &target_str);
316        }
317        if let Ok(success) = tests_runner.run_merged_tests(
318            rustdoc_test_options,
319            edition,
320            &opts,
321            &test_args,
322            rustdoc_options,
323        ) {
324            ran_edition_tests += 1;
325            if !success {
326                nb_errors += 1;
327            }
328            continue;
329        }
330        // We failed to compile all compatible tests as one so we push them into the
331        // `standalone_tests` doctests.
332        debug!("Failed to compile compatible doctests for edition {} all at once", edition);
333        for (doctest, scraped_test) in doctests {
334            doctest.generate_unique_doctest(
335                &scraped_test.text,
336                scraped_test.langstr.test_harness,
337                &opts,
338                Some(&opts.crate_name),
339            );
340            standalone_tests.push(generate_test_desc_and_fn(
341                doctest,
342                scraped_test,
343                opts.clone(),
344                Arc::clone(rustdoc_options),
345                unused_extern_reports.clone(),
346            ));
347        }
348    }
349
350    // We need to call `test_main` even if there is no doctest to run to get the output
351    // `running 0 tests...`.
352    if ran_edition_tests == 0 || !standalone_tests.is_empty() {
353        standalone_tests.sort_by(|a, b| a.desc.name.as_slice().cmp(b.desc.name.as_slice()));
354        test::test_main_with_exit_callback(&test_args, standalone_tests, None, || {
355            // We ensure temp dir destructor is called.
356            std::mem::drop(temp_dir.take());
357        });
358    }
359    if nb_errors != 0 {
360        // We ensure temp dir destructor is called.
361        std::mem::drop(temp_dir);
362        // libtest::ERROR_EXIT_CODE is not public but it's the same value.
363        std::process::exit(101);
364    }
365}
366
367// Look for `#![doc(test(no_crate_inject))]`, used by crates in the std facade.
368fn scrape_test_config(
369    crate_name: String,
370    attrs: &[hir::Attribute],
371    args_file: PathBuf,
372) -> GlobalTestOptions {
373    let mut opts = GlobalTestOptions {
374        crate_name,
375        no_crate_inject: false,
376        insert_indent_space: false,
377        args_file,
378    };
379
380    let test_attrs: Vec<_> = attrs
381        .iter()
382        .filter(|a| a.has_name(sym::doc))
383        .flat_map(|a| a.meta_item_list().unwrap_or_default())
384        .filter(|a| a.has_name(sym::test))
385        .collect();
386    let attrs = test_attrs.iter().flat_map(|a| a.meta_item_list().unwrap_or(&[]));
387
388    for attr in attrs {
389        if attr.has_name(sym::no_crate_inject) {
390            opts.no_crate_inject = true;
391        }
392        // NOTE: `test(attr(..))` is handled when discovering the individual tests
393    }
394
395    opts
396}
397
398/// Documentation test failure modes.
399enum TestFailure {
400    /// The test failed to compile.
401    CompileError,
402    /// The test is marked `compile_fail` but compiled successfully.
403    UnexpectedCompilePass,
404    /// The test failed to compile (as expected) but the compiler output did not contain all
405    /// expected error codes.
406    MissingErrorCodes(Vec<String>),
407    /// The test binary was unable to be executed.
408    ExecutionError(io::Error),
409    /// The test binary exited with a non-zero exit code.
410    ///
411    /// This typically means an assertion in the test failed or another form of panic occurred.
412    ExecutionFailure(process::Output),
413    /// The test is marked `should_panic` but the test binary executed successfully.
414    UnexpectedRunPass,
415}
416
417enum DirState {
418    Temp(TempDir),
419    Perm(PathBuf),
420}
421
422impl DirState {
423    fn path(&self) -> &std::path::Path {
424        match self {
425            DirState::Temp(t) => t.path(),
426            DirState::Perm(p) => p.as_path(),
427        }
428    }
429}
430
431// NOTE: Keep this in sync with the equivalent structs in rustc
432// and cargo.
433// We could unify this struct the one in rustc but they have different
434// ownership semantics, so doing so would create wasteful allocations.
435#[derive(serde::Serialize, serde::Deserialize)]
436pub(crate) struct UnusedExterns {
437    /// Lint level of the unused_crate_dependencies lint
438    lint_level: String,
439    /// List of unused externs by their names.
440    unused_extern_names: Vec<String>,
441}
442
443fn add_exe_suffix(input: String, target: &TargetTuple) -> String {
444    let exe_suffix = match target {
445        TargetTuple::TargetTuple(_) => Target::expect_builtin(target).options.exe_suffix,
446        TargetTuple::TargetJson { contents, .. } => {
447            Target::from_json(contents.parse().unwrap()).unwrap().0.options.exe_suffix
448        }
449    };
450    input + &exe_suffix
451}
452
453fn wrapped_rustc_command(rustc_wrappers: &[PathBuf], rustc_binary: &Path) -> Command {
454    let mut args = rustc_wrappers.iter().map(PathBuf::as_path).chain([rustc_binary]);
455
456    let exe = args.next().expect("unable to create rustc command");
457    let mut command = Command::new(exe);
458    for arg in args {
459        command.arg(arg);
460    }
461
462    command
463}
464
465/// Information needed for running a bundle of doctests.
466///
467/// This data structure contains the "full" test code, including the wrappers
468/// (if multiple doctests are merged), `main` function,
469/// and everything needed to calculate the compiler's command-line arguments.
470/// The `# ` prefix on boring lines has also been stripped.
471pub(crate) struct RunnableDocTest {
472    full_test_code: String,
473    full_test_line_offset: usize,
474    test_opts: IndividualTestOptions,
475    global_opts: GlobalTestOptions,
476    langstr: LangString,
477    line: usize,
478    edition: Edition,
479    no_run: bool,
480    merged_test_code: Option<String>,
481}
482
483impl RunnableDocTest {
484    fn path_for_merged_doctest_bundle(&self) -> PathBuf {
485        self.test_opts.outdir.path().join(format!("doctest_bundle_{}.rs", self.edition))
486    }
487    fn path_for_merged_doctest_runner(&self) -> PathBuf {
488        self.test_opts.outdir.path().join(format!("doctest_runner_{}.rs", self.edition))
489    }
490    fn is_multiple_tests(&self) -> bool {
491        self.merged_test_code.is_some()
492    }
493}
494
495/// Execute a `RunnableDoctest`.
496///
497/// This is the function that calculates the compiler command line, invokes the compiler, then
498/// invokes the test or tests in a separate executable (if applicable).
499fn run_test(
500    doctest: RunnableDocTest,
501    rustdoc_options: &RustdocOptions,
502    supports_color: bool,
503    report_unused_externs: impl Fn(UnusedExterns),
504) -> Result<(), TestFailure> {
505    let langstr = &doctest.langstr;
506    // Make sure we emit well-formed executable names for our target.
507    let rust_out = add_exe_suffix("rust_out".to_owned(), &rustdoc_options.target);
508    let output_file = doctest.test_opts.outdir.path().join(rust_out);
509
510    // Common arguments used for compiling the doctest runner.
511    // On merged doctests, the compiler is invoked twice: once for the test code itself,
512    // and once for the runner wrapper (which needs to use `#![feature]` on stable).
513    let mut compiler_args = vec![];
514
515    compiler_args.push(format!("@{}", doctest.global_opts.args_file.display()));
516
517    let sysroot = &rustdoc_options.sysroot;
518    if let Some(explicit_sysroot) = &sysroot.explicit {
519        compiler_args.push(format!("--sysroot={}", explicit_sysroot.display()));
520    }
521
522    compiler_args.extend_from_slice(&["--edition".to_owned(), doctest.edition.to_string()]);
523    if langstr.test_harness {
524        compiler_args.push("--test".to_owned());
525    }
526    if rustdoc_options.json_unused_externs.is_enabled() && !langstr.compile_fail {
527        compiler_args.push("--error-format=json".to_owned());
528        compiler_args.extend_from_slice(&["--json".to_owned(), "unused-externs".to_owned()]);
529        compiler_args.extend_from_slice(&["-W".to_owned(), "unused_crate_dependencies".to_owned()]);
530        compiler_args.extend_from_slice(&["-Z".to_owned(), "unstable-options".to_owned()]);
531    }
532
533    if doctest.no_run && !langstr.compile_fail && rustdoc_options.persist_doctests.is_none() {
534        // FIXME: why does this code check if it *shouldn't* persist doctests
535        //        -- shouldn't it be the negation?
536        compiler_args.push("--emit=metadata".to_owned());
537    }
538    compiler_args.extend_from_slice(&[
539        "--target".to_owned(),
540        match &rustdoc_options.target {
541            TargetTuple::TargetTuple(s) => s.clone(),
542            TargetTuple::TargetJson { path_for_rustdoc, .. } => {
543                path_for_rustdoc.to_str().expect("target path must be valid unicode").to_owned()
544            }
545        },
546    ]);
547    if let ErrorOutputType::HumanReadable { kind, color_config } = rustdoc_options.error_format {
548        let short = kind.short();
549        let unicode = kind == HumanReadableErrorType::Unicode;
550
551        if short {
552            compiler_args.extend_from_slice(&["--error-format".to_owned(), "short".to_owned()]);
553        }
554        if unicode {
555            compiler_args
556                .extend_from_slice(&["--error-format".to_owned(), "human-unicode".to_owned()]);
557        }
558
559        match color_config {
560            ColorConfig::Never => {
561                compiler_args.extend_from_slice(&["--color".to_owned(), "never".to_owned()]);
562            }
563            ColorConfig::Always => {
564                compiler_args.extend_from_slice(&["--color".to_owned(), "always".to_owned()]);
565            }
566            ColorConfig::Auto => {
567                compiler_args.extend_from_slice(&[
568                    "--color".to_owned(),
569                    if supports_color { "always" } else { "never" }.to_owned(),
570                ]);
571            }
572        }
573    }
574
575    let rustc_binary = rustdoc_options
576        .test_builder
577        .as_deref()
578        .unwrap_or_else(|| rustc_interface::util::rustc_path(sysroot).expect("found rustc"));
579    let mut compiler = wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
580
581    compiler.args(&compiler_args);
582
583    // If this is a merged doctest, we need to write it into a file instead of using stdin
584    // because if the size of the merged doctests is too big, it'll simply break stdin.
585    if doctest.is_multiple_tests() {
586        // It makes the compilation failure much faster if it is for a combined doctest.
587        compiler.arg("--error-format=short");
588        let input_file = doctest.path_for_merged_doctest_bundle();
589        if std::fs::write(&input_file, &doctest.full_test_code).is_err() {
590            // If we cannot write this file for any reason, we leave. All combined tests will be
591            // tested as standalone tests.
592            return Err(TestFailure::CompileError);
593        }
594        if !rustdoc_options.nocapture {
595            // If `nocapture` is disabled, then we don't display rustc's output when compiling
596            // the merged doctests.
597            compiler.stderr(Stdio::null());
598        }
599        // bundled tests are an rlib, loaded by a separate runner executable
600        compiler
601            .arg("--crate-type=lib")
602            .arg("--out-dir")
603            .arg(doctest.test_opts.outdir.path())
604            .arg(input_file);
605    } else {
606        compiler.arg("--crate-type=bin").arg("-o").arg(&output_file);
607        // Setting these environment variables is unneeded if this is a merged doctest.
608        compiler.env("UNSTABLE_RUSTDOC_TEST_PATH", &doctest.test_opts.path);
609        compiler.env(
610            "UNSTABLE_RUSTDOC_TEST_LINE",
611            format!("{}", doctest.line as isize - doctest.full_test_line_offset as isize),
612        );
613        compiler.arg("-");
614        compiler.stdin(Stdio::piped());
615        compiler.stderr(Stdio::piped());
616    }
617
618    debug!("compiler invocation for doctest: {compiler:?}");
619
620    let mut child = compiler.spawn().expect("Failed to spawn rustc process");
621    let output = if let Some(merged_test_code) = &doctest.merged_test_code {
622        // compile-fail tests never get merged, so this should always pass
623        let status = child.wait().expect("Failed to wait");
624
625        // the actual test runner is a separate component, built with nightly-only features;
626        // build it now
627        let runner_input_file = doctest.path_for_merged_doctest_runner();
628
629        let mut runner_compiler =
630            wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
631        // the test runner does not contain any user-written code, so this doesn't allow
632        // the user to exploit nightly-only features on stable
633        runner_compiler.env("RUSTC_BOOTSTRAP", "1");
634        runner_compiler.args(compiler_args);
635        runner_compiler.args(&["--crate-type=bin", "-o"]).arg(&output_file);
636        let mut extern_path = std::ffi::OsString::from(format!(
637            "--extern=doctest_bundle_{edition}=",
638            edition = doctest.edition
639        ));
640        for extern_str in &rustdoc_options.extern_strs {
641            if let Some((_cratename, path)) = extern_str.split_once('=') {
642                // Direct dependencies of the tests themselves are
643                // indirect dependencies of the test runner.
644                // They need to be in the library search path.
645                let dir = Path::new(path)
646                    .parent()
647                    .filter(|x| x.components().count() > 0)
648                    .unwrap_or(Path::new("."));
649                runner_compiler.arg("-L").arg(dir);
650            }
651        }
652        let output_bundle_file = doctest
653            .test_opts
654            .outdir
655            .path()
656            .join(format!("libdoctest_bundle_{edition}.rlib", edition = doctest.edition));
657        extern_path.push(&output_bundle_file);
658        runner_compiler.arg(extern_path);
659        runner_compiler.arg(&runner_input_file);
660        if std::fs::write(&runner_input_file, &merged_test_code).is_err() {
661            // If we cannot write this file for any reason, we leave. All combined tests will be
662            // tested as standalone tests.
663            return Err(TestFailure::CompileError);
664        }
665        if !rustdoc_options.nocapture {
666            // If `nocapture` is disabled, then we don't display rustc's output when compiling
667            // the merged doctests.
668            runner_compiler.stderr(Stdio::null());
669        }
670        runner_compiler.arg("--error-format=short");
671        debug!("compiler invocation for doctest runner: {runner_compiler:?}");
672
673        let status = if !status.success() {
674            status
675        } else {
676            let mut child_runner = runner_compiler.spawn().expect("Failed to spawn rustc process");
677            child_runner.wait().expect("Failed to wait")
678        };
679
680        process::Output { status, stdout: Vec::new(), stderr: Vec::new() }
681    } else {
682        let stdin = child.stdin.as_mut().expect("Failed to open stdin");
683        stdin.write_all(doctest.full_test_code.as_bytes()).expect("could write out test sources");
684        child.wait_with_output().expect("Failed to read stdout")
685    };
686
687    struct Bomb<'a>(&'a str);
688    impl Drop for Bomb<'_> {
689        fn drop(&mut self) {
690            eprint!("{}", self.0);
691        }
692    }
693    let mut out = str::from_utf8(&output.stderr)
694        .unwrap()
695        .lines()
696        .filter(|l| {
697            if let Ok(uext) = serde_json::from_str::<UnusedExterns>(l) {
698                report_unused_externs(uext);
699                false
700            } else {
701                true
702            }
703        })
704        .intersperse_with(|| "\n")
705        .collect::<String>();
706
707    // Add a \n to the end to properly terminate the last line,
708    // but only if there was output to be printed
709    if !out.is_empty() {
710        out.push('\n');
711    }
712
713    let _bomb = Bomb(&out);
714    match (output.status.success(), langstr.compile_fail) {
715        (true, true) => {
716            return Err(TestFailure::UnexpectedCompilePass);
717        }
718        (true, false) => {}
719        (false, true) => {
720            if !langstr.error_codes.is_empty() {
721                // We used to check if the output contained "error[{}]: " but since we added the
722                // colored output, we can't anymore because of the color escape characters before
723                // the ":".
724                let missing_codes: Vec<String> = langstr
725                    .error_codes
726                    .iter()
727                    .filter(|err| !out.contains(&format!("error[{err}]")))
728                    .cloned()
729                    .collect();
730
731                if !missing_codes.is_empty() {
732                    return Err(TestFailure::MissingErrorCodes(missing_codes));
733                }
734            }
735        }
736        (false, false) => {
737            return Err(TestFailure::CompileError);
738        }
739    }
740
741    if doctest.no_run {
742        return Ok(());
743    }
744
745    // Run the code!
746    let mut cmd;
747
748    let output_file = make_maybe_absolute_path(output_file);
749    if let Some(tool) = &rustdoc_options.test_runtool {
750        let tool = make_maybe_absolute_path(tool.into());
751        cmd = Command::new(tool);
752        cmd.args(&rustdoc_options.test_runtool_args);
753        cmd.arg(&output_file);
754    } else {
755        cmd = Command::new(&output_file);
756        if doctest.is_multiple_tests() {
757            cmd.env("RUSTDOC_DOCTEST_BIN_PATH", &output_file);
758        }
759    }
760    if let Some(run_directory) = &rustdoc_options.test_run_directory {
761        cmd.current_dir(run_directory);
762    }
763
764    let result = if doctest.is_multiple_tests() || rustdoc_options.nocapture {
765        cmd.status().map(|status| process::Output {
766            status,
767            stdout: Vec::new(),
768            stderr: Vec::new(),
769        })
770    } else {
771        cmd.output()
772    };
773    match result {
774        Err(e) => return Err(TestFailure::ExecutionError(e)),
775        Ok(out) => {
776            if langstr.should_panic && out.status.success() {
777                return Err(TestFailure::UnexpectedRunPass);
778            } else if !langstr.should_panic && !out.status.success() {
779                return Err(TestFailure::ExecutionFailure(out));
780            }
781        }
782    }
783
784    Ok(())
785}
786
787/// Converts a path intended to use as a command to absolute if it is
788/// relative, and not a single component.
789///
790/// This is needed to deal with relative paths interacting with
791/// `Command::current_dir` in a platform-specific way.
792fn make_maybe_absolute_path(path: PathBuf) -> PathBuf {
793    if path.components().count() == 1 {
794        // Look up process via PATH.
795        path
796    } else {
797        std::env::current_dir().map(|c| c.join(&path)).unwrap_or_else(|_| path)
798    }
799}
800struct IndividualTestOptions {
801    outdir: DirState,
802    path: PathBuf,
803}
804
805impl IndividualTestOptions {
806    fn new(options: &RustdocOptions, test_id: &Option<String>, test_path: PathBuf) -> Self {
807        let outdir = if let Some(ref path) = options.persist_doctests {
808            let mut path = path.clone();
809            path.push(test_id.as_deref().unwrap_or("<doctest>"));
810
811            if let Err(err) = std::fs::create_dir_all(&path) {
812                eprintln!("Couldn't create directory for doctest executables: {err}");
813                panic::resume_unwind(Box::new(()));
814            }
815
816            DirState::Perm(path)
817        } else {
818            DirState::Temp(get_doctest_dir().expect("rustdoc needs a tempdir"))
819        };
820
821        Self { outdir, path: test_path }
822    }
823}
824
825/// A doctest scraped from the code, ready to be turned into a runnable test.
826///
827/// The pipeline goes: [`clean`] AST -> `ScrapedDoctest` -> `RunnableDoctest`.
828/// [`run_merged_tests`] converts a bunch of scraped doctests to a single runnable doctest,
829/// while [`generate_unique_doctest`] does the standalones.
830///
831/// [`clean`]: crate::clean
832/// [`run_merged_tests`]: crate::doctest::runner::DocTestRunner::run_merged_tests
833/// [`generate_unique_doctest`]: crate::doctest::make::DocTestBuilder::generate_unique_doctest
834#[derive(Debug)]
835pub(crate) struct ScrapedDocTest {
836    filename: FileName,
837    line: usize,
838    langstr: LangString,
839    text: String,
840    name: String,
841    span: Span,
842    global_crate_attrs: Vec<String>,
843}
844
845impl ScrapedDocTest {
846    fn new(
847        filename: FileName,
848        line: usize,
849        logical_path: Vec<String>,
850        langstr: LangString,
851        text: String,
852        span: Span,
853        global_crate_attrs: Vec<String>,
854    ) -> Self {
855        let mut item_path = logical_path.join("::");
856        item_path.retain(|c| c != ' ');
857        if !item_path.is_empty() {
858            item_path.push(' ');
859        }
860        let name =
861            format!("{} - {item_path}(line {line})", filename.prefer_remapped_unconditionaly());
862
863        Self { filename, line, langstr, text, name, span, global_crate_attrs }
864    }
865    fn edition(&self, opts: &RustdocOptions) -> Edition {
866        self.langstr.edition.unwrap_or(opts.edition)
867    }
868
869    fn no_run(&self, opts: &RustdocOptions) -> bool {
870        self.langstr.no_run || opts.no_run
871    }
872    fn path(&self) -> PathBuf {
873        match &self.filename {
874            FileName::Real(path) => {
875                if let Some(local_path) = path.local_path() {
876                    local_path.to_path_buf()
877                } else {
878                    // Somehow we got the filename from the metadata of another crate, should never happen
879                    unreachable!("doctest from a different crate");
880                }
881            }
882            _ => PathBuf::from(r"doctest.rs"),
883        }
884    }
885}
886
887pub(crate) trait DocTestVisitor {
888    fn visit_test(&mut self, test: String, config: LangString, rel_line: MdRelLine);
889    fn visit_header(&mut self, _name: &str, _level: u32) {}
890}
891
892#[derive(Clone, Debug, Hash, Eq, PartialEq)]
893pub(crate) struct MergeableTestKey {
894    edition: Edition,
895    global_crate_attrs_hash: u64,
896}
897
898struct CreateRunnableDocTests {
899    standalone_tests: Vec<test::TestDescAndFn>,
900    mergeable_tests: FxIndexMap<MergeableTestKey, Vec<(DocTestBuilder, ScrapedDocTest)>>,
901
902    rustdoc_options: Arc<RustdocOptions>,
903    opts: GlobalTestOptions,
904    visited_tests: FxHashMap<(String, usize), usize>,
905    unused_extern_reports: Arc<Mutex<Vec<UnusedExterns>>>,
906    compiling_test_count: AtomicUsize,
907    can_merge_doctests: bool,
908}
909
910impl CreateRunnableDocTests {
911    fn new(rustdoc_options: RustdocOptions, opts: GlobalTestOptions) -> CreateRunnableDocTests {
912        let can_merge_doctests = rustdoc_options.edition >= Edition::Edition2024;
913        CreateRunnableDocTests {
914            standalone_tests: Vec::new(),
915            mergeable_tests: FxIndexMap::default(),
916            rustdoc_options: Arc::new(rustdoc_options),
917            opts,
918            visited_tests: FxHashMap::default(),
919            unused_extern_reports: Default::default(),
920            compiling_test_count: AtomicUsize::new(0),
921            can_merge_doctests,
922        }
923    }
924
925    fn add_test(&mut self, scraped_test: ScrapedDocTest, dcx: Option<DiagCtxtHandle<'_>>) {
926        // For example `module/file.rs` would become `module_file_rs`
927        let file = scraped_test
928            .filename
929            .prefer_local()
930            .to_string_lossy()
931            .chars()
932            .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
933            .collect::<String>();
934        let test_id = format!(
935            "{file}_{line}_{number}",
936            file = file,
937            line = scraped_test.line,
938            number = {
939                // Increases the current test number, if this file already
940                // exists or it creates a new entry with a test number of 0.
941                self.visited_tests
942                    .entry((file.clone(), scraped_test.line))
943                    .and_modify(|v| *v += 1)
944                    .or_insert(0)
945            },
946        );
947
948        let edition = scraped_test.edition(&self.rustdoc_options);
949        let doctest = BuildDocTestBuilder::new(&scraped_test.text)
950            .crate_name(&self.opts.crate_name)
951            .global_crate_attrs(scraped_test.global_crate_attrs.clone())
952            .edition(edition)
953            .can_merge_doctests(self.can_merge_doctests)
954            .test_id(test_id)
955            .lang_str(&scraped_test.langstr)
956            .span(scraped_test.span)
957            .build(dcx);
958        let is_standalone = !doctest.can_be_merged
959            || scraped_test.langstr.compile_fail
960            || scraped_test.langstr.test_harness
961            || scraped_test.langstr.standalone_crate
962            || self.rustdoc_options.nocapture
963            || self.rustdoc_options.test_args.iter().any(|arg| arg == "--show-output");
964        if is_standalone {
965            let test_desc = self.generate_test_desc_and_fn(doctest, scraped_test);
966            self.standalone_tests.push(test_desc);
967        } else {
968            self.mergeable_tests
969                .entry(MergeableTestKey {
970                    edition,
971                    global_crate_attrs_hash: {
972                        let mut hasher = FxHasher::default();
973                        scraped_test.global_crate_attrs.hash(&mut hasher);
974                        hasher.finish()
975                    },
976                })
977                .or_default()
978                .push((doctest, scraped_test));
979        }
980    }
981
982    fn generate_test_desc_and_fn(
983        &mut self,
984        test: DocTestBuilder,
985        scraped_test: ScrapedDocTest,
986    ) -> test::TestDescAndFn {
987        if !scraped_test.langstr.compile_fail {
988            self.compiling_test_count.fetch_add(1, Ordering::SeqCst);
989        }
990
991        generate_test_desc_and_fn(
992            test,
993            scraped_test,
994            self.opts.clone(),
995            Arc::clone(&self.rustdoc_options),
996            self.unused_extern_reports.clone(),
997        )
998    }
999}
1000
1001fn generate_test_desc_and_fn(
1002    test: DocTestBuilder,
1003    scraped_test: ScrapedDocTest,
1004    opts: GlobalTestOptions,
1005    rustdoc_options: Arc<RustdocOptions>,
1006    unused_externs: Arc<Mutex<Vec<UnusedExterns>>>,
1007) -> test::TestDescAndFn {
1008    let target_str = rustdoc_options.target.to_string();
1009    let rustdoc_test_options =
1010        IndividualTestOptions::new(&rustdoc_options, &test.test_id, scraped_test.path());
1011
1012    debug!("creating test {}: {}", scraped_test.name, scraped_test.text);
1013    test::TestDescAndFn {
1014        desc: test::TestDesc {
1015            name: test::DynTestName(scraped_test.name.clone()),
1016            ignore: match scraped_test.langstr.ignore {
1017                Ignore::All => true,
1018                Ignore::None => false,
1019                Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)),
1020            },
1021            ignore_message: None,
1022            source_file: "",
1023            start_line: 0,
1024            start_col: 0,
1025            end_line: 0,
1026            end_col: 0,
1027            // compiler failures are test failures
1028            should_panic: test::ShouldPanic::No,
1029            compile_fail: scraped_test.langstr.compile_fail,
1030            no_run: scraped_test.no_run(&rustdoc_options),
1031            test_type: test::TestType::DocTest,
1032        },
1033        testfn: test::DynTestFn(Box::new(move || {
1034            doctest_run_fn(
1035                rustdoc_test_options,
1036                opts,
1037                test,
1038                scraped_test,
1039                rustdoc_options,
1040                unused_externs,
1041            )
1042        })),
1043    }
1044}
1045
1046fn doctest_run_fn(
1047    test_opts: IndividualTestOptions,
1048    global_opts: GlobalTestOptions,
1049    doctest: DocTestBuilder,
1050    scraped_test: ScrapedDocTest,
1051    rustdoc_options: Arc<RustdocOptions>,
1052    unused_externs: Arc<Mutex<Vec<UnusedExterns>>>,
1053) -> Result<(), String> {
1054    let report_unused_externs = |uext| {
1055        unused_externs.lock().unwrap().push(uext);
1056    };
1057    let (wrapped, full_test_line_offset) = doctest.generate_unique_doctest(
1058        &scraped_test.text,
1059        scraped_test.langstr.test_harness,
1060        &global_opts,
1061        Some(&global_opts.crate_name),
1062    );
1063    let runnable_test = RunnableDocTest {
1064        full_test_code: wrapped.to_string(),
1065        full_test_line_offset,
1066        test_opts,
1067        global_opts,
1068        langstr: scraped_test.langstr.clone(),
1069        line: scraped_test.line,
1070        edition: scraped_test.edition(&rustdoc_options),
1071        no_run: scraped_test.no_run(&rustdoc_options),
1072        merged_test_code: None,
1073    };
1074    let res =
1075        run_test(runnable_test, &rustdoc_options, doctest.supports_color, report_unused_externs);
1076
1077    if let Err(err) = res {
1078        match err {
1079            TestFailure::CompileError => {
1080                eprint!("Couldn't compile the test.");
1081            }
1082            TestFailure::UnexpectedCompilePass => {
1083                eprint!("Test compiled successfully, but it's marked `compile_fail`.");
1084            }
1085            TestFailure::UnexpectedRunPass => {
1086                eprint!("Test executable succeeded, but it's marked `should_panic`.");
1087            }
1088            TestFailure::MissingErrorCodes(codes) => {
1089                eprint!("Some expected error codes were not found: {codes:?}");
1090            }
1091            TestFailure::ExecutionError(err) => {
1092                eprint!("Couldn't run the test: {err}");
1093                if err.kind() == io::ErrorKind::PermissionDenied {
1094                    eprint!(" - maybe your tempdir is mounted with noexec?");
1095                }
1096            }
1097            TestFailure::ExecutionFailure(out) => {
1098                eprintln!("Test executable failed ({reason}).", reason = out.status);
1099
1100                // FIXME(#12309): An unfortunate side-effect of capturing the test
1101                // executable's output is that the relative ordering between the test's
1102                // stdout and stderr is lost. However, this is better than the
1103                // alternative: if the test executable inherited the parent's I/O
1104                // handles the output wouldn't be captured at all, even on success.
1105                //
1106                // The ordering could be preserved if the test process' stderr was
1107                // redirected to stdout, but that functionality does not exist in the
1108                // standard library, so it may not be portable enough.
1109                let stdout = str::from_utf8(&out.stdout).unwrap_or_default();
1110                let stderr = str::from_utf8(&out.stderr).unwrap_or_default();
1111
1112                if !stdout.is_empty() || !stderr.is_empty() {
1113                    eprintln!();
1114
1115                    if !stdout.is_empty() {
1116                        eprintln!("stdout:\n{stdout}");
1117                    }
1118
1119                    if !stderr.is_empty() {
1120                        eprintln!("stderr:\n{stderr}");
1121                    }
1122                }
1123            }
1124        }
1125
1126        panic::resume_unwind(Box::new(()));
1127    }
1128    Ok(())
1129}
1130
1131#[cfg(test)] // used in tests
1132impl DocTestVisitor for Vec<usize> {
1133    fn visit_test(&mut self, _test: String, _config: LangString, rel_line: MdRelLine) {
1134        self.push(1 + rel_line.offset());
1135    }
1136}
1137
1138#[cfg(test)]
1139mod tests;