tidy/
ext_tool_checks.rs

1//! Optional checks for file types other than Rust source
2//!
3//! Handles python tool version management via a virtual environment in
4//! `build/venv`.
5//!
6//! # Functional outline
7//!
8//! 1. Run tidy with an extra option: `--extra-checks=py,shell`,
9//!    `--extra-checks=py:lint`, or similar. Optionally provide specific
10//!    configuration after a double dash (`--extra-checks=py -- foo.py`)
11//! 2. Build configuration based on args/environment:
12//!    - Formatters by default are in check only mode
13//!    - If in CI (TIDY_PRINT_DIFF=1 is set), check and print the diff
14//!    - If `--bless` is provided, formatters may run
15//!    - Pass any additional config after the `--`. If no files are specified,
16//!      use a default.
17//! 3. Print the output of the given command. If it fails and `TIDY_PRINT_DIFF`
18//!    is set, rerun the tool to print a suggestion diff (for e.g. CI)
19
20use std::ffi::OsStr;
21use std::path::{Path, PathBuf};
22use std::process::Command;
23use std::{fmt, fs, io};
24
25const MIN_PY_REV: (u32, u32) = (3, 9);
26const MIN_PY_REV_STR: &str = "≥3.9";
27
28/// Path to find the python executable within a virtual environment
29#[cfg(target_os = "windows")]
30const REL_PY_PATH: &[&str] = &["Scripts", "python3.exe"];
31#[cfg(not(target_os = "windows"))]
32const REL_PY_PATH: &[&str] = &["bin", "python3"];
33
34const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"];
35/// Location within build directory
36const RUFF_CACHE_PATH: &[&str] = &["cache", "ruff_cache"];
37const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"];
38
39pub fn check(
40    root_path: &Path,
41    outdir: &Path,
42    bless: bool,
43    extra_checks: Option<&str>,
44    pos_args: &[String],
45    bad: &mut bool,
46) {
47    if let Err(e) = check_impl(root_path, outdir, bless, extra_checks, pos_args) {
48        tidy_error!(bad, "{e}");
49    }
50}
51
52fn check_impl(
53    root_path: &Path,
54    outdir: &Path,
55    bless: bool,
56    extra_checks: Option<&str>,
57    pos_args: &[String],
58) -> Result<(), Error> {
59    let show_diff = std::env::var("TIDY_PRINT_DIFF")
60        .map_or(false, |v| v.eq_ignore_ascii_case("true") || v == "1");
61
62    // Split comma-separated args up
63    let lint_args = match extra_checks {
64        Some(s) => s.strip_prefix("--extra-checks=").unwrap().split(',').collect(),
65        None => vec![],
66    };
67
68    if lint_args.contains(&"spellcheck:fix") {
69        return Err(Error::Generic(
70            "`spellcheck:fix` is no longer valid, use `--extra=check=spellcheck --bless`"
71                .to_string(),
72        ));
73    }
74
75    let python_all = lint_args.contains(&"py");
76    let python_lint = lint_args.contains(&"py:lint") || python_all;
77    let python_fmt = lint_args.contains(&"py:fmt") || python_all;
78    let shell_all = lint_args.contains(&"shell");
79    let shell_lint = lint_args.contains(&"shell:lint") || shell_all;
80    let cpp_all = lint_args.contains(&"cpp");
81    let cpp_fmt = lint_args.contains(&"cpp:fmt") || cpp_all;
82    let spellcheck = lint_args.contains(&"spellcheck");
83
84    let mut py_path = None;
85
86    let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args
87        .iter()
88        .map(OsStr::new)
89        .partition(|arg| arg.to_str().is_some_and(|s| s.starts_with('-')));
90
91    if python_lint || python_fmt || cpp_fmt {
92        let venv_path = outdir.join("venv");
93        let mut reqs_path = root_path.to_owned();
94        reqs_path.extend(PIP_REQ_PATH);
95        py_path = Some(get_or_create_venv(&venv_path, &reqs_path)?);
96    }
97
98    if python_lint {
99        eprintln!("linting python files");
100        let py_path = py_path.as_ref().unwrap();
101        let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &["check".as_ref()]);
102
103        if res.is_err() && show_diff {
104            eprintln!("\npython linting failed! Printing diff suggestions:");
105
106            let _ = run_ruff(
107                root_path,
108                outdir,
109                py_path,
110                &cfg_args,
111                &file_args,
112                &["check".as_ref(), "--diff".as_ref()],
113            );
114        }
115        // Rethrow error
116        let _ = res?;
117    }
118
119    if python_fmt {
120        let mut args: Vec<&OsStr> = vec!["format".as_ref()];
121        if bless {
122            eprintln!("formatting python files");
123        } else {
124            eprintln!("checking python file formatting");
125            args.push("--check".as_ref());
126        }
127
128        let py_path = py_path.as_ref().unwrap();
129        let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &args);
130
131        if res.is_err() && !bless {
132            if show_diff {
133                eprintln!("\npython formatting does not match! Printing diff:");
134
135                let _ = run_ruff(
136                    root_path,
137                    outdir,
138                    py_path,
139                    &cfg_args,
140                    &file_args,
141                    &["format".as_ref(), "--diff".as_ref()],
142                );
143            }
144            eprintln!("rerun tidy with `--extra-checks=py:fmt --bless` to reformat Python code");
145        }
146
147        // Rethrow error
148        let _ = res?;
149    }
150
151    if cpp_fmt {
152        let mut cfg_args_clang_format = cfg_args.clone();
153        let mut file_args_clang_format = file_args.clone();
154        let config_path = root_path.join(".clang-format");
155        let config_file_arg = format!("file:{}", config_path.display());
156        cfg_args_clang_format.extend(&["--style".as_ref(), config_file_arg.as_ref()]);
157        if bless {
158            eprintln!("formatting C++ files");
159            cfg_args_clang_format.push("-i".as_ref());
160        } else {
161            eprintln!("checking C++ file formatting");
162            cfg_args_clang_format.extend(&["--dry-run".as_ref(), "--Werror".as_ref()]);
163        }
164        let files;
165        if file_args_clang_format.is_empty() {
166            let llvm_wrapper = root_path.join("compiler/rustc_llvm/llvm-wrapper");
167            files = find_with_extension(
168                root_path,
169                Some(llvm_wrapper.as_path()),
170                &[OsStr::new("h"), OsStr::new("cpp")],
171            )?;
172            file_args_clang_format.extend(files.iter().map(|p| p.as_os_str()));
173        }
174        let args = merge_args(&cfg_args_clang_format, &file_args_clang_format);
175        let res = py_runner(py_path.as_ref().unwrap(), false, None, "clang-format", &args);
176
177        if res.is_err() && show_diff {
178            eprintln!("\nclang-format linting failed! Printing diff suggestions:");
179
180            let mut cfg_args_clang_format_diff = cfg_args.clone();
181            cfg_args_clang_format_diff.extend(&["--style".as_ref(), config_file_arg.as_ref()]);
182            for file in file_args_clang_format {
183                let mut formatted = String::new();
184                let mut diff_args = cfg_args_clang_format_diff.clone();
185                diff_args.push(file);
186                let _ = py_runner(
187                    py_path.as_ref().unwrap(),
188                    false,
189                    Some(&mut formatted),
190                    "clang-format",
191                    &diff_args,
192                );
193                if formatted.is_empty() {
194                    eprintln!(
195                        "failed to obtain the formatted content for '{}'",
196                        file.to_string_lossy()
197                    );
198                    continue;
199                }
200                let actual = std::fs::read_to_string(file).unwrap_or_else(|e| {
201                    panic!(
202                        "failed to read the C++ file at '{}' due to '{e}'",
203                        file.to_string_lossy()
204                    )
205                });
206                if formatted != actual {
207                    let diff = similar::TextDiff::from_lines(&actual, &formatted);
208                    eprintln!(
209                        "{}",
210                        diff.unified_diff().context_radius(4).header(
211                            &format!("{} (actual)", file.to_string_lossy()),
212                            &format!("{} (formatted)", file.to_string_lossy())
213                        )
214                    );
215                }
216            }
217        }
218        // Rethrow error
219        let _ = res?;
220    }
221
222    if shell_lint {
223        eprintln!("linting shell files");
224
225        let mut file_args_shc = file_args.clone();
226        let files;
227        if file_args_shc.is_empty() {
228            files = find_with_extension(root_path, None, &[OsStr::new("sh")])?;
229            file_args_shc.extend(files.iter().map(|p| p.as_os_str()));
230        }
231
232        shellcheck_runner(&merge_args(&cfg_args, &file_args_shc))?;
233    }
234
235    if spellcheck {
236        let config_path = root_path.join("typos.toml");
237        // sync target files with .github/workflows/spellcheck.yml
238        let mut args = vec![
239            "-c",
240            config_path.as_os_str().to_str().unwrap(),
241            "./compiler",
242            "./library",
243            "./src/bootstrap",
244            "./src/librustdoc",
245        ];
246
247        if bless {
248            eprintln!("spellcheck files and fix");
249            args.push("--write-changes");
250        } else {
251            eprintln!("spellcheck files");
252        }
253        spellcheck_runner(&args)?;
254    }
255
256    Ok(())
257}
258
259fn run_ruff(
260    root_path: &Path,
261    outdir: &Path,
262    py_path: &Path,
263    cfg_args: &[&OsStr],
264    file_args: &[&OsStr],
265    ruff_args: &[&OsStr],
266) -> Result<(), Error> {
267    let mut cfg_args_ruff = cfg_args.into_iter().copied().collect::<Vec<_>>();
268    let mut file_args_ruff = file_args.into_iter().copied().collect::<Vec<_>>();
269
270    let mut cfg_path = root_path.to_owned();
271    cfg_path.extend(RUFF_CONFIG_PATH);
272    let mut cache_dir = outdir.to_owned();
273    cache_dir.extend(RUFF_CACHE_PATH);
274
275    cfg_args_ruff.extend([
276        "--config".as_ref(),
277        cfg_path.as_os_str(),
278        "--cache-dir".as_ref(),
279        cache_dir.as_os_str(),
280    ]);
281
282    if file_args_ruff.is_empty() {
283        file_args_ruff.push(root_path.as_os_str());
284    }
285
286    let mut args: Vec<&OsStr> = ruff_args.into_iter().copied().collect();
287    args.extend(merge_args(&cfg_args_ruff, &file_args_ruff));
288    py_runner(py_path, true, None, "ruff", &args)
289}
290
291/// Helper to create `cfg1 cfg2 -- file1 file2` output
292fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> {
293    let mut args = cfg_args.to_owned();
294    args.push("--".as_ref());
295    args.extend(file_args);
296    args
297}
298
299/// Run a python command with given arguments. `py_path` should be a virtualenv.
300///
301/// Captures `stdout` to a string if provided, otherwise prints the output.
302fn py_runner(
303    py_path: &Path,
304    as_module: bool,
305    stdout: Option<&mut String>,
306    bin: &'static str,
307    args: &[&OsStr],
308) -> Result<(), Error> {
309    let mut cmd = Command::new(py_path);
310    if as_module {
311        cmd.arg("-m").arg(bin).args(args);
312    } else {
313        let bin_path = py_path.with_file_name(bin);
314        cmd.arg(bin_path).args(args);
315    }
316    let status = if let Some(stdout) = stdout {
317        let output = cmd.output()?;
318        if let Ok(s) = std::str::from_utf8(&output.stdout) {
319            stdout.push_str(s);
320        }
321        output.status
322    } else {
323        cmd.status()?
324    };
325    if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) }
326}
327
328/// Create a virtuaenv at a given path if it doesn't already exist, or validate
329/// the install if it does. Returns the path to that venv's python executable.
330fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result<PathBuf, Error> {
331    let mut should_create = true;
332    let dst_reqs_path = venv_path.join("requirements.txt");
333    let mut py_path = venv_path.to_owned();
334    py_path.extend(REL_PY_PATH);
335
336    if let Ok(req) = fs::read_to_string(&dst_reqs_path) {
337        if req == fs::read_to_string(src_reqs_path)? {
338            // found existing environment
339            should_create = false;
340        } else {
341            eprintln!("requirements.txt file mismatch, recreating environment");
342        }
343    }
344
345    if should_create {
346        eprintln!("removing old virtual environment");
347        if venv_path.is_dir() {
348            fs::remove_dir_all(venv_path).unwrap_or_else(|_| {
349                panic!("failed to remove directory at {}", venv_path.display())
350            });
351        }
352        create_venv_at_path(venv_path)?;
353        install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?;
354    }
355
356    verify_py_version(&py_path)?;
357    Ok(py_path)
358}
359
360/// Attempt to create a virtualenv at this path. Cycles through all expected
361/// valid python versions to find one that is installed.
362fn create_venv_at_path(path: &Path) -> Result<(), Error> {
363    /// Preferred python versions in order. Newest to oldest then current
364    /// development versions
365    const TRY_PY: &[&str] = &[
366        "python3.13",
367        "python3.12",
368        "python3.11",
369        "python3.10",
370        "python3.9",
371        "python3",
372        "python",
373        "python3.14",
374    ];
375
376    let mut sys_py = None;
377    let mut found = Vec::new();
378
379    for py in TRY_PY {
380        match verify_py_version(Path::new(py)) {
381            Ok(_) => {
382                sys_py = Some(*py);
383                break;
384            }
385            // Skip not found errors
386            Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (),
387            // Skip insufficient version errors
388            Err(Error::Version { installed, .. }) => found.push(installed),
389            // just log and skip unrecognized errors
390            Err(e) => eprintln!("note: error running '{py}': {e}"),
391        }
392    }
393
394    let Some(sys_py) = sys_py else {
395        let ret = if found.is_empty() {
396            Error::MissingReq("python3", "python file checks", None)
397        } else {
398            found.sort();
399            found.dedup();
400            Error::Version {
401                program: "python3",
402                required: MIN_PY_REV_STR,
403                installed: found.join(", "),
404            }
405        };
406        return Err(ret);
407    };
408
409    // First try venv, which should be packaged in the Python3 standard library.
410    // If it is not available, try to create the virtual environment using the
411    // virtualenv package.
412    if try_create_venv(sys_py, path, "venv").is_ok() {
413        return Ok(());
414    }
415    try_create_venv(sys_py, path, "virtualenv")
416}
417
418fn try_create_venv(python: &str, path: &Path, module: &str) -> Result<(), Error> {
419    eprintln!(
420        "creating virtual environment at '{}' using '{python}' and '{module}'",
421        path.display()
422    );
423    let out = Command::new(python).args(["-m", module]).arg(path).output().unwrap();
424
425    if out.status.success() {
426        return Ok(());
427    }
428
429    let stderr = String::from_utf8_lossy(&out.stderr);
430    let err = if stderr.contains(&format!("No module named {module}")) {
431        Error::Generic(format!(
432            r#"{module} not found: you may need to install it:
433`{python} -m pip install {module}`
434If you see an error about "externally managed environment" when running the above command,
435either install `{module}` using your system package manager
436(e.g. `sudo apt-get install {python}-{module}`) or create a virtual environment manually, install
437`{module}` in it and then activate it before running tidy.
438"#
439        ))
440    } else {
441        Error::Generic(format!(
442            "failed to create venv at '{}' using {python} -m {module}: {stderr}",
443            path.display()
444        ))
445    };
446    Err(err)
447}
448
449/// Parse python's version output (`Python x.y.z`) and ensure we have a
450/// suitable version.
451fn verify_py_version(py_path: &Path) -> Result<(), Error> {
452    let out = Command::new(py_path).arg("--version").output()?;
453    let outstr = String::from_utf8_lossy(&out.stdout);
454    let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim();
455    let mut vers_comps = vers.split('.');
456    let major: u32 = vers_comps.next().unwrap().parse().unwrap();
457    let minor: u32 = vers_comps.next().unwrap().parse().unwrap();
458
459    if (major, minor) < MIN_PY_REV {
460        Err(Error::Version {
461            program: "python",
462            required: MIN_PY_REV_STR,
463            installed: vers.to_owned(),
464        })
465    } else {
466        Ok(())
467    }
468}
469
470fn install_requirements(
471    py_path: &Path,
472    src_reqs_path: &Path,
473    dst_reqs_path: &Path,
474) -> Result<(), Error> {
475    let stat = Command::new(py_path)
476        .args(["-m", "pip", "install", "--upgrade", "pip"])
477        .status()
478        .expect("failed to launch pip");
479    if !stat.success() {
480        return Err(Error::Generic(format!("pip install failed with status {stat}")));
481    }
482
483    let stat = Command::new(py_path)
484        .args(["-m", "pip", "install", "--quiet", "--require-hashes", "-r"])
485        .arg(src_reqs_path)
486        .status()?;
487    if !stat.success() {
488        return Err(Error::Generic(format!(
489            "failed to install requirements at {}",
490            src_reqs_path.display()
491        )));
492    }
493    fs::copy(src_reqs_path, dst_reqs_path)?;
494    assert_eq!(
495        fs::read_to_string(src_reqs_path).unwrap(),
496        fs::read_to_string(dst_reqs_path).unwrap()
497    );
498    Ok(())
499}
500
501/// Check that shellcheck is installed then run it at the given path
502fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> {
503    match Command::new("shellcheck").arg("--version").status() {
504        Ok(_) => (),
505        Err(e) if e.kind() == io::ErrorKind::NotFound => {
506            return Err(Error::MissingReq(
507                "shellcheck",
508                "shell file checks",
509                Some(
510                    "see <https://github.com/koalaman/shellcheck#installing> \
511                    for installation instructions"
512                        .to_owned(),
513                ),
514            ));
515        }
516        Err(e) => return Err(e.into()),
517    }
518
519    let status = Command::new("shellcheck").args(args).status()?;
520    if status.success() { Ok(()) } else { Err(Error::FailedCheck("shellcheck")) }
521}
522
523/// Check that spellchecker is installed then run it at the given path
524fn spellcheck_runner(args: &[&str]) -> Result<(), Error> {
525    // sync version with .github/workflows/spellcheck.yml
526    let expected_version = "typos-cli 1.34.0";
527    match Command::new("typos").arg("--version").output() {
528        Ok(o) => {
529            let stdout = String::from_utf8_lossy(&o.stdout);
530            if stdout.trim() != expected_version {
531                return Err(Error::Version {
532                    program: "typos",
533                    required: expected_version,
534                    installed: stdout.trim().to_string(),
535                });
536            }
537        }
538        Err(e) if e.kind() == io::ErrorKind::NotFound => {
539            return Err(Error::MissingReq(
540                "typos",
541                "spellcheck file checks",
542                // sync version with .github/workflows/spellcheck.yml
543                Some("install tool via `cargo install typos-cli@1.34.0`".to_owned()),
544            ));
545        }
546        Err(e) => return Err(e.into()),
547    }
548
549    let status = Command::new("typos").args(args).status()?;
550    if status.success() { Ok(()) } else { Err(Error::FailedCheck("typos")) }
551}
552
553/// Check git for tracked files matching an extension
554fn find_with_extension(
555    root_path: &Path,
556    find_dir: Option<&Path>,
557    extensions: &[&OsStr],
558) -> Result<Vec<PathBuf>, Error> {
559    // Untracked files show up for short status and are indicated with a leading `?`
560    // -C changes git to be as if run from that directory
561    let stat_output =
562        Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout;
563
564    if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 {
565        eprintln!("found untracked files, ignoring");
566    }
567
568    let mut output = Vec::new();
569    let binding = {
570        let mut command = Command::new("git");
571        command.arg("-C").arg(root_path).args(["ls-files"]);
572        if let Some(find_dir) = find_dir {
573            command.arg(find_dir);
574        }
575        command.output()?
576    };
577    let tracked = String::from_utf8_lossy(&binding.stdout);
578
579    for line in tracked.lines() {
580        let line = line.trim();
581        let path = Path::new(line);
582
583        let Some(ref extension) = path.extension() else {
584            continue;
585        };
586        if extensions.contains(extension) {
587            output.push(root_path.join(path));
588        }
589    }
590
591    Ok(output)
592}
593
594#[derive(Debug)]
595enum Error {
596    Io(io::Error),
597    /// a is required to run b. c is extra info
598    MissingReq(&'static str, &'static str, Option<String>),
599    /// Tool x failed the check
600    FailedCheck(&'static str),
601    /// Any message, just print it
602    Generic(String),
603    /// Installed but wrong version
604    Version {
605        program: &'static str,
606        required: &'static str,
607        installed: String,
608    },
609}
610
611impl fmt::Display for Error {
612    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
613        match self {
614            Self::MissingReq(a, b, ex) => {
615                write!(
616                    f,
617                    "{a} is required to run {b} but it could not be located. Is it installed?"
618                )?;
619                if let Some(s) = ex {
620                    write!(f, "\n{s}")?;
621                };
622                Ok(())
623            }
624            Self::Version { program, required, installed } => write!(
625                f,
626                "insufficient version of '{program}' to run external tools: \
627                {required} required but found {installed}",
628            ),
629            Self::Generic(s) => f.write_str(s),
630            Self::Io(e) => write!(f, "IO error: {e}"),
631            Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"),
632        }
633    }
634}
635
636impl From<io::Error> for Error {
637    fn from(value: io::Error) -> Self {
638        Self::Io(value)
639    }
640}