rustc_mir_transform/
cross_crate_inline.rs

1use rustc_hir::attrs::InlineAttr;
2use rustc_hir::def::DefKind;
3use rustc_hir::def_id::LocalDefId;
4use rustc_middle::mir::visit::Visitor;
5use rustc_middle::mir::*;
6use rustc_middle::query::Providers;
7use rustc_middle::ty::TyCtxt;
8use rustc_session::config::{InliningThreshold, OptLevel};
9use rustc_span::sym;
10
11use crate::{inline, pass_manager as pm};
12
13pub(super) fn provide(providers: &mut Providers) {
14    providers.cross_crate_inlinable = cross_crate_inlinable;
15}
16
17fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
18    let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
19    // If this has an extern indicator, then this function is globally shared and thus will not
20    // generate cgu-internal copies which would make it cross-crate inlinable.
21    if codegen_fn_attrs.contains_extern_indicator() {
22        return false;
23    }
24
25    // This just reproduces the logic from Instance::requires_inline.
26    match tcx.def_kind(def_id) {
27        DefKind::Ctor(..) | DefKind::Closure | DefKind::SyntheticCoroutineBody => return true,
28        DefKind::Fn | DefKind::AssocFn => {}
29        _ => return false,
30    }
31
32    // From this point on, it is valid to return true or false.
33    if tcx.sess.opts.unstable_opts.cross_crate_inline_threshold == InliningThreshold::Always {
34        return true;
35    }
36
37    // FIXME(autodiff): replace this as per discussion in https://github.com/rust-lang/rust/pull/149033#discussion_r2535465880
38    if tcx.has_attr(def_id, sym::autodiff_forward)
39        || tcx.has_attr(def_id, sym::autodiff_reverse)
40        || tcx.has_attr(def_id, sym::rustc_autodiff)
41    {
42        return true;
43    }
44
45    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
46        // Intrinsic fallback bodies are always cross-crate inlineable.
47        // To ensure that the MIR inliner doesn't cluelessly try to inline fallback
48        // bodies even when the backend would implement something better, we stop
49        // the MIR inliner from ever inlining an intrinsic.
50        return true;
51    }
52
53    // Obey source annotations first; this is important because it means we can use
54    // #[inline(never)] to force code generation.
55    match codegen_fn_attrs.inline {
56        InlineAttr::Never => return false,
57        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
58        _ => {}
59    }
60
61    // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until
62    // the function is referenced, in order to skip codegen for unused functions. This is
63    // intentionally after the check for `inline(never)`, so that `inline(never)` wins.
64    if tcx.sess.opts.unstable_opts.hint_mostly_unused {
65        return true;
66    }
67
68    let sig = tcx.fn_sig(def_id).instantiate_identity();
69    for ty in sig.inputs().skip_binder().iter().chain(std::iter::once(&sig.output().skip_binder()))
70    {
71        // FIXME(f16_f128): in order to avoid crashes building `core`, always inline to skip
72        // codegen if the function is not used.
73        if ty == &tcx.types.f16 || ty == &tcx.types.f128 {
74            return true;
75        }
76    }
77
78    // Don't do any inference when incremental compilation is enabled; the additional inlining that
79    // inference permits also creates more work for small edits.
80    if tcx.sess.opts.incremental.is_some() {
81        return false;
82    }
83
84    // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
85    // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
86    // which is less confusing than having to also enable -Copt-level=1.
87    let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline, pm::Optimizations::Allowed)
88        || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
89    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
90        return false;
91    }
92
93    if !tcx.is_mir_available(def_id) {
94        return false;
95    }
96
97    let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold {
98        InliningThreshold::Always => return true,
99        InliningThreshold::Sometimes(threshold) => threshold,
100        InliningThreshold::Never => return false,
101    };
102
103    let mir = tcx.optimized_mir(def_id);
104    let mut checker =
105        CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };
106    checker.visit_body(mir);
107    checker.calls == 0
108        && checker.resumes == 0
109        && checker.landing_pads == 0
110        && checker.statements <= threshold
111}
112
113struct CostChecker<'b, 'tcx> {
114    tcx: TyCtxt<'tcx>,
115    callee_body: &'b Body<'tcx>,
116    calls: usize,
117    statements: usize,
118    landing_pads: usize,
119    resumes: usize,
120}
121
122impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
123    fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
124        // Don't count StorageLive/StorageDead in the inlining cost.
125        match statement.kind {
126            StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Nop => {}
127            _ => self.statements += 1,
128        }
129    }
130
131    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
132        let tcx = self.tcx;
133        match terminator.kind {
134            TerminatorKind::Drop { ref place, unwind, .. } => {
135                let ty = place.ty(self.callee_body, tcx).ty;
136                if !ty.is_trivially_pure_clone_copy() {
137                    self.calls += 1;
138                    if let UnwindAction::Cleanup(_) = unwind {
139                        self.landing_pads += 1;
140                    }
141                }
142            }
143            TerminatorKind::Call { ref func, unwind, .. } => {
144                // We track calls because they make our function not a leaf (and in theory, the
145                // number of calls indicates how likely this function is to perturb other CGUs).
146                // But intrinsics don't have a body that gets assigned to a CGU, so they are
147                // ignored.
148                if let Some((fn_def_id, _)) = func.const_fn_def()
149                    && self.tcx.has_attr(fn_def_id, sym::rustc_intrinsic)
150                {
151                    return;
152                }
153                self.calls += 1;
154                if let UnwindAction::Cleanup(_) = unwind {
155                    self.landing_pads += 1;
156                }
157            }
158            TerminatorKind::Assert { unwind, .. } => {
159                self.calls += 1;
160                if let UnwindAction::Cleanup(_) = unwind {
161                    self.landing_pads += 1;
162                }
163            }
164            TerminatorKind::UnwindResume => self.resumes += 1,
165            TerminatorKind::InlineAsm { unwind, .. } => {
166                self.statements += 1;
167                if let UnwindAction::Cleanup(_) = unwind {
168                    self.landing_pads += 1;
169                }
170            }
171            TerminatorKind::Return => {}
172            _ => self.statements += 1,
173        }
174    }
175}
rustc_mir_transform/cross_crate_inline.rs

rustc_mir_transform/
cross_crate_inline.rs
