1use std::assert_matches::debug_assert_matches;
4use std::iter;
5use std::ops::{Range, RangeFrom};
6
7use rustc_abi::{ExternAbi, FieldIdx};
8use rustc_attr_data_structures::{InlineAttr, OptimizeAttr};
9use rustc_hir::def::DefKind;
10use rustc_hir::def_id::DefId;
11use rustc_index::Idx;
12use rustc_index::bit_set::DenseBitSet;
13use rustc_middle::bug;
14use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
15use rustc_middle::mir::visit::*;
16use rustc_middle::mir::*;
17use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
18use rustc_session::config::{DebugInfo, OptLevel};
19use rustc_span::source_map::Spanned;
20use tracing::{debug, instrument, trace, trace_span};
21
22use crate::cost_checker::{CostChecker, is_call_like};
23use crate::deref_separator::deref_finder;
24use crate::simplify::simplify_cfg;
25use crate::validate::validate_types;
26use crate::{check_inline, util};
27
28pub(crate) mod cycle;
29
30const HISTORY_DEPTH_LIMIT: usize = 20;
31const TOP_DOWN_DEPTH_LIMIT: usize = 5;
32
33#[derive(Clone, Debug)]
34struct CallSite<'tcx> {
35 callee: Instance<'tcx>,
36 fn_sig: ty::PolyFnSig<'tcx>,
37 block: BasicBlock,
38 source_info: SourceInfo,
39}
40
41pub struct Inline;
44
45impl<'tcx> crate::MirPass<'tcx> for Inline {
46 fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
47 if let Some(enabled) = sess.opts.unstable_opts.inline_mir {
48 return enabled;
49 }
50
51 match sess.mir_opt_level() {
52 0 | 1 => false,
53 2 => {
54 (sess.opts.optimize == OptLevel::More || sess.opts.optimize == OptLevel::Aggressive)
55 && sess.opts.incremental == None
56 }
57 _ => true,
58 }
59 }
60
61 fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
62 let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id()));
63 let _guard = span.enter();
64 if inline::<NormalInliner<'tcx>>(tcx, body) {
65 debug!("running simplify cfg on {:?}", body.source);
66 simplify_cfg(tcx, body);
67 deref_finder(tcx, body);
68 }
69 }
70
71 fn is_required(&self) -> bool {
72 false
73 }
74}
75
76pub struct ForceInline;
77
78impl ForceInline {
79 pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
80 matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
81 }
82}
83
84impl<'tcx> crate::MirPass<'tcx> for ForceInline {
85 fn is_enabled(&self, _: &rustc_session::Session) -> bool {
86 true
87 }
88
89 fn can_be_overridden(&self) -> bool {
90 false
91 }
92
93 fn is_required(&self) -> bool {
94 true
95 }
96
97 fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
98 let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id()));
99 let _guard = span.enter();
100 if inline::<ForceInliner<'tcx>>(tcx, body) {
101 debug!("running simplify cfg on {:?}", body.source);
102 simplify_cfg(tcx, body);
103 deref_finder(tcx, body);
104 }
105 }
106}
107
108trait Inliner<'tcx> {
109 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self;
110
111 fn tcx(&self) -> TyCtxt<'tcx>;
112 fn typing_env(&self) -> ty::TypingEnv<'tcx>;
113 fn history(&self) -> &[DefId];
114 fn caller_def_id(&self) -> DefId;
115
116 fn changed(self) -> bool;
118
119 fn should_inline_for_callee(&self, def_id: DefId) -> bool;
121
122 fn check_codegen_attributes_extra(
123 &self,
124 callee_attrs: &CodegenFnAttrs,
125 ) -> Result<(), &'static str>;
126
127 fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool;
128
129 fn check_callee_mir_body(
132 &self,
133 callsite: &CallSite<'tcx>,
134 callee_body: &Body<'tcx>,
135 callee_attrs: &CodegenFnAttrs,
136 ) -> Result<(), &'static str>;
137
138 fn on_inline_success(
140 &mut self,
141 callsite: &CallSite<'tcx>,
142 caller_body: &mut Body<'tcx>,
143 new_blocks: std::ops::Range<BasicBlock>,
144 );
145
146 fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str);
148}
149
150struct ForceInliner<'tcx> {
151 tcx: TyCtxt<'tcx>,
152 typing_env: ty::TypingEnv<'tcx>,
153 def_id: DefId,
155 history: Vec<DefId>,
161 changed: bool,
163}
164
165impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
166 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
167 Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false }
168 }
169
170 fn tcx(&self) -> TyCtxt<'tcx> {
171 self.tcx
172 }
173
174 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
175 self.typing_env
176 }
177
178 fn history(&self) -> &[DefId] {
179 &self.history
180 }
181
182 fn caller_def_id(&self) -> DefId {
183 self.def_id
184 }
185
186 fn changed(self) -> bool {
187 self.changed
188 }
189
190 fn should_inline_for_callee(&self, def_id: DefId) -> bool {
191 ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
192 }
193
194 fn check_codegen_attributes_extra(
195 &self,
196 callee_attrs: &CodegenFnAttrs,
197 ) -> Result<(), &'static str> {
198 debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
199 Ok(())
200 }
201
202 fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool {
203 true
204 }
205
206 #[instrument(level = "debug", skip(self, callee_body))]
207 fn check_callee_mir_body(
208 &self,
209 _: &CallSite<'tcx>,
210 callee_body: &Body<'tcx>,
211 callee_attrs: &CodegenFnAttrs,
212 ) -> Result<(), &'static str> {
213 if callee_body.tainted_by_errors.is_some() {
214 return Err("body has errors");
215 }
216
217 let caller_attrs = self.tcx().codegen_fn_attrs(self.caller_def_id());
218 if callee_attrs.instruction_set != caller_attrs.instruction_set
219 && callee_body
220 .basic_blocks
221 .iter()
222 .any(|bb| matches!(bb.terminator().kind, TerminatorKind::InlineAsm { .. }))
223 {
224 Err("cannot move inline-asm across instruction sets")
230 } else {
231 Ok(())
232 }
233 }
234
235 fn on_inline_success(
236 &mut self,
237 callsite: &CallSite<'tcx>,
238 caller_body: &mut Body<'tcx>,
239 new_blocks: std::ops::Range<BasicBlock>,
240 ) {
241 self.changed = true;
242
243 self.history.push(callsite.callee.def_id());
244 process_blocks(self, caller_body, new_blocks);
245 self.history.pop();
246 }
247
248 fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
249 let tcx = self.tcx();
250 let InlineAttr::Force { attr_span, reason: justification } =
251 tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
252 else {
253 bug!("called on item without required inlining");
254 };
255
256 let call_span = callsite.source_info.span;
257 tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
258 call_span,
259 attr_span,
260 caller_span: tcx.def_span(self.def_id),
261 caller: tcx.def_path_str(self.def_id),
262 callee_span: tcx.def_span(callsite.callee.def_id()),
263 callee: tcx.def_path_str(callsite.callee.def_id()),
264 reason,
265 justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
266 });
267 }
268}
269
270struct NormalInliner<'tcx> {
271 tcx: TyCtxt<'tcx>,
272 typing_env: ty::TypingEnv<'tcx>,
273 def_id: DefId,
275 history: Vec<DefId>,
281 top_down_counter: usize,
285 changed: bool,
287 caller_is_inline_forwarder: bool,
290}
291
292impl<'tcx> NormalInliner<'tcx> {
293 fn past_depth_limit(&self) -> bool {
294 self.history.len() > HISTORY_DEPTH_LIMIT || self.top_down_counter > TOP_DOWN_DEPTH_LIMIT
295 }
296}
297
298impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
299 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
300 let typing_env = body.typing_env(tcx);
301 let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
302
303 Self {
304 tcx,
305 typing_env,
306 def_id,
307 history: Vec::new(),
308 top_down_counter: 0,
309 changed: false,
310 caller_is_inline_forwarder: matches!(
311 codegen_fn_attrs.inline,
312 InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. }
313 ) && body_is_forwarder(body),
314 }
315 }
316
317 fn tcx(&self) -> TyCtxt<'tcx> {
318 self.tcx
319 }
320
321 fn caller_def_id(&self) -> DefId {
322 self.def_id
323 }
324
325 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
326 self.typing_env
327 }
328
329 fn history(&self) -> &[DefId] {
330 &self.history
331 }
332
333 fn changed(self) -> bool {
334 self.changed
335 }
336
337 fn should_inline_for_callee(&self, _: DefId) -> bool {
338 true
339 }
340
341 fn check_codegen_attributes_extra(
342 &self,
343 callee_attrs: &CodegenFnAttrs,
344 ) -> Result<(), &'static str> {
345 if self.past_depth_limit() && matches!(callee_attrs.inline, InlineAttr::None) {
346 Err("Past depth limit so not inspecting unmarked callee")
347 } else {
348 Ok(())
349 }
350 }
351
352 fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool {
353 if body.coroutine.is_some() {
357 return false;
358 }
359
360 true
361 }
362
363 #[instrument(level = "debug", skip(self, callee_body))]
364 fn check_callee_mir_body(
365 &self,
366 callsite: &CallSite<'tcx>,
367 callee_body: &Body<'tcx>,
368 callee_attrs: &CodegenFnAttrs,
369 ) -> Result<(), &'static str> {
370 let tcx = self.tcx();
371
372 if let Some(_) = callee_body.tainted_by_errors {
373 return Err("body has errors");
374 }
375
376 if self.past_depth_limit() && callee_body.basic_blocks.len() > 1 {
377 return Err("Not inlining multi-block body as we're past a depth limit");
378 }
379
380 let mut threshold = if self.caller_is_inline_forwarder || self.past_depth_limit() {
381 tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
382 } else if tcx.cross_crate_inlinable(callsite.callee.def_id()) {
383 tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
384 } else {
385 tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
386 };
387
388 if callee_body.basic_blocks.len() <= 3 {
392 threshold += threshold / 4;
393 }
394 debug!(" final inline threshold = {}", threshold);
395
396 let mut checker =
399 CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body);
400
401 checker.add_function_level_costs();
402
403 let mut work_list = vec![START_BLOCK];
405 let mut visited = DenseBitSet::new_empty(callee_body.basic_blocks.len());
406 while let Some(bb) = work_list.pop() {
407 if !visited.insert(bb.index()) {
408 continue;
409 }
410
411 let blk = &callee_body.basic_blocks[bb];
412 checker.visit_basic_block_data(bb, blk);
413
414 let term = blk.terminator();
415 let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id());
416 if let TerminatorKind::Drop {
417 ref place,
418 target,
419 unwind,
420 replace: _,
421 drop: _,
422 async_fut: _,
423 } = term.kind
424 {
425 work_list.push(target);
426
427 let ty = callsite
429 .callee
430 .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty));
431 if ty.needs_drop(tcx, self.typing_env())
432 && let UnwindAction::Cleanup(unwind) = unwind
433 {
434 work_list.push(unwind);
435 }
436 } else if callee_attrs.instruction_set != caller_attrs.instruction_set
437 && matches!(term.kind, TerminatorKind::InlineAsm { .. })
438 {
439 return Err("cannot move inline-asm across instruction sets");
445 } else if let TerminatorKind::TailCall { .. } = term.kind {
446 return Err("can't inline functions with tail calls");
449 } else {
450 work_list.extend(term.successors())
451 }
452 }
453
454 let cost = checker.cost();
459 if cost <= threshold {
460 debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
461 Ok(())
462 } else {
463 debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
464 Err("cost above threshold")
465 }
466 }
467
468 fn on_inline_success(
469 &mut self,
470 callsite: &CallSite<'tcx>,
471 caller_body: &mut Body<'tcx>,
472 new_blocks: std::ops::Range<BasicBlock>,
473 ) {
474 self.changed = true;
475
476 let new_calls_count = new_blocks
477 .clone()
478 .filter(|&bb| is_call_like(caller_body.basic_blocks[bb].terminator()))
479 .count();
480 if new_calls_count > 1 {
481 self.top_down_counter += 1;
482 }
483
484 self.history.push(callsite.callee.def_id());
485 process_blocks(self, caller_body, new_blocks);
486 self.history.pop();
487
488 if self.history.is_empty() {
489 self.top_down_counter = 0;
490 }
491 }
492
493 fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {}
494}
495
496fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
497 let def_id = body.source.def_id();
498
499 if !tcx.hir_body_owner_kind(def_id).is_fn_or_closure() {
501 return false;
502 }
503
504 let mut inliner = T::new(tcx, def_id, body);
505 if !inliner.check_caller_mir_body(body) {
506 return false;
507 }
508
509 let blocks = START_BLOCK..body.basic_blocks.next_index();
510 process_blocks(&mut inliner, body, blocks);
511 inliner.changed()
512}
513
514fn process_blocks<'tcx, I: Inliner<'tcx>>(
515 inliner: &mut I,
516 caller_body: &mut Body<'tcx>,
517 blocks: Range<BasicBlock>,
518) {
519 for bb in blocks {
520 let bb_data = &caller_body[bb];
521 if bb_data.is_cleanup {
522 continue;
523 }
524
525 let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else {
526 continue;
527 };
528
529 let span = trace_span!("process_blocks", %callsite.callee, ?bb);
530 let _guard = span.enter();
531
532 match try_inlining(inliner, caller_body, &callsite) {
533 Err(reason) => {
534 debug!("not-inlined {} [{}]", callsite.callee, reason);
535 inliner.on_inline_failure(&callsite, reason);
536 }
537 Ok(new_blocks) => {
538 debug!("inlined {}", callsite.callee);
539 inliner.on_inline_success(&callsite, caller_body, new_blocks);
540 }
541 }
542 }
543}
544
545fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
546 inliner: &I,
547 caller_body: &Body<'tcx>,
548 bb: BasicBlock,
549 bb_data: &BasicBlockData<'tcx>,
550) -> Option<CallSite<'tcx>> {
551 let tcx = inliner.tcx();
552 let terminator = bb_data.terminator();
554
555 if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
557 let func_ty = func.ty(caller_body, tcx);
558 if let ty::FnDef(def_id, args) = *func_ty.kind() {
559 if !inliner.should_inline_for_callee(def_id) {
560 debug!("not enabled");
561 return None;
562 }
563
564 let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?;
566 let callee =
567 Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
568
569 if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
570 return None;
571 }
572
573 if inliner.history().contains(&callee.def_id()) {
574 return None;
575 }
576
577 let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args);
578
579 if let InstanceKind::Item(instance_def_id) = callee.def
582 && tcx.def_kind(instance_def_id) == DefKind::AssocFn
583 && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder()
584 && instance_fn_sig.abi() != fn_sig.abi()
585 {
586 return None;
587 }
588
589 let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
590
591 return Some(CallSite { callee, fn_sig, block: bb, source_info });
592 }
593 }
594
595 None
596}
597
598fn try_inlining<'tcx, I: Inliner<'tcx>>(
602 inliner: &I,
603 caller_body: &mut Body<'tcx>,
604 callsite: &CallSite<'tcx>,
605) -> Result<std::ops::Range<BasicBlock>, &'static str> {
606 let tcx = inliner.tcx();
607 check_mir_is_available(inliner, caller_body, callsite.callee)?;
608
609 let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id());
610 check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?;
611 check_codegen_attributes(inliner, callsite, callee_attrs)?;
612 inliner.check_codegen_attributes_extra(callee_attrs)?;
613
614 let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
615 let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
616 let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty;
617 for arg in args {
618 if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) {
619 return Err("call has unsized argument");
622 }
623 }
624
625 let callee_body = try_instance_mir(tcx, callsite.callee.def)?;
626 check_inline::is_inline_valid_on_body(tcx, callee_body)?;
627 inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?;
628
629 let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions(
630 tcx,
631 inliner.typing_env(),
632 ty::EarlyBinder::bind(callee_body.clone()),
633 ) else {
634 debug!("failed to normalize callee body");
635 return Err("implementation limitation -- could not normalize callee body");
636 };
637
638 if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() {
641 debug!("failed to validate callee body");
642 return Err("implementation limitation -- callee body failed validation");
643 }
644
645 let output_type = callee_body.return_ty();
649 if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) {
650 trace!(?output_type, ?destination_ty);
651 return Err("implementation limitation -- return type mismatch");
652 }
653 if callsite.fn_sig.abi() == ExternAbi::RustCall {
654 let (self_arg, arg_tuple) = match &args[..] {
655 [arg_tuple] => (None, arg_tuple),
656 [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
657 _ => bug!("Expected `rust-call` to have 1 or 2 args"),
658 };
659
660 let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx));
661
662 let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx);
663 let arg_tys = if callee_body.spread_arg.is_some() {
664 std::slice::from_ref(&arg_tuple_ty)
665 } else {
666 let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
667 bug!("Closure arguments are not passed as a tuple");
668 };
669 arg_tuple_tys.as_slice()
670 };
671
672 for (arg_ty, input) in
673 self_arg_ty.into_iter().chain(arg_tys.iter().copied()).zip(callee_body.args_iter())
674 {
675 let input_type = callee_body.local_decls[input].ty;
676 if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
677 trace!(?arg_ty, ?input_type);
678 debug!("failed to normalize tuple argument type");
679 return Err("implementation limitation");
680 }
681 }
682 } else {
683 for (arg, input) in args.iter().zip(callee_body.args_iter()) {
684 let input_type = callee_body.local_decls[input].ty;
685 let arg_ty = arg.node.ty(&caller_body.local_decls, tcx);
686 if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
687 trace!(?arg_ty, ?input_type);
688 debug!("failed to normalize argument type");
689 return Err("implementation limitation -- arg mismatch");
690 }
691 }
692 }
693
694 let old_blocks = caller_body.basic_blocks.next_index();
695 inline_call(inliner, caller_body, callsite, callee_body);
696 let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
697
698 Ok(new_blocks)
699}
700
701fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
702 inliner: &I,
703 caller_body: &Body<'tcx>,
704 callee: Instance<'tcx>,
705) -> Result<(), &'static str> {
706 let caller_def_id = caller_body.source.def_id();
707 let callee_def_id = callee.def_id();
708 if callee_def_id == caller_def_id {
709 return Err("self-recursion");
710 }
711
712 match callee.def {
713 InstanceKind::Item(_) => {
714 if !inliner.tcx().is_mir_available(callee_def_id) {
718 debug!("item MIR unavailable");
719 return Err("implementation limitation -- MIR unavailable");
720 }
721 }
722 InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
724 debug!("instance without MIR (intrinsic / virtual)");
725 return Err("implementation limitation -- cannot inline intrinsic");
726 }
727
728 InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => {
734 debug!("still needs substitution");
735 return Err("implementation limitation -- HACK for dropping polymorphic type");
736 }
737 InstanceKind::AsyncDropGlue(_, ty) | InstanceKind::AsyncDropGlueCtorShim(_, ty) => {
738 return if ty.still_further_specializable() {
739 Err("still needs substitution")
740 } else {
741 Ok(())
742 };
743 }
744 InstanceKind::FutureDropPollShim(_, ty, ty2) => {
745 return if ty.still_further_specializable() || ty2.still_further_specializable() {
746 Err("still needs substitution")
747 } else {
748 Ok(())
749 };
750 }
751
752 InstanceKind::VTableShim(_)
757 | InstanceKind::ReifyShim(..)
758 | InstanceKind::FnPtrShim(..)
759 | InstanceKind::ClosureOnceShim { .. }
760 | InstanceKind::ConstructCoroutineInClosureShim { .. }
761 | InstanceKind::DropGlue(..)
762 | InstanceKind::CloneShim(..)
763 | InstanceKind::ThreadLocalShim(..)
764 | InstanceKind::FnPtrAddrShim(..) => return Ok(()),
765 }
766
767 if inliner.tcx().is_constructor(callee_def_id) {
768 trace!("constructors always have MIR");
769 return Ok(());
771 }
772
773 if let Some(callee_def_id) = callee_def_id.as_local()
774 && !inliner
775 .tcx()
776 .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce)
777 {
778 if inliner.tcx().mir_callgraph_cyclic(caller_def_id.expect_local()).contains(&callee_def_id)
781 {
782 debug!("query cycle avoidance");
783 return Err("caller might be reachable from callee");
784 }
785
786 Ok(())
787 } else {
788 trace!("functions from other crates always have MIR");
793 Ok(())
794 }
795}
796
797fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>(
800 inliner: &I,
801 callsite: &CallSite<'tcx>,
802 callee_attrs: &CodegenFnAttrs,
803) -> Result<(), &'static str> {
804 let tcx = inliner.tcx();
805 if let InlineAttr::Never = callee_attrs.inline {
806 return Err("never inline attribute");
807 }
808
809 if let OptimizeAttr::DoNotOptimize = callee_attrs.optimize {
810 return Err("has DoNotOptimize attribute");
811 }
812
813 inliner.check_codegen_attributes_extra(callee_attrs)?;
814
815 let is_generic = callsite.callee.args.non_erasable_generics().next().is_some();
818 if !is_generic && !tcx.cross_crate_inlinable(callsite.callee.def_id()) {
819 return Err("not exported");
820 }
821
822 let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id());
823 if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize {
824 return Err("incompatible sanitizer set");
825 }
826
827 if callee_attrs.instruction_set.is_some()
831 && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set
832 {
833 return Err("incompatible instruction set");
834 }
835
836 let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name);
837 let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name);
838 if callee_feature_names.ne(this_feature_names) {
839 return Err("incompatible target features");
845 }
846
847 Ok(())
848}
849
850fn inline_call<'tcx, I: Inliner<'tcx>>(
851 inliner: &I,
852 caller_body: &mut Body<'tcx>,
853 callsite: &CallSite<'tcx>,
854 mut callee_body: Body<'tcx>,
855) {
856 let tcx = inliner.tcx();
857 let terminator = caller_body[callsite.block].terminator.take().unwrap();
858 let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind
859 else {
860 bug!("unexpected terminator kind {:?}", terminator.kind);
861 };
862
863 let return_block = if let Some(block) = target {
864 let data = BasicBlockData::new(
867 Some(Terminator {
868 source_info: terminator.source_info,
869 kind: TerminatorKind::Goto { target: block },
870 }),
871 caller_body[block].is_cleanup,
872 );
873 Some(caller_body.basic_blocks_mut().push(data))
874 } else {
875 None
876 };
877
878 fn dest_needs_borrow(place: Place<'_>) -> bool {
884 for elem in place.projection.iter() {
885 match elem {
886 ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
887 _ => {}
888 }
889 }
890
891 false
892 }
893
894 let dest = if dest_needs_borrow(destination) {
895 trace!("creating temp for return destination");
896 let dest = Rvalue::Ref(
897 tcx.lifetimes.re_erased,
898 BorrowKind::Mut { kind: MutBorrowKind::Default },
899 destination,
900 );
901 let dest_ty = dest.ty(caller_body, tcx);
902 let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block));
903 caller_body[callsite.block].statements.push(Statement {
904 source_info: callsite.source_info,
905 kind: StatementKind::Assign(Box::new((temp, dest))),
906 });
907 tcx.mk_place_deref(temp)
908 } else {
909 destination
910 };
911
912 let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
915 (false, d)
916 } else {
917 (
918 true,
919 new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block),
920 )
921 };
922
923 let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block);
925
926 let mut integrator = Integrator {
927 args: &args,
928 new_locals: caller_body.local_decls.next_index()..,
929 new_scopes: caller_body.source_scopes.next_index()..,
930 new_blocks: caller_body.basic_blocks.next_index()..,
931 destination: destination_local,
932 callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
933 callsite,
934 cleanup_block: unwind,
935 in_cleanup_block: false,
936 return_block,
937 tcx,
938 always_live_locals: DenseBitSet::new_filled(callee_body.local_decls.len()),
939 };
940
941 integrator.visit_body(&mut callee_body);
944
945 for local in callee_body.vars_and_temps_iter() {
948 if integrator.always_live_locals.contains(local) {
949 let new_local = integrator.map_local(local);
950 caller_body[callsite.block].statements.push(Statement {
951 source_info: callsite.source_info,
952 kind: StatementKind::StorageLive(new_local),
953 });
954 }
955 }
956 if let Some(block) = return_block {
957 let mut n = 0;
960 if remap_destination {
961 caller_body[block].statements.push(Statement {
962 source_info: callsite.source_info,
963 kind: StatementKind::Assign(Box::new((
964 dest,
965 Rvalue::Use(Operand::Move(destination_local.into())),
966 ))),
967 });
968 n += 1;
969 }
970 for local in callee_body.vars_and_temps_iter().rev() {
971 if integrator.always_live_locals.contains(local) {
972 let new_local = integrator.map_local(local);
973 caller_body[block].statements.push(Statement {
974 source_info: callsite.source_info,
975 kind: StatementKind::StorageDead(new_local),
976 });
977 n += 1;
978 }
979 }
980 caller_body[block].statements.rotate_right(n);
981 }
982
983 caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
985 caller_body.source_scopes.append(&mut callee_body.source_scopes);
986 if tcx
987 .sess
988 .opts
989 .unstable_opts
990 .inline_mir_preserve_debug
991 .unwrap_or(tcx.sess.opts.debuginfo != DebugInfo::None)
992 {
993 caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
997 }
998 caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut());
999
1000 caller_body[callsite.block].terminator = Some(Terminator {
1001 source_info: callsite.source_info,
1002 kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
1003 });
1004
1005 caller_body.required_consts.as_mut().unwrap().extend(
1009 callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()),
1010 );
1011 let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx));
1019 let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap();
1020 if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) {
1021 caller_mentioned_items.remove(idx);
1023 caller_mentioned_items.extend(callee_body.mentioned_items());
1024 } else {
1025 }
1029}
1030
1031fn make_call_args<'tcx, I: Inliner<'tcx>>(
1032 inliner: &I,
1033 args: Box<[Spanned<Operand<'tcx>>]>,
1034 callsite: &CallSite<'tcx>,
1035 caller_body: &mut Body<'tcx>,
1036 callee_body: &Body<'tcx>,
1037 return_block: Option<BasicBlock>,
1038) -> Box<[Local]> {
1039 let tcx = inliner.tcx();
1040
1041 if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() {
1065 let mut args = <_>::into_iter(args);
1067 let self_ = create_temp_if_necessary(
1068 inliner,
1069 args.next().unwrap().node,
1070 callsite,
1071 caller_body,
1072 return_block,
1073 );
1074 let tuple = create_temp_if_necessary(
1075 inliner,
1076 args.next().unwrap().node,
1077 callsite,
1078 caller_body,
1079 return_block,
1080 );
1081 assert!(args.next().is_none());
1082
1083 let tuple = Place::from(tuple);
1084 let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else {
1085 bug!("Closure arguments are not passed as a tuple");
1086 };
1087
1088 let closure_ref_arg = iter::once(self_);
1090
1091 let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
1093 let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
1095
1096 create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block)
1098 });
1099
1100 closure_ref_arg.chain(tuple_tmp_args).collect()
1101 } else {
1102 args.into_iter()
1103 .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block))
1104 .collect()
1105 }
1106}
1107
1108fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>(
1111 inliner: &I,
1112 arg: Operand<'tcx>,
1113 callsite: &CallSite<'tcx>,
1114 caller_body: &mut Body<'tcx>,
1115 return_block: Option<BasicBlock>,
1116) -> Local {
1117 if let Operand::Move(place) = &arg
1119 && let Some(local) = place.as_local()
1120 && caller_body.local_kind(local) == LocalKind::Temp
1121 {
1122 return local;
1123 }
1124
1125 trace!("creating temp for argument {:?}", arg);
1127 let arg_ty = arg.ty(caller_body, inliner.tcx());
1128 let local = new_call_temp(caller_body, callsite, arg_ty, return_block);
1129 caller_body[callsite.block].statements.push(Statement {
1130 source_info: callsite.source_info,
1131 kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))),
1132 });
1133 local
1134}
1135
1136fn new_call_temp<'tcx>(
1138 caller_body: &mut Body<'tcx>,
1139 callsite: &CallSite<'tcx>,
1140 ty: Ty<'tcx>,
1141 return_block: Option<BasicBlock>,
1142) -> Local {
1143 let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
1144
1145 caller_body[callsite.block].statements.push(Statement {
1146 source_info: callsite.source_info,
1147 kind: StatementKind::StorageLive(local),
1148 });
1149
1150 if let Some(block) = return_block {
1151 caller_body[block].statements.insert(
1152 0,
1153 Statement {
1154 source_info: callsite.source_info,
1155 kind: StatementKind::StorageDead(local),
1156 },
1157 );
1158 }
1159
1160 local
1161}
1162
1163struct Integrator<'a, 'tcx> {
1171 args: &'a [Local],
1172 new_locals: RangeFrom<Local>,
1173 new_scopes: RangeFrom<SourceScope>,
1174 new_blocks: RangeFrom<BasicBlock>,
1175 destination: Local,
1176 callsite_scope: SourceScopeData<'tcx>,
1177 callsite: &'a CallSite<'tcx>,
1178 cleanup_block: UnwindAction,
1179 in_cleanup_block: bool,
1180 return_block: Option<BasicBlock>,
1181 tcx: TyCtxt<'tcx>,
1182 always_live_locals: DenseBitSet<Local>,
1183}
1184
1185impl Integrator<'_, '_> {
1186 fn map_local(&self, local: Local) -> Local {
1187 let new = if local == RETURN_PLACE {
1188 self.destination
1189 } else {
1190 let idx = local.index() - 1;
1191 if idx < self.args.len() {
1192 self.args[idx]
1193 } else {
1194 self.new_locals.start + (idx - self.args.len())
1195 }
1196 };
1197 trace!("mapping local `{:?}` to `{:?}`", local, new);
1198 new
1199 }
1200
1201 fn map_scope(&self, scope: SourceScope) -> SourceScope {
1202 let new = self.new_scopes.start + scope.index();
1203 trace!("mapping scope `{:?}` to `{:?}`", scope, new);
1204 new
1205 }
1206
1207 fn map_block(&self, block: BasicBlock) -> BasicBlock {
1208 let new = self.new_blocks.start + block.index();
1209 trace!("mapping block `{:?}` to `{:?}`", block, new);
1210 new
1211 }
1212
1213 fn map_unwind(&self, unwind: UnwindAction) -> UnwindAction {
1214 if self.in_cleanup_block {
1215 match unwind {
1216 UnwindAction::Cleanup(_) | UnwindAction::Continue => {
1217 bug!("cleanup on cleanup block");
1218 }
1219 UnwindAction::Unreachable | UnwindAction::Terminate(_) => return unwind,
1220 }
1221 }
1222
1223 match unwind {
1224 UnwindAction::Unreachable | UnwindAction::Terminate(_) => unwind,
1225 UnwindAction::Cleanup(target) => UnwindAction::Cleanup(self.map_block(target)),
1226 UnwindAction::Continue => self.cleanup_block,
1228 }
1229 }
1230}
1231
1232impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
1233 fn tcx(&self) -> TyCtxt<'tcx> {
1234 self.tcx
1235 }
1236
1237 fn visit_local(&mut self, local: &mut Local, _ctxt: PlaceContext, _location: Location) {
1238 *local = self.map_local(*local);
1239 }
1240
1241 fn visit_source_scope_data(&mut self, scope_data: &mut SourceScopeData<'tcx>) {
1242 self.super_source_scope_data(scope_data);
1243 if scope_data.parent_scope.is_none() {
1244 scope_data.parent_scope = Some(self.callsite.source_info.scope);
1247 assert_eq!(scope_data.inlined_parent_scope, None);
1248 scope_data.inlined_parent_scope = if self.callsite_scope.inlined.is_some() {
1249 Some(self.callsite.source_info.scope)
1250 } else {
1251 self.callsite_scope.inlined_parent_scope
1252 };
1253
1254 assert_eq!(scope_data.inlined, None);
1256 scope_data.inlined = Some((self.callsite.callee, self.callsite.source_info.span));
1257 } else if scope_data.inlined_parent_scope.is_none() {
1258 scope_data.inlined_parent_scope = Some(self.map_scope(OUTERMOST_SOURCE_SCOPE));
1260 }
1261 }
1262
1263 fn visit_source_scope(&mut self, scope: &mut SourceScope) {
1264 *scope = self.map_scope(*scope);
1265 }
1266
1267 fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) {
1268 self.in_cleanup_block = data.is_cleanup;
1269 self.super_basic_block_data(block, data);
1270 self.in_cleanup_block = false;
1271 }
1272
1273 fn visit_retag(&mut self, kind: &mut RetagKind, place: &mut Place<'tcx>, loc: Location) {
1274 self.super_retag(kind, place, loc);
1275
1276 if *kind == RetagKind::FnEntry {
1279 *kind = RetagKind::Default;
1280 }
1281 }
1282
1283 fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
1284 if let StatementKind::StorageLive(local) | StatementKind::StorageDead(local) =
1285 statement.kind
1286 {
1287 self.always_live_locals.remove(local);
1288 }
1289 self.super_statement(statement, location);
1290 }
1291
1292 fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, loc: Location) {
1293 if !matches!(terminator.kind, TerminatorKind::Return) {
1296 self.super_terminator(terminator, loc);
1297 } else {
1298 self.visit_source_info(&mut terminator.source_info);
1299 }
1300
1301 match terminator.kind {
1302 TerminatorKind::CoroutineDrop | TerminatorKind::Yield { .. } => bug!(),
1303 TerminatorKind::Goto { ref mut target } => {
1304 *target = self.map_block(*target);
1305 }
1306 TerminatorKind::SwitchInt { ref mut targets, .. } => {
1307 for tgt in targets.all_targets_mut() {
1308 *tgt = self.map_block(*tgt);
1309 }
1310 }
1311 TerminatorKind::Drop { ref mut target, ref mut unwind, .. } => {
1312 *target = self.map_block(*target);
1313 *unwind = self.map_unwind(*unwind);
1314 }
1315 TerminatorKind::TailCall { .. } => {
1316 unreachable!()
1318 }
1319 TerminatorKind::Call { ref mut target, ref mut unwind, .. } => {
1320 if let Some(ref mut tgt) = *target {
1321 *tgt = self.map_block(*tgt);
1322 }
1323 *unwind = self.map_unwind(*unwind);
1324 }
1325 TerminatorKind::Assert { ref mut target, ref mut unwind, .. } => {
1326 *target = self.map_block(*target);
1327 *unwind = self.map_unwind(*unwind);
1328 }
1329 TerminatorKind::Return => {
1330 terminator.kind = if let Some(tgt) = self.return_block {
1331 TerminatorKind::Goto { target: tgt }
1332 } else {
1333 TerminatorKind::Unreachable
1334 }
1335 }
1336 TerminatorKind::UnwindResume => {
1337 terminator.kind = match self.cleanup_block {
1338 UnwindAction::Cleanup(tgt) => TerminatorKind::Goto { target: tgt },
1339 UnwindAction::Continue => TerminatorKind::UnwindResume,
1340 UnwindAction::Unreachable => TerminatorKind::Unreachable,
1341 UnwindAction::Terminate(reason) => TerminatorKind::UnwindTerminate(reason),
1342 };
1343 }
1344 TerminatorKind::UnwindTerminate(_) => {}
1345 TerminatorKind::Unreachable => {}
1346 TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => {
1347 *real_target = self.map_block(*real_target);
1348 *imaginary_target = self.map_block(*imaginary_target);
1349 }
1350 TerminatorKind::FalseUnwind { real_target: _, unwind: _ } =>
1351 {
1353 bug!("False unwinds should have been removed before inlining")
1354 }
1355 TerminatorKind::InlineAsm { ref mut targets, ref mut unwind, .. } => {
1356 for tgt in targets.iter_mut() {
1357 *tgt = self.map_block(*tgt);
1358 }
1359 *unwind = self.map_unwind(*unwind);
1360 }
1361 }
1362 }
1363}
1364
1365#[instrument(skip(tcx), level = "debug")]
1366fn try_instance_mir<'tcx>(
1367 tcx: TyCtxt<'tcx>,
1368 instance: InstanceKind<'tcx>,
1369) -> Result<&'tcx Body<'tcx>, &'static str> {
1370 if let ty::InstanceKind::DropGlue(_, Some(ty)) | ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) =
1371 instance
1372 && let ty::Adt(def, args) = ty.kind()
1373 {
1374 let fields = def.all_fields();
1375 for field in fields {
1376 let field_ty = field.ty(tcx, args);
1377 if field_ty.has_param() && field_ty.has_aliases() {
1378 return Err("cannot build drop shim for polymorphic type");
1379 }
1380 }
1381 }
1382 Ok(tcx.instance_mir(instance))
1383}
1384
1385fn body_is_forwarder(body: &Body<'_>) -> bool {
1386 let TerminatorKind::Call { target, .. } = body.basic_blocks[START_BLOCK].terminator().kind
1387 else {
1388 return false;
1389 };
1390 if let Some(target) = target {
1391 let TerminatorKind::Return = body.basic_blocks[target].terminator().kind else {
1392 return false;
1393 };
1394 }
1395
1396 let max_blocks = if !body.is_polymorphic {
1397 2
1398 } else if target.is_none() {
1399 3
1400 } else {
1401 4
1402 };
1403 if body.basic_blocks.len() > max_blocks {
1404 return false;
1405 }
1406
1407 body.basic_blocks.iter_enumerated().all(|(bb, bb_data)| {
1408 bb == START_BLOCK
1409 || matches!(
1410 bb_data.terminator().kind,
1411 TerminatorKind::Return
1412 | TerminatorKind::Drop { .. }
1413 | TerminatorKind::UnwindResume
1414 | TerminatorKind::UnwindTerminate(_)
1415 )
1416 })
1417}