1#![feature(array_windows)]
3#![feature(assert_matches)]
4#![feature(box_patterns)]
5#![feature(const_type_name)]
6#![feature(cow_is_borrowed)]
7#![feature(file_buffered)]
8#![feature(if_let_guard)]
9#![feature(impl_trait_in_assoc_type)]
10#![feature(try_blocks)]
11#![feature(yeet_expr)]
12use hir::ConstContext;
15use required_consts::RequiredConstsVisitor;
16use rustc_const_eval::check_consts::{self, ConstCx};
17use rustc_const_eval::util;
18use rustc_data_structures::fx::FxIndexSet;
19use rustc_data_structures::steal::Steal;
20use rustc_hir as hir;
21use rustc_hir::def::{CtorKind, DefKind};
22use rustc_hir::def_id::LocalDefId;
23use rustc_index::IndexVec;
24use rustc_middle::mir::{
25 AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstOperand, ConstQualifs, LocalDecl,
26 MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, START_BLOCK,
27 SourceInfo, Statement, StatementKind, TerminatorKind,
28};
29use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
30use rustc_middle::util::Providers;
31use rustc_middle::{bug, query, span_bug};
32use rustc_mir_build::builder::build_mir;
33use rustc_span::source_map::Spanned;
34use rustc_span::{DUMMY_SP, sym};
35use tracing::debug;
36
37#[macro_use]
38mod pass_manager;
39
40use std::sync::LazyLock;
41
42use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel};
43
44mod check_pointers;
45mod cost_checker;
46mod cross_crate_inline;
47mod deduce_param_attrs;
48mod elaborate_drop;
49mod errors;
50mod ffi_unwind_calls;
51mod lint;
52mod lint_tail_expr_drop_order;
53mod patch;
54mod shim;
55mod ssa;
56
57macro_rules! declare_passes {
80 (
81 $(
82 $vis:vis mod $mod_name:ident : $($pass_name:ident $( { $($ident:ident),* } )?),+ $(,)?;
83 )*
84 ) => {
85 $(
86 $vis mod $mod_name;
87 $(
88 #[allow(unused_imports)]
90 use $mod_name::$pass_name as _;
91 )+
92 )*
93
94 static PASS_NAMES: LazyLock<FxIndexSet<&str>> = LazyLock::new(|| [
95 "PreCodegen",
97 $(
98 $(
99 stringify!($pass_name),
100 $(
101 $(
102 $mod_name::$pass_name::$ident.name(),
103 )*
104 )?
105 )+
106 )*
107 ].into_iter().collect());
108 };
109}
110
111declare_passes! {
112 mod abort_unwinding_calls : AbortUnwindingCalls;
113 mod add_call_guards : AddCallGuards { AllCallEdges, CriticalCallEdges };
114 mod add_moves_for_packed_drops : AddMovesForPackedDrops;
115 mod add_retag : AddRetag;
116 mod add_subtyping_projections : Subtyper;
117 mod check_inline : CheckForceInline;
118 mod check_call_recursion : CheckCallRecursion, CheckDropRecursion;
119 mod check_alignment : CheckAlignment;
120 mod check_enums : CheckEnums;
121 mod check_const_item_mutation : CheckConstItemMutation;
122 mod check_null : CheckNull;
123 mod check_packed_ref : CheckPackedRef;
124 pub mod cleanup_post_borrowck : CleanupPostBorrowck;
126
127 mod copy_prop : CopyProp;
128 mod coroutine : StateTransform;
129 mod coverage : InstrumentCoverage;
130 mod ctfe_limit : CtfeLimit;
131 mod dataflow_const_prop : DataflowConstProp;
132 mod dead_store_elimination : DeadStoreElimination {
133 Initial,
134 Final
135 };
136 mod deref_separator : Derefer;
137 mod dest_prop : DestinationPropagation;
138 pub mod dump_mir : Marker;
139 mod early_otherwise_branch : EarlyOtherwiseBranch;
140 mod elaborate_box_derefs : ElaborateBoxDerefs;
141 mod elaborate_drops : ElaborateDrops;
142 mod function_item_references : FunctionItemReferences;
143 mod gvn : GVN;
144 pub mod inline : Inline, ForceInline;
147 mod impossible_predicates : ImpossiblePredicates;
148 mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg };
149 mod jump_threading : JumpThreading;
150 mod known_panics_lint : KnownPanicsLint;
151 mod large_enums : EnumSizeOpt;
152 mod lower_intrinsics : LowerIntrinsics;
153 mod lower_slice_len : LowerSliceLenCalls;
154 mod match_branches : MatchBranchSimplification;
155 mod mentioned_items : MentionedItems;
156 mod multiple_return_terminators : MultipleReturnTerminators;
157 mod nrvo : RenameReturnPlace;
158 mod post_drop_elaboration : CheckLiveDrops;
159 mod prettify : ReorderBasicBlocks, ReorderLocals;
160 mod promote_consts : PromoteTemps;
161 mod ref_prop : ReferencePropagation;
162 mod remove_noop_landing_pads : RemoveNoopLandingPads;
163 mod remove_place_mention : RemovePlaceMention;
164 mod remove_storage_markers : RemoveStorageMarkers;
165 mod remove_uninit_drops : RemoveUninitDrops;
166 mod remove_unneeded_drops : RemoveUnneededDrops;
167 mod remove_zsts : RemoveZsts;
168 mod required_consts : RequiredConstsVisitor;
169 mod post_analysis_normalize : PostAnalysisNormalize;
170 mod sanity_check : SanityCheck;
171 pub mod simplify :
173 SimplifyCfg {
174 Initial,
175 PromoteConsts,
176 RemoveFalseEdges,
177 PostAnalysis,
178 PreOptimizations,
179 Final,
180 MakeShim,
181 AfterUnreachableEnumBranching
182 },
183 SimplifyLocals {
184 BeforeConstProp,
185 AfterGVN,
186 Final
187 };
188 mod simplify_branches : SimplifyConstCondition {
189 AfterConstProp,
190 Final
191 };
192 mod simplify_comparison_integral : SimplifyComparisonIntegral;
193 mod single_use_consts : SingleUseConsts;
194 mod sroa : ScalarReplacementOfAggregates;
195 mod strip_debuginfo : StripDebugInfo;
196 mod unreachable_enum_branching : UnreachableEnumBranching;
197 mod unreachable_prop : UnreachablePropagation;
198 mod validate : Validator;
199}
200
201rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
202
203pub fn provide(providers: &mut Providers) {
204 coverage::query::provide(providers);
205 ffi_unwind_calls::provide(providers);
206 shim::provide(providers);
207 cross_crate_inline::provide(providers);
208 providers.queries = query::Providers {
209 mir_keys,
210 mir_built,
211 mir_const_qualif,
212 mir_promoted,
213 mir_drops_elaborated_and_const_checked,
214 mir_for_ctfe,
215 mir_coroutine_witnesses: coroutine::mir_coroutine_witnesses,
216 optimized_mir,
217 is_mir_available,
218 is_ctfe_mir_available: is_mir_available,
219 mir_callgraph_cyclic: inline::cycle::mir_callgraph_cyclic,
220 mir_inliner_callees: inline::cycle::mir_inliner_callees,
221 promoted_mir,
222 deduced_param_attrs: deduce_param_attrs::deduced_param_attrs,
223 coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id,
224 ..providers.queries
225 };
226}
227
228fn remap_mir_for_const_eval_select<'tcx>(
229 tcx: TyCtxt<'tcx>,
230 mut body: Body<'tcx>,
231 context: hir::Constness,
232) -> Body<'tcx> {
233 for bb in body.basic_blocks.as_mut().iter_mut() {
234 let terminator = bb.terminator.as_mut().expect("invalid terminator");
235 match terminator.kind {
236 TerminatorKind::Call {
237 func: Operand::Constant(box ConstOperand { ref const_, .. }),
238 ref mut args,
239 destination,
240 target,
241 unwind,
242 fn_span,
243 ..
244 } if let ty::FnDef(def_id, _) = *const_.ty().kind()
245 && tcx.is_intrinsic(def_id, sym::const_eval_select) =>
246 {
247 let Ok([tupled_args, called_in_const, called_at_rt]) = take_array(args) else {
248 unreachable!()
249 };
250 let ty = tupled_args.node.ty(&body.local_decls, tcx);
251 let fields = ty.tuple_fields();
252 let num_args = fields.len();
253 let func =
254 if context == hir::Constness::Const { called_in_const } else { called_at_rt };
255 let (method, place): (fn(Place<'tcx>) -> Operand<'tcx>, Place<'tcx>) =
256 match tupled_args.node {
257 Operand::Constant(_) => {
258 let local = body.local_decls.push(LocalDecl::new(ty, fn_span));
262 bb.statements.push(Statement::new(
263 SourceInfo::outermost(fn_span),
264 StatementKind::Assign(Box::new((
265 local.into(),
266 Rvalue::Use(tupled_args.node.clone()),
267 ))),
268 ));
269 (Operand::Move, local.into())
270 }
271 Operand::Move(place) => (Operand::Move, place),
272 Operand::Copy(place) => (Operand::Copy, place),
273 };
274 let place_elems = place.projection;
275 let arguments = (0..num_args)
276 .map(|x| {
277 let mut place_elems = place_elems.to_vec();
278 place_elems.push(ProjectionElem::Field(x.into(), fields[x]));
279 let projection = tcx.mk_place_elems(&place_elems);
280 let place = Place { local: place.local, projection };
281 Spanned { node: method(place), span: DUMMY_SP }
282 })
283 .collect();
284 terminator.kind = TerminatorKind::Call {
285 func: func.node,
286 args: arguments,
287 destination,
288 target,
289 unwind,
290 call_source: CallSource::Misc,
291 fn_span,
292 };
293 }
294 _ => {}
295 }
296 }
297 body
298}
299
300fn take_array<T, const N: usize>(b: &mut Box<[T]>) -> Result<[T; N], Box<[T]>> {
301 let b: Box<[T; N]> = std::mem::take(b).try_into()?;
302 Ok(*b)
303}
304
305fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
306 tcx.mir_keys(()).contains(&def_id)
307}
308
309fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> {
312 let mut set: FxIndexSet<_> = tcx.hir_body_owners().collect();
314
315 set.retain(|&def_id| !matches!(tcx.def_kind(def_id), DefKind::GlobalAsm));
318
319 for body_owner in tcx.hir_body_owners() {
322 if let DefKind::Closure = tcx.def_kind(body_owner)
323 && tcx.needs_coroutine_by_move_body_def_id(body_owner.to_def_id())
324 {
325 set.insert(tcx.coroutine_by_move_body_def_id(body_owner).expect_local());
326 }
327 }
328
329 for item in tcx.hir_crate_items(()).free_items() {
332 if let DefKind::Struct | DefKind::Enum = tcx.def_kind(item.owner_id) {
333 for variant in tcx.adt_def(item.owner_id).variants() {
334 if let Some((CtorKind::Fn, ctor_def_id)) = variant.ctor {
335 set.insert(ctor_def_id.expect_local());
336 }
337 }
338 }
339 }
340
341 set
342}
343
344fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs {
345 let body = &tcx.mir_built(def).borrow();
350 let ccx = check_consts::ConstCx::new(tcx, body);
351 match ccx.const_kind {
353 Some(ConstContext::Const { .. } | ConstContext::Static(_) | ConstContext::ConstFn) => {}
354 None => span_bug!(
355 tcx.def_span(def),
356 "`mir_const_qualif` should only be called on const fns and const items"
357 ),
358 }
359
360 if body.return_ty().references_error() {
361 tcx.dcx().span_delayed_bug(body.span, "mir_const_qualif: MIR had errors");
363 return Default::default();
364 }
365
366 let mut validator = check_consts::check::Checker::new(&ccx);
367 validator.check_body();
368
369 validator.qualifs_in_return_place()
372}
373
374fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
375 let mut body = build_mir(tcx, def);
376
377 pass_manager::dump_mir_for_phase_change(tcx, &body);
378
379 pm::run_passes(
380 tcx,
381 &mut body,
382 &[
383 &Lint(check_inline::CheckForceInline),
385 &Lint(check_call_recursion::CheckCallRecursion),
386 &Lint(check_packed_ref::CheckPackedRef),
387 &Lint(check_const_item_mutation::CheckConstItemMutation),
388 &Lint(function_item_references::FunctionItemReferences),
389 &simplify::SimplifyCfg::Initial,
391 &Lint(sanity_check::SanityCheck),
392 ],
393 None,
394 pm::Optimizations::Allowed,
395 );
396 tcx.alloc_steal_mir(body)
397}
398
399fn mir_promoted(
401 tcx: TyCtxt<'_>,
402 def: LocalDefId,
403) -> (&Steal<Body<'_>>, &Steal<IndexVec<Promoted, Body<'_>>>) {
404 let const_qualifs = match tcx.def_kind(def) {
409 DefKind::Fn | DefKind::AssocFn | DefKind::Closure
410 if tcx.constness(def) == hir::Constness::Const
411 || tcx.is_const_default_method(def.to_def_id()) =>
412 {
413 tcx.mir_const_qualif(def)
414 }
415 DefKind::AssocConst
416 | DefKind::Const
417 | DefKind::Static { .. }
418 | DefKind::InlineConst
419 | DefKind::AnonConst => tcx.mir_const_qualif(def),
420 _ => ConstQualifs::default(),
421 };
422
423 tcx.ensure_done().has_ffi_unwind_calls(def);
425
426 if tcx.needs_coroutine_by_move_body_def_id(def.to_def_id()) {
428 tcx.ensure_done().coroutine_by_move_body_def_id(def);
429 }
430
431 let mut body = tcx.mir_built(def).steal();
432 if let Some(error_reported) = const_qualifs.tainted_by_errors {
433 body.tainted_by_errors = Some(error_reported);
434 }
435
436 RequiredConstsVisitor::compute_required_consts(&mut body);
439
440 let promote_pass = promote_consts::PromoteTemps::default();
442 pm::run_passes(
443 tcx,
444 &mut body,
445 &[&promote_pass, &simplify::SimplifyCfg::PromoteConsts, &coverage::InstrumentCoverage],
446 Some(MirPhase::Analysis(AnalysisPhase::Initial)),
447 pm::Optimizations::Allowed,
448 );
449
450 lint_tail_expr_drop_order::run_lint(tcx, def, &body);
451
452 let promoted = promote_pass.promoted_fragments.into_inner();
453 (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted))
454}
455
456fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &Body<'_> {
458 tcx.arena.alloc(inner_mir_for_ctfe(tcx, def_id))
459}
460
461fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
462 if tcx.is_constructor(def.to_def_id()) {
464 return shim::build_adt_ctor(tcx, def.to_def_id());
469 }
470
471 let body = tcx.mir_drops_elaborated_and_const_checked(def);
472 let body = match tcx.hir_body_const_context(def) {
473 Some(hir::ConstContext::Const { .. } | hir::ConstContext::Static(_)) => body.steal(),
476 Some(hir::ConstContext::ConstFn) => body.borrow().clone(),
477 None => bug!("`mir_for_ctfe` called on non-const {def:?}"),
478 };
479
480 let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::Const);
481 pm::run_passes(tcx, &mut body, &[&ctfe_limit::CtfeLimit], None, pm::Optimizations::Allowed);
482
483 body
484}
485
486fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
490 if tcx.is_coroutine(def.to_def_id()) {
491 tcx.ensure_done().mir_coroutine_witnesses(def);
492 }
493
494 let tainted_by_errors = if !tcx.is_synthetic_mir(def) {
496 tcx.mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local()).err()
497 } else {
498 None
499 };
500
501 let is_fn_like = tcx.def_kind(def).is_fn_like();
502 if is_fn_like {
503 if pm::should_run_pass(tcx, &inline::Inline, pm::Optimizations::Allowed)
505 || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id())
506 {
507 tcx.ensure_done().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id()));
508 }
509 }
510
511 let (body, _) = tcx.mir_promoted(def);
512 let mut body = body.steal();
513
514 if let Some(error_reported) = tainted_by_errors {
515 body.tainted_by_errors = Some(error_reported);
516 }
517
518 let root = tcx.typeck_root_def_id(def.to_def_id());
523 match tcx.def_kind(root) {
524 DefKind::Fn
525 | DefKind::AssocFn
526 | DefKind::Static { .. }
527 | DefKind::Const
528 | DefKind::AssocConst => {
529 if let Err(guar) = tcx.ensure_ok().check_well_formed(root.expect_local()) {
530 body.tainted_by_errors = Some(guar);
531 }
532 }
533 _ => {}
534 }
535
536 run_analysis_to_runtime_passes(tcx, &mut body);
537
538 tcx.alloc_steal_mir(body)
539}
540
541pub fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
544 assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial));
545 let did = body.source.def_id();
546
547 debug!("analysis_mir_cleanup({:?})", did);
548 run_analysis_cleanup_passes(tcx, body);
549 assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup));
550
551 if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, body)) {
553 pm::run_passes(
554 tcx,
555 body,
556 &[
557 &remove_uninit_drops::RemoveUninitDrops,
558 &simplify::SimplifyCfg::RemoveFalseEdges,
559 &Lint(post_drop_elaboration::CheckLiveDrops),
560 ],
561 None,
562 pm::Optimizations::Allowed,
563 );
564 }
565
566 debug!("runtime_mir_lowering({:?})", did);
567 run_runtime_lowering_passes(tcx, body);
568 assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial));
569
570 debug!("runtime_mir_cleanup({:?})", did);
571 run_runtime_cleanup_passes(tcx, body);
572 assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup));
573}
574
575fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
579 let passes: &[&dyn MirPass<'tcx>] = &[
580 &impossible_predicates::ImpossiblePredicates,
581 &cleanup_post_borrowck::CleanupPostBorrowck,
582 &remove_noop_landing_pads::RemoveNoopLandingPads,
583 &simplify::SimplifyCfg::PostAnalysis,
584 &deref_separator::Derefer,
585 ];
586
587 pm::run_passes(
588 tcx,
589 body,
590 passes,
591 Some(MirPhase::Analysis(AnalysisPhase::PostCleanup)),
592 pm::Optimizations::Allowed,
593 );
594}
595
596fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
598 let passes: &[&dyn MirPass<'tcx>] = &[
599 &add_call_guards::CriticalCallEdges,
601 &post_analysis_normalize::PostAnalysisNormalize,
603 &add_subtyping_projections::Subtyper,
605 &elaborate_drops::ElaborateDrops,
606 &Lint(check_call_recursion::CheckDropRecursion),
608 &abort_unwinding_calls::AbortUnwindingCalls,
612 &add_moves_for_packed_drops::AddMovesForPackedDrops,
615 &add_retag::AddRetag,
618 &elaborate_box_derefs::ElaborateBoxDerefs,
619 &coroutine::StateTransform,
620 &Lint(known_panics_lint::KnownPanicsLint),
621 ];
622 pm::run_passes_no_validate(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::Initial)));
623}
624
625fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
627 let passes: &[&dyn MirPass<'tcx>] = &[
628 &lower_intrinsics::LowerIntrinsics,
629 &remove_place_mention::RemovePlaceMention,
630 &simplify::SimplifyCfg::PreOptimizations,
631 ];
632
633 pm::run_passes(
634 tcx,
635 body,
636 passes,
637 Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
638 pm::Optimizations::Allowed,
639 );
640
641 for decl in &mut body.local_decls {
644 decl.local_info = ClearCrossCrate::Clear;
645 }
646}
647
648pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
649 fn o1<T>(x: T) -> WithMinOptLevel<T> {
650 WithMinOptLevel(1, x)
651 }
652
653 let def_id = body.source.def_id();
654 let optimizations = if tcx.def_kind(def_id).has_codegen_attrs()
655 && tcx.codegen_fn_attrs(def_id).optimize.do_not_optimize()
656 {
657 pm::Optimizations::Suppressed
658 } else {
659 pm::Optimizations::Allowed
660 };
661
662 pm::run_passes(
664 tcx,
665 body,
666 &[
667 &check_alignment::CheckAlignment,
669 &check_null::CheckNull,
670 &check_enums::CheckEnums,
671 &lower_slice_len::LowerSliceLenCalls,
676 &instsimplify::InstSimplify::BeforeInline,
679 &inline::ForceInline,
681 &inline::Inline,
683 &remove_storage_markers::RemoveStorageMarkers,
686 &remove_zsts::RemoveZsts,
688 &remove_unneeded_drops::RemoveUnneededDrops,
689 &unreachable_enum_branching::UnreachableEnumBranching,
692 &unreachable_prop::UnreachablePropagation,
693 &o1(simplify::SimplifyCfg::AfterUnreachableEnumBranching),
694 &ref_prop::ReferencePropagation,
697 &sroa::ScalarReplacementOfAggregates,
698 &multiple_return_terminators::MultipleReturnTerminators,
699 &instsimplify::InstSimplify::AfterSimplifyCfg,
702 &simplify::SimplifyLocals::BeforeConstProp,
703 &dead_store_elimination::DeadStoreElimination::Initial,
704 &gvn::GVN,
705 &simplify::SimplifyLocals::AfterGVN,
706 &match_branches::MatchBranchSimplification,
707 &dataflow_const_prop::DataflowConstProp,
708 &single_use_consts::SingleUseConsts,
709 &o1(simplify_branches::SimplifyConstCondition::AfterConstProp),
710 &jump_threading::JumpThreading,
711 &early_otherwise_branch::EarlyOtherwiseBranch,
712 &simplify_comparison_integral::SimplifyComparisonIntegral,
713 &dest_prop::DestinationPropagation,
714 &o1(simplify_branches::SimplifyConstCondition::Final),
715 &o1(remove_noop_landing_pads::RemoveNoopLandingPads),
716 &o1(simplify::SimplifyCfg::Final),
717 &strip_debuginfo::StripDebugInfo,
719 ©_prop::CopyProp,
720 &dead_store_elimination::DeadStoreElimination::Final,
721 &nrvo::RenameReturnPlace,
722 &simplify::SimplifyLocals::Final,
723 &multiple_return_terminators::MultipleReturnTerminators,
724 &large_enums::EnumSizeOpt { discrepancy: 128 },
725 &add_call_guards::CriticalCallEdges,
727 &prettify::ReorderBasicBlocks,
729 &prettify::ReorderLocals,
730 &dump_mir::Marker("PreCodegen"),
732 ],
733 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
734 optimizations,
735 );
736}
737
738fn optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> &Body<'_> {
740 tcx.arena.alloc(inner_optimized_mir(tcx, did))
741}
742
743fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
744 if tcx.is_constructor(did.to_def_id()) {
745 return shim::build_adt_ctor(tcx, did.to_def_id());
750 }
751
752 match tcx.hir_body_const_context(did) {
753 Some(hir::ConstContext::ConstFn) => tcx.ensure_done().mir_for_ctfe(did),
757 None => {}
758 Some(other) => panic!("do not use `optimized_mir` for constants: {other:?}"),
759 }
760 debug!("about to call mir_drops_elaborated...");
761 let body = tcx.mir_drops_elaborated_and_const_checked(did).steal();
762 let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst);
763
764 if body.tainted_by_errors.is_some() {
765 return body;
766 }
767
768 mentioned_items::MentionedItems.run_pass(tcx, &mut body);
772
773 if let TerminatorKind::Unreachable = body.basic_blocks[START_BLOCK].terminator().kind
777 && body.basic_blocks[START_BLOCK].statements.is_empty()
778 {
779 return body;
780 }
781
782 run_optimization_passes(tcx, &mut body);
783
784 body
785}
786
787fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_>> {
790 if tcx.is_constructor(def.to_def_id()) {
791 return tcx.arena.alloc(IndexVec::new());
792 }
793
794 if !tcx.is_synthetic_mir(def) {
795 tcx.ensure_done().mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local());
796 }
797 let mut promoted = tcx.mir_promoted(def).1.steal();
798
799 for body in &mut promoted {
800 run_analysis_to_runtime_passes(tcx, body);
801 }
802
803 tcx.arena.alloc(promoted)
804}