1use std::assert_matches::assert_matches;
2use std::{fmt, iter};
3
4use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx, VariantIdx};
5use rustc_hir as hir;
6use rustc_hir::def_id::DefId;
7use rustc_hir::lang_items::LangItem;
8use rustc_index::{Idx, IndexVec};
9use rustc_middle::mir::visit::{MutVisitor, PlaceContext};
10use rustc_middle::mir::*;
11use rustc_middle::query::Providers;
12use rustc_middle::ty::{
13 self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt,
14};
15use rustc_middle::{bug, span_bug};
16use rustc_span::source_map::{Spanned, dummy_spanned};
17use rustc_span::{DUMMY_SP, Span};
18use tracing::{debug, instrument};
19
20use crate::elaborate_drop::{DropElaborator, DropFlagMode, DropStyle, Unwind, elaborate_drop};
21use crate::patch::MirPatch;
22use crate::{
23 abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline,
24 instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads,
25 run_optimization_passes, simplify,
26};
27
28mod async_destructor_ctor;
29
30pub(super) fn provide(providers: &mut Providers) {
31 providers.mir_shims = make_shim;
32}
33
34struct FixProxyFutureDropVisitor<'tcx> {
36 tcx: TyCtxt<'tcx>,
37 replace_to: Local,
38}
39
40impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> {
41 fn tcx(&self) -> TyCtxt<'tcx> {
42 self.tcx
43 }
44
45 fn visit_place(
46 &mut self,
47 place: &mut Place<'tcx>,
48 _context: PlaceContext,
49 _location: Location,
50 ) {
51 if place.local == Local::from_u32(1) {
52 if place.projection.len() == 1 {
53 assert!(matches!(
54 place.projection.first(),
55 Some(ProjectionElem::Field(FieldIdx::ZERO, _))
56 ));
57 *place = Place::from(self.replace_to);
58 } else if place.projection.len() == 2 {
59 assert!(matches!(place.projection[0], ProjectionElem::Field(FieldIdx::ZERO, _)));
60 assert!(matches!(place.projection[1], ProjectionElem::Deref));
61 *place =
62 Place::from(self.replace_to).project_deeper(&[ProjectionElem::Deref], self.tcx);
63 }
64 }
65 }
66}
67
68fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> {
69 debug!("make_shim({:?})", instance);
70
71 let mut result = match instance {
72 ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance),
73 ty::InstanceKind::VTableShim(def_id) => {
74 let adjustment = Adjustment::Deref { source: DerefSource::MutPtr };
75 build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id))
76 }
77 ty::InstanceKind::FnPtrShim(def_id, ty) => {
78 let trait_ = tcx.trait_of_item(def_id).unwrap();
79 let adjustment = match tcx
81 .fn_trait_kind_from_def_id(trait_)
82 .or_else(|| tcx.async_fn_trait_kind_from_def_id(trait_))
83 {
84 Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
85 Some(ty::ClosureKind::Fn) => Adjustment::Deref { source: DerefSource::ImmRef },
86 Some(ty::ClosureKind::FnMut) => Adjustment::Deref { source: DerefSource::MutRef },
87 None => bug!("fn pointer {:?} is not an fn", ty),
88 };
89
90 build_call_shim(tcx, instance, Some(adjustment), CallKind::Indirect(ty))
91 }
92 ty::InstanceKind::ReifyShim(def_id, _) => {
98 build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
99 }
100 ty::InstanceKind::ClosureOnceShim { call_once: _, track_caller: _ } => {
101 let fn_mut = tcx.require_lang_item(LangItem::FnMut, None);
102 let call_mut = tcx
103 .associated_items(fn_mut)
104 .in_definition_order()
105 .find(|it| it.is_fn())
106 .unwrap()
107 .def_id;
108
109 build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut))
110 }
111
112 ty::InstanceKind::ConstructCoroutineInClosureShim {
113 coroutine_closure_def_id,
114 receiver_by_ref,
115 } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
116
117 ty::InstanceKind::DropGlue(def_id, ty) => {
118 if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
121 let coroutine_body = tcx.optimized_mir(coroutine_def_id);
122
123 let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
124 else {
125 bug!()
126 };
127
128 let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() {
134 coroutine_body.coroutine_drop().unwrap()
135 } else {
136 assert_eq!(
137 args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
138 ty::ClosureKind::FnOnce
139 );
140 tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
141 .coroutine_drop()
142 .unwrap()
143 };
144
145 let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);
146 debug!("make_shim({:?}) = {:?}", instance, body);
147
148 pm::run_passes(
149 tcx,
150 &mut body,
151 &[
152 &mentioned_items::MentionedItems,
153 &abort_unwinding_calls::AbortUnwindingCalls,
154 &add_call_guards::CriticalCallEdges,
155 ],
156 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
157 pm::Optimizations::Allowed,
158 );
159
160 return body;
161 }
162
163 build_drop_shim(tcx, def_id, ty)
164 }
165 ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
166 ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
167 ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
168 ty::InstanceKind::FutureDropPollShim(def_id, proxy_ty, impl_ty) => {
169 let mut body =
170 async_destructor_ctor::build_future_drop_poll_shim(tcx, def_id, proxy_ty, impl_ty);
171
172 pm::run_passes(
173 tcx,
174 &mut body,
175 &[
176 &mentioned_items::MentionedItems,
177 &abort_unwinding_calls::AbortUnwindingCalls,
178 &add_call_guards::CriticalCallEdges,
179 ],
180 Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
181 pm::Optimizations::Allowed,
182 );
183 run_optimization_passes(tcx, &mut body);
184 debug!("make_shim({:?}) = {:?}", instance, body);
185 return body;
186 }
187 ty::InstanceKind::AsyncDropGlue(def_id, ty) => {
188 let mut body = async_destructor_ctor::build_async_drop_shim(tcx, def_id, ty);
189
190 pm::run_passes(
194 tcx,
195 &mut body,
196 &[
197 &mentioned_items::MentionedItems,
198 &abort_unwinding_calls::AbortUnwindingCalls,
199 &add_call_guards::CriticalCallEdges,
200 &simplify::SimplifyCfg::MakeShim,
201 &crate::coroutine::StateTransform,
202 ],
203 Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
204 pm::Optimizations::Allowed,
205 );
206 run_optimization_passes(tcx, &mut body);
207 debug!("make_shim({:?}) = {:?}", instance, body);
208 return body;
209 }
210
211 ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) => {
212 let body = async_destructor_ctor::build_async_destructor_ctor_shim(tcx, def_id, ty);
213 debug!("make_shim({:?}) = {:?}", instance, body);
214 return body;
215 }
216 ty::InstanceKind::Virtual(..) => {
217 bug!("InstanceKind::Virtual ({:?}) is for direct calls only", instance)
218 }
219 ty::InstanceKind::Intrinsic(_) => {
220 bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
221 }
222 };
223 debug!("make_shim({:?}) = untransformed {:?}", instance, result);
224
225 pm::run_passes_no_validate(
230 tcx,
231 &mut result,
232 &[
233 &mentioned_items::MentionedItems,
234 &add_moves_for_packed_drops::AddMovesForPackedDrops,
235 &deref_separator::Derefer,
236 &remove_noop_landing_pads::RemoveNoopLandingPads,
237 &simplify::SimplifyCfg::MakeShim,
238 &instsimplify::InstSimplify::BeforeInline,
239 &inline::ForceInline,
241 &abort_unwinding_calls::AbortUnwindingCalls,
242 &add_call_guards::CriticalCallEdges,
243 ],
244 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
245 );
246
247 debug!("make_shim({:?}) = {:?}", instance, result);
248
249 result
250}
251
252#[derive(Copy, Clone, Debug, PartialEq)]
253enum DerefSource {
254 ImmRef,
256 MutRef,
258 MutPtr,
260}
261
262#[derive(Copy, Clone, Debug, PartialEq)]
263enum Adjustment {
264 Identity,
266
267 Deref { source: DerefSource },
272
273 RefMut,
278}
279
280#[derive(Copy, Clone, Debug, PartialEq)]
281enum CallKind<'tcx> {
282 Indirect(Ty<'tcx>),
284
285 Direct(DefId),
287}
288
289fn local_decls_for_sig<'tcx>(
290 sig: &ty::FnSig<'tcx>,
291 span: Span,
292) -> IndexVec<Local, LocalDecl<'tcx>> {
293 iter::once(LocalDecl::new(sig.output(), span))
294 .chain(sig.inputs().iter().map(|ity| LocalDecl::new(*ity, span).immutable()))
295 .collect()
296}
297
298fn dropee_emit_retag<'tcx>(
299 tcx: TyCtxt<'tcx>,
300 body: &mut Body<'tcx>,
301 mut dropee_ptr: Place<'tcx>,
302 span: Span,
303) -> Place<'tcx> {
304 if tcx.sess.opts.unstable_opts.mir_emit_retag {
305 let source_info = SourceInfo::outermost(span);
306 let reborrow = Rvalue::Ref(
315 tcx.lifetimes.re_erased,
316 BorrowKind::Mut { kind: MutBorrowKind::Default },
317 tcx.mk_place_deref(dropee_ptr),
318 );
319 let ref_ty = reborrow.ty(body.local_decls(), tcx);
320 dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into();
321 let new_statements = [
322 StatementKind::Assign(Box::new((dropee_ptr, reborrow))),
323 StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)),
324 ];
325 for s in new_statements {
326 body.basic_blocks_mut()[START_BLOCK]
327 .statements
328 .push(Statement { source_info, kind: s });
329 }
330 }
331 dropee_ptr
332}
333
334fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
335 debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
336
337 assert!(!matches!(ty, Some(ty) if ty.is_coroutine()));
338
339 let args = if let Some(ty) = ty {
340 tcx.mk_args(&[ty.into()])
341 } else {
342 GenericArgs::identity_for_item(tcx, def_id)
343 };
344 let sig = tcx.fn_sig(def_id).instantiate(tcx, args);
345 let sig = tcx.instantiate_bound_regions_with_erased(sig);
346 let span = tcx.def_span(def_id);
347
348 let source_info = SourceInfo::outermost(span);
349
350 let return_block = BasicBlock::new(1);
351 let mut blocks = IndexVec::with_capacity(2);
352 let block = |blocks: &mut IndexVec<_, _>, kind| {
353 blocks.push(BasicBlockData {
354 statements: vec![],
355 terminator: Some(Terminator { source_info, kind }),
356 is_cleanup: false,
357 })
358 };
359 block(&mut blocks, TerminatorKind::Goto { target: return_block });
360 block(&mut blocks, TerminatorKind::Return);
361
362 let source = MirSource::from_instance(ty::InstanceKind::DropGlue(def_id, ty));
363 let mut body =
364 new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span);
365
366 let dropee_ptr = Place::from(Local::new(1 + 0));
368 let dropee_ptr = dropee_emit_retag(tcx, &mut body, dropee_ptr, span);
369
370 if ty.is_some() {
371 let patch = {
372 let typing_env = ty::TypingEnv::post_analysis(tcx, def_id);
373 let mut elaborator = DropShimElaborator {
374 body: &body,
375 patch: MirPatch::new(&body),
376 tcx,
377 typing_env,
378 produce_async_drops: false,
379 };
380 let dropee = tcx.mk_place_deref(dropee_ptr);
381 let resume_block = elaborator.patch.resume_block();
382 elaborate_drop(
383 &mut elaborator,
384 source_info,
385 dropee,
386 (),
387 return_block,
388 Unwind::To(resume_block),
389 START_BLOCK,
390 None,
391 );
392 elaborator.patch
393 };
394 patch.apply(&mut body);
395 }
396
397 body
398}
399
400fn new_body<'tcx>(
401 source: MirSource<'tcx>,
402 basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
403 local_decls: IndexVec<Local, LocalDecl<'tcx>>,
404 arg_count: usize,
405 span: Span,
406) -> Body<'tcx> {
407 let mut body = Body::new(
408 source,
409 basic_blocks,
410 IndexVec::from_elem_n(
411 SourceScopeData {
412 span,
413 parent_scope: None,
414 inlined: None,
415 inlined_parent_scope: None,
416 local_data: ClearCrossCrate::Clear,
417 },
418 1,
419 ),
420 local_decls,
421 IndexVec::new(),
422 arg_count,
423 vec![],
424 span,
425 None,
426 None,
428 );
429 body.set_required_consts(Vec::new());
431 body
432}
433
434pub(super) struct DropShimElaborator<'a, 'tcx> {
435 pub body: &'a Body<'tcx>,
436 pub patch: MirPatch<'tcx>,
437 pub tcx: TyCtxt<'tcx>,
438 pub typing_env: ty::TypingEnv<'tcx>,
439 pub produce_async_drops: bool,
440}
441
442impl fmt::Debug for DropShimElaborator<'_, '_> {
443 fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
444 Ok(())
445 }
446}
447
448impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
449 type Path = ();
450
451 fn patch_ref(&self) -> &MirPatch<'tcx> {
452 &self.patch
453 }
454 fn patch(&mut self) -> &mut MirPatch<'tcx> {
455 &mut self.patch
456 }
457 fn body(&self) -> &'a Body<'tcx> {
458 self.body
459 }
460 fn tcx(&self) -> TyCtxt<'tcx> {
461 self.tcx
462 }
463 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
464 self.typing_env
465 }
466
467 fn terminator_loc(&self, bb: BasicBlock) -> Location {
468 self.patch.terminator_loc(self.body, bb)
469 }
470 fn allow_async_drops(&self) -> bool {
471 self.produce_async_drops
472 }
473
474 fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle {
475 match mode {
476 DropFlagMode::Shallow => {
477 DropStyle::Static
480 }
481 DropFlagMode::Deep => {
482 DropStyle::Open
485 }
486 }
487 }
488
489 fn get_drop_flag(&mut self, _path: Self::Path) -> Option<Operand<'tcx>> {
490 None
491 }
492
493 fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
494
495 fn field_subpath(&self, _path: Self::Path, _field: FieldIdx) -> Option<Self::Path> {
496 None
497 }
498 fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
499 None
500 }
501 fn downcast_subpath(&self, _path: Self::Path, _variant: VariantIdx) -> Option<Self::Path> {
502 Some(())
503 }
504 fn array_subpath(&self, _path: Self::Path, _index: u64, _size: u64) -> Option<Self::Path> {
505 None
506 }
507}
508
509fn build_thread_local_shim<'tcx>(
510 tcx: TyCtxt<'tcx>,
511 instance: ty::InstanceKind<'tcx>,
512) -> Body<'tcx> {
513 let def_id = instance.def_id();
514
515 let span = tcx.def_span(def_id);
516 let source_info = SourceInfo::outermost(span);
517
518 let blocks = IndexVec::from_raw(vec![BasicBlockData {
519 statements: vec![Statement {
520 source_info,
521 kind: StatementKind::Assign(Box::new((
522 Place::return_place(),
523 Rvalue::ThreadLocalRef(def_id),
524 ))),
525 }],
526 terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
527 is_cleanup: false,
528 }]);
529
530 new_body(
531 MirSource::from_instance(instance),
532 blocks,
533 IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]),
534 0,
535 span,
536 )
537}
538
539fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
541 debug!("build_clone_shim(def_id={:?})", def_id);
542
543 let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty);
544
545 let dest = Place::return_place();
546 let src = tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
547
548 match self_ty.kind() {
549 ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(),
550 ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
551 ty::CoroutineClosure(_, args) => {
552 builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys())
553 }
554 ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
555 ty::Coroutine(coroutine_def_id, args) => {
556 assert_eq!(tcx.coroutine_movability(*coroutine_def_id), hir::Movability::Movable);
557 builder.coroutine_shim(dest, src, *coroutine_def_id, args.as_coroutine())
558 }
559 _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
560 };
561
562 builder.into_mir()
563}
564
565struct CloneShimBuilder<'tcx> {
566 tcx: TyCtxt<'tcx>,
567 def_id: DefId,
568 local_decls: IndexVec<Local, LocalDecl<'tcx>>,
569 blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
570 span: Span,
571 sig: ty::FnSig<'tcx>,
572}
573
574impl<'tcx> CloneShimBuilder<'tcx> {
575 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self {
576 let sig = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]);
580 let sig = tcx.instantiate_bound_regions_with_erased(sig);
581 let span = tcx.def_span(def_id);
582
583 CloneShimBuilder {
584 tcx,
585 def_id,
586 local_decls: local_decls_for_sig(&sig, span),
587 blocks: IndexVec::new(),
588 span,
589 sig,
590 }
591 }
592
593 fn into_mir(self) -> Body<'tcx> {
594 let source = MirSource::from_instance(ty::InstanceKind::CloneShim(
595 self.def_id,
596 self.sig.inputs_and_output[0],
597 ));
598 new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span)
599 }
600
601 fn source_info(&self) -> SourceInfo {
602 SourceInfo::outermost(self.span)
603 }
604
605 fn block(
606 &mut self,
607 statements: Vec<Statement<'tcx>>,
608 kind: TerminatorKind<'tcx>,
609 is_cleanup: bool,
610 ) -> BasicBlock {
611 let source_info = self.source_info();
612 self.blocks.push(BasicBlockData {
613 statements,
614 terminator: Some(Terminator { source_info, kind }),
615 is_cleanup,
616 })
617 }
618
619 fn block_index_offset(&self, offset: usize) -> BasicBlock {
624 BasicBlock::new(self.blocks.len() + offset)
625 }
626
627 fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> {
628 Statement { source_info: self.source_info(), kind }
629 }
630
631 fn copy_shim(&mut self) {
632 let rcvr = self.tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
633 let ret_statement = self.make_statement(StatementKind::Assign(Box::new((
634 Place::return_place(),
635 Rvalue::Use(Operand::Copy(rcvr)),
636 ))));
637 self.block(vec![ret_statement], TerminatorKind::Return, false);
638 }
639
640 fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
641 let span = self.span;
642 let mut local = LocalDecl::new(ty, span);
643 if mutability.is_not() {
644 local = local.immutable();
645 }
646 Place::from(self.local_decls.push(local))
647 }
648
649 fn make_clone_call(
650 &mut self,
651 dest: Place<'tcx>,
652 src: Place<'tcx>,
653 ty: Ty<'tcx>,
654 next: BasicBlock,
655 cleanup: BasicBlock,
656 ) {
657 let tcx = self.tcx;
658
659 let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]);
661 let func = Operand::Constant(Box::new(ConstOperand {
662 span: self.span,
663 user_ty: None,
664 const_: Const::zero_sized(func_ty),
665 }));
666
667 let ref_loc =
668 self.make_place(Mutability::Not, Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty));
669
670 let statement = self.make_statement(StatementKind::Assign(Box::new((
672 ref_loc,
673 Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, src),
674 ))));
675
676 self.block(
678 vec![statement],
679 TerminatorKind::Call {
680 func,
681 args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(),
682 destination: dest,
683 target: Some(next),
684 unwind: UnwindAction::Cleanup(cleanup),
685 call_source: CallSource::Normal,
686 fn_span: self.span,
687 },
688 false,
689 );
690 }
691
692 fn clone_fields<I>(
693 &mut self,
694 dest: Place<'tcx>,
695 src: Place<'tcx>,
696 target: BasicBlock,
697 mut unwind: BasicBlock,
698 tys: I,
699 ) -> BasicBlock
700 where
701 I: IntoIterator<Item = Ty<'tcx>>,
702 {
703 for (i, ity) in tys.into_iter().enumerate() {
705 let field = FieldIdx::new(i);
716 let src_field = self.tcx.mk_place_field(src, field, ity);
717
718 let dest_field = self.tcx.mk_place_field(dest, field, ity);
719
720 let next_unwind = self.block_index_offset(1);
721 let next_block = self.block_index_offset(2);
722 self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
723 self.block(
724 vec![],
725 TerminatorKind::Drop {
726 place: dest_field,
727 target: unwind,
728 unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
729 replace: false,
730 drop: None,
731 async_fut: None,
732 },
733 true,
734 );
735 unwind = next_unwind;
736 }
737 self.block(vec![], TerminatorKind::Goto { target }, false);
739 unwind
740 }
741
742 fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
743 where
744 I: IntoIterator<Item = Ty<'tcx>>,
745 {
746 self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
747 let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
748 let target = self.block(vec![], TerminatorKind::Return, false);
749
750 let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
751 }
752
753 fn coroutine_shim(
754 &mut self,
755 dest: Place<'tcx>,
756 src: Place<'tcx>,
757 coroutine_def_id: DefId,
758 args: CoroutineArgs<TyCtxt<'tcx>>,
759 ) {
760 self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
761 let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
762 let switch = self.block(vec![], TerminatorKind::Unreachable, false);
764 let unwind = self.clone_fields(dest, src, switch, unwind, args.upvar_tys());
765 let target = self.block(vec![], TerminatorKind::Return, false);
766 let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
767 let mut cases = Vec::with_capacity(args.state_tys(coroutine_def_id, self.tcx).count());
768 for (index, state_tys) in args.state_tys(coroutine_def_id, self.tcx).enumerate() {
769 let variant_index = VariantIdx::new(index);
770 let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
771 let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
772 let clone_block = self.block_index_offset(1);
773 let start_block = self.block(
774 vec![self.make_statement(StatementKind::SetDiscriminant {
775 place: Box::new(Place::return_place()),
776 variant_index,
777 })],
778 TerminatorKind::Goto { target: clone_block },
779 false,
780 );
781 cases.push((index as u128, start_block));
782 let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
783 }
784 let discr_ty = args.discr_ty(self.tcx);
785 let temp = self.make_place(Mutability::Mut, discr_ty);
786 let rvalue = Rvalue::Discriminant(src);
787 let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
788 match &mut self.blocks[switch] {
789 BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
790 statements.push(statement);
791 *kind = TerminatorKind::SwitchInt {
792 discr: Operand::Move(temp),
793 targets: SwitchTargets::new(cases.into_iter(), unreachable),
794 };
795 }
796 BasicBlockData { terminator: None, .. } => unreachable!(),
797 }
798 }
799}
800
801#[instrument(level = "debug", skip(tcx), ret)]
804fn build_call_shim<'tcx>(
805 tcx: TyCtxt<'tcx>,
806 instance: ty::InstanceKind<'tcx>,
807 rcvr_adjustment: Option<Adjustment>,
808 call_kind: CallKind<'tcx>,
809) -> Body<'tcx> {
810 let (sig_args, untuple_args) = if let ty::InstanceKind::FnPtrShim(_, ty) = instance {
814 let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx));
815
816 let untuple_args = sig.inputs();
817
818 let arg_tup = Ty::new_tup(tcx, untuple_args);
820
821 (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
822 } else {
823 (None, None)
824 };
825
826 let def_id = instance.def_id();
827
828 let sig = tcx.fn_sig(def_id);
829 let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
830
831 assert_eq!(sig_args.is_some(), !instance.has_polymorphic_mir_body());
832 let mut sig = if let Some(sig_args) = sig_args {
833 sig.instantiate(tcx, &sig_args)
834 } else {
835 sig.instantiate_identity()
836 };
837
838 if let CallKind::Indirect(fnty) = call_kind {
839 let mut inputs_and_output = sig.inputs_and_output.to_vec();
845
846 assert_eq!(inputs_and_output.len(), 3);
849
850 let self_arg = &mut inputs_and_output[0];
853 *self_arg = match rcvr_adjustment.unwrap() {
854 Adjustment::Identity => fnty,
855 Adjustment::Deref { source } => match source {
856 DerefSource::ImmRef => Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fnty),
857 DerefSource::MutRef => Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fnty),
858 DerefSource::MutPtr => Ty::new_mut_ptr(tcx, fnty),
859 },
860 Adjustment::RefMut => bug!("`RefMut` is never used with indirect calls: {instance:?}"),
861 };
862 sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
863 }
864
865 if let ty::InstanceKind::VTableShim(..) = instance {
868 let mut inputs_and_output = sig.inputs_and_output.to_vec();
870 let self_arg = &mut inputs_and_output[0];
871 debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
872 *self_arg = Ty::new_mut_ptr(tcx, *self_arg);
873 sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
874 }
875
876 let span = tcx.def_span(def_id);
877
878 debug!(?sig);
879
880 let mut local_decls = local_decls_for_sig(&sig, span);
881 let source_info = SourceInfo::outermost(span);
882
883 let destination = Place::return_place();
884
885 let rcvr_place = || {
886 assert!(rcvr_adjustment.is_some());
887 Place::from(Local::new(1))
888 };
889 let mut statements = vec![];
890
891 let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment {
892 Adjustment::Identity => Operand::Move(rcvr_place()),
893 Adjustment::Deref { source: _ } => Operand::Move(tcx.mk_place_deref(rcvr_place())),
894 Adjustment::RefMut => {
895 let ref_rcvr = local_decls.push(
897 LocalDecl::new(
898 Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, sig.inputs()[0]),
899 span,
900 )
901 .immutable(),
902 );
903 let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default };
904 statements.push(Statement {
905 source_info,
906 kind: StatementKind::Assign(Box::new((
907 Place::from(ref_rcvr),
908 Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()),
909 ))),
910 });
911 Operand::Move(Place::from(ref_rcvr))
912 }
913 });
914
915 let (callee, mut args) = match call_kind {
916 CallKind::Indirect(_) => (rcvr.unwrap(), vec![]),
918
919 CallKind::Direct(def_id) => {
921 let ty = tcx.type_of(def_id).instantiate_identity();
922 (
923 Operand::Constant(Box::new(ConstOperand {
924 span,
925 user_ty: None,
926 const_: Const::zero_sized(ty),
927 })),
928 rcvr.into_iter().collect::<Vec<_>>(),
929 )
930 }
931 };
932
933 let mut arg_range = 0..sig.inputs().len();
934
935 if rcvr_adjustment.is_some() {
937 arg_range.start += 1;
938 }
939
940 if untuple_args.is_some() {
942 arg_range.end -= 1;
943 }
944
945 args.extend(arg_range.map(|i| Operand::Move(Place::from(Local::new(1 + i)))));
947
948 if let Some(untuple_args) = untuple_args {
950 let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
951 args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
952 Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), FieldIdx::new(i), *ity))
953 }));
954 }
955
956 let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 };
957 let mut blocks = IndexVec::with_capacity(n_blocks);
958 let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| {
959 blocks.push(BasicBlockData {
960 statements,
961 terminator: Some(Terminator { source_info, kind }),
962 is_cleanup,
963 })
964 };
965
966 let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect();
968 block(
969 &mut blocks,
970 statements,
971 TerminatorKind::Call {
972 func: callee,
973 args,
974 destination,
975 target: Some(BasicBlock::new(1)),
976 unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
977 UnwindAction::Cleanup(BasicBlock::new(3))
978 } else {
979 UnwindAction::Continue
980 },
981 call_source: CallSource::Misc,
982 fn_span: span,
983 },
984 false,
985 );
986
987 if let Some(Adjustment::RefMut) = rcvr_adjustment {
988 block(
990 &mut blocks,
991 vec![],
992 TerminatorKind::Drop {
993 place: rcvr_place(),
994 target: BasicBlock::new(2),
995 unwind: UnwindAction::Continue,
996 replace: false,
997 drop: None,
998 async_fut: None,
999 },
1000 false,
1001 );
1002 }
1003 let stmts = vec![];
1005 block(&mut blocks, stmts, TerminatorKind::Return, false);
1006 if let Some(Adjustment::RefMut) = rcvr_adjustment {
1007 block(
1009 &mut blocks,
1010 vec![],
1011 TerminatorKind::Drop {
1012 place: rcvr_place(),
1013 target: BasicBlock::new(4),
1014 unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
1015 replace: false,
1016 drop: None,
1017 async_fut: None,
1018 },
1019 true,
1020 );
1021
1022 block(&mut blocks, vec![], TerminatorKind::UnwindResume, true);
1024 }
1025
1026 let mut body =
1027 new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span);
1028
1029 if let ExternAbi::RustCall = sig.abi {
1030 body.spread_arg = Some(Local::new(sig.inputs().len()));
1031 }
1032
1033 body
1034}
1035
1036pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
1037 debug_assert!(tcx.is_constructor(ctor_id));
1038
1039 let typing_env = ty::TypingEnv::post_analysis(tcx, ctor_id);
1040
1041 let sig = tcx
1043 .fn_sig(ctor_id)
1044 .instantiate_identity()
1045 .no_bound_vars()
1046 .expect("LBR in ADT constructor signature");
1047 let sig = tcx.normalize_erasing_regions(typing_env, sig);
1048
1049 let ty::Adt(adt_def, args) = sig.output().kind() else {
1050 bug!("unexpected type for ADT ctor {:?}", sig.output());
1051 };
1052
1053 debug!("build_ctor: ctor_id={:?} sig={:?}", ctor_id, sig);
1054
1055 let span = tcx.def_span(ctor_id);
1056
1057 let local_decls = local_decls_for_sig(&sig, span);
1058
1059 let source_info = SourceInfo::outermost(span);
1060
1061 let variant_index =
1062 if adt_def.is_enum() { adt_def.variant_index_with_ctor_id(ctor_id) } else { FIRST_VARIANT };
1063
1064 debug!("build_ctor: variant_index={:?}", variant_index);
1071
1072 let kind = AggregateKind::Adt(adt_def.did(), variant_index, args, None, None);
1073 let variant = adt_def.variant(variant_index);
1074 let statement = Statement {
1075 kind: StatementKind::Assign(Box::new((
1076 Place::return_place(),
1077 Rvalue::Aggregate(
1078 Box::new(kind),
1079 (0..variant.fields.len())
1080 .map(|idx| Operand::Move(Place::from(Local::new(idx + 1))))
1081 .collect(),
1082 ),
1083 ))),
1084 source_info,
1085 };
1086
1087 let start_block = BasicBlockData {
1088 statements: vec![statement],
1089 terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1090 is_cleanup: false,
1091 };
1092
1093 let source = MirSource::item(ctor_id);
1094 let mut body = new_body(
1095 source,
1096 IndexVec::from_elem_n(start_block, 1),
1097 local_decls,
1098 sig.inputs().len(),
1099 span,
1100 );
1101 body.set_mentioned_items(Vec::new());
1104
1105 crate::pass_manager::dump_mir_for_phase_change(tcx, &body);
1106
1107 body
1108}
1109
1110fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
1118 assert_matches!(self_ty.kind(), ty::FnPtr(..), "expected fn ptr, found {self_ty}");
1119 let span = tcx.def_span(def_id);
1120 let Some(sig) = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]).no_bound_vars() else {
1121 span_bug!(span, "FnPtr::addr with bound vars for `{self_ty}`");
1122 };
1123 let locals = local_decls_for_sig(&sig, span);
1124
1125 let source_info = SourceInfo::outermost(span);
1126 let rvalue = Rvalue::Cast(
1129 CastKind::FnPtrToPtr,
1130 Operand::Move(Place::from(Local::new(1))),
1131 Ty::new_imm_ptr(tcx, tcx.types.unit),
1132 );
1133 let stmt = Statement {
1134 source_info,
1135 kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1136 };
1137 let statements = vec![stmt];
1138 let start_block = BasicBlockData {
1139 statements,
1140 terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1141 is_cleanup: false,
1142 };
1143 let source = MirSource::from_instance(ty::InstanceKind::FnPtrAddrShim(def_id, self_ty));
1144 new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span)
1145}
1146
1147fn build_construct_coroutine_by_move_shim<'tcx>(
1148 tcx: TyCtxt<'tcx>,
1149 coroutine_closure_def_id: DefId,
1150 receiver_by_ref: bool,
1151) -> Body<'tcx> {
1152 let mut self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity();
1153 let mut self_local: Place<'tcx> = Local::from_usize(1).into();
1154 let ty::CoroutineClosure(_, args) = *self_ty.kind() else {
1155 bug!();
1156 };
1157
1158 if receiver_by_ref {
1164 self_local = tcx.mk_place_deref(self_local);
1165 self_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, self_ty);
1166 }
1167
1168 let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| {
1169 tcx.mk_fn_sig(
1170 [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()),
1171 sig.to_coroutine_given_kind_and_upvars(
1172 tcx,
1173 args.as_coroutine_closure().parent_args(),
1174 tcx.coroutine_for_closure(coroutine_closure_def_id),
1175 ty::ClosureKind::FnOnce,
1176 tcx.lifetimes.re_erased,
1177 args.as_coroutine_closure().tupled_upvars_ty(),
1178 args.as_coroutine_closure().coroutine_captures_by_ref_ty(),
1179 ),
1180 sig.c_variadic,
1181 sig.safety,
1182 sig.abi,
1183 )
1184 });
1185 let sig = tcx.liberate_late_bound_regions(coroutine_closure_def_id, poly_sig);
1186 let ty::Coroutine(coroutine_def_id, coroutine_args) = *sig.output().kind() else {
1187 bug!();
1188 };
1189
1190 let span = tcx.def_span(coroutine_closure_def_id);
1191 let locals = local_decls_for_sig(&sig, span);
1192
1193 let mut fields = vec![];
1194
1195 for idx in 1..sig.inputs().len() {
1197 fields.push(Operand::Move(Local::from_usize(idx + 1).into()));
1198 }
1199
1200 for (idx, ty) in args.as_coroutine_closure().upvar_tys().iter().enumerate() {
1201 if receiver_by_ref {
1202 if !matches!(ty.kind(), ty::Ref(_, _, hir::Mutability::Not)) {
1206 tcx.dcx().delayed_bug(format!(
1210 "field should be captured by immutable ref if we have \
1211 an `Fn` instance, but it was: {ty}"
1212 ));
1213 }
1214 fields.push(Operand::Copy(tcx.mk_place_field(
1215 self_local,
1216 FieldIdx::from_usize(idx),
1217 ty,
1218 )));
1219 } else {
1220 fields.push(Operand::Move(tcx.mk_place_field(
1221 self_local,
1222 FieldIdx::from_usize(idx),
1223 ty,
1224 )));
1225 }
1226 }
1227
1228 let source_info = SourceInfo::outermost(span);
1229 let rvalue = Rvalue::Aggregate(
1230 Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)),
1231 IndexVec::from_raw(fields),
1232 );
1233 let stmt = Statement {
1234 source_info,
1235 kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1236 };
1237 let statements = vec![stmt];
1238 let start_block = BasicBlockData {
1239 statements,
1240 terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1241 is_cleanup: false,
1242 };
1243
1244 let source = MirSource::from_instance(ty::InstanceKind::ConstructCoroutineInClosureShim {
1245 coroutine_closure_def_id,
1246 receiver_by_ref,
1247 });
1248
1249 let body =
1250 new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span);
1251 dump_mir(
1252 tcx,
1253 false,
1254 if receiver_by_ref { "coroutine_closure_by_ref" } else { "coroutine_closure_by_move" },
1255 &0,
1256 &body,
1257 |_, _| Ok(()),
1258 );
1259
1260 body
1261}