rustc_mir_transform/
shim.rs

1use std::assert_matches::assert_matches;
2use std::{fmt, iter};
3
4use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx, VariantIdx};
5use rustc_hir as hir;
6use rustc_hir::def_id::DefId;
7use rustc_hir::lang_items::LangItem;
8use rustc_index::{Idx, IndexVec};
9use rustc_middle::mir::visit::{MutVisitor, PlaceContext};
10use rustc_middle::mir::*;
11use rustc_middle::query::Providers;
12use rustc_middle::ty::{
13    self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt,
14};
15use rustc_middle::{bug, span_bug};
16use rustc_span::source_map::{Spanned, dummy_spanned};
17use rustc_span::{DUMMY_SP, Span};
18use tracing::{debug, instrument};
19
20use crate::elaborate_drop::{DropElaborator, DropFlagMode, DropStyle, Unwind, elaborate_drop};
21use crate::patch::MirPatch;
22use crate::{
23    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline,
24    instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads,
25    run_optimization_passes, simplify,
26};
27
28mod async_destructor_ctor;
29
30pub(super) fn provide(providers: &mut Providers) {
31    providers.mir_shims = make_shim;
32}
33
34// Replace Pin<&mut ImplCoroutine> accesses (_1.0) into Pin<&mut ProxyCoroutine> acceses
35struct FixProxyFutureDropVisitor<'tcx> {
36    tcx: TyCtxt<'tcx>,
37    replace_to: Local,
38}
39
40impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> {
41    fn tcx(&self) -> TyCtxt<'tcx> {
42        self.tcx
43    }
44
45    fn visit_place(
46        &mut self,
47        place: &mut Place<'tcx>,
48        _context: PlaceContext,
49        _location: Location,
50    ) {
51        if place.local == Local::from_u32(1) {
52            if place.projection.len() == 1 {
53                assert!(matches!(
54                    place.projection.first(),
55                    Some(ProjectionElem::Field(FieldIdx::ZERO, _))
56                ));
57                *place = Place::from(self.replace_to);
58            } else if place.projection.len() == 2 {
59                assert!(matches!(place.projection[0], ProjectionElem::Field(FieldIdx::ZERO, _)));
60                assert!(matches!(place.projection[1], ProjectionElem::Deref));
61                *place =
62                    Place::from(self.replace_to).project_deeper(&[ProjectionElem::Deref], self.tcx);
63            }
64        }
65    }
66}
67
68fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> {
69    debug!("make_shim({:?})", instance);
70
71    let mut result = match instance {
72        ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance),
73        ty::InstanceKind::VTableShim(def_id) => {
74            let adjustment = Adjustment::Deref { source: DerefSource::MutPtr };
75            build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id))
76        }
77        ty::InstanceKind::FnPtrShim(def_id, ty) => {
78            let trait_ = tcx.trait_of_item(def_id).unwrap();
79            // Supports `Fn` or `async Fn` traits.
80            let adjustment = match tcx
81                .fn_trait_kind_from_def_id(trait_)
82                .or_else(|| tcx.async_fn_trait_kind_from_def_id(trait_))
83            {
84                Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
85                Some(ty::ClosureKind::Fn) => Adjustment::Deref { source: DerefSource::ImmRef },
86                Some(ty::ClosureKind::FnMut) => Adjustment::Deref { source: DerefSource::MutRef },
87                None => bug!("fn pointer {:?} is not an fn", ty),
88            };
89
90            build_call_shim(tcx, instance, Some(adjustment), CallKind::Indirect(ty))
91        }
92        // We are generating a call back to our def-id, which the
93        // codegen backend knows to turn to an actual call, be it
94        // a virtual call, or a direct call to a function for which
95        // indirect calls must be codegen'd differently than direct ones
96        // (such as `#[track_caller]`).
97        ty::InstanceKind::ReifyShim(def_id, _) => {
98            build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
99        }
100        ty::InstanceKind::ClosureOnceShim { call_once: _, track_caller: _ } => {
101            let fn_mut = tcx.require_lang_item(LangItem::FnMut, None);
102            let call_mut = tcx
103                .associated_items(fn_mut)
104                .in_definition_order()
105                .find(|it| it.is_fn())
106                .unwrap()
107                .def_id;
108
109            build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut))
110        }
111
112        ty::InstanceKind::ConstructCoroutineInClosureShim {
113            coroutine_closure_def_id,
114            receiver_by_ref,
115        } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
116
117        ty::InstanceKind::DropGlue(def_id, ty) => {
118            // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end
119            // of this function. Is this intentional?
120            if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
121                let coroutine_body = tcx.optimized_mir(coroutine_def_id);
122
123                let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
124                else {
125                    bug!()
126                };
127
128                // If this is a regular coroutine, grab its drop shim. If this is a coroutine
129                // that comes from a coroutine-closure, and the kind ty differs from the "maximum"
130                // kind that it supports, then grab the appropriate drop shim. This ensures that
131                // the future returned by `<[coroutine-closure] as AsyncFnOnce>::call_once` will
132                // drop the coroutine-closure's upvars.
133                let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() {
134                    coroutine_body.coroutine_drop().unwrap()
135                } else {
136                    assert_eq!(
137                        args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
138                        ty::ClosureKind::FnOnce
139                    );
140                    tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
141                        .coroutine_drop()
142                        .unwrap()
143                };
144
145                let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);
146                debug!("make_shim({:?}) = {:?}", instance, body);
147
148                pm::run_passes(
149                    tcx,
150                    &mut body,
151                    &[
152                        &mentioned_items::MentionedItems,
153                        &abort_unwinding_calls::AbortUnwindingCalls,
154                        &add_call_guards::CriticalCallEdges,
155                    ],
156                    Some(MirPhase::Runtime(RuntimePhase::Optimized)),
157                    pm::Optimizations::Allowed,
158                );
159
160                return body;
161            }
162
163            build_drop_shim(tcx, def_id, ty)
164        }
165        ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
166        ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
167        ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
168        ty::InstanceKind::FutureDropPollShim(def_id, proxy_ty, impl_ty) => {
169            let mut body =
170                async_destructor_ctor::build_future_drop_poll_shim(tcx, def_id, proxy_ty, impl_ty);
171
172            pm::run_passes(
173                tcx,
174                &mut body,
175                &[
176                    &mentioned_items::MentionedItems,
177                    &abort_unwinding_calls::AbortUnwindingCalls,
178                    &add_call_guards::CriticalCallEdges,
179                ],
180                Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
181                pm::Optimizations::Allowed,
182            );
183            run_optimization_passes(tcx, &mut body);
184            debug!("make_shim({:?}) = {:?}", instance, body);
185            return body;
186        }
187        ty::InstanceKind::AsyncDropGlue(def_id, ty) => {
188            let mut body = async_destructor_ctor::build_async_drop_shim(tcx, def_id, ty);
189
190            // Main pass required here is StateTransform to convert sync drop ladder
191            // into coroutine.
192            // Others are minimal passes as for sync drop glue shim
193            pm::run_passes(
194                tcx,
195                &mut body,
196                &[
197                    &mentioned_items::MentionedItems,
198                    &abort_unwinding_calls::AbortUnwindingCalls,
199                    &add_call_guards::CriticalCallEdges,
200                    &simplify::SimplifyCfg::MakeShim,
201                    &crate::coroutine::StateTransform,
202                ],
203                Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
204                pm::Optimizations::Allowed,
205            );
206            run_optimization_passes(tcx, &mut body);
207            debug!("make_shim({:?}) = {:?}", instance, body);
208            return body;
209        }
210
211        ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) => {
212            let body = async_destructor_ctor::build_async_destructor_ctor_shim(tcx, def_id, ty);
213            debug!("make_shim({:?}) = {:?}", instance, body);
214            return body;
215        }
216        ty::InstanceKind::Virtual(..) => {
217            bug!("InstanceKind::Virtual ({:?}) is for direct calls only", instance)
218        }
219        ty::InstanceKind::Intrinsic(_) => {
220            bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
221        }
222    };
223    debug!("make_shim({:?}) = untransformed {:?}", instance, result);
224
225    // We don't validate MIR here because the shims may generate code that's
226    // only valid in a `PostAnalysis` param-env. However, since we do initial
227    // validation with the MirBuilt phase, which uses a user-facing param-env.
228    // This causes validation errors when TAITs are involved.
229    pm::run_passes_no_validate(
230        tcx,
231        &mut result,
232        &[
233            &mentioned_items::MentionedItems,
234            &add_moves_for_packed_drops::AddMovesForPackedDrops,
235            &deref_separator::Derefer,
236            &remove_noop_landing_pads::RemoveNoopLandingPads,
237            &simplify::SimplifyCfg::MakeShim,
238            &instsimplify::InstSimplify::BeforeInline,
239            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
240            &inline::ForceInline,
241            &abort_unwinding_calls::AbortUnwindingCalls,
242            &add_call_guards::CriticalCallEdges,
243        ],
244        Some(MirPhase::Runtime(RuntimePhase::Optimized)),
245    );
246
247    debug!("make_shim({:?}) = {:?}", instance, result);
248
249    result
250}
251
252#[derive(Copy, Clone, Debug, PartialEq)]
253enum DerefSource {
254    /// `fn shim(&self) { inner(*self )}`.
255    ImmRef,
256    /// `fn shim(&mut self) { inner(*self )}`.
257    MutRef,
258    /// `fn shim(*mut self) { inner(*self )}`.
259    MutPtr,
260}
261
262#[derive(Copy, Clone, Debug, PartialEq)]
263enum Adjustment {
264    /// Pass the receiver as-is.
265    Identity,
266
267    /// We get passed a reference or a raw pointer to `self` and call the target with `*self`.
268    ///
269    /// This either copies `self` (if `Self: Copy`, eg. for function items), or moves out of it
270    /// (for `VTableShim`, which effectively is passed `&own Self`).
271    Deref { source: DerefSource },
272
273    /// We get passed `self: Self` and call the target with `&mut self`.
274    ///
275    /// In this case we need to ensure that the `Self` is dropped after the call, as the callee
276    /// won't do it for us.
277    RefMut,
278}
279
280#[derive(Copy, Clone, Debug, PartialEq)]
281enum CallKind<'tcx> {
282    /// Call the `FnPtr` that was passed as the receiver.
283    Indirect(Ty<'tcx>),
284
285    /// Call a known `FnDef`.
286    Direct(DefId),
287}
288
289fn local_decls_for_sig<'tcx>(
290    sig: &ty::FnSig<'tcx>,
291    span: Span,
292) -> IndexVec<Local, LocalDecl<'tcx>> {
293    iter::once(LocalDecl::new(sig.output(), span))
294        .chain(sig.inputs().iter().map(|ity| LocalDecl::new(*ity, span).immutable()))
295        .collect()
296}
297
298fn dropee_emit_retag<'tcx>(
299    tcx: TyCtxt<'tcx>,
300    body: &mut Body<'tcx>,
301    mut dropee_ptr: Place<'tcx>,
302    span: Span,
303) -> Place<'tcx> {
304    if tcx.sess.opts.unstable_opts.mir_emit_retag {
305        let source_info = SourceInfo::outermost(span);
306        // We want to treat the function argument as if it was passed by `&mut`. As such, we
307        // generate
308        // ```
309        // temp = &mut *arg;
310        // Retag(temp, FnEntry)
311        // ```
312        // It's important that we do this first, before anything that depends on `dropee_ptr`
313        // has been put into the body.
314        let reborrow = Rvalue::Ref(
315            tcx.lifetimes.re_erased,
316            BorrowKind::Mut { kind: MutBorrowKind::Default },
317            tcx.mk_place_deref(dropee_ptr),
318        );
319        let ref_ty = reborrow.ty(body.local_decls(), tcx);
320        dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into();
321        let new_statements = [
322            StatementKind::Assign(Box::new((dropee_ptr, reborrow))),
323            StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)),
324        ];
325        for s in new_statements {
326            body.basic_blocks_mut()[START_BLOCK]
327                .statements
328                .push(Statement { source_info, kind: s });
329        }
330    }
331    dropee_ptr
332}
333
334fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
335    debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
336
337    assert!(!matches!(ty, Some(ty) if ty.is_coroutine()));
338
339    let args = if let Some(ty) = ty {
340        tcx.mk_args(&[ty.into()])
341    } else {
342        GenericArgs::identity_for_item(tcx, def_id)
343    };
344    let sig = tcx.fn_sig(def_id).instantiate(tcx, args);
345    let sig = tcx.instantiate_bound_regions_with_erased(sig);
346    let span = tcx.def_span(def_id);
347
348    let source_info = SourceInfo::outermost(span);
349
350    let return_block = BasicBlock::new(1);
351    let mut blocks = IndexVec::with_capacity(2);
352    let block = |blocks: &mut IndexVec<_, _>, kind| {
353        blocks.push(BasicBlockData {
354            statements: vec![],
355            terminator: Some(Terminator { source_info, kind }),
356            is_cleanup: false,
357        })
358    };
359    block(&mut blocks, TerminatorKind::Goto { target: return_block });
360    block(&mut blocks, TerminatorKind::Return);
361
362    let source = MirSource::from_instance(ty::InstanceKind::DropGlue(def_id, ty));
363    let mut body =
364        new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span);
365
366    // The first argument (index 0), but add 1 for the return value.
367    let dropee_ptr = Place::from(Local::new(1 + 0));
368    let dropee_ptr = dropee_emit_retag(tcx, &mut body, dropee_ptr, span);
369
370    if ty.is_some() {
371        let patch = {
372            let typing_env = ty::TypingEnv::post_analysis(tcx, def_id);
373            let mut elaborator = DropShimElaborator {
374                body: &body,
375                patch: MirPatch::new(&body),
376                tcx,
377                typing_env,
378                produce_async_drops: false,
379            };
380            let dropee = tcx.mk_place_deref(dropee_ptr);
381            let resume_block = elaborator.patch.resume_block();
382            elaborate_drop(
383                &mut elaborator,
384                source_info,
385                dropee,
386                (),
387                return_block,
388                Unwind::To(resume_block),
389                START_BLOCK,
390                None,
391            );
392            elaborator.patch
393        };
394        patch.apply(&mut body);
395    }
396
397    body
398}
399
400fn new_body<'tcx>(
401    source: MirSource<'tcx>,
402    basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
403    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
404    arg_count: usize,
405    span: Span,
406) -> Body<'tcx> {
407    let mut body = Body::new(
408        source,
409        basic_blocks,
410        IndexVec::from_elem_n(
411            SourceScopeData {
412                span,
413                parent_scope: None,
414                inlined: None,
415                inlined_parent_scope: None,
416                local_data: ClearCrossCrate::Clear,
417            },
418            1,
419        ),
420        local_decls,
421        IndexVec::new(),
422        arg_count,
423        vec![],
424        span,
425        None,
426        // FIXME(compiler-errors): is this correct?
427        None,
428    );
429    // Shims do not directly mention any consts.
430    body.set_required_consts(Vec::new());
431    body
432}
433
434pub(super) struct DropShimElaborator<'a, 'tcx> {
435    pub body: &'a Body<'tcx>,
436    pub patch: MirPatch<'tcx>,
437    pub tcx: TyCtxt<'tcx>,
438    pub typing_env: ty::TypingEnv<'tcx>,
439    pub produce_async_drops: bool,
440}
441
442impl fmt::Debug for DropShimElaborator<'_, '_> {
443    fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
444        Ok(())
445    }
446}
447
448impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
449    type Path = ();
450
451    fn patch_ref(&self) -> &MirPatch<'tcx> {
452        &self.patch
453    }
454    fn patch(&mut self) -> &mut MirPatch<'tcx> {
455        &mut self.patch
456    }
457    fn body(&self) -> &'a Body<'tcx> {
458        self.body
459    }
460    fn tcx(&self) -> TyCtxt<'tcx> {
461        self.tcx
462    }
463    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
464        self.typing_env
465    }
466
467    fn terminator_loc(&self, bb: BasicBlock) -> Location {
468        self.patch.terminator_loc(self.body, bb)
469    }
470    fn allow_async_drops(&self) -> bool {
471        self.produce_async_drops
472    }
473
474    fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle {
475        match mode {
476            DropFlagMode::Shallow => {
477                // Drops for the contained fields are "shallow" and "static" - they will simply call
478                // the field's own drop glue.
479                DropStyle::Static
480            }
481            DropFlagMode::Deep => {
482                // The top-level drop is "deep" and "open" - it will be elaborated to a drop ladder
483                // dropping each field contained in the value.
484                DropStyle::Open
485            }
486        }
487    }
488
489    fn get_drop_flag(&mut self, _path: Self::Path) -> Option<Operand<'tcx>> {
490        None
491    }
492
493    fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
494
495    fn field_subpath(&self, _path: Self::Path, _field: FieldIdx) -> Option<Self::Path> {
496        None
497    }
498    fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
499        None
500    }
501    fn downcast_subpath(&self, _path: Self::Path, _variant: VariantIdx) -> Option<Self::Path> {
502        Some(())
503    }
504    fn array_subpath(&self, _path: Self::Path, _index: u64, _size: u64) -> Option<Self::Path> {
505        None
506    }
507}
508
509fn build_thread_local_shim<'tcx>(
510    tcx: TyCtxt<'tcx>,
511    instance: ty::InstanceKind<'tcx>,
512) -> Body<'tcx> {
513    let def_id = instance.def_id();
514
515    let span = tcx.def_span(def_id);
516    let source_info = SourceInfo::outermost(span);
517
518    let blocks = IndexVec::from_raw(vec![BasicBlockData {
519        statements: vec![Statement {
520            source_info,
521            kind: StatementKind::Assign(Box::new((
522                Place::return_place(),
523                Rvalue::ThreadLocalRef(def_id),
524            ))),
525        }],
526        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
527        is_cleanup: false,
528    }]);
529
530    new_body(
531        MirSource::from_instance(instance),
532        blocks,
533        IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]),
534        0,
535        span,
536    )
537}
538
539/// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`.
540fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
541    debug!("build_clone_shim(def_id={:?})", def_id);
542
543    let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty);
544
545    let dest = Place::return_place();
546    let src = tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
547
548    match self_ty.kind() {
549        ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(),
550        ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
551        ty::CoroutineClosure(_, args) => {
552            builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys())
553        }
554        ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
555        ty::Coroutine(coroutine_def_id, args) => {
556            assert_eq!(tcx.coroutine_movability(*coroutine_def_id), hir::Movability::Movable);
557            builder.coroutine_shim(dest, src, *coroutine_def_id, args.as_coroutine())
558        }
559        _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
560    };
561
562    builder.into_mir()
563}
564
565struct CloneShimBuilder<'tcx> {
566    tcx: TyCtxt<'tcx>,
567    def_id: DefId,
568    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
569    blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
570    span: Span,
571    sig: ty::FnSig<'tcx>,
572}
573
574impl<'tcx> CloneShimBuilder<'tcx> {
575    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self {
576        // we must instantiate the self_ty because it's
577        // otherwise going to be TySelf and we can't index
578        // or access fields of a Place of type TySelf.
579        let sig = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]);
580        let sig = tcx.instantiate_bound_regions_with_erased(sig);
581        let span = tcx.def_span(def_id);
582
583        CloneShimBuilder {
584            tcx,
585            def_id,
586            local_decls: local_decls_for_sig(&sig, span),
587            blocks: IndexVec::new(),
588            span,
589            sig,
590        }
591    }
592
593    fn into_mir(self) -> Body<'tcx> {
594        let source = MirSource::from_instance(ty::InstanceKind::CloneShim(
595            self.def_id,
596            self.sig.inputs_and_output[0],
597        ));
598        new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span)
599    }
600
601    fn source_info(&self) -> SourceInfo {
602        SourceInfo::outermost(self.span)
603    }
604
605    fn block(
606        &mut self,
607        statements: Vec<Statement<'tcx>>,
608        kind: TerminatorKind<'tcx>,
609        is_cleanup: bool,
610    ) -> BasicBlock {
611        let source_info = self.source_info();
612        self.blocks.push(BasicBlockData {
613            statements,
614            terminator: Some(Terminator { source_info, kind }),
615            is_cleanup,
616        })
617    }
618
619    /// Gives the index of an upcoming BasicBlock, with an offset.
620    /// offset=0 will give you the index of the next BasicBlock,
621    /// offset=1 will give the index of the next-to-next block,
622    /// offset=-1 will give you the index of the last-created block
623    fn block_index_offset(&self, offset: usize) -> BasicBlock {
624        BasicBlock::new(self.blocks.len() + offset)
625    }
626
627    fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> {
628        Statement { source_info: self.source_info(), kind }
629    }
630
631    fn copy_shim(&mut self) {
632        let rcvr = self.tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
633        let ret_statement = self.make_statement(StatementKind::Assign(Box::new((
634            Place::return_place(),
635            Rvalue::Use(Operand::Copy(rcvr)),
636        ))));
637        self.block(vec![ret_statement], TerminatorKind::Return, false);
638    }
639
640    fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
641        let span = self.span;
642        let mut local = LocalDecl::new(ty, span);
643        if mutability.is_not() {
644            local = local.immutable();
645        }
646        Place::from(self.local_decls.push(local))
647    }
648
649    fn make_clone_call(
650        &mut self,
651        dest: Place<'tcx>,
652        src: Place<'tcx>,
653        ty: Ty<'tcx>,
654        next: BasicBlock,
655        cleanup: BasicBlock,
656    ) {
657        let tcx = self.tcx;
658
659        // `func == Clone::clone(&ty) -> ty`
660        let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]);
661        let func = Operand::Constant(Box::new(ConstOperand {
662            span: self.span,
663            user_ty: None,
664            const_: Const::zero_sized(func_ty),
665        }));
666
667        let ref_loc =
668            self.make_place(Mutability::Not, Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty));
669
670        // `let ref_loc: &ty = &src;`
671        let statement = self.make_statement(StatementKind::Assign(Box::new((
672            ref_loc,
673            Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, src),
674        ))));
675
676        // `let loc = Clone::clone(ref_loc);`
677        self.block(
678            vec![statement],
679            TerminatorKind::Call {
680                func,
681                args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(),
682                destination: dest,
683                target: Some(next),
684                unwind: UnwindAction::Cleanup(cleanup),
685                call_source: CallSource::Normal,
686                fn_span: self.span,
687            },
688            false,
689        );
690    }
691
692    fn clone_fields<I>(
693        &mut self,
694        dest: Place<'tcx>,
695        src: Place<'tcx>,
696        target: BasicBlock,
697        mut unwind: BasicBlock,
698        tys: I,
699    ) -> BasicBlock
700    where
701        I: IntoIterator<Item = Ty<'tcx>>,
702    {
703        // For an iterator of length n, create 2*n + 1 blocks.
704        for (i, ity) in tys.into_iter().enumerate() {
705            // Each iteration creates two blocks, referred to here as block 2*i and block 2*i + 1.
706            //
707            // Block 2*i attempts to clone the field. If successful it branches to 2*i + 2 (the
708            // next clone block). If unsuccessful it branches to the previous unwind block, which
709            // is initially the `unwind` argument passed to this function.
710            //
711            // Block 2*i + 1 is the unwind block for this iteration. It drops the cloned value
712            // created by block 2*i. We store this block in `unwind` so that the next clone block
713            // will unwind to it if cloning fails.
714
715            let field = FieldIdx::new(i);
716            let src_field = self.tcx.mk_place_field(src, field, ity);
717
718            let dest_field = self.tcx.mk_place_field(dest, field, ity);
719
720            let next_unwind = self.block_index_offset(1);
721            let next_block = self.block_index_offset(2);
722            self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
723            self.block(
724                vec![],
725                TerminatorKind::Drop {
726                    place: dest_field,
727                    target: unwind,
728                    unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
729                    replace: false,
730                    drop: None,
731                    async_fut: None,
732                },
733                /* is_cleanup */ true,
734            );
735            unwind = next_unwind;
736        }
737        // If all clones succeed then we end up here.
738        self.block(vec![], TerminatorKind::Goto { target }, false);
739        unwind
740    }
741
742    fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
743    where
744        I: IntoIterator<Item = Ty<'tcx>>,
745    {
746        self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
747        let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
748        let target = self.block(vec![], TerminatorKind::Return, false);
749
750        let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
751    }
752
753    fn coroutine_shim(
754        &mut self,
755        dest: Place<'tcx>,
756        src: Place<'tcx>,
757        coroutine_def_id: DefId,
758        args: CoroutineArgs<TyCtxt<'tcx>>,
759    ) {
760        self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
761        let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
762        // This will get overwritten with a switch once we know the target blocks
763        let switch = self.block(vec![], TerminatorKind::Unreachable, false);
764        let unwind = self.clone_fields(dest, src, switch, unwind, args.upvar_tys());
765        let target = self.block(vec![], TerminatorKind::Return, false);
766        let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
767        let mut cases = Vec::with_capacity(args.state_tys(coroutine_def_id, self.tcx).count());
768        for (index, state_tys) in args.state_tys(coroutine_def_id, self.tcx).enumerate() {
769            let variant_index = VariantIdx::new(index);
770            let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
771            let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
772            let clone_block = self.block_index_offset(1);
773            let start_block = self.block(
774                vec![self.make_statement(StatementKind::SetDiscriminant {
775                    place: Box::new(Place::return_place()),
776                    variant_index,
777                })],
778                TerminatorKind::Goto { target: clone_block },
779                false,
780            );
781            cases.push((index as u128, start_block));
782            let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
783        }
784        let discr_ty = args.discr_ty(self.tcx);
785        let temp = self.make_place(Mutability::Mut, discr_ty);
786        let rvalue = Rvalue::Discriminant(src);
787        let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
788        match &mut self.blocks[switch] {
789            BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
790                statements.push(statement);
791                *kind = TerminatorKind::SwitchInt {
792                    discr: Operand::Move(temp),
793                    targets: SwitchTargets::new(cases.into_iter(), unreachable),
794                };
795            }
796            BasicBlockData { terminator: None, .. } => unreachable!(),
797        }
798    }
799}
800
801/// Builds a "call" shim for `instance`. The shim calls the function specified by `call_kind`,
802/// first adjusting its first argument according to `rcvr_adjustment`.
803#[instrument(level = "debug", skip(tcx), ret)]
804fn build_call_shim<'tcx>(
805    tcx: TyCtxt<'tcx>,
806    instance: ty::InstanceKind<'tcx>,
807    rcvr_adjustment: Option<Adjustment>,
808    call_kind: CallKind<'tcx>,
809) -> Body<'tcx> {
810    // `FnPtrShim` contains the fn pointer type that a call shim is being built for - this is used
811    // to instantiate into the signature of the shim. It is not necessary for users of this
812    // MIR body to perform further instantiations (see `InstanceKind::has_polymorphic_mir_body`).
813    let (sig_args, untuple_args) = if let ty::InstanceKind::FnPtrShim(_, ty) = instance {
814        let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx));
815
816        let untuple_args = sig.inputs();
817
818        // Create substitutions for the `Self` and `Args` generic parameters of the shim body.
819        let arg_tup = Ty::new_tup(tcx, untuple_args);
820
821        (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
822    } else {
823        (None, None)
824    };
825
826    let def_id = instance.def_id();
827
828    let sig = tcx.fn_sig(def_id);
829    let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
830
831    assert_eq!(sig_args.is_some(), !instance.has_polymorphic_mir_body());
832    let mut sig = if let Some(sig_args) = sig_args {
833        sig.instantiate(tcx, &sig_args)
834    } else {
835        sig.instantiate_identity()
836    };
837
838    if let CallKind::Indirect(fnty) = call_kind {
839        // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This
840        // can only be an `FnDef` or `FnPtr`, but currently will be `Self` since the types come from
841        // the implemented `FnX` trait.
842
843        // Apply the opposite adjustment to the MIR input.
844        let mut inputs_and_output = sig.inputs_and_output.to_vec();
845
846        // Initial signature is `fn(&? Self, Args) -> Self::Output` where `Args` is a tuple of the
847        // fn arguments. `Self` may be passed via (im)mutable reference or by-value.
848        assert_eq!(inputs_and_output.len(), 3);
849
850        // `Self` is always the original fn type `ty`. The MIR call terminator is only defined for
851        // `FnDef` and `FnPtr` callees, not the `Self` type param.
852        let self_arg = &mut inputs_and_output[0];
853        *self_arg = match rcvr_adjustment.unwrap() {
854            Adjustment::Identity => fnty,
855            Adjustment::Deref { source } => match source {
856                DerefSource::ImmRef => Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fnty),
857                DerefSource::MutRef => Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fnty),
858                DerefSource::MutPtr => Ty::new_mut_ptr(tcx, fnty),
859            },
860            Adjustment::RefMut => bug!("`RefMut` is never used with indirect calls: {instance:?}"),
861        };
862        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
863    }
864
865    // FIXME: Avoid having to adjust the signature both here and in
866    // `fn_sig_for_fn_abi`.
867    if let ty::InstanceKind::VTableShim(..) = instance {
868        // Modify fn(self, ...) to fn(self: *mut Self, ...)
869        let mut inputs_and_output = sig.inputs_and_output.to_vec();
870        let self_arg = &mut inputs_and_output[0];
871        debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
872        *self_arg = Ty::new_mut_ptr(tcx, *self_arg);
873        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
874    }
875
876    let span = tcx.def_span(def_id);
877
878    debug!(?sig);
879
880    let mut local_decls = local_decls_for_sig(&sig, span);
881    let source_info = SourceInfo::outermost(span);
882
883    let destination = Place::return_place();
884
885    let rcvr_place = || {
886        assert!(rcvr_adjustment.is_some());
887        Place::from(Local::new(1))
888    };
889    let mut statements = vec![];
890
891    let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment {
892        Adjustment::Identity => Operand::Move(rcvr_place()),
893        Adjustment::Deref { source: _ } => Operand::Move(tcx.mk_place_deref(rcvr_place())),
894        Adjustment::RefMut => {
895            // let rcvr = &mut rcvr;
896            let ref_rcvr = local_decls.push(
897                LocalDecl::new(
898                    Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, sig.inputs()[0]),
899                    span,
900                )
901                .immutable(),
902            );
903            let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default };
904            statements.push(Statement {
905                source_info,
906                kind: StatementKind::Assign(Box::new((
907                    Place::from(ref_rcvr),
908                    Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()),
909                ))),
910            });
911            Operand::Move(Place::from(ref_rcvr))
912        }
913    });
914
915    let (callee, mut args) = match call_kind {
916        // `FnPtr` call has no receiver. Args are untupled below.
917        CallKind::Indirect(_) => (rcvr.unwrap(), vec![]),
918
919        // `FnDef` call with optional receiver.
920        CallKind::Direct(def_id) => {
921            let ty = tcx.type_of(def_id).instantiate_identity();
922            (
923                Operand::Constant(Box::new(ConstOperand {
924                    span,
925                    user_ty: None,
926                    const_: Const::zero_sized(ty),
927                })),
928                rcvr.into_iter().collect::<Vec<_>>(),
929            )
930        }
931    };
932
933    let mut arg_range = 0..sig.inputs().len();
934
935    // Take the `self` ("receiver") argument out of the range (it's adjusted above).
936    if rcvr_adjustment.is_some() {
937        arg_range.start += 1;
938    }
939
940    // Take the last argument, if we need to untuple it (handled below).
941    if untuple_args.is_some() {
942        arg_range.end -= 1;
943    }
944
945    // Pass all of the non-special arguments directly.
946    args.extend(arg_range.map(|i| Operand::Move(Place::from(Local::new(1 + i)))));
947
948    // Untuple the last argument, if we have to.
949    if let Some(untuple_args) = untuple_args {
950        let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
951        args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
952            Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), FieldIdx::new(i), *ity))
953        }));
954    }
955
956    let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 };
957    let mut blocks = IndexVec::with_capacity(n_blocks);
958    let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| {
959        blocks.push(BasicBlockData {
960            statements,
961            terminator: Some(Terminator { source_info, kind }),
962            is_cleanup,
963        })
964    };
965
966    // BB #0
967    let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect();
968    block(
969        &mut blocks,
970        statements,
971        TerminatorKind::Call {
972            func: callee,
973            args,
974            destination,
975            target: Some(BasicBlock::new(1)),
976            unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
977                UnwindAction::Cleanup(BasicBlock::new(3))
978            } else {
979                UnwindAction::Continue
980            },
981            call_source: CallSource::Misc,
982            fn_span: span,
983        },
984        false,
985    );
986
987    if let Some(Adjustment::RefMut) = rcvr_adjustment {
988        // BB #1 - drop for Self
989        block(
990            &mut blocks,
991            vec![],
992            TerminatorKind::Drop {
993                place: rcvr_place(),
994                target: BasicBlock::new(2),
995                unwind: UnwindAction::Continue,
996                replace: false,
997                drop: None,
998                async_fut: None,
999            },
1000            false,
1001        );
1002    }
1003    // BB #1/#2 - return
1004    let stmts = vec![];
1005    block(&mut blocks, stmts, TerminatorKind::Return, false);
1006    if let Some(Adjustment::RefMut) = rcvr_adjustment {
1007        // BB #3 - drop if closure panics
1008        block(
1009            &mut blocks,
1010            vec![],
1011            TerminatorKind::Drop {
1012                place: rcvr_place(),
1013                target: BasicBlock::new(4),
1014                unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
1015                replace: false,
1016                drop: None,
1017                async_fut: None,
1018            },
1019            /* is_cleanup */ true,
1020        );
1021
1022        // BB #4 - resume
1023        block(&mut blocks, vec![], TerminatorKind::UnwindResume, true);
1024    }
1025
1026    let mut body =
1027        new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span);
1028
1029    if let ExternAbi::RustCall = sig.abi {
1030        body.spread_arg = Some(Local::new(sig.inputs().len()));
1031    }
1032
1033    body
1034}
1035
1036pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
1037    debug_assert!(tcx.is_constructor(ctor_id));
1038
1039    let typing_env = ty::TypingEnv::post_analysis(tcx, ctor_id);
1040
1041    // Normalize the sig.
1042    let sig = tcx
1043        .fn_sig(ctor_id)
1044        .instantiate_identity()
1045        .no_bound_vars()
1046        .expect("LBR in ADT constructor signature");
1047    let sig = tcx.normalize_erasing_regions(typing_env, sig);
1048
1049    let ty::Adt(adt_def, args) = sig.output().kind() else {
1050        bug!("unexpected type for ADT ctor {:?}", sig.output());
1051    };
1052
1053    debug!("build_ctor: ctor_id={:?} sig={:?}", ctor_id, sig);
1054
1055    let span = tcx.def_span(ctor_id);
1056
1057    let local_decls = local_decls_for_sig(&sig, span);
1058
1059    let source_info = SourceInfo::outermost(span);
1060
1061    let variant_index =
1062        if adt_def.is_enum() { adt_def.variant_index_with_ctor_id(ctor_id) } else { FIRST_VARIANT };
1063
1064    // Generate the following MIR:
1065    //
1066    // (return as Variant).field0 = arg0;
1067    // (return as Variant).field1 = arg1;
1068    //
1069    // return;
1070    debug!("build_ctor: variant_index={:?}", variant_index);
1071
1072    let kind = AggregateKind::Adt(adt_def.did(), variant_index, args, None, None);
1073    let variant = adt_def.variant(variant_index);
1074    let statement = Statement {
1075        kind: StatementKind::Assign(Box::new((
1076            Place::return_place(),
1077            Rvalue::Aggregate(
1078                Box::new(kind),
1079                (0..variant.fields.len())
1080                    .map(|idx| Operand::Move(Place::from(Local::new(idx + 1))))
1081                    .collect(),
1082            ),
1083        ))),
1084        source_info,
1085    };
1086
1087    let start_block = BasicBlockData {
1088        statements: vec![statement],
1089        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1090        is_cleanup: false,
1091    };
1092
1093    let source = MirSource::item(ctor_id);
1094    let mut body = new_body(
1095        source,
1096        IndexVec::from_elem_n(start_block, 1),
1097        local_decls,
1098        sig.inputs().len(),
1099        span,
1100    );
1101    // A constructor doesn't mention any other items (and we don't run the usual optimization passes
1102    // so this would otherwise not get filled).
1103    body.set_mentioned_items(Vec::new());
1104
1105    crate::pass_manager::dump_mir_for_phase_change(tcx, &body);
1106
1107    body
1108}
1109
1110/// ```ignore (pseudo-impl)
1111/// impl FnPtr for fn(u32) {
1112///     fn addr(self) -> usize {
1113///         self as usize
1114///     }
1115/// }
1116/// ```
1117fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
1118    assert_matches!(self_ty.kind(), ty::FnPtr(..), "expected fn ptr, found {self_ty}");
1119    let span = tcx.def_span(def_id);
1120    let Some(sig) = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]).no_bound_vars() else {
1121        span_bug!(span, "FnPtr::addr with bound vars for `{self_ty}`");
1122    };
1123    let locals = local_decls_for_sig(&sig, span);
1124
1125    let source_info = SourceInfo::outermost(span);
1126    // FIXME: use `expose_provenance` once we figure out whether function pointers have meaningful
1127    // provenance.
1128    let rvalue = Rvalue::Cast(
1129        CastKind::FnPtrToPtr,
1130        Operand::Move(Place::from(Local::new(1))),
1131        Ty::new_imm_ptr(tcx, tcx.types.unit),
1132    );
1133    let stmt = Statement {
1134        source_info,
1135        kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1136    };
1137    let statements = vec![stmt];
1138    let start_block = BasicBlockData {
1139        statements,
1140        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1141        is_cleanup: false,
1142    };
1143    let source = MirSource::from_instance(ty::InstanceKind::FnPtrAddrShim(def_id, self_ty));
1144    new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span)
1145}
1146
1147fn build_construct_coroutine_by_move_shim<'tcx>(
1148    tcx: TyCtxt<'tcx>,
1149    coroutine_closure_def_id: DefId,
1150    receiver_by_ref: bool,
1151) -> Body<'tcx> {
1152    let mut self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity();
1153    let mut self_local: Place<'tcx> = Local::from_usize(1).into();
1154    let ty::CoroutineClosure(_, args) = *self_ty.kind() else {
1155        bug!();
1156    };
1157
1158    // We use `&Self` here because we only need to emit an ABI-compatible shim body,
1159    // rather than match the signature exactly (which might take `&mut self` instead).
1160    //
1161    // We adjust the `self_local` to be a deref since we want to copy fields out of
1162    // a reference to the closure.
1163    if receiver_by_ref {
1164        self_local = tcx.mk_place_deref(self_local);
1165        self_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, self_ty);
1166    }
1167
1168    let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| {
1169        tcx.mk_fn_sig(
1170            [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()),
1171            sig.to_coroutine_given_kind_and_upvars(
1172                tcx,
1173                args.as_coroutine_closure().parent_args(),
1174                tcx.coroutine_for_closure(coroutine_closure_def_id),
1175                ty::ClosureKind::FnOnce,
1176                tcx.lifetimes.re_erased,
1177                args.as_coroutine_closure().tupled_upvars_ty(),
1178                args.as_coroutine_closure().coroutine_captures_by_ref_ty(),
1179            ),
1180            sig.c_variadic,
1181            sig.safety,
1182            sig.abi,
1183        )
1184    });
1185    let sig = tcx.liberate_late_bound_regions(coroutine_closure_def_id, poly_sig);
1186    let ty::Coroutine(coroutine_def_id, coroutine_args) = *sig.output().kind() else {
1187        bug!();
1188    };
1189
1190    let span = tcx.def_span(coroutine_closure_def_id);
1191    let locals = local_decls_for_sig(&sig, span);
1192
1193    let mut fields = vec![];
1194
1195    // Move all of the closure args.
1196    for idx in 1..sig.inputs().len() {
1197        fields.push(Operand::Move(Local::from_usize(idx + 1).into()));
1198    }
1199
1200    for (idx, ty) in args.as_coroutine_closure().upvar_tys().iter().enumerate() {
1201        if receiver_by_ref {
1202            // The only situation where it's possible is when we capture immuatable references,
1203            // since those don't need to be reborrowed with the closure's env lifetime. Since
1204            // references are always `Copy`, just emit a copy.
1205            if !matches!(ty.kind(), ty::Ref(_, _, hir::Mutability::Not)) {
1206                // This copy is only sound if it's a `&T`. This may be
1207                // reachable e.g. when eagerly computing the `Fn` instance
1208                // of an async closure that doesn't borrowck.
1209                tcx.dcx().delayed_bug(format!(
1210                    "field should be captured by immutable ref if we have \
1211                    an `Fn` instance, but it was: {ty}"
1212                ));
1213            }
1214            fields.push(Operand::Copy(tcx.mk_place_field(
1215                self_local,
1216                FieldIdx::from_usize(idx),
1217                ty,
1218            )));
1219        } else {
1220            fields.push(Operand::Move(tcx.mk_place_field(
1221                self_local,
1222                FieldIdx::from_usize(idx),
1223                ty,
1224            )));
1225        }
1226    }
1227
1228    let source_info = SourceInfo::outermost(span);
1229    let rvalue = Rvalue::Aggregate(
1230        Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)),
1231        IndexVec::from_raw(fields),
1232    );
1233    let stmt = Statement {
1234        source_info,
1235        kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1236    };
1237    let statements = vec![stmt];
1238    let start_block = BasicBlockData {
1239        statements,
1240        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1241        is_cleanup: false,
1242    };
1243
1244    let source = MirSource::from_instance(ty::InstanceKind::ConstructCoroutineInClosureShim {
1245        coroutine_closure_def_id,
1246        receiver_by_ref,
1247    });
1248
1249    let body =
1250        new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span);
1251    dump_mir(
1252        tcx,
1253        false,
1254        if receiver_by_ref { "coroutine_closure_by_ref" } else { "coroutine_closure_by_move" },
1255        &0,
1256        &body,
1257        |_, _| Ok(()),
1258    );
1259
1260    body
1261}