rustc_mir_transform/
elaborate_drop.rs

1use std::{fmt, iter, mem};
2
3use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
4use rustc_hir::def::DefKind;
5use rustc_hir::lang_items::LangItem;
6use rustc_index::Idx;
7use rustc_middle::mir::*;
8use rustc_middle::ty::adjustment::PointerCoercion;
9use rustc_middle::ty::util::IntTypeExt;
10use rustc_middle::ty::{self, GenericArg, GenericArgsRef, Ty, TyCtxt};
11use rustc_middle::{bug, span_bug, traits};
12use rustc_span::DUMMY_SP;
13use rustc_span::source_map::{Spanned, dummy_spanned};
14use tracing::{debug, instrument};
15
16use crate::patch::MirPatch;
17
18/// Describes how/if a value should be dropped.
19#[derive(Debug)]
20pub(crate) enum DropStyle {
21    /// The value is already dead at the drop location, no drop will be executed.
22    Dead,
23
24    /// The value is known to always be initialized at the drop location, drop will always be
25    /// executed.
26    Static,
27
28    /// Whether the value needs to be dropped depends on its drop flag.
29    Conditional,
30
31    /// An "open" drop is one where only the fields of a value are dropped.
32    ///
33    /// For example, this happens when moving out of a struct field: The rest of the struct will be
34    /// dropped in such an "open" drop. It is also used to generate drop glue for the individual
35    /// components of a value, for example for dropping array elements.
36    Open,
37}
38
39/// Which drop flags to affect/check with an operation.
40#[derive(Debug)]
41pub(crate) enum DropFlagMode {
42    /// Only affect the top-level drop flag, not that of any contained fields.
43    Shallow,
44    /// Affect all nested drop flags in addition to the top-level one.
45    Deep,
46}
47
48/// Describes if unwinding is necessary and where to unwind to if a panic occurs.
49#[derive(Copy, Clone, Debug)]
50pub(crate) enum Unwind {
51    /// Unwind to this block.
52    To(BasicBlock),
53    /// Already in an unwind path, any panic will cause an abort.
54    InCleanup,
55}
56
57impl Unwind {
58    fn is_cleanup(self) -> bool {
59        match self {
60            Unwind::To(..) => false,
61            Unwind::InCleanup => true,
62        }
63    }
64
65    fn into_action(self) -> UnwindAction {
66        match self {
67            Unwind::To(bb) => UnwindAction::Cleanup(bb),
68            Unwind::InCleanup => UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
69        }
70    }
71
72    fn map<F>(self, f: F) -> Self
73    where
74        F: FnOnce(BasicBlock) -> BasicBlock,
75    {
76        match self {
77            Unwind::To(bb) => Unwind::To(f(bb)),
78            Unwind::InCleanup => Unwind::InCleanup,
79        }
80    }
81}
82
83pub(crate) trait DropElaborator<'a, 'tcx>: fmt::Debug {
84    /// The type representing paths that can be moved out of.
85    ///
86    /// Users can move out of individual fields of a struct, such as `a.b.c`. This type is used to
87    /// represent such move paths. Sometimes tracking individual move paths is not necessary, in
88    /// which case this may be set to (for example) `()`.
89    type Path: Copy + fmt::Debug;
90
91    // Accessors
92
93    fn patch_ref(&self) -> &MirPatch<'tcx>;
94    fn patch(&mut self) -> &mut MirPatch<'tcx>;
95    fn body(&self) -> &'a Body<'tcx>;
96    fn tcx(&self) -> TyCtxt<'tcx>;
97    fn typing_env(&self) -> ty::TypingEnv<'tcx>;
98    fn allow_async_drops(&self) -> bool;
99
100    fn terminator_loc(&self, bb: BasicBlock) -> Location;
101
102    // Drop logic
103
104    /// Returns how `path` should be dropped, given `mode`.
105    fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle;
106
107    /// Returns the drop flag of `path` as a MIR `Operand` (or `None` if `path` has no drop flag).
108    fn get_drop_flag(&mut self, path: Self::Path) -> Option<Operand<'tcx>>;
109
110    /// Modifies the MIR patch so that the drop flag of `path` (if any) is cleared at `location`.
111    ///
112    /// If `mode` is deep, drop flags of all child paths should also be cleared by inserting
113    /// additional statements.
114    fn clear_drop_flag(&mut self, location: Location, path: Self::Path, mode: DropFlagMode);
115
116    // Subpaths
117
118    /// Returns the subpath of a field of `path` (or `None` if there is no dedicated subpath).
119    ///
120    /// If this returns `None`, `field` will not get a dedicated drop flag.
121    fn field_subpath(&self, path: Self::Path, field: FieldIdx) -> Option<Self::Path>;
122
123    /// Returns the subpath of a dereference of `path` (or `None` if there is no dedicated subpath).
124    ///
125    /// If this returns `None`, `*path` will not get a dedicated drop flag.
126    ///
127    /// This is only relevant for `Box<T>`, where the contained `T` can be moved out of the box.
128    fn deref_subpath(&self, path: Self::Path) -> Option<Self::Path>;
129
130    /// Returns the subpath of downcasting `path` to one of its variants.
131    ///
132    /// If this returns `None`, the downcast of `path` will not get a dedicated drop flag.
133    fn downcast_subpath(&self, path: Self::Path, variant: VariantIdx) -> Option<Self::Path>;
134
135    /// Returns the subpath of indexing a fixed-size array `path`.
136    ///
137    /// If this returns `None`, elements of `path` will not get a dedicated drop flag.
138    ///
139    /// This is only relevant for array patterns, which can move out of individual array elements.
140    fn array_subpath(&self, path: Self::Path, index: u64, size: u64) -> Option<Self::Path>;
141}
142
143#[derive(Debug)]
144struct DropCtxt<'a, 'b, 'tcx, D>
145where
146    D: DropElaborator<'b, 'tcx>,
147{
148    elaborator: &'a mut D,
149
150    source_info: SourceInfo,
151
152    place: Place<'tcx>,
153    path: D::Path,
154    succ: BasicBlock,
155    unwind: Unwind,
156    dropline: Option<BasicBlock>,
157}
158
159/// "Elaborates" a drop of `place`/`path` and patches `bb`'s terminator to execute it.
160///
161/// The passed `elaborator` is used to determine what should happen at the drop terminator. It
162/// decides whether the drop can be statically determined or whether it needs a dynamic drop flag,
163/// and whether the drop is "open", ie. should be expanded to drop all subfields of the dropped
164/// value.
165///
166/// When this returns, the MIR patch in the `elaborator` contains the necessary changes.
167pub(crate) fn elaborate_drop<'b, 'tcx, D>(
168    elaborator: &mut D,
169    source_info: SourceInfo,
170    place: Place<'tcx>,
171    path: D::Path,
172    succ: BasicBlock,
173    unwind: Unwind,
174    bb: BasicBlock,
175    dropline: Option<BasicBlock>,
176) where
177    D: DropElaborator<'b, 'tcx>,
178    'tcx: 'b,
179{
180    DropCtxt { elaborator, source_info, place, path, succ, unwind, dropline }.elaborate_drop(bb)
181}
182
183impl<'a, 'b, 'tcx, D> DropCtxt<'a, 'b, 'tcx, D>
184where
185    D: DropElaborator<'b, 'tcx>,
186    'tcx: 'b,
187{
188    #[instrument(level = "trace", skip(self), ret)]
189    fn place_ty(&self, place: Place<'tcx>) -> Ty<'tcx> {
190        if place.local < self.elaborator.body().local_decls.next_index() {
191            place.ty(self.elaborator.body(), self.tcx()).ty
192        } else {
193            // We don't have a slice with all the locals, since some are in the patch.
194            PlaceTy::from_ty(self.elaborator.patch_ref().local_ty(place.local))
195                .multi_projection_ty(self.elaborator.tcx(), place.projection)
196                .ty
197        }
198    }
199
200    fn tcx(&self) -> TyCtxt<'tcx> {
201        self.elaborator.tcx()
202    }
203
204    // Generates three blocks:
205    // * #1:pin_obj_bb:   call Pin<ObjTy>::new_unchecked(&mut obj)
206    // * #2:call_drop_bb: fut = call obj.<AsyncDrop::drop>() OR call async_drop_in_place<T>(obj)
207    // * #3:drop_term_bb: drop (obj, fut, ...)
208    // We keep async drop unexpanded to poll-loop here, to expand it later, at StateTransform -
209    //   into states expand.
210    // call_destructor_only - to call only AsyncDrop::drop, not full async_drop_in_place glue
211    fn build_async_drop(
212        &mut self,
213        place: Place<'tcx>,
214        drop_ty: Ty<'tcx>,
215        bb: Option<BasicBlock>,
216        succ: BasicBlock,
217        unwind: Unwind,
218        dropline: Option<BasicBlock>,
219        call_destructor_only: bool,
220    ) -> BasicBlock {
221        let tcx = self.tcx();
222        let span = self.source_info.span;
223
224        let pin_obj_bb = bb.unwrap_or_else(|| {
225            self.elaborator.patch().new_block(BasicBlockData::new(
226                Some(Terminator {
227                    // Temporary terminator, will be replaced by patch
228                    source_info: self.source_info,
229                    kind: TerminatorKind::Return,
230                }),
231                false,
232            ))
233        });
234
235        let (fut_ty, drop_fn_def_id, trait_args) = if call_destructor_only {
236            // Resolving obj.<AsyncDrop::drop>()
237            let trait_ref =
238                ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::AsyncDrop, span), [drop_ty]);
239            let (drop_trait, trait_args) = match tcx.codegen_select_candidate(
240                ty::TypingEnv::fully_monomorphized().as_query_input(trait_ref),
241            ) {
242                Ok(traits::ImplSource::UserDefined(traits::ImplSourceUserDefinedData {
243                    impl_def_id,
244                    args,
245                    ..
246                })) => (*impl_def_id, *args),
247                impl_source => {
248                    span_bug!(span, "invalid `AsyncDrop` impl_source: {:?}", impl_source);
249                }
250            };
251            // impl_item_refs may be empty if drop fn is not implemented in 'impl AsyncDrop for ...'
252            // (#140974).
253            // Such code will report error, so just generate sync drop here and return
254            let Some(drop_fn_def_id) = tcx
255                .associated_item_def_ids(drop_trait)
256                .first()
257                .and_then(|def_id| {
258                    if tcx.def_kind(def_id) == DefKind::AssocFn
259                        && tcx.check_args_compatible(*def_id, trait_args)
260                    {
261                        Some(def_id)
262                    } else {
263                        None
264                    }
265                })
266                .copied()
267            else {
268                tcx.dcx().span_delayed_bug(
269                    self.elaborator.body().span,
270                    "AsyncDrop type without correct `async fn drop(...)`.",
271                );
272                self.elaborator.patch().patch_terminator(
273                    pin_obj_bb,
274                    TerminatorKind::Drop {
275                        place,
276                        target: succ,
277                        unwind: unwind.into_action(),
278                        replace: false,
279                        drop: None,
280                        async_fut: None,
281                    },
282                );
283                return pin_obj_bb;
284            };
285            let drop_fn = Ty::new_fn_def(tcx, drop_fn_def_id, trait_args);
286            let sig = drop_fn.fn_sig(tcx);
287            let sig = tcx.instantiate_bound_regions_with_erased(sig);
288            (sig.output(), drop_fn_def_id, trait_args)
289        } else {
290            // Resolving async_drop_in_place<T> function for drop_ty
291            let drop_fn_def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, span);
292            let trait_args = tcx.mk_args(&[drop_ty.into()]);
293            let sig = tcx.fn_sig(drop_fn_def_id).instantiate(tcx, trait_args);
294            let sig = tcx.instantiate_bound_regions_with_erased(sig);
295            (sig.output(), drop_fn_def_id, trait_args)
296        };
297
298        let fut = Place::from(self.new_temp(fut_ty));
299
300        // #1:pin_obj_bb >>> obj_ref = &mut obj
301        let obj_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, drop_ty);
302        let obj_ref_place = Place::from(self.new_temp(obj_ref_ty));
303
304        let term_loc = self.elaborator.terminator_loc(pin_obj_bb);
305        self.elaborator.patch().add_assign(
306            term_loc,
307            obj_ref_place,
308            Rvalue::Ref(
309                tcx.lifetimes.re_erased,
310                BorrowKind::Mut { kind: MutBorrowKind::Default },
311                place,
312            ),
313        );
314
315        // pin_obj_place preparation
316        let pin_obj_new_unchecked_fn = Ty::new_fn_def(
317            tcx,
318            tcx.require_lang_item(LangItem::PinNewUnchecked, span),
319            [GenericArg::from(obj_ref_ty)],
320        );
321        let pin_obj_ty = pin_obj_new_unchecked_fn.fn_sig(tcx).output().no_bound_vars().unwrap();
322        let pin_obj_place = Place::from(self.new_temp(pin_obj_ty));
323        let pin_obj_new_unchecked_fn = Operand::Constant(Box::new(ConstOperand {
324            span,
325            user_ty: None,
326            const_: Const::zero_sized(pin_obj_new_unchecked_fn),
327        }));
328
329        // #3:drop_term_bb
330        let drop_term_bb = self.new_block(
331            unwind,
332            TerminatorKind::Drop {
333                place,
334                target: succ,
335                unwind: unwind.into_action(),
336                replace: false,
337                drop: dropline,
338                async_fut: Some(fut.local),
339            },
340        );
341
342        // #2:call_drop_bb
343        let mut call_statements = Vec::new();
344        let drop_arg = if call_destructor_only {
345            pin_obj_place
346        } else {
347            let ty::Adt(adt_def, adt_args) = pin_obj_ty.kind() else {
348                bug!();
349            };
350            let obj_ptr_ty = Ty::new_mut_ptr(tcx, drop_ty);
351            let unwrap_ty = adt_def.non_enum_variant().fields[FieldIdx::ZERO].ty(tcx, adt_args);
352            let obj_ref_place = Place::from(self.new_temp(unwrap_ty));
353            call_statements.push(self.assign(
354                obj_ref_place,
355                Rvalue::Use(Operand::Copy(tcx.mk_place_field(
356                    pin_obj_place,
357                    FieldIdx::ZERO,
358                    unwrap_ty,
359                ))),
360            ));
361
362            let obj_ptr_place = Place::from(self.new_temp(obj_ptr_ty));
363
364            let addr = Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_deref(obj_ref_place));
365            call_statements.push(self.assign(obj_ptr_place, addr));
366            obj_ptr_place
367        };
368        call_statements
369            .push(Statement::new(self.source_info, StatementKind::StorageLive(fut.local)));
370
371        let call_drop_bb = self.new_block_with_statements(
372            unwind,
373            call_statements,
374            TerminatorKind::Call {
375                func: Operand::function_handle(tcx, drop_fn_def_id, trait_args, span),
376                args: [Spanned { node: Operand::Move(drop_arg), span: DUMMY_SP }].into(),
377                destination: fut,
378                target: Some(drop_term_bb),
379                unwind: unwind.into_action(),
380                call_source: CallSource::Misc,
381                fn_span: self.source_info.span,
382            },
383        );
384
385        // StorageDead(fut) in self.succ block (at the begin)
386        self.elaborator.patch().add_statement(
387            Location { block: self.succ, statement_index: 0 },
388            StatementKind::StorageDead(fut.local),
389        );
390        // StorageDead(fut) in unwind block (at the begin)
391        if let Unwind::To(block) = unwind {
392            self.elaborator.patch().add_statement(
393                Location { block, statement_index: 0 },
394                StatementKind::StorageDead(fut.local),
395            );
396        }
397        // StorageDead(fut) in dropline block (at the begin)
398        if let Some(block) = dropline {
399            self.elaborator.patch().add_statement(
400                Location { block, statement_index: 0 },
401                StatementKind::StorageDead(fut.local),
402            );
403        }
404
405        // #1:pin_obj_bb >>> call Pin<ObjTy>::new_unchecked(&mut obj)
406        self.elaborator.patch().patch_terminator(
407            pin_obj_bb,
408            TerminatorKind::Call {
409                func: pin_obj_new_unchecked_fn,
410                args: [dummy_spanned(Operand::Move(obj_ref_place))].into(),
411                destination: pin_obj_place,
412                target: Some(call_drop_bb),
413                unwind: unwind.into_action(),
414                call_source: CallSource::Misc,
415                fn_span: span,
416            },
417        );
418        pin_obj_bb
419    }
420
421    fn build_drop(&mut self, bb: BasicBlock) {
422        let drop_ty = self.place_ty(self.place);
423        if self.tcx().features().async_drop()
424            && self.elaborator.body().coroutine.is_some()
425            && self.elaborator.allow_async_drops()
426            && !self.elaborator.patch_ref().block(self.elaborator.body(), bb).is_cleanup
427            && drop_ty.needs_async_drop(self.tcx(), self.elaborator.typing_env())
428        {
429            self.build_async_drop(
430                self.place,
431                drop_ty,
432                Some(bb),
433                self.succ,
434                self.unwind,
435                self.dropline,
436                false,
437            );
438        } else {
439            self.elaborator.patch().patch_terminator(
440                bb,
441                TerminatorKind::Drop {
442                    place: self.place,
443                    target: self.succ,
444                    unwind: self.unwind.into_action(),
445                    replace: false,
446                    drop: None,
447                    async_fut: None,
448                },
449            );
450        }
451    }
452
453    /// This elaborates a single drop instruction, located at `bb`, and
454    /// patches over it.
455    ///
456    /// The elaborated drop checks the drop flags to only drop what
457    /// is initialized.
458    ///
459    /// In addition, the relevant drop flags also need to be cleared
460    /// to avoid double-drops. However, in the middle of a complex
461    /// drop, one must avoid clearing some of the flags before they
462    /// are read, as that would cause a memory leak.
463    ///
464    /// In particular, when dropping an ADT, multiple fields may be
465    /// joined together under the `rest` subpath. They are all controlled
466    /// by the primary drop flag, but only the last rest-field dropped
467    /// should clear it (and it must also not clear anything else).
468    //
469    // FIXME: I think we should just control the flags externally,
470    // and then we do not need this machinery.
471    #[instrument(level = "debug")]
472    fn elaborate_drop(&mut self, bb: BasicBlock) {
473        match self.elaborator.drop_style(self.path, DropFlagMode::Deep) {
474            DropStyle::Dead => {
475                self.elaborator
476                    .patch()
477                    .patch_terminator(bb, TerminatorKind::Goto { target: self.succ });
478            }
479            DropStyle::Static => {
480                self.build_drop(bb);
481            }
482            DropStyle::Conditional => {
483                let drop_bb = self.complete_drop(self.succ, self.unwind);
484                self.elaborator
485                    .patch()
486                    .patch_terminator(bb, TerminatorKind::Goto { target: drop_bb });
487            }
488            DropStyle::Open => {
489                let drop_bb = self.open_drop();
490                self.elaborator
491                    .patch()
492                    .patch_terminator(bb, TerminatorKind::Goto { target: drop_bb });
493            }
494        }
495    }
496
497    /// Returns the place and move path for each field of `variant`,
498    /// (the move path is `None` if the field is a rest field).
499    fn move_paths_for_fields(
500        &self,
501        base_place: Place<'tcx>,
502        variant_path: D::Path,
503        variant: &'tcx ty::VariantDef,
504        args: GenericArgsRef<'tcx>,
505    ) -> Vec<(Place<'tcx>, Option<D::Path>)> {
506        variant
507            .fields
508            .iter_enumerated()
509            .map(|(field_idx, field)| {
510                let subpath = self.elaborator.field_subpath(variant_path, field_idx);
511                let tcx = self.tcx();
512
513                assert_eq!(self.elaborator.typing_env().typing_mode, ty::TypingMode::PostAnalysis);
514                let field_ty = match tcx.try_normalize_erasing_regions(
515                    self.elaborator.typing_env(),
516                    field.ty(tcx, args),
517                ) {
518                    Ok(t) => t,
519                    Err(_) => Ty::new_error(
520                        self.tcx(),
521                        self.tcx().dcx().span_delayed_bug(
522                            self.elaborator.body().span,
523                            "Error normalizing in drop elaboration.",
524                        ),
525                    ),
526                };
527
528                (tcx.mk_place_field(base_place, field_idx, field_ty), subpath)
529            })
530            .collect()
531    }
532
533    fn drop_subpath(
534        &mut self,
535        place: Place<'tcx>,
536        path: Option<D::Path>,
537        succ: BasicBlock,
538        unwind: Unwind,
539        dropline: Option<BasicBlock>,
540    ) -> BasicBlock {
541        if let Some(path) = path {
542            debug!("drop_subpath: for std field {:?}", place);
543
544            DropCtxt {
545                elaborator: self.elaborator,
546                source_info: self.source_info,
547                path,
548                place,
549                succ,
550                unwind,
551                dropline,
552            }
553            .elaborated_drop_block()
554        } else {
555            debug!("drop_subpath: for rest field {:?}", place);
556
557            DropCtxt {
558                elaborator: self.elaborator,
559                source_info: self.source_info,
560                place,
561                succ,
562                unwind,
563                dropline,
564                // Using `self.path` here to condition the drop on
565                // our own drop flag.
566                path: self.path,
567            }
568            .complete_drop(succ, unwind)
569        }
570    }
571
572    /// Creates one-half of the drop ladder for a list of fields, and return
573    /// the list of steps in it in reverse order, with the first step
574    /// dropping 0 fields and so on.
575    ///
576    /// `unwind_ladder` is such a list of steps in reverse order,
577    /// which is called if the matching step of the drop glue panics.
578    ///
579    /// `dropline_ladder` is a similar list of steps in reverse order,
580    /// which is called if the matching step of the drop glue will contain async drop
581    /// (expanded later to Yield) and the containing coroutine will be dropped at this point.
582    fn drop_halfladder(
583        &mut self,
584        unwind_ladder: &[Unwind],
585        dropline_ladder: &[Option<BasicBlock>],
586        mut succ: BasicBlock,
587        fields: &[(Place<'tcx>, Option<D::Path>)],
588    ) -> Vec<BasicBlock> {
589        iter::once(succ)
590            .chain(itertools::izip!(fields.iter().rev(), unwind_ladder, dropline_ladder).map(
591                |(&(place, path), &unwind_succ, &dropline_to)| {
592                    succ = self.drop_subpath(place, path, succ, unwind_succ, dropline_to);
593                    succ
594                },
595            ))
596            .collect()
597    }
598
599    fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind, Option<BasicBlock>) {
600        // Clear the "master" drop flag at the end. This is needed
601        // because the "master" drop protects the ADT's discriminant,
602        // which is invalidated after the ADT is dropped.
603        (
604            self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind),
605            self.unwind,
606            self.dropline,
607        )
608    }
609
610    /// Creates a full drop ladder, consisting of 2 connected half-drop-ladders
611    ///
612    /// For example, with 3 fields, the drop ladder is
613    ///
614    /// .d0:
615    ///     ELAB(drop location.0 [target=.d1, unwind=.c1])
616    /// .d1:
617    ///     ELAB(drop location.1 [target=.d2, unwind=.c2])
618    /// .d2:
619    ///     ELAB(drop location.2 [target=`self.succ`, unwind=`self.unwind`])
620    /// .c1:
621    ///     ELAB(drop location.1 [target=.c2])
622    /// .c2:
623    ///     ELAB(drop location.2 [target=`self.unwind`])
624    ///
625    /// For possible-async drops in coroutines we also need dropline ladder
626    /// .d0 (mainline):
627    ///     ELAB(drop location.0 [target=.d1, unwind=.c1, drop=.e1])
628    /// .d1 (mainline):
629    ///     ELAB(drop location.1 [target=.d2, unwind=.c2, drop=.e2])
630    /// .d2 (mainline):
631    ///     ELAB(drop location.2 [target=`self.succ`, unwind=`self.unwind`, drop=`self.drop`])
632    /// .c1 (unwind):
633    ///     ELAB(drop location.1 [target=.c2])
634    /// .c2 (unwind):
635    ///     ELAB(drop location.2 [target=`self.unwind`])
636    /// .e1 (dropline):
637    ///     ELAB(drop location.1 [target=.e2, unwind=.c2])
638    /// .e2 (dropline):
639    ///     ELAB(drop location.2 [target=`self.drop`, unwind=`self.unwind`])
640    ///
641    /// NOTE: this does not clear the master drop flag, so you need
642    /// to point succ/unwind on a `drop_ladder_bottom`.
643    fn drop_ladder(
644        &mut self,
645        fields: Vec<(Place<'tcx>, Option<D::Path>)>,
646        succ: BasicBlock,
647        unwind: Unwind,
648        dropline: Option<BasicBlock>,
649    ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
650        debug!("drop_ladder({:?}, {:?})", self, fields);
651        assert!(
652            if unwind.is_cleanup() { dropline.is_none() } else { true },
653            "Dropline is set for cleanup drop ladder"
654        );
655
656        let mut fields = fields;
657        fields.retain(|&(place, _)| {
658            self.place_ty(place).needs_drop(self.tcx(), self.elaborator.typing_env())
659        });
660
661        debug!("drop_ladder - fields needing drop: {:?}", fields);
662
663        let dropline_ladder: Vec<Option<BasicBlock>> = vec![None; fields.len() + 1];
664        let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1];
665        let unwind_ladder: Vec<_> = if let Unwind::To(succ) = unwind {
666            let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
667            halfladder.into_iter().map(Unwind::To).collect()
668        } else {
669            unwind_ladder
670        };
671        let dropline_ladder: Vec<_> = if let Some(succ) = dropline {
672            let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
673            halfladder.into_iter().map(Some).collect()
674        } else {
675            dropline_ladder
676        };
677
678        let normal_ladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
679
680        (
681            *normal_ladder.last().unwrap(),
682            *unwind_ladder.last().unwrap(),
683            *dropline_ladder.last().unwrap(),
684        )
685    }
686
687    fn open_drop_for_tuple(&mut self, tys: &[Ty<'tcx>]) -> BasicBlock {
688        debug!("open_drop_for_tuple({:?}, {:?})", self, tys);
689
690        let fields = tys
691            .iter()
692            .enumerate()
693            .map(|(i, &ty)| {
694                (
695                    self.tcx().mk_place_field(self.place, FieldIdx::new(i), ty),
696                    self.elaborator.field_subpath(self.path, FieldIdx::new(i)),
697                )
698            })
699            .collect();
700
701        let (succ, unwind, dropline) = self.drop_ladder_bottom();
702        self.drop_ladder(fields, succ, unwind, dropline).0
703    }
704
705    /// Drops the T contained in a `Box<T>` if it has not been moved out of
706    #[instrument(level = "debug", ret)]
707    fn open_drop_for_box_contents(
708        &mut self,
709        adt: ty::AdtDef<'tcx>,
710        args: GenericArgsRef<'tcx>,
711        succ: BasicBlock,
712        unwind: Unwind,
713        dropline: Option<BasicBlock>,
714    ) -> BasicBlock {
715        // drop glue is sent straight to codegen
716        // box cannot be directly dereferenced
717        let unique_ty = adt.non_enum_variant().fields[FieldIdx::ZERO].ty(self.tcx(), args);
718        let unique_variant = unique_ty.ty_adt_def().unwrap().non_enum_variant();
719        let nonnull_ty = unique_variant.fields[FieldIdx::ZERO].ty(self.tcx(), args);
720        let ptr_ty = Ty::new_imm_ptr(self.tcx(), args[0].expect_ty());
721
722        let unique_place = self.tcx().mk_place_field(self.place, FieldIdx::ZERO, unique_ty);
723        let nonnull_place = self.tcx().mk_place_field(unique_place, FieldIdx::ZERO, nonnull_ty);
724
725        let ptr_local = self.new_temp(ptr_ty);
726
727        let interior = self.tcx().mk_place_deref(Place::from(ptr_local));
728        let interior_path = self.elaborator.deref_subpath(self.path);
729
730        let do_drop_bb = self.drop_subpath(interior, interior_path, succ, unwind, dropline);
731
732        let setup_bbd = BasicBlockData::new_stmts(
733            vec![self.assign(
734                Place::from(ptr_local),
735                Rvalue::Cast(CastKind::Transmute, Operand::Copy(nonnull_place), ptr_ty),
736            )],
737            Some(Terminator {
738                kind: TerminatorKind::Goto { target: do_drop_bb },
739                source_info: self.source_info,
740            }),
741            unwind.is_cleanup(),
742        );
743        self.elaborator.patch().new_block(setup_bbd)
744    }
745
746    #[instrument(level = "debug", ret)]
747    fn open_drop_for_adt(
748        &mut self,
749        adt: ty::AdtDef<'tcx>,
750        args: GenericArgsRef<'tcx>,
751    ) -> BasicBlock {
752        if adt.variants().is_empty() {
753            return self.elaborator.patch().new_block(BasicBlockData::new(
754                Some(Terminator {
755                    source_info: self.source_info,
756                    kind: TerminatorKind::Unreachable,
757                }),
758                self.unwind.is_cleanup(),
759            ));
760        }
761
762        let skip_contents = adt.is_union() || adt.is_manually_drop();
763        let contents_drop = if skip_contents {
764            (self.succ, self.unwind, self.dropline)
765        } else {
766            self.open_drop_for_adt_contents(adt, args)
767        };
768
769        if adt.is_box() {
770            // we need to drop the inside of the box before running the destructor
771            let succ = self.destructor_call_block_sync((contents_drop.0, contents_drop.1));
772            let unwind = contents_drop
773                .1
774                .map(|unwind| self.destructor_call_block_sync((unwind, Unwind::InCleanup)));
775            let dropline = contents_drop
776                .2
777                .map(|dropline| self.destructor_call_block_sync((dropline, contents_drop.1)));
778
779            self.open_drop_for_box_contents(adt, args, succ, unwind, dropline)
780        } else if adt.has_dtor(self.tcx()) {
781            self.destructor_call_block(contents_drop)
782        } else {
783            contents_drop.0
784        }
785    }
786
787    fn open_drop_for_adt_contents(
788        &mut self,
789        adt: ty::AdtDef<'tcx>,
790        args: GenericArgsRef<'tcx>,
791    ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
792        let (succ, unwind, dropline) = self.drop_ladder_bottom();
793        if !adt.is_enum() {
794            let fields =
795                self.move_paths_for_fields(self.place, self.path, adt.variant(FIRST_VARIANT), args);
796            self.drop_ladder(fields, succ, unwind, dropline)
797        } else {
798            self.open_drop_for_multivariant(adt, args, succ, unwind, dropline)
799        }
800    }
801
802    fn open_drop_for_multivariant(
803        &mut self,
804        adt: ty::AdtDef<'tcx>,
805        args: GenericArgsRef<'tcx>,
806        succ: BasicBlock,
807        unwind: Unwind,
808        dropline: Option<BasicBlock>,
809    ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
810        let mut values = Vec::with_capacity(adt.variants().len());
811        let mut normal_blocks = Vec::with_capacity(adt.variants().len());
812        let mut unwind_blocks =
813            if unwind.is_cleanup() { None } else { Some(Vec::with_capacity(adt.variants().len())) };
814        let mut dropline_blocks =
815            if dropline.is_none() { None } else { Some(Vec::with_capacity(adt.variants().len())) };
816
817        let mut have_otherwise_with_drop_glue = false;
818        let mut have_otherwise = false;
819        let tcx = self.tcx();
820
821        for (variant_index, discr) in adt.discriminants(tcx) {
822            let variant = &adt.variant(variant_index);
823            let subpath = self.elaborator.downcast_subpath(self.path, variant_index);
824
825            if let Some(variant_path) = subpath {
826                let base_place = tcx.mk_place_elem(
827                    self.place,
828                    ProjectionElem::Downcast(Some(variant.name), variant_index),
829                );
830                let fields = self.move_paths_for_fields(base_place, variant_path, variant, args);
831                values.push(discr.val);
832                if let Unwind::To(unwind) = unwind {
833                    // We can't use the half-ladder from the original
834                    // drop ladder, because this breaks the
835                    // "funclet can't have 2 successor funclets"
836                    // requirement from MSVC:
837                    //
838                    //           switch       unwind-switch
839                    //          /      \         /        \
840                    //         v1.0    v2.0  v2.0-unwind  v1.0-unwind
841                    //         |        |      /             |
842                    //    v1.1-unwind  v2.1-unwind           |
843                    //      ^                                |
844                    //       \-------------------------------/
845                    //
846                    // Create a duplicate half-ladder to avoid that. We
847                    // could technically only do this on MSVC, but I
848                    // I want to minimize the divergence between MSVC
849                    // and non-MSVC.
850
851                    let unwind_blocks = unwind_blocks.as_mut().unwrap();
852                    let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1];
853                    let dropline_ladder: Vec<Option<BasicBlock>> = vec![None; fields.len() + 1];
854                    let halfladder =
855                        self.drop_halfladder(&unwind_ladder, &dropline_ladder, unwind, &fields);
856                    unwind_blocks.push(halfladder.last().cloned().unwrap());
857                }
858                let (normal, _, drop_bb) = self.drop_ladder(fields, succ, unwind, dropline);
859                normal_blocks.push(normal);
860                if dropline.is_some() {
861                    dropline_blocks.as_mut().unwrap().push(drop_bb.unwrap());
862                }
863            } else {
864                have_otherwise = true;
865
866                let typing_env = self.elaborator.typing_env();
867                let have_field_with_drop_glue = variant
868                    .fields
869                    .iter()
870                    .any(|field| field.ty(tcx, args).needs_drop(tcx, typing_env));
871                if have_field_with_drop_glue {
872                    have_otherwise_with_drop_glue = true;
873                }
874            }
875        }
876
877        if !have_otherwise {
878            values.pop();
879        } else if !have_otherwise_with_drop_glue {
880            normal_blocks.push(self.goto_block(succ, unwind));
881            if let Unwind::To(unwind) = unwind {
882                unwind_blocks.as_mut().unwrap().push(self.goto_block(unwind, Unwind::InCleanup));
883            }
884        } else {
885            normal_blocks.push(self.drop_block(succ, unwind));
886            if let Unwind::To(unwind) = unwind {
887                unwind_blocks.as_mut().unwrap().push(self.drop_block(unwind, Unwind::InCleanup));
888            }
889        }
890
891        (
892            self.adt_switch_block(adt, normal_blocks, &values, succ, unwind),
893            unwind.map(|unwind| {
894                self.adt_switch_block(
895                    adt,
896                    unwind_blocks.unwrap(),
897                    &values,
898                    unwind,
899                    Unwind::InCleanup,
900                )
901            }),
902            dropline.map(|dropline| {
903                self.adt_switch_block(adt, dropline_blocks.unwrap(), &values, dropline, unwind)
904            }),
905        )
906    }
907
908    fn adt_switch_block(
909        &mut self,
910        adt: ty::AdtDef<'tcx>,
911        blocks: Vec<BasicBlock>,
912        values: &[u128],
913        succ: BasicBlock,
914        unwind: Unwind,
915    ) -> BasicBlock {
916        // If there are multiple variants, then if something
917        // is present within the enum the discriminant, tracked
918        // by the rest path, must be initialized.
919        //
920        // Additionally, we do not want to switch on the
921        // discriminant after it is free-ed, because that
922        // way lies only trouble.
923        let discr_ty = adt.repr().discr_type().to_ty(self.tcx());
924        let discr = Place::from(self.new_temp(discr_ty));
925        let discr_rv = Rvalue::Discriminant(self.place);
926        let switch_block = BasicBlockData::new_stmts(
927            vec![self.assign(discr, discr_rv)],
928            Some(Terminator {
929                source_info: self.source_info,
930                kind: TerminatorKind::SwitchInt {
931                    discr: Operand::Move(discr),
932                    targets: SwitchTargets::new(
933                        values.iter().copied().zip(blocks.iter().copied()),
934                        *blocks.last().unwrap(),
935                    ),
936                },
937            }),
938            unwind.is_cleanup(),
939        );
940        let switch_block = self.elaborator.patch().new_block(switch_block);
941        self.drop_flag_test_block(switch_block, succ, unwind)
942    }
943
944    fn destructor_call_block_sync(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock {
945        debug!("destructor_call_block_sync({:?}, {:?})", self, succ);
946        let tcx = self.tcx();
947        let drop_trait = tcx.require_lang_item(LangItem::Drop, DUMMY_SP);
948        let drop_fn = tcx.associated_item_def_ids(drop_trait)[0];
949        let ty = self.place_ty(self.place);
950
951        let ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, ty);
952        let ref_place = self.new_temp(ref_ty);
953        let unit_temp = Place::from(self.new_temp(tcx.types.unit));
954
955        let result = BasicBlockData::new_stmts(
956            vec![self.assign(
957                Place::from(ref_place),
958                Rvalue::Ref(
959                    tcx.lifetimes.re_erased,
960                    BorrowKind::Mut { kind: MutBorrowKind::Default },
961                    self.place,
962                ),
963            )],
964            Some(Terminator {
965                kind: TerminatorKind::Call {
966                    func: Operand::function_handle(
967                        tcx,
968                        drop_fn,
969                        [ty.into()],
970                        self.source_info.span,
971                    ),
972                    args: [Spanned { node: Operand::Move(Place::from(ref_place)), span: DUMMY_SP }]
973                        .into(),
974                    destination: unit_temp,
975                    target: Some(succ),
976                    unwind: unwind.into_action(),
977                    call_source: CallSource::Misc,
978                    fn_span: self.source_info.span,
979                },
980                source_info: self.source_info,
981            }),
982            unwind.is_cleanup(),
983        );
984
985        let destructor_block = self.elaborator.patch().new_block(result);
986
987        let block_start = Location { block: destructor_block, statement_index: 0 };
988        self.elaborator.clear_drop_flag(block_start, self.path, DropFlagMode::Shallow);
989
990        self.drop_flag_test_block(destructor_block, succ, unwind)
991    }
992
993    fn destructor_call_block(
994        &mut self,
995        (succ, unwind, dropline): (BasicBlock, Unwind, Option<BasicBlock>),
996    ) -> BasicBlock {
997        debug!("destructor_call_block({:?}, {:?})", self, succ);
998        let ty = self.place_ty(self.place);
999        if self.tcx().features().async_drop()
1000            && self.elaborator.body().coroutine.is_some()
1001            && self.elaborator.allow_async_drops()
1002            && !unwind.is_cleanup()
1003            && ty.is_async_drop(self.tcx(), self.elaborator.typing_env())
1004        {
1005            let destructor_block =
1006                self.build_async_drop(self.place, ty, None, succ, unwind, dropline, true);
1007
1008            let block_start = Location { block: destructor_block, statement_index: 0 };
1009            self.elaborator.clear_drop_flag(block_start, self.path, DropFlagMode::Shallow);
1010
1011            self.drop_flag_test_block(destructor_block, succ, unwind)
1012        } else {
1013            self.destructor_call_block_sync((succ, unwind))
1014        }
1015    }
1016
1017    /// Create a loop that drops an array:
1018    ///
1019    /// ```text
1020    /// loop-block:
1021    ///    can_go = cur == len
1022    ///    if can_go then succ else drop-block
1023    /// drop-block:
1024    ///    ptr = &raw mut P[cur]
1025    ///    cur = cur + 1
1026    ///    drop(ptr)
1027    /// ```
1028    fn drop_loop(
1029        &mut self,
1030        succ: BasicBlock,
1031        cur: Local,
1032        len: Local,
1033        ety: Ty<'tcx>,
1034        unwind: Unwind,
1035        dropline: Option<BasicBlock>,
1036    ) -> BasicBlock {
1037        let copy = |place: Place<'tcx>| Operand::Copy(place);
1038        let move_ = |place: Place<'tcx>| Operand::Move(place);
1039        let tcx = self.tcx();
1040
1041        let ptr_ty = Ty::new_mut_ptr(tcx, ety);
1042        let ptr = Place::from(self.new_temp(ptr_ty));
1043        let can_go = Place::from(self.new_temp(tcx.types.bool));
1044        let one = self.constant_usize(1);
1045
1046        let drop_block = BasicBlockData::new_stmts(
1047            vec![
1048                self.assign(
1049                    ptr,
1050                    Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_index(self.place, cur)),
1051                ),
1052                self.assign(
1053                    cur.into(),
1054                    Rvalue::BinaryOp(BinOp::Add, Box::new((move_(cur.into()), one))),
1055                ),
1056            ],
1057            Some(Terminator {
1058                source_info: self.source_info,
1059                // this gets overwritten by drop elaboration.
1060                kind: TerminatorKind::Unreachable,
1061            }),
1062            unwind.is_cleanup(),
1063        );
1064        let drop_block = self.elaborator.patch().new_block(drop_block);
1065
1066        let loop_block = BasicBlockData::new_stmts(
1067            vec![self.assign(
1068                can_go,
1069                Rvalue::BinaryOp(BinOp::Eq, Box::new((copy(Place::from(cur)), copy(len.into())))),
1070            )],
1071            Some(Terminator {
1072                source_info: self.source_info,
1073                kind: TerminatorKind::if_(move_(can_go), succ, drop_block),
1074            }),
1075            unwind.is_cleanup(),
1076        );
1077        let loop_block = self.elaborator.patch().new_block(loop_block);
1078
1079        let place = tcx.mk_place_deref(ptr);
1080        if self.tcx().features().async_drop()
1081            && self.elaborator.body().coroutine.is_some()
1082            && self.elaborator.allow_async_drops()
1083            && !unwind.is_cleanup()
1084            && ety.needs_async_drop(self.tcx(), self.elaborator.typing_env())
1085        {
1086            self.build_async_drop(
1087                place,
1088                ety,
1089                Some(drop_block),
1090                loop_block,
1091                unwind,
1092                dropline,
1093                false,
1094            );
1095        } else {
1096            self.elaborator.patch().patch_terminator(
1097                drop_block,
1098                TerminatorKind::Drop {
1099                    place,
1100                    target: loop_block,
1101                    unwind: unwind.into_action(),
1102                    replace: false,
1103                    drop: None,
1104                    async_fut: None,
1105                },
1106            );
1107        }
1108        loop_block
1109    }
1110
1111    fn open_drop_for_array(
1112        &mut self,
1113        array_ty: Ty<'tcx>,
1114        ety: Ty<'tcx>,
1115        opt_size: Option<u64>,
1116    ) -> BasicBlock {
1117        debug!("open_drop_for_array({:?}, {:?}, {:?})", array_ty, ety, opt_size);
1118        let tcx = self.tcx();
1119
1120        if let Some(size) = opt_size {
1121            enum ProjectionKind<Path> {
1122                Drop(std::ops::Range<u64>),
1123                Keep(u64, Path),
1124            }
1125            // Previously, we'd make a projection for every element in the array and create a drop
1126            // ladder if any `array_subpath` was `Some`, i.e. moving out with an array pattern.
1127            // This caused huge memory usage when generating the drops for large arrays, so we instead
1128            // record the *subslices* which are dropped and the *indexes* which are kept
1129            let mut drop_ranges = vec![];
1130            let mut dropping = true;
1131            let mut start = 0;
1132            for i in 0..size {
1133                let path = self.elaborator.array_subpath(self.path, i, size);
1134                if dropping && path.is_some() {
1135                    drop_ranges.push(ProjectionKind::Drop(start..i));
1136                    dropping = false;
1137                } else if !dropping && path.is_none() {
1138                    dropping = true;
1139                    start = i;
1140                }
1141                if let Some(path) = path {
1142                    drop_ranges.push(ProjectionKind::Keep(i, path));
1143                }
1144            }
1145            if !drop_ranges.is_empty() {
1146                if dropping {
1147                    drop_ranges.push(ProjectionKind::Drop(start..size));
1148                }
1149                let fields = drop_ranges
1150                    .iter()
1151                    .rev()
1152                    .map(|p| {
1153                        let (project, path) = match p {
1154                            ProjectionKind::Drop(r) => (
1155                                ProjectionElem::Subslice {
1156                                    from: r.start,
1157                                    to: r.end,
1158                                    from_end: false,
1159                                },
1160                                None,
1161                            ),
1162                            &ProjectionKind::Keep(offset, path) => (
1163                                ProjectionElem::ConstantIndex {
1164                                    offset,
1165                                    min_length: size,
1166                                    from_end: false,
1167                                },
1168                                Some(path),
1169                            ),
1170                        };
1171                        (tcx.mk_place_elem(self.place, project), path)
1172                    })
1173                    .collect::<Vec<_>>();
1174                let (succ, unwind, dropline) = self.drop_ladder_bottom();
1175                return self.drop_ladder(fields, succ, unwind, dropline).0;
1176            }
1177        }
1178
1179        let array_ptr_ty = Ty::new_mut_ptr(tcx, array_ty);
1180        let array_ptr = self.new_temp(array_ptr_ty);
1181
1182        let slice_ty = Ty::new_slice(tcx, ety);
1183        let slice_ptr_ty = Ty::new_mut_ptr(tcx, slice_ty);
1184        let slice_ptr = self.new_temp(slice_ptr_ty);
1185
1186        let mut delegate_block = BasicBlockData::new_stmts(
1187            vec![
1188                self.assign(Place::from(array_ptr), Rvalue::RawPtr(RawPtrKind::Mut, self.place)),
1189                self.assign(
1190                    Place::from(slice_ptr),
1191                    Rvalue::Cast(
1192                        CastKind::PointerCoercion(
1193                            PointerCoercion::Unsize,
1194                            CoercionSource::Implicit,
1195                        ),
1196                        Operand::Move(Place::from(array_ptr)),
1197                        slice_ptr_ty,
1198                    ),
1199                ),
1200            ],
1201            None,
1202            self.unwind.is_cleanup(),
1203        );
1204
1205        let array_place = mem::replace(
1206            &mut self.place,
1207            Place::from(slice_ptr).project_deeper(&[PlaceElem::Deref], tcx),
1208        );
1209        let slice_block = self.drop_loop_trio_for_slice(ety);
1210        self.place = array_place;
1211
1212        delegate_block.terminator = Some(Terminator {
1213            source_info: self.source_info,
1214            kind: TerminatorKind::Goto { target: slice_block },
1215        });
1216        self.elaborator.patch().new_block(delegate_block)
1217    }
1218
1219    /// Creates a trio of drop-loops of `place`, which drops its contents, even
1220    /// in the case of 1 panic or in the case of coroutine drop
1221    fn drop_loop_trio_for_slice(&mut self, ety: Ty<'tcx>) -> BasicBlock {
1222        debug!("drop_loop_trio_for_slice({:?})", ety);
1223        let tcx = self.tcx();
1224        let len = self.new_temp(tcx.types.usize);
1225        let cur = self.new_temp(tcx.types.usize);
1226
1227        let unwind = self
1228            .unwind
1229            .map(|unwind| self.drop_loop(unwind, cur, len, ety, Unwind::InCleanup, None));
1230
1231        let dropline =
1232            self.dropline.map(|dropline| self.drop_loop(dropline, cur, len, ety, unwind, None));
1233
1234        let loop_block = self.drop_loop(self.succ, cur, len, ety, unwind, dropline);
1235
1236        let [PlaceElem::Deref] = self.place.projection.as_slice() else {
1237            span_bug!(
1238                self.source_info.span,
1239                "Expected place for slice drop shim to be *_n, but it's {:?}",
1240                self.place,
1241            );
1242        };
1243
1244        let zero = self.constant_usize(0);
1245        let block = BasicBlockData::new_stmts(
1246            vec![
1247                self.assign(
1248                    len.into(),
1249                    Rvalue::UnaryOp(
1250                        UnOp::PtrMetadata,
1251                        Operand::Copy(Place::from(self.place.local)),
1252                    ),
1253                ),
1254                self.assign(cur.into(), Rvalue::Use(zero)),
1255            ],
1256            Some(Terminator {
1257                source_info: self.source_info,
1258                kind: TerminatorKind::Goto { target: loop_block },
1259            }),
1260            unwind.is_cleanup(),
1261        );
1262
1263        let drop_block = self.elaborator.patch().new_block(block);
1264        // FIXME(#34708): handle partially-dropped array/slice elements.
1265        let reset_block = self.drop_flag_reset_block(DropFlagMode::Deep, drop_block, unwind);
1266        self.drop_flag_test_block(reset_block, self.succ, unwind)
1267    }
1268
1269    /// The slow-path - create an "open", elaborated drop for a type
1270    /// which is moved-out-of only partially, and patch `bb` to a jump
1271    /// to it. This must not be called on ADTs with a destructor,
1272    /// as these can't be moved-out-of, except for `Box<T>`, which is
1273    /// special-cased.
1274    ///
1275    /// This creates a "drop ladder" that drops the needed fields of the
1276    /// ADT, both in the success case or if one of the destructors fail.
1277    fn open_drop(&mut self) -> BasicBlock {
1278        let ty = self.place_ty(self.place);
1279        match ty.kind() {
1280            ty::Closure(_, args) => self.open_drop_for_tuple(args.as_closure().upvar_tys()),
1281            ty::CoroutineClosure(_, args) => {
1282                self.open_drop_for_tuple(args.as_coroutine_closure().upvar_tys())
1283            }
1284            // Note that `elaborate_drops` only drops the upvars of a coroutine,
1285            // and this is ok because `open_drop` here can only be reached
1286            // within that own coroutine's resume function.
1287            // This should only happen for the self argument on the resume function.
1288            // It effectively only contains upvars until the coroutine transformation runs.
1289            // See librustc_body/transform/coroutine.rs for more details.
1290            ty::Coroutine(_, args) => self.open_drop_for_tuple(args.as_coroutine().upvar_tys()),
1291            ty::Tuple(fields) => self.open_drop_for_tuple(fields),
1292            ty::Adt(def, args) => self.open_drop_for_adt(*def, args),
1293            ty::Dynamic(..) => self.complete_drop(self.succ, self.unwind),
1294            ty::Array(ety, size) => {
1295                let size = size.try_to_target_usize(self.tcx());
1296                self.open_drop_for_array(ty, *ety, size)
1297            }
1298            ty::Slice(ety) => self.drop_loop_trio_for_slice(*ety),
1299
1300            ty::UnsafeBinder(_) => {
1301                // Unsafe binders may elaborate drops if their inner type isn't copy.
1302                // This is enforced in typeck, so this should never happen.
1303                self.tcx().dcx().span_delayed_bug(
1304                    self.source_info.span,
1305                    "open drop for unsafe binder shouldn't be encountered",
1306                );
1307                self.elaborator.patch().new_block(BasicBlockData::new(
1308                    Some(Terminator {
1309                        source_info: self.source_info,
1310                        kind: TerminatorKind::Unreachable,
1311                    }),
1312                    self.unwind.is_cleanup(),
1313                ))
1314            }
1315
1316            _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty),
1317        }
1318    }
1319
1320    fn complete_drop(&mut self, succ: BasicBlock, unwind: Unwind) -> BasicBlock {
1321        debug!("complete_drop(succ={:?}, unwind={:?})", succ, unwind);
1322
1323        let drop_block = self.drop_block(succ, unwind);
1324
1325        self.drop_flag_test_block(drop_block, succ, unwind)
1326    }
1327
1328    /// Creates a block that resets the drop flag. If `mode` is deep, all children drop flags will
1329    /// also be cleared.
1330    fn drop_flag_reset_block(
1331        &mut self,
1332        mode: DropFlagMode,
1333        succ: BasicBlock,
1334        unwind: Unwind,
1335    ) -> BasicBlock {
1336        debug!("drop_flag_reset_block({:?},{:?})", self, mode);
1337
1338        if unwind.is_cleanup() {
1339            // The drop flag isn't read again on the unwind path, so don't
1340            // bother setting it.
1341            return succ;
1342        }
1343        let block = self.new_block(unwind, TerminatorKind::Goto { target: succ });
1344        let block_start = Location { block, statement_index: 0 };
1345        self.elaborator.clear_drop_flag(block_start, self.path, mode);
1346        block
1347    }
1348
1349    fn elaborated_drop_block(&mut self) -> BasicBlock {
1350        debug!("elaborated_drop_block({:?})", self);
1351        let blk = self.drop_block_simple(self.succ, self.unwind);
1352        self.elaborate_drop(blk);
1353        blk
1354    }
1355
1356    fn drop_block_simple(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
1357        let block = TerminatorKind::Drop {
1358            place: self.place,
1359            target,
1360            unwind: unwind.into_action(),
1361            replace: false,
1362            drop: self.dropline,
1363            async_fut: None,
1364        };
1365        self.new_block(unwind, block)
1366    }
1367
1368    fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
1369        let drop_ty = self.place_ty(self.place);
1370        if self.tcx().features().async_drop()
1371            && self.elaborator.body().coroutine.is_some()
1372            && self.elaborator.allow_async_drops()
1373            && !unwind.is_cleanup()
1374            && drop_ty.needs_async_drop(self.tcx(), self.elaborator.typing_env())
1375        {
1376            self.build_async_drop(
1377                self.place,
1378                drop_ty,
1379                None,
1380                self.succ,
1381                unwind,
1382                self.dropline,
1383                false,
1384            )
1385        } else {
1386            let block = TerminatorKind::Drop {
1387                place: self.place,
1388                target,
1389                unwind: unwind.into_action(),
1390                replace: false,
1391                drop: None,
1392                async_fut: None,
1393            };
1394            self.new_block(unwind, block)
1395        }
1396    }
1397
1398    fn goto_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
1399        let block = TerminatorKind::Goto { target };
1400        self.new_block(unwind, block)
1401    }
1402
1403    /// Returns the block to jump to in order to test the drop flag and execute the drop.
1404    ///
1405    /// Depending on the required `DropStyle`, this might be a generated block with an `if`
1406    /// terminator (for dynamic/open drops), or it might be `on_set` or `on_unset` itself, in case
1407    /// the drop can be statically determined.
1408    fn drop_flag_test_block(
1409        &mut self,
1410        on_set: BasicBlock,
1411        on_unset: BasicBlock,
1412        unwind: Unwind,
1413    ) -> BasicBlock {
1414        let style = self.elaborator.drop_style(self.path, DropFlagMode::Shallow);
1415        debug!(
1416            "drop_flag_test_block({:?},{:?},{:?},{:?}) - {:?}",
1417            self, on_set, on_unset, unwind, style
1418        );
1419
1420        match style {
1421            DropStyle::Dead => on_unset,
1422            DropStyle::Static => on_set,
1423            DropStyle::Conditional | DropStyle::Open => {
1424                let flag = self.elaborator.get_drop_flag(self.path).unwrap();
1425                let term = TerminatorKind::if_(flag, on_set, on_unset);
1426                self.new_block(unwind, term)
1427            }
1428        }
1429    }
1430
1431    fn new_block(&mut self, unwind: Unwind, k: TerminatorKind<'tcx>) -> BasicBlock {
1432        self.elaborator.patch().new_block(BasicBlockData::new(
1433            Some(Terminator { source_info: self.source_info, kind: k }),
1434            unwind.is_cleanup(),
1435        ))
1436    }
1437
1438    fn new_block_with_statements(
1439        &mut self,
1440        unwind: Unwind,
1441        statements: Vec<Statement<'tcx>>,
1442        k: TerminatorKind<'tcx>,
1443    ) -> BasicBlock {
1444        self.elaborator.patch().new_block(BasicBlockData::new_stmts(
1445            statements,
1446            Some(Terminator { source_info: self.source_info, kind: k }),
1447            unwind.is_cleanup(),
1448        ))
1449    }
1450
1451    fn new_temp(&mut self, ty: Ty<'tcx>) -> Local {
1452        self.elaborator.patch().new_temp(ty, self.source_info.span)
1453    }
1454
1455    fn constant_usize(&self, val: u16) -> Operand<'tcx> {
1456        Operand::Constant(Box::new(ConstOperand {
1457            span: self.source_info.span,
1458            user_ty: None,
1459            const_: Const::from_usize(self.tcx(), val.into()),
1460        }))
1461    }
1462
1463    fn assign(&self, lhs: Place<'tcx>, rhs: Rvalue<'tcx>) -> Statement<'tcx> {
1464        Statement::new(self.source_info, StatementKind::Assign(Box::new((lhs, rhs))))
1465    }
1466}