1use std::{fmt, iter, mem};
2
3use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
4use rustc_hir::def::DefKind;
5use rustc_hir::lang_items::LangItem;
6use rustc_index::Idx;
7use rustc_middle::mir::*;
8use rustc_middle::ty::adjustment::PointerCoercion;
9use rustc_middle::ty::util::IntTypeExt;
10use rustc_middle::ty::{self, GenericArg, GenericArgsRef, Ty, TyCtxt};
11use rustc_middle::{bug, span_bug, traits};
12use rustc_span::DUMMY_SP;
13use rustc_span::source_map::{Spanned, dummy_spanned};
14use tracing::{debug, instrument};
15
16use crate::patch::MirPatch;
17
18#[derive(Debug)]
20pub(crate) enum DropStyle {
21 Dead,
23
24 Static,
27
28 Conditional,
30
31 Open,
37}
38
39#[derive(Debug)]
41pub(crate) enum DropFlagMode {
42 Shallow,
44 Deep,
46}
47
48#[derive(Copy, Clone, Debug)]
50pub(crate) enum Unwind {
51 To(BasicBlock),
53 InCleanup,
55}
56
57impl Unwind {
58 fn is_cleanup(self) -> bool {
59 match self {
60 Unwind::To(..) => false,
61 Unwind::InCleanup => true,
62 }
63 }
64
65 fn into_action(self) -> UnwindAction {
66 match self {
67 Unwind::To(bb) => UnwindAction::Cleanup(bb),
68 Unwind::InCleanup => UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
69 }
70 }
71
72 fn map<F>(self, f: F) -> Self
73 where
74 F: FnOnce(BasicBlock) -> BasicBlock,
75 {
76 match self {
77 Unwind::To(bb) => Unwind::To(f(bb)),
78 Unwind::InCleanup => Unwind::InCleanup,
79 }
80 }
81}
82
83pub(crate) trait DropElaborator<'a, 'tcx>: fmt::Debug {
84 type Path: Copy + fmt::Debug;
90
91 fn patch_ref(&self) -> &MirPatch<'tcx>;
94 fn patch(&mut self) -> &mut MirPatch<'tcx>;
95 fn body(&self) -> &'a Body<'tcx>;
96 fn tcx(&self) -> TyCtxt<'tcx>;
97 fn typing_env(&self) -> ty::TypingEnv<'tcx>;
98 fn allow_async_drops(&self) -> bool;
99
100 fn terminator_loc(&self, bb: BasicBlock) -> Location;
101
102 fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle;
106
107 fn get_drop_flag(&mut self, path: Self::Path) -> Option<Operand<'tcx>>;
109
110 fn clear_drop_flag(&mut self, location: Location, path: Self::Path, mode: DropFlagMode);
115
116 fn field_subpath(&self, path: Self::Path, field: FieldIdx) -> Option<Self::Path>;
122
123 fn deref_subpath(&self, path: Self::Path) -> Option<Self::Path>;
129
130 fn downcast_subpath(&self, path: Self::Path, variant: VariantIdx) -> Option<Self::Path>;
134
135 fn array_subpath(&self, path: Self::Path, index: u64, size: u64) -> Option<Self::Path>;
141}
142
143#[derive(Debug)]
144struct DropCtxt<'a, 'b, 'tcx, D>
145where
146 D: DropElaborator<'b, 'tcx>,
147{
148 elaborator: &'a mut D,
149
150 source_info: SourceInfo,
151
152 place: Place<'tcx>,
153 path: D::Path,
154 succ: BasicBlock,
155 unwind: Unwind,
156 dropline: Option<BasicBlock>,
157}
158
159pub(crate) fn elaborate_drop<'b, 'tcx, D>(
168 elaborator: &mut D,
169 source_info: SourceInfo,
170 place: Place<'tcx>,
171 path: D::Path,
172 succ: BasicBlock,
173 unwind: Unwind,
174 bb: BasicBlock,
175 dropline: Option<BasicBlock>,
176) where
177 D: DropElaborator<'b, 'tcx>,
178 'tcx: 'b,
179{
180 DropCtxt { elaborator, source_info, place, path, succ, unwind, dropline }.elaborate_drop(bb)
181}
182
183impl<'a, 'b, 'tcx, D> DropCtxt<'a, 'b, 'tcx, D>
184where
185 D: DropElaborator<'b, 'tcx>,
186 'tcx: 'b,
187{
188 #[instrument(level = "trace", skip(self), ret)]
189 fn place_ty(&self, place: Place<'tcx>) -> Ty<'tcx> {
190 if place.local < self.elaborator.body().local_decls.next_index() {
191 place.ty(self.elaborator.body(), self.tcx()).ty
192 } else {
193 PlaceTy::from_ty(self.elaborator.patch_ref().local_ty(place.local))
195 .multi_projection_ty(self.elaborator.tcx(), place.projection)
196 .ty
197 }
198 }
199
200 fn tcx(&self) -> TyCtxt<'tcx> {
201 self.elaborator.tcx()
202 }
203
204 fn build_async_drop(
212 &mut self,
213 place: Place<'tcx>,
214 drop_ty: Ty<'tcx>,
215 bb: Option<BasicBlock>,
216 succ: BasicBlock,
217 unwind: Unwind,
218 dropline: Option<BasicBlock>,
219 call_destructor_only: bool,
220 ) -> BasicBlock {
221 let tcx = self.tcx();
222 let span = self.source_info.span;
223
224 let pin_obj_bb = bb.unwrap_or_else(|| {
225 self.elaborator.patch().new_block(BasicBlockData::new(
226 Some(Terminator {
227 source_info: self.source_info,
229 kind: TerminatorKind::Return,
230 }),
231 false,
232 ))
233 });
234
235 let (fut_ty, drop_fn_def_id, trait_args) = if call_destructor_only {
236 let trait_ref =
238 ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::AsyncDrop, span), [drop_ty]);
239 let (drop_trait, trait_args) = match tcx.codegen_select_candidate(
240 ty::TypingEnv::fully_monomorphized().as_query_input(trait_ref),
241 ) {
242 Ok(traits::ImplSource::UserDefined(traits::ImplSourceUserDefinedData {
243 impl_def_id,
244 args,
245 ..
246 })) => (*impl_def_id, *args),
247 impl_source => {
248 span_bug!(span, "invalid `AsyncDrop` impl_source: {:?}", impl_source);
249 }
250 };
251 let Some(drop_fn_def_id) = tcx
255 .associated_item_def_ids(drop_trait)
256 .first()
257 .and_then(|def_id| {
258 if tcx.def_kind(def_id) == DefKind::AssocFn
259 && tcx.check_args_compatible(*def_id, trait_args)
260 {
261 Some(def_id)
262 } else {
263 None
264 }
265 })
266 .copied()
267 else {
268 tcx.dcx().span_delayed_bug(
269 self.elaborator.body().span,
270 "AsyncDrop type without correct `async fn drop(...)`.",
271 );
272 self.elaborator.patch().patch_terminator(
273 pin_obj_bb,
274 TerminatorKind::Drop {
275 place,
276 target: succ,
277 unwind: unwind.into_action(),
278 replace: false,
279 drop: None,
280 async_fut: None,
281 },
282 );
283 return pin_obj_bb;
284 };
285 let drop_fn = Ty::new_fn_def(tcx, drop_fn_def_id, trait_args);
286 let sig = drop_fn.fn_sig(tcx);
287 let sig = tcx.instantiate_bound_regions_with_erased(sig);
288 (sig.output(), drop_fn_def_id, trait_args)
289 } else {
290 let drop_fn_def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, span);
292 let trait_args = tcx.mk_args(&[drop_ty.into()]);
293 let sig = tcx.fn_sig(drop_fn_def_id).instantiate(tcx, trait_args);
294 let sig = tcx.instantiate_bound_regions_with_erased(sig);
295 (sig.output(), drop_fn_def_id, trait_args)
296 };
297
298 let fut = Place::from(self.new_temp(fut_ty));
299
300 let obj_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, drop_ty);
302 let obj_ref_place = Place::from(self.new_temp(obj_ref_ty));
303
304 let term_loc = self.elaborator.terminator_loc(pin_obj_bb);
305 self.elaborator.patch().add_assign(
306 term_loc,
307 obj_ref_place,
308 Rvalue::Ref(
309 tcx.lifetimes.re_erased,
310 BorrowKind::Mut { kind: MutBorrowKind::Default },
311 place,
312 ),
313 );
314
315 let pin_obj_new_unchecked_fn = Ty::new_fn_def(
317 tcx,
318 tcx.require_lang_item(LangItem::PinNewUnchecked, span),
319 [GenericArg::from(obj_ref_ty)],
320 );
321 let pin_obj_ty = pin_obj_new_unchecked_fn.fn_sig(tcx).output().no_bound_vars().unwrap();
322 let pin_obj_place = Place::from(self.new_temp(pin_obj_ty));
323 let pin_obj_new_unchecked_fn = Operand::Constant(Box::new(ConstOperand {
324 span,
325 user_ty: None,
326 const_: Const::zero_sized(pin_obj_new_unchecked_fn),
327 }));
328
329 let drop_term_bb = self.new_block(
331 unwind,
332 TerminatorKind::Drop {
333 place,
334 target: succ,
335 unwind: unwind.into_action(),
336 replace: false,
337 drop: dropline,
338 async_fut: Some(fut.local),
339 },
340 );
341
342 let mut call_statements = Vec::new();
344 let drop_arg = if call_destructor_only {
345 pin_obj_place
346 } else {
347 let ty::Adt(adt_def, adt_args) = pin_obj_ty.kind() else {
348 bug!();
349 };
350 let obj_ptr_ty = Ty::new_mut_ptr(tcx, drop_ty);
351 let unwrap_ty = adt_def.non_enum_variant().fields[FieldIdx::ZERO].ty(tcx, adt_args);
352 let obj_ref_place = Place::from(self.new_temp(unwrap_ty));
353 call_statements.push(self.assign(
354 obj_ref_place,
355 Rvalue::Use(Operand::Copy(tcx.mk_place_field(
356 pin_obj_place,
357 FieldIdx::ZERO,
358 unwrap_ty,
359 ))),
360 ));
361
362 let obj_ptr_place = Place::from(self.new_temp(obj_ptr_ty));
363
364 let addr = Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_deref(obj_ref_place));
365 call_statements.push(self.assign(obj_ptr_place, addr));
366 obj_ptr_place
367 };
368 call_statements
369 .push(Statement::new(self.source_info, StatementKind::StorageLive(fut.local)));
370
371 let call_drop_bb = self.new_block_with_statements(
372 unwind,
373 call_statements,
374 TerminatorKind::Call {
375 func: Operand::function_handle(tcx, drop_fn_def_id, trait_args, span),
376 args: [Spanned { node: Operand::Move(drop_arg), span: DUMMY_SP }].into(),
377 destination: fut,
378 target: Some(drop_term_bb),
379 unwind: unwind.into_action(),
380 call_source: CallSource::Misc,
381 fn_span: self.source_info.span,
382 },
383 );
384
385 self.elaborator.patch().add_statement(
387 Location { block: self.succ, statement_index: 0 },
388 StatementKind::StorageDead(fut.local),
389 );
390 if let Unwind::To(block) = unwind {
392 self.elaborator.patch().add_statement(
393 Location { block, statement_index: 0 },
394 StatementKind::StorageDead(fut.local),
395 );
396 }
397 if let Some(block) = dropline {
399 self.elaborator.patch().add_statement(
400 Location { block, statement_index: 0 },
401 StatementKind::StorageDead(fut.local),
402 );
403 }
404
405 self.elaborator.patch().patch_terminator(
407 pin_obj_bb,
408 TerminatorKind::Call {
409 func: pin_obj_new_unchecked_fn,
410 args: [dummy_spanned(Operand::Move(obj_ref_place))].into(),
411 destination: pin_obj_place,
412 target: Some(call_drop_bb),
413 unwind: unwind.into_action(),
414 call_source: CallSource::Misc,
415 fn_span: span,
416 },
417 );
418 pin_obj_bb
419 }
420
421 fn build_drop(&mut self, bb: BasicBlock) {
422 let drop_ty = self.place_ty(self.place);
423 if self.tcx().features().async_drop()
424 && self.elaborator.body().coroutine.is_some()
425 && self.elaborator.allow_async_drops()
426 && !self.elaborator.patch_ref().block(self.elaborator.body(), bb).is_cleanup
427 && drop_ty.needs_async_drop(self.tcx(), self.elaborator.typing_env())
428 {
429 self.build_async_drop(
430 self.place,
431 drop_ty,
432 Some(bb),
433 self.succ,
434 self.unwind,
435 self.dropline,
436 false,
437 );
438 } else {
439 self.elaborator.patch().patch_terminator(
440 bb,
441 TerminatorKind::Drop {
442 place: self.place,
443 target: self.succ,
444 unwind: self.unwind.into_action(),
445 replace: false,
446 drop: None,
447 async_fut: None,
448 },
449 );
450 }
451 }
452
453 #[instrument(level = "debug")]
472 fn elaborate_drop(&mut self, bb: BasicBlock) {
473 match self.elaborator.drop_style(self.path, DropFlagMode::Deep) {
474 DropStyle::Dead => {
475 self.elaborator
476 .patch()
477 .patch_terminator(bb, TerminatorKind::Goto { target: self.succ });
478 }
479 DropStyle::Static => {
480 self.build_drop(bb);
481 }
482 DropStyle::Conditional => {
483 let drop_bb = self.complete_drop(self.succ, self.unwind);
484 self.elaborator
485 .patch()
486 .patch_terminator(bb, TerminatorKind::Goto { target: drop_bb });
487 }
488 DropStyle::Open => {
489 let drop_bb = self.open_drop();
490 self.elaborator
491 .patch()
492 .patch_terminator(bb, TerminatorKind::Goto { target: drop_bb });
493 }
494 }
495 }
496
497 fn move_paths_for_fields(
500 &self,
501 base_place: Place<'tcx>,
502 variant_path: D::Path,
503 variant: &'tcx ty::VariantDef,
504 args: GenericArgsRef<'tcx>,
505 ) -> Vec<(Place<'tcx>, Option<D::Path>)> {
506 variant
507 .fields
508 .iter_enumerated()
509 .map(|(field_idx, field)| {
510 let subpath = self.elaborator.field_subpath(variant_path, field_idx);
511 let tcx = self.tcx();
512
513 assert_eq!(self.elaborator.typing_env().typing_mode, ty::TypingMode::PostAnalysis);
514 let field_ty = match tcx.try_normalize_erasing_regions(
515 self.elaborator.typing_env(),
516 field.ty(tcx, args),
517 ) {
518 Ok(t) => t,
519 Err(_) => Ty::new_error(
520 self.tcx(),
521 self.tcx().dcx().span_delayed_bug(
522 self.elaborator.body().span,
523 "Error normalizing in drop elaboration.",
524 ),
525 ),
526 };
527
528 (tcx.mk_place_field(base_place, field_idx, field_ty), subpath)
529 })
530 .collect()
531 }
532
533 fn drop_subpath(
534 &mut self,
535 place: Place<'tcx>,
536 path: Option<D::Path>,
537 succ: BasicBlock,
538 unwind: Unwind,
539 dropline: Option<BasicBlock>,
540 ) -> BasicBlock {
541 if let Some(path) = path {
542 debug!("drop_subpath: for std field {:?}", place);
543
544 DropCtxt {
545 elaborator: self.elaborator,
546 source_info: self.source_info,
547 path,
548 place,
549 succ,
550 unwind,
551 dropline,
552 }
553 .elaborated_drop_block()
554 } else {
555 debug!("drop_subpath: for rest field {:?}", place);
556
557 DropCtxt {
558 elaborator: self.elaborator,
559 source_info: self.source_info,
560 place,
561 succ,
562 unwind,
563 dropline,
564 path: self.path,
567 }
568 .complete_drop(succ, unwind)
569 }
570 }
571
572 fn drop_halfladder(
583 &mut self,
584 unwind_ladder: &[Unwind],
585 dropline_ladder: &[Option<BasicBlock>],
586 mut succ: BasicBlock,
587 fields: &[(Place<'tcx>, Option<D::Path>)],
588 ) -> Vec<BasicBlock> {
589 iter::once(succ)
590 .chain(itertools::izip!(fields.iter().rev(), unwind_ladder, dropline_ladder).map(
591 |(&(place, path), &unwind_succ, &dropline_to)| {
592 succ = self.drop_subpath(place, path, succ, unwind_succ, dropline_to);
593 succ
594 },
595 ))
596 .collect()
597 }
598
599 fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind, Option<BasicBlock>) {
600 (
604 self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind),
605 self.unwind,
606 self.dropline,
607 )
608 }
609
610 fn drop_ladder(
644 &mut self,
645 fields: Vec<(Place<'tcx>, Option<D::Path>)>,
646 succ: BasicBlock,
647 unwind: Unwind,
648 dropline: Option<BasicBlock>,
649 ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
650 debug!("drop_ladder({:?}, {:?})", self, fields);
651 assert!(
652 if unwind.is_cleanup() { dropline.is_none() } else { true },
653 "Dropline is set for cleanup drop ladder"
654 );
655
656 let mut fields = fields;
657 fields.retain(|&(place, _)| {
658 self.place_ty(place).needs_drop(self.tcx(), self.elaborator.typing_env())
659 });
660
661 debug!("drop_ladder - fields needing drop: {:?}", fields);
662
663 let dropline_ladder: Vec<Option<BasicBlock>> = vec![None; fields.len() + 1];
664 let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1];
665 let unwind_ladder: Vec<_> = if let Unwind::To(succ) = unwind {
666 let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
667 halfladder.into_iter().map(Unwind::To).collect()
668 } else {
669 unwind_ladder
670 };
671 let dropline_ladder: Vec<_> = if let Some(succ) = dropline {
672 let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
673 halfladder.into_iter().map(Some).collect()
674 } else {
675 dropline_ladder
676 };
677
678 let normal_ladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
679
680 (
681 *normal_ladder.last().unwrap(),
682 *unwind_ladder.last().unwrap(),
683 *dropline_ladder.last().unwrap(),
684 )
685 }
686
687 fn open_drop_for_tuple(&mut self, tys: &[Ty<'tcx>]) -> BasicBlock {
688 debug!("open_drop_for_tuple({:?}, {:?})", self, tys);
689
690 let fields = tys
691 .iter()
692 .enumerate()
693 .map(|(i, &ty)| {
694 (
695 self.tcx().mk_place_field(self.place, FieldIdx::new(i), ty),
696 self.elaborator.field_subpath(self.path, FieldIdx::new(i)),
697 )
698 })
699 .collect();
700
701 let (succ, unwind, dropline) = self.drop_ladder_bottom();
702 self.drop_ladder(fields, succ, unwind, dropline).0
703 }
704
705 #[instrument(level = "debug", ret)]
707 fn open_drop_for_box_contents(
708 &mut self,
709 adt: ty::AdtDef<'tcx>,
710 args: GenericArgsRef<'tcx>,
711 succ: BasicBlock,
712 unwind: Unwind,
713 dropline: Option<BasicBlock>,
714 ) -> BasicBlock {
715 let unique_ty = adt.non_enum_variant().fields[FieldIdx::ZERO].ty(self.tcx(), args);
718 let unique_variant = unique_ty.ty_adt_def().unwrap().non_enum_variant();
719 let nonnull_ty = unique_variant.fields[FieldIdx::ZERO].ty(self.tcx(), args);
720 let ptr_ty = Ty::new_imm_ptr(self.tcx(), args[0].expect_ty());
721
722 let unique_place = self.tcx().mk_place_field(self.place, FieldIdx::ZERO, unique_ty);
723 let nonnull_place = self.tcx().mk_place_field(unique_place, FieldIdx::ZERO, nonnull_ty);
724
725 let ptr_local = self.new_temp(ptr_ty);
726
727 let interior = self.tcx().mk_place_deref(Place::from(ptr_local));
728 let interior_path = self.elaborator.deref_subpath(self.path);
729
730 let do_drop_bb = self.drop_subpath(interior, interior_path, succ, unwind, dropline);
731
732 let setup_bbd = BasicBlockData::new_stmts(
733 vec![self.assign(
734 Place::from(ptr_local),
735 Rvalue::Cast(CastKind::Transmute, Operand::Copy(nonnull_place), ptr_ty),
736 )],
737 Some(Terminator {
738 kind: TerminatorKind::Goto { target: do_drop_bb },
739 source_info: self.source_info,
740 }),
741 unwind.is_cleanup(),
742 );
743 self.elaborator.patch().new_block(setup_bbd)
744 }
745
746 #[instrument(level = "debug", ret)]
747 fn open_drop_for_adt(
748 &mut self,
749 adt: ty::AdtDef<'tcx>,
750 args: GenericArgsRef<'tcx>,
751 ) -> BasicBlock {
752 if adt.variants().is_empty() {
753 return self.elaborator.patch().new_block(BasicBlockData::new(
754 Some(Terminator {
755 source_info: self.source_info,
756 kind: TerminatorKind::Unreachable,
757 }),
758 self.unwind.is_cleanup(),
759 ));
760 }
761
762 let skip_contents = adt.is_union() || adt.is_manually_drop();
763 let contents_drop = if skip_contents {
764 (self.succ, self.unwind, self.dropline)
765 } else {
766 self.open_drop_for_adt_contents(adt, args)
767 };
768
769 if adt.is_box() {
770 let succ = self.destructor_call_block_sync((contents_drop.0, contents_drop.1));
772 let unwind = contents_drop
773 .1
774 .map(|unwind| self.destructor_call_block_sync((unwind, Unwind::InCleanup)));
775 let dropline = contents_drop
776 .2
777 .map(|dropline| self.destructor_call_block_sync((dropline, contents_drop.1)));
778
779 self.open_drop_for_box_contents(adt, args, succ, unwind, dropline)
780 } else if adt.has_dtor(self.tcx()) {
781 self.destructor_call_block(contents_drop)
782 } else {
783 contents_drop.0
784 }
785 }
786
787 fn open_drop_for_adt_contents(
788 &mut self,
789 adt: ty::AdtDef<'tcx>,
790 args: GenericArgsRef<'tcx>,
791 ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
792 let (succ, unwind, dropline) = self.drop_ladder_bottom();
793 if !adt.is_enum() {
794 let fields =
795 self.move_paths_for_fields(self.place, self.path, adt.variant(FIRST_VARIANT), args);
796 self.drop_ladder(fields, succ, unwind, dropline)
797 } else {
798 self.open_drop_for_multivariant(adt, args, succ, unwind, dropline)
799 }
800 }
801
802 fn open_drop_for_multivariant(
803 &mut self,
804 adt: ty::AdtDef<'tcx>,
805 args: GenericArgsRef<'tcx>,
806 succ: BasicBlock,
807 unwind: Unwind,
808 dropline: Option<BasicBlock>,
809 ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
810 let mut values = Vec::with_capacity(adt.variants().len());
811 let mut normal_blocks = Vec::with_capacity(adt.variants().len());
812 let mut unwind_blocks =
813 if unwind.is_cleanup() { None } else { Some(Vec::with_capacity(adt.variants().len())) };
814 let mut dropline_blocks =
815 if dropline.is_none() { None } else { Some(Vec::with_capacity(adt.variants().len())) };
816
817 let mut have_otherwise_with_drop_glue = false;
818 let mut have_otherwise = false;
819 let tcx = self.tcx();
820
821 for (variant_index, discr) in adt.discriminants(tcx) {
822 let variant = &adt.variant(variant_index);
823 let subpath = self.elaborator.downcast_subpath(self.path, variant_index);
824
825 if let Some(variant_path) = subpath {
826 let base_place = tcx.mk_place_elem(
827 self.place,
828 ProjectionElem::Downcast(Some(variant.name), variant_index),
829 );
830 let fields = self.move_paths_for_fields(base_place, variant_path, variant, args);
831 values.push(discr.val);
832 if let Unwind::To(unwind) = unwind {
833 let unwind_blocks = unwind_blocks.as_mut().unwrap();
852 let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1];
853 let dropline_ladder: Vec<Option<BasicBlock>> = vec![None; fields.len() + 1];
854 let halfladder =
855 self.drop_halfladder(&unwind_ladder, &dropline_ladder, unwind, &fields);
856 unwind_blocks.push(halfladder.last().cloned().unwrap());
857 }
858 let (normal, _, drop_bb) = self.drop_ladder(fields, succ, unwind, dropline);
859 normal_blocks.push(normal);
860 if dropline.is_some() {
861 dropline_blocks.as_mut().unwrap().push(drop_bb.unwrap());
862 }
863 } else {
864 have_otherwise = true;
865
866 let typing_env = self.elaborator.typing_env();
867 let have_field_with_drop_glue = variant
868 .fields
869 .iter()
870 .any(|field| field.ty(tcx, args).needs_drop(tcx, typing_env));
871 if have_field_with_drop_glue {
872 have_otherwise_with_drop_glue = true;
873 }
874 }
875 }
876
877 if !have_otherwise {
878 values.pop();
879 } else if !have_otherwise_with_drop_glue {
880 normal_blocks.push(self.goto_block(succ, unwind));
881 if let Unwind::To(unwind) = unwind {
882 unwind_blocks.as_mut().unwrap().push(self.goto_block(unwind, Unwind::InCleanup));
883 }
884 } else {
885 normal_blocks.push(self.drop_block(succ, unwind));
886 if let Unwind::To(unwind) = unwind {
887 unwind_blocks.as_mut().unwrap().push(self.drop_block(unwind, Unwind::InCleanup));
888 }
889 }
890
891 (
892 self.adt_switch_block(adt, normal_blocks, &values, succ, unwind),
893 unwind.map(|unwind| {
894 self.adt_switch_block(
895 adt,
896 unwind_blocks.unwrap(),
897 &values,
898 unwind,
899 Unwind::InCleanup,
900 )
901 }),
902 dropline.map(|dropline| {
903 self.adt_switch_block(adt, dropline_blocks.unwrap(), &values, dropline, unwind)
904 }),
905 )
906 }
907
908 fn adt_switch_block(
909 &mut self,
910 adt: ty::AdtDef<'tcx>,
911 blocks: Vec<BasicBlock>,
912 values: &[u128],
913 succ: BasicBlock,
914 unwind: Unwind,
915 ) -> BasicBlock {
916 let discr_ty = adt.repr().discr_type().to_ty(self.tcx());
924 let discr = Place::from(self.new_temp(discr_ty));
925 let discr_rv = Rvalue::Discriminant(self.place);
926 let switch_block = BasicBlockData::new_stmts(
927 vec![self.assign(discr, discr_rv)],
928 Some(Terminator {
929 source_info: self.source_info,
930 kind: TerminatorKind::SwitchInt {
931 discr: Operand::Move(discr),
932 targets: SwitchTargets::new(
933 values.iter().copied().zip(blocks.iter().copied()),
934 *blocks.last().unwrap(),
935 ),
936 },
937 }),
938 unwind.is_cleanup(),
939 );
940 let switch_block = self.elaborator.patch().new_block(switch_block);
941 self.drop_flag_test_block(switch_block, succ, unwind)
942 }
943
944 fn destructor_call_block_sync(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock {
945 debug!("destructor_call_block_sync({:?}, {:?})", self, succ);
946 let tcx = self.tcx();
947 let drop_trait = tcx.require_lang_item(LangItem::Drop, DUMMY_SP);
948 let drop_fn = tcx.associated_item_def_ids(drop_trait)[0];
949 let ty = self.place_ty(self.place);
950
951 let ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, ty);
952 let ref_place = self.new_temp(ref_ty);
953 let unit_temp = Place::from(self.new_temp(tcx.types.unit));
954
955 let result = BasicBlockData::new_stmts(
956 vec![self.assign(
957 Place::from(ref_place),
958 Rvalue::Ref(
959 tcx.lifetimes.re_erased,
960 BorrowKind::Mut { kind: MutBorrowKind::Default },
961 self.place,
962 ),
963 )],
964 Some(Terminator {
965 kind: TerminatorKind::Call {
966 func: Operand::function_handle(
967 tcx,
968 drop_fn,
969 [ty.into()],
970 self.source_info.span,
971 ),
972 args: [Spanned { node: Operand::Move(Place::from(ref_place)), span: DUMMY_SP }]
973 .into(),
974 destination: unit_temp,
975 target: Some(succ),
976 unwind: unwind.into_action(),
977 call_source: CallSource::Misc,
978 fn_span: self.source_info.span,
979 },
980 source_info: self.source_info,
981 }),
982 unwind.is_cleanup(),
983 );
984
985 let destructor_block = self.elaborator.patch().new_block(result);
986
987 let block_start = Location { block: destructor_block, statement_index: 0 };
988 self.elaborator.clear_drop_flag(block_start, self.path, DropFlagMode::Shallow);
989
990 self.drop_flag_test_block(destructor_block, succ, unwind)
991 }
992
993 fn destructor_call_block(
994 &mut self,
995 (succ, unwind, dropline): (BasicBlock, Unwind, Option<BasicBlock>),
996 ) -> BasicBlock {
997 debug!("destructor_call_block({:?}, {:?})", self, succ);
998 let ty = self.place_ty(self.place);
999 if self.tcx().features().async_drop()
1000 && self.elaborator.body().coroutine.is_some()
1001 && self.elaborator.allow_async_drops()
1002 && !unwind.is_cleanup()
1003 && ty.is_async_drop(self.tcx(), self.elaborator.typing_env())
1004 {
1005 let destructor_block =
1006 self.build_async_drop(self.place, ty, None, succ, unwind, dropline, true);
1007
1008 let block_start = Location { block: destructor_block, statement_index: 0 };
1009 self.elaborator.clear_drop_flag(block_start, self.path, DropFlagMode::Shallow);
1010
1011 self.drop_flag_test_block(destructor_block, succ, unwind)
1012 } else {
1013 self.destructor_call_block_sync((succ, unwind))
1014 }
1015 }
1016
1017 fn drop_loop(
1029 &mut self,
1030 succ: BasicBlock,
1031 cur: Local,
1032 len: Local,
1033 ety: Ty<'tcx>,
1034 unwind: Unwind,
1035 dropline: Option<BasicBlock>,
1036 ) -> BasicBlock {
1037 let copy = |place: Place<'tcx>| Operand::Copy(place);
1038 let move_ = |place: Place<'tcx>| Operand::Move(place);
1039 let tcx = self.tcx();
1040
1041 let ptr_ty = Ty::new_mut_ptr(tcx, ety);
1042 let ptr = Place::from(self.new_temp(ptr_ty));
1043 let can_go = Place::from(self.new_temp(tcx.types.bool));
1044 let one = self.constant_usize(1);
1045
1046 let drop_block = BasicBlockData::new_stmts(
1047 vec![
1048 self.assign(
1049 ptr,
1050 Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_index(self.place, cur)),
1051 ),
1052 self.assign(
1053 cur.into(),
1054 Rvalue::BinaryOp(BinOp::Add, Box::new((move_(cur.into()), one))),
1055 ),
1056 ],
1057 Some(Terminator {
1058 source_info: self.source_info,
1059 kind: TerminatorKind::Unreachable,
1061 }),
1062 unwind.is_cleanup(),
1063 );
1064 let drop_block = self.elaborator.patch().new_block(drop_block);
1065
1066 let loop_block = BasicBlockData::new_stmts(
1067 vec![self.assign(
1068 can_go,
1069 Rvalue::BinaryOp(BinOp::Eq, Box::new((copy(Place::from(cur)), copy(len.into())))),
1070 )],
1071 Some(Terminator {
1072 source_info: self.source_info,
1073 kind: TerminatorKind::if_(move_(can_go), succ, drop_block),
1074 }),
1075 unwind.is_cleanup(),
1076 );
1077 let loop_block = self.elaborator.patch().new_block(loop_block);
1078
1079 let place = tcx.mk_place_deref(ptr);
1080 if self.tcx().features().async_drop()
1081 && self.elaborator.body().coroutine.is_some()
1082 && self.elaborator.allow_async_drops()
1083 && !unwind.is_cleanup()
1084 && ety.needs_async_drop(self.tcx(), self.elaborator.typing_env())
1085 {
1086 self.build_async_drop(
1087 place,
1088 ety,
1089 Some(drop_block),
1090 loop_block,
1091 unwind,
1092 dropline,
1093 false,
1094 );
1095 } else {
1096 self.elaborator.patch().patch_terminator(
1097 drop_block,
1098 TerminatorKind::Drop {
1099 place,
1100 target: loop_block,
1101 unwind: unwind.into_action(),
1102 replace: false,
1103 drop: None,
1104 async_fut: None,
1105 },
1106 );
1107 }
1108 loop_block
1109 }
1110
1111 fn open_drop_for_array(
1112 &mut self,
1113 array_ty: Ty<'tcx>,
1114 ety: Ty<'tcx>,
1115 opt_size: Option<u64>,
1116 ) -> BasicBlock {
1117 debug!("open_drop_for_array({:?}, {:?}, {:?})", array_ty, ety, opt_size);
1118 let tcx = self.tcx();
1119
1120 if let Some(size) = opt_size {
1121 enum ProjectionKind<Path> {
1122 Drop(std::ops::Range<u64>),
1123 Keep(u64, Path),
1124 }
1125 let mut drop_ranges = vec![];
1130 let mut dropping = true;
1131 let mut start = 0;
1132 for i in 0..size {
1133 let path = self.elaborator.array_subpath(self.path, i, size);
1134 if dropping && path.is_some() {
1135 drop_ranges.push(ProjectionKind::Drop(start..i));
1136 dropping = false;
1137 } else if !dropping && path.is_none() {
1138 dropping = true;
1139 start = i;
1140 }
1141 if let Some(path) = path {
1142 drop_ranges.push(ProjectionKind::Keep(i, path));
1143 }
1144 }
1145 if !drop_ranges.is_empty() {
1146 if dropping {
1147 drop_ranges.push(ProjectionKind::Drop(start..size));
1148 }
1149 let fields = drop_ranges
1150 .iter()
1151 .rev()
1152 .map(|p| {
1153 let (project, path) = match p {
1154 ProjectionKind::Drop(r) => (
1155 ProjectionElem::Subslice {
1156 from: r.start,
1157 to: r.end,
1158 from_end: false,
1159 },
1160 None,
1161 ),
1162 &ProjectionKind::Keep(offset, path) => (
1163 ProjectionElem::ConstantIndex {
1164 offset,
1165 min_length: size,
1166 from_end: false,
1167 },
1168 Some(path),
1169 ),
1170 };
1171 (tcx.mk_place_elem(self.place, project), path)
1172 })
1173 .collect::<Vec<_>>();
1174 let (succ, unwind, dropline) = self.drop_ladder_bottom();
1175 return self.drop_ladder(fields, succ, unwind, dropline).0;
1176 }
1177 }
1178
1179 let array_ptr_ty = Ty::new_mut_ptr(tcx, array_ty);
1180 let array_ptr = self.new_temp(array_ptr_ty);
1181
1182 let slice_ty = Ty::new_slice(tcx, ety);
1183 let slice_ptr_ty = Ty::new_mut_ptr(tcx, slice_ty);
1184 let slice_ptr = self.new_temp(slice_ptr_ty);
1185
1186 let mut delegate_block = BasicBlockData::new_stmts(
1187 vec![
1188 self.assign(Place::from(array_ptr), Rvalue::RawPtr(RawPtrKind::Mut, self.place)),
1189 self.assign(
1190 Place::from(slice_ptr),
1191 Rvalue::Cast(
1192 CastKind::PointerCoercion(
1193 PointerCoercion::Unsize,
1194 CoercionSource::Implicit,
1195 ),
1196 Operand::Move(Place::from(array_ptr)),
1197 slice_ptr_ty,
1198 ),
1199 ),
1200 ],
1201 None,
1202 self.unwind.is_cleanup(),
1203 );
1204
1205 let array_place = mem::replace(
1206 &mut self.place,
1207 Place::from(slice_ptr).project_deeper(&[PlaceElem::Deref], tcx),
1208 );
1209 let slice_block = self.drop_loop_trio_for_slice(ety);
1210 self.place = array_place;
1211
1212 delegate_block.terminator = Some(Terminator {
1213 source_info: self.source_info,
1214 kind: TerminatorKind::Goto { target: slice_block },
1215 });
1216 self.elaborator.patch().new_block(delegate_block)
1217 }
1218
1219 fn drop_loop_trio_for_slice(&mut self, ety: Ty<'tcx>) -> BasicBlock {
1222 debug!("drop_loop_trio_for_slice({:?})", ety);
1223 let tcx = self.tcx();
1224 let len = self.new_temp(tcx.types.usize);
1225 let cur = self.new_temp(tcx.types.usize);
1226
1227 let unwind = self
1228 .unwind
1229 .map(|unwind| self.drop_loop(unwind, cur, len, ety, Unwind::InCleanup, None));
1230
1231 let dropline =
1232 self.dropline.map(|dropline| self.drop_loop(dropline, cur, len, ety, unwind, None));
1233
1234 let loop_block = self.drop_loop(self.succ, cur, len, ety, unwind, dropline);
1235
1236 let [PlaceElem::Deref] = self.place.projection.as_slice() else {
1237 span_bug!(
1238 self.source_info.span,
1239 "Expected place for slice drop shim to be *_n, but it's {:?}",
1240 self.place,
1241 );
1242 };
1243
1244 let zero = self.constant_usize(0);
1245 let block = BasicBlockData::new_stmts(
1246 vec![
1247 self.assign(
1248 len.into(),
1249 Rvalue::UnaryOp(
1250 UnOp::PtrMetadata,
1251 Operand::Copy(Place::from(self.place.local)),
1252 ),
1253 ),
1254 self.assign(cur.into(), Rvalue::Use(zero)),
1255 ],
1256 Some(Terminator {
1257 source_info: self.source_info,
1258 kind: TerminatorKind::Goto { target: loop_block },
1259 }),
1260 unwind.is_cleanup(),
1261 );
1262
1263 let drop_block = self.elaborator.patch().new_block(block);
1264 let reset_block = self.drop_flag_reset_block(DropFlagMode::Deep, drop_block, unwind);
1266 self.drop_flag_test_block(reset_block, self.succ, unwind)
1267 }
1268
1269 fn open_drop(&mut self) -> BasicBlock {
1278 let ty = self.place_ty(self.place);
1279 match ty.kind() {
1280 ty::Closure(_, args) => self.open_drop_for_tuple(args.as_closure().upvar_tys()),
1281 ty::CoroutineClosure(_, args) => {
1282 self.open_drop_for_tuple(args.as_coroutine_closure().upvar_tys())
1283 }
1284 ty::Coroutine(_, args) => self.open_drop_for_tuple(args.as_coroutine().upvar_tys()),
1291 ty::Tuple(fields) => self.open_drop_for_tuple(fields),
1292 ty::Adt(def, args) => self.open_drop_for_adt(*def, args),
1293 ty::Dynamic(..) => self.complete_drop(self.succ, self.unwind),
1294 ty::Array(ety, size) => {
1295 let size = size.try_to_target_usize(self.tcx());
1296 self.open_drop_for_array(ty, *ety, size)
1297 }
1298 ty::Slice(ety) => self.drop_loop_trio_for_slice(*ety),
1299
1300 ty::UnsafeBinder(_) => {
1301 self.tcx().dcx().span_delayed_bug(
1304 self.source_info.span,
1305 "open drop for unsafe binder shouldn't be encountered",
1306 );
1307 self.elaborator.patch().new_block(BasicBlockData::new(
1308 Some(Terminator {
1309 source_info: self.source_info,
1310 kind: TerminatorKind::Unreachable,
1311 }),
1312 self.unwind.is_cleanup(),
1313 ))
1314 }
1315
1316 _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty),
1317 }
1318 }
1319
1320 fn complete_drop(&mut self, succ: BasicBlock, unwind: Unwind) -> BasicBlock {
1321 debug!("complete_drop(succ={:?}, unwind={:?})", succ, unwind);
1322
1323 let drop_block = self.drop_block(succ, unwind);
1324
1325 self.drop_flag_test_block(drop_block, succ, unwind)
1326 }
1327
1328 fn drop_flag_reset_block(
1331 &mut self,
1332 mode: DropFlagMode,
1333 succ: BasicBlock,
1334 unwind: Unwind,
1335 ) -> BasicBlock {
1336 debug!("drop_flag_reset_block({:?},{:?})", self, mode);
1337
1338 if unwind.is_cleanup() {
1339 return succ;
1342 }
1343 let block = self.new_block(unwind, TerminatorKind::Goto { target: succ });
1344 let block_start = Location { block, statement_index: 0 };
1345 self.elaborator.clear_drop_flag(block_start, self.path, mode);
1346 block
1347 }
1348
1349 fn elaborated_drop_block(&mut self) -> BasicBlock {
1350 debug!("elaborated_drop_block({:?})", self);
1351 let blk = self.drop_block_simple(self.succ, self.unwind);
1352 self.elaborate_drop(blk);
1353 blk
1354 }
1355
1356 fn drop_block_simple(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
1357 let block = TerminatorKind::Drop {
1358 place: self.place,
1359 target,
1360 unwind: unwind.into_action(),
1361 replace: false,
1362 drop: self.dropline,
1363 async_fut: None,
1364 };
1365 self.new_block(unwind, block)
1366 }
1367
1368 fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
1369 let drop_ty = self.place_ty(self.place);
1370 if self.tcx().features().async_drop()
1371 && self.elaborator.body().coroutine.is_some()
1372 && self.elaborator.allow_async_drops()
1373 && !unwind.is_cleanup()
1374 && drop_ty.needs_async_drop(self.tcx(), self.elaborator.typing_env())
1375 {
1376 self.build_async_drop(
1377 self.place,
1378 drop_ty,
1379 None,
1380 self.succ,
1381 unwind,
1382 self.dropline,
1383 false,
1384 )
1385 } else {
1386 let block = TerminatorKind::Drop {
1387 place: self.place,
1388 target,
1389 unwind: unwind.into_action(),
1390 replace: false,
1391 drop: None,
1392 async_fut: None,
1393 };
1394 self.new_block(unwind, block)
1395 }
1396 }
1397
1398 fn goto_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
1399 let block = TerminatorKind::Goto { target };
1400 self.new_block(unwind, block)
1401 }
1402
1403 fn drop_flag_test_block(
1409 &mut self,
1410 on_set: BasicBlock,
1411 on_unset: BasicBlock,
1412 unwind: Unwind,
1413 ) -> BasicBlock {
1414 let style = self.elaborator.drop_style(self.path, DropFlagMode::Shallow);
1415 debug!(
1416 "drop_flag_test_block({:?},{:?},{:?},{:?}) - {:?}",
1417 self, on_set, on_unset, unwind, style
1418 );
1419
1420 match style {
1421 DropStyle::Dead => on_unset,
1422 DropStyle::Static => on_set,
1423 DropStyle::Conditional | DropStyle::Open => {
1424 let flag = self.elaborator.get_drop_flag(self.path).unwrap();
1425 let term = TerminatorKind::if_(flag, on_set, on_unset);
1426 self.new_block(unwind, term)
1427 }
1428 }
1429 }
1430
1431 fn new_block(&mut self, unwind: Unwind, k: TerminatorKind<'tcx>) -> BasicBlock {
1432 self.elaborator.patch().new_block(BasicBlockData::new(
1433 Some(Terminator { source_info: self.source_info, kind: k }),
1434 unwind.is_cleanup(),
1435 ))
1436 }
1437
1438 fn new_block_with_statements(
1439 &mut self,
1440 unwind: Unwind,
1441 statements: Vec<Statement<'tcx>>,
1442 k: TerminatorKind<'tcx>,
1443 ) -> BasicBlock {
1444 self.elaborator.patch().new_block(BasicBlockData::new_stmts(
1445 statements,
1446 Some(Terminator { source_info: self.source_info, kind: k }),
1447 unwind.is_cleanup(),
1448 ))
1449 }
1450
1451 fn new_temp(&mut self, ty: Ty<'tcx>) -> Local {
1452 self.elaborator.patch().new_temp(ty, self.source_info.span)
1453 }
1454
1455 fn constant_usize(&self, val: u16) -> Operand<'tcx> {
1456 Operand::Constant(Box::new(ConstOperand {
1457 span: self.source_info.span,
1458 user_ty: None,
1459 const_: Const::from_usize(self.tcx(), val.into()),
1460 }))
1461 }
1462
1463 fn assign(&self, lhs: Place<'tcx>, rhs: Rvalue<'tcx>) -> Statement<'tcx> {
1464 Statement::new(self.source_info, StatementKind::Assign(Box::new((lhs, rhs))))
1465 }
1466}