1use std::iter;
2
3use rustc_index::IndexVec;
4use rustc_index::bit_set::DenseBitSet;
5use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
6use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal};
7use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout};
8use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
9use rustc_middle::{bug, mir, span_bug};
10use rustc_target::callconv::{FnAbi, PassMode};
11use tracing::{debug, instrument};
12
13use crate::base;
14use crate::traits::*;
15
16mod analyze;
17mod block;
18mod constant;
19mod coverageinfo;
20pub mod debuginfo;
21mod intrinsic;
22mod locals;
23pub mod naked_asm;
24pub mod operand;
25pub mod place;
26mod rvalue;
27mod statement;
28
29pub use self::block::store_cast;
30use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
31use self::operand::{OperandRef, OperandValue};
32use self::place::PlaceRef;
33
34enum CachedLlbb<T> {
36 None,
38
39 Some(T),
41
42 Skip,
44}
45
46type PerLocalVarDebugInfoIndexVec<'tcx, V> =
47 IndexVec<mir::Local, Vec<PerLocalVarDebugInfo<'tcx, V>>>;
48
49pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
51 instance: Instance<'tcx>,
52
53 mir: &'tcx mir::Body<'tcx>,
54
55 debug_context: Option<FunctionDebugContext<'tcx, Bx::DIScope, Bx::DILocation>>,
56
57 llfn: Bx::Function,
58
59 cx: &'a Bx::CodegenCx,
60
61 fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
62
63 personality_slot: Option<PlaceRef<'tcx, Bx::Value>>,
71
72 cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>>,
77
78 cleanup_kinds: Option<IndexVec<mir::BasicBlock, analyze::CleanupKind>>,
80
81 funclets: IndexVec<mir::BasicBlock, Option<Bx::Funclet>>,
85
86 landing_pads: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,
89
90 unreachable_block: Option<Bx::BasicBlock>,
92
93 terminate_block: Option<(Bx::BasicBlock, UnwindTerminateReason)>,
95
96 cold_blocks: IndexVec<mir::BasicBlock, bool>,
99
100 locals: locals::Locals<'tcx, Bx::Value>,
116
117 per_local_var_debug_info: Option<PerLocalVarDebugInfoIndexVec<'tcx, Bx::DIVariable>>,
120
121 caller_location: Option<OperandRef<'tcx, Bx::Value>>,
123}
124
125impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
126 pub fn monomorphize<T>(&self, value: T) -> T
127 where
128 T: Copy + TypeFoldable<TyCtxt<'tcx>>,
129 {
130 debug!("monomorphize: self.instance={:?}", self.instance);
131 self.instance.instantiate_mir_and_normalize_erasing_regions(
132 self.cx.tcx(),
133 self.cx.typing_env(),
134 ty::EarlyBinder::bind(value),
135 )
136 }
137}
138
139enum LocalRef<'tcx, V> {
140 Place(PlaceRef<'tcx, V>),
141 UnsizedPlace(PlaceRef<'tcx, V>),
146 Operand(OperandRef<'tcx, V>),
148 PendingOperand,
150}
151
152impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> {
153 fn new_operand(layout: TyAndLayout<'tcx>) -> LocalRef<'tcx, V> {
154 if layout.is_zst() {
155 LocalRef::Operand(OperandRef::zero_sized(layout))
159 } else {
160 LocalRef::PendingOperand
161 }
162 }
163}
164
165#[instrument(level = "debug", skip(cx))]
168pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
169 cx: &'a Bx::CodegenCx,
170 instance: Instance<'tcx>,
171) {
172 assert!(!instance.args.has_infer());
173
174 let tcx = cx.tcx();
175 let llfn = cx.get_fn(instance);
176
177 let mut mir = tcx.instance_mir(instance.def);
178
179 let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
180 debug!("fn_abi: {:?}", fn_abi);
181
182 if tcx.features().ergonomic_clones() {
183 let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions(
184 tcx,
185 ty::TypingEnv::fully_monomorphized(),
186 ty::EarlyBinder::bind(mir.clone()),
187 );
188 mir = tcx.arena.alloc(optimize_use_clone::<Bx>(cx, monomorphized_mir));
189 }
190
191 let debug_context = cx.create_function_debug_context(instance, fn_abi, llfn, &mir);
192
193 let start_llbb = Bx::append_block(cx, llfn, "start");
194 let mut start_bx = Bx::build(cx, start_llbb);
195
196 if mir.basic_blocks.iter().any(|bb| {
197 bb.is_cleanup || matches!(bb.terminator().unwind(), Some(mir::UnwindAction::Terminate(_)))
198 }) {
199 start_bx.set_personality_fn(cx.eh_personality());
200 }
201
202 let cleanup_kinds =
203 base::wants_new_eh_instructions(tcx.sess).then(|| analyze::cleanup_kinds(&mir));
204
205 let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> =
206 mir.basic_blocks
207 .indices()
208 .map(|bb| {
209 if bb == mir::START_BLOCK { CachedLlbb::Some(start_llbb) } else { CachedLlbb::None }
210 })
211 .collect();
212
213 let mut fx = FunctionCx {
214 instance,
215 mir,
216 llfn,
217 fn_abi,
218 cx,
219 personality_slot: None,
220 cached_llbbs,
221 unreachable_block: None,
222 terminate_block: None,
223 cleanup_kinds,
224 landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
225 funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
226 cold_blocks: find_cold_blocks(tcx, mir),
227 locals: locals::Locals::empty(),
228 debug_context,
229 per_local_var_debug_info: None,
230 caller_location: None,
231 };
232
233 let (per_local_var_debug_info, consts_debug_info) =
239 fx.compute_per_local_var_debug_info(&mut start_bx).unzip();
240 fx.per_local_var_debug_info = per_local_var_debug_info;
241
242 let traversal_order = traversal::mono_reachable_reverse_postorder(mir, tcx, instance);
243 let memory_locals = analyze::non_ssa_locals(&fx, &traversal_order);
244
245 let local_values = {
247 let args = arg_local_refs(&mut start_bx, &mut fx, &memory_locals);
248
249 let mut allocate_local = |local: Local| {
250 let decl = &mir.local_decls[local];
251 let layout = start_bx.layout_of(fx.monomorphize(decl.ty));
252 assert!(!layout.ty.has_erasable_regions());
253
254 if local == mir::RETURN_PLACE {
255 match fx.fn_abi.ret.mode {
256 PassMode::Indirect { .. } => {
257 debug!("alloc: {:?} (return place) -> place", local);
258 let llretptr = start_bx.get_param(0);
259 return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
260 }
261 PassMode::Cast { ref cast, .. } => {
262 debug!("alloc: {:?} (return place) -> place", local);
263 let size = cast.size(&start_bx).max(layout.size);
264 return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
265 }
266 _ => {}
267 };
268 }
269
270 if memory_locals.contains(local) {
271 debug!("alloc: {:?} -> place", local);
272 if layout.is_unsized() {
273 LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut start_bx, layout))
274 } else {
275 LocalRef::Place(PlaceRef::alloca(&mut start_bx, layout))
276 }
277 } else {
278 debug!("alloc: {:?} -> operand", local);
279 LocalRef::new_operand(layout)
280 }
281 };
282
283 let retptr = allocate_local(mir::RETURN_PLACE);
284 iter::once(retptr)
285 .chain(args.into_iter())
286 .chain(mir.vars_and_temps_iter().map(allocate_local))
287 .collect()
288 };
289 fx.initialize_locals(local_values);
290
291 fx.debug_introduce_locals(&mut start_bx, consts_debug_info.unwrap_or_default());
293
294 start_bx.init_coverage(instance);
297
298 drop(start_bx);
301
302 let mut unreached_blocks = DenseBitSet::new_filled(mir.basic_blocks.len());
303 for bb in traversal_order {
305 fx.codegen_block(bb);
306 unreached_blocks.remove(bb);
307 }
308
309 for bb in unreached_blocks.iter() {
315 fx.codegen_block_as_unreachable(bb);
316 }
317}
318
319fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
321 cx: &'a Bx::CodegenCx,
322 mut mir: Body<'tcx>,
323) -> Body<'tcx> {
324 let tcx = cx.tcx();
325
326 if tcx.features().ergonomic_clones() {
327 for bb in mir.basic_blocks.as_mut() {
328 let mir::TerminatorKind::Call {
329 args,
330 destination,
331 target,
332 call_source: mir::CallSource::Use,
333 ..
334 } = &bb.terminator().kind
335 else {
336 continue;
337 };
338
339 assert_eq!(args.len(), 1);
341 let arg = &args[0];
342
343 let arg_ty = arg.node.ty(&mir.local_decls, tcx);
346
347 let ty::Ref(_region, inner_ty, mir::Mutability::Not) = *arg_ty.kind() else { continue };
348
349 if !tcx.type_is_copy_modulo_regions(cx.typing_env(), inner_ty) {
350 continue;
351 }
352
353 let Some(arg_place) = arg.node.place() else { continue };
354
355 let destination_block = target.unwrap();
356
357 bb.statements.push(mir::Statement {
358 source_info: bb.terminator().source_info,
359 kind: mir::StatementKind::Assign(Box::new((
360 *destination,
361 mir::Rvalue::Use(mir::Operand::Copy(
362 arg_place.project_deeper(&[mir::ProjectionElem::Deref], tcx),
363 )),
364 ))),
365 });
366
367 bb.terminator_mut().kind = mir::TerminatorKind::Goto { target: destination_block };
368 }
369 }
370
371 mir
372}
373
374fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
378 bx: &mut Bx,
379 fx: &mut FunctionCx<'a, 'tcx, Bx>,
380 memory_locals: &DenseBitSet<mir::Local>,
381) -> Vec<LocalRef<'tcx, Bx::Value>> {
382 let mir = fx.mir;
383 let mut idx = 0;
384 let mut llarg_idx = fx.fn_abi.ret.is_indirect() as usize;
385
386 let mut num_untupled = None;
387
388 let codegen_fn_attrs = bx.tcx().codegen_fn_attrs(fx.instance.def_id());
389 let naked = codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED);
390 if naked {
391 return vec![];
392 }
393
394 let args = mir
395 .args_iter()
396 .enumerate()
397 .map(|(arg_index, local)| {
398 let arg_decl = &mir.local_decls[local];
399 let arg_ty = fx.monomorphize(arg_decl.ty);
400
401 if Some(local) == mir.spread_arg {
402 let ty::Tuple(tupled_arg_tys) = arg_ty.kind() else {
407 bug!("spread argument isn't a tuple?!");
408 };
409
410 let layout = bx.layout_of(arg_ty);
411
412 if layout.is_unsized() {
414 span_bug!(
415 arg_decl.source_info.span,
416 "\"rust-call\" ABI does not support unsized params",
417 );
418 }
419
420 let place = PlaceRef::alloca(bx, layout);
421 for i in 0..tupled_arg_tys.len() {
422 let arg = &fx.fn_abi.args[idx];
423 idx += 1;
424 if let PassMode::Cast { pad_i32: true, .. } = arg.mode {
425 llarg_idx += 1;
426 }
427 let pr_field = place.project_field(bx, i);
428 bx.store_fn_arg(arg, &mut llarg_idx, pr_field);
429 }
430 assert_eq!(
431 None,
432 num_untupled.replace(tupled_arg_tys.len()),
433 "Replaced existing num_tupled"
434 );
435
436 return LocalRef::Place(place);
437 }
438
439 if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() {
440 let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty));
441 bx.va_start(va_list.val.llval);
442
443 return LocalRef::Place(va_list);
444 }
445
446 let arg = &fx.fn_abi.args[idx];
447 idx += 1;
448 if let PassMode::Cast { pad_i32: true, .. } = arg.mode {
449 llarg_idx += 1;
450 }
451
452 if !memory_locals.contains(local) {
453 let local = |op| LocalRef::Operand(op);
457 match arg.mode {
458 PassMode::Ignore => {
459 return local(OperandRef::zero_sized(arg.layout));
460 }
461 PassMode::Direct(_) => {
462 let llarg = bx.get_param(llarg_idx);
463 llarg_idx += 1;
464 return local(OperandRef::from_immediate_or_packed_pair(
465 bx, llarg, arg.layout,
466 ));
467 }
468 PassMode::Pair(..) => {
469 let (a, b) = (bx.get_param(llarg_idx), bx.get_param(llarg_idx + 1));
470 llarg_idx += 2;
471
472 return local(OperandRef {
473 val: OperandValue::Pair(a, b),
474 layout: arg.layout,
475 });
476 }
477 _ => {}
478 }
479 }
480
481 match arg.mode {
482 PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
484 if let Some(pointee_align) = attrs.pointee_align
488 && pointee_align < arg.layout.align.abi
489 {
490 let tmp = PlaceRef::alloca(bx, arg.layout);
493 bx.store_fn_arg(arg, &mut llarg_idx, tmp);
494 LocalRef::Place(tmp)
495 } else {
496 let llarg = bx.get_param(llarg_idx);
497 llarg_idx += 1;
498 LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout))
499 }
500 }
501 PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
503 let llarg = bx.get_param(llarg_idx);
506 llarg_idx += 1;
507 let llextra = bx.get_param(llarg_idx);
508 llarg_idx += 1;
509 let indirect_operand = OperandValue::Pair(llarg, llextra);
510
511 let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout);
512 indirect_operand.store(bx, tmp);
513 LocalRef::UnsizedPlace(tmp)
514 }
515 _ => {
516 let tmp = PlaceRef::alloca(bx, arg.layout);
517 bx.store_fn_arg(arg, &mut llarg_idx, tmp);
518 LocalRef::Place(tmp)
519 }
520 }
521 })
522 .collect::<Vec<_>>();
523
524 if fx.instance.def.requires_caller_location(bx.tcx()) {
525 let mir_args = if let Some(num_untupled) = num_untupled {
526 args.len() - 1 + num_untupled
528 } else {
529 args.len()
530 };
531 assert_eq!(
532 fx.fn_abi.args.len(),
533 mir_args + 1,
534 "#[track_caller] instance {:?} must have 1 more argument in their ABI than in their MIR",
535 fx.instance
536 );
537
538 let arg = fx.fn_abi.args.last().unwrap();
539 match arg.mode {
540 PassMode::Direct(_) => (),
541 _ => bug!("caller location must be PassMode::Direct, found {:?}", arg.mode),
542 }
543
544 fx.caller_location = Some(OperandRef {
545 val: OperandValue::Immediate(bx.get_param(llarg_idx)),
546 layout: arg.layout,
547 });
548 }
549
550 args
551}
552
553fn find_cold_blocks<'tcx>(
554 tcx: TyCtxt<'tcx>,
555 mir: &mir::Body<'tcx>,
556) -> IndexVec<mir::BasicBlock, bool> {
557 let local_decls = &mir.local_decls;
558
559 let mut cold_blocks: IndexVec<mir::BasicBlock, bool> =
560 IndexVec::from_elem(false, &mir.basic_blocks);
561
562 for (bb, bb_data) in traversal::postorder(mir) {
564 let terminator = bb_data.terminator();
565
566 match terminator.kind {
567 mir::TerminatorKind::Call { ref func, .. }
569 | mir::TerminatorKind::TailCall { ref func, .. }
570 if let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
571 && let attrs = tcx.codegen_fn_attrs(def_id)
572 && attrs.flags.contains(CodegenFnAttrFlags::COLD) =>
573 {
574 cold_blocks[bb] = true;
575 continue;
576 }
577
578 mir::TerminatorKind::Unreachable => {
580 cold_blocks[bb] = true;
581 continue;
582 }
583
584 _ => {}
585 }
586
587 let mut succ = terminator.successors();
589 if let Some(first) = succ.next()
590 && cold_blocks[first]
591 && succ.all(|s| cold_blocks[s])
592 {
593 cold_blocks[bb] = true;
594 }
595 }
596
597 cold_blocks
598}