rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
18use rustc_middle::ty::layout::{
19    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
20    TyAndLayout,
21};
22use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
23use rustc_sanitizers::{cfi, kcfi};
24use rustc_session::config::OptLevel;
25use rustc_span::Span;
26use rustc_target::callconv::FnAbi;
27use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
28use smallvec::SmallVec;
29use tracing::{debug, instrument};
30
31use crate::abi::FnAbiLlvmExt;
32use crate::attributes;
33use crate::common::Funclet;
34use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
35use crate::llvm::{
36    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
37};
38use crate::type_::Type;
39use crate::type_of::LayoutLlvmExt;
40use crate::value::Value;
41
42#[must_use]
43pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
44    pub llbuilder: &'ll mut llvm::Builder<'ll>,
45    pub cx: &'a GenericCx<'ll, CX>,
46}
47
48pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
49pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
50
51impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
52    fn drop(&mut self) {
53        unsafe {
54            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
55        }
56    }
57}
58
59impl<'a, 'll> SBuilder<'a, 'll> {
60    pub(crate) fn call(
61        &mut self,
62        llty: &'ll Type,
63        llfn: &'ll Value,
64        args: &[&'ll Value],
65        funclet: Option<&Funclet<'ll>>,
66    ) -> &'ll Value {
67        debug!("call {:?} with args ({:?})", llfn, args);
68
69        let args = self.check_call("call", llty, llfn, args);
70        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
71        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
72        if let Some(funclet_bundle) = funclet_bundle {
73            bundles.push(funclet_bundle);
74        }
75
76        let call = unsafe {
77            llvm::LLVMBuildCallWithOperandBundles(
78                self.llbuilder,
79                llty,
80                llfn,
81                args.as_ptr() as *const &llvm::Value,
82                args.len() as c_uint,
83                bundles.as_ptr(),
84                bundles.len() as c_uint,
85                c"".as_ptr(),
86            )
87        };
88        call
89    }
90}
91
92impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
93    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
94        // Create a fresh builder from the simple context.
95        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
96        GenericBuilder { llbuilder, cx: scx }
97    }
98
99    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
100        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
101    }
102
103    pub(crate) fn ret_void(&mut self) {
104        llvm::LLVMBuildRetVoid(self.llbuilder);
105    }
106
107    pub(crate) fn ret(&mut self, v: &'ll Value) {
108        unsafe {
109            llvm::LLVMBuildRet(self.llbuilder, v);
110        }
111    }
112
113    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
114        let bx = Self::with_cx(cx);
115        unsafe {
116            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
117        }
118        bx
119    }
120}
121
122/// Empty string, to be used where LLVM expects an instruction name, indicating
123/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
124// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
125pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
126
127impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
128    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
129    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
130    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
131    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
132    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
133    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
134
135    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
136    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
137    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
138}
139
140impl abi::HasDataLayout for Builder<'_, '_, '_> {
141    fn data_layout(&self) -> &abi::TargetDataLayout {
142        self.cx.data_layout()
143    }
144}
145
146impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
147    #[inline]
148    fn tcx(&self) -> TyCtxt<'tcx> {
149        self.cx.tcx
150    }
151}
152
153impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
154    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
155        self.cx.typing_env()
156    }
157}
158
159impl HasTargetSpec for Builder<'_, '_, '_> {
160    #[inline]
161    fn target_spec(&self) -> &Target {
162        self.cx.target_spec()
163    }
164}
165
166impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
167    #[inline]
168    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
169        self.cx.handle_layout_err(err, span, ty)
170    }
171}
172
173impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
174    #[inline]
175    fn handle_fn_abi_err(
176        &self,
177        err: FnAbiError<'tcx>,
178        span: Span,
179        fn_abi_request: FnAbiRequest<'tcx>,
180    ) -> ! {
181        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
182    }
183}
184
185impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
186    type Target = CodegenCx<'ll, 'tcx>;
187
188    #[inline]
189    fn deref(&self) -> &Self::Target {
190        self.cx
191    }
192}
193
194macro_rules! math_builder_methods {
195    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
196        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
197            unsafe {
198                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
199            }
200        })+
201    }
202}
203
204macro_rules! set_math_builder_methods {
205    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
206        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
207            unsafe {
208                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
209                llvm::$llvm_set_math(instr);
210                instr
211            }
212        })+
213    }
214}
215
216impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
217    type CodegenCx = CodegenCx<'ll, 'tcx>;
218
219    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
220        let bx = Builder::with_cx(cx);
221        unsafe {
222            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
223        }
224        bx
225    }
226
227    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
228        self.cx
229    }
230
231    fn llbb(&self) -> &'ll BasicBlock {
232        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
233    }
234
235    fn set_span(&mut self, _span: Span) {}
236
237    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
238        unsafe {
239            let name = SmallCStr::new(name);
240            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
241        }
242    }
243
244    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
245        Self::append_block(self.cx, self.llfn(), name)
246    }
247
248    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
249        *self = Self::build(self.cx, llbb)
250    }
251
252    fn ret_void(&mut self) {
253        llvm::LLVMBuildRetVoid(self.llbuilder);
254    }
255
256    fn ret(&mut self, v: &'ll Value) {
257        unsafe {
258            llvm::LLVMBuildRet(self.llbuilder, v);
259        }
260    }
261
262    fn br(&mut self, dest: &'ll BasicBlock) {
263        unsafe {
264            llvm::LLVMBuildBr(self.llbuilder, dest);
265        }
266    }
267
268    fn cond_br(
269        &mut self,
270        cond: &'ll Value,
271        then_llbb: &'ll BasicBlock,
272        else_llbb: &'ll BasicBlock,
273    ) {
274        unsafe {
275            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
276        }
277    }
278
279    fn switch(
280        &mut self,
281        v: &'ll Value,
282        else_llbb: &'ll BasicBlock,
283        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
284    ) {
285        let switch =
286            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
287        for (on_val, dest) in cases {
288            let on_val = self.const_uint_big(self.val_ty(v), on_val);
289            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
290        }
291    }
292
293    fn switch_with_weights(
294        &mut self,
295        v: Self::Value,
296        else_llbb: Self::BasicBlock,
297        else_is_cold: bool,
298        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
299    ) {
300        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
301            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
302            return;
303        }
304
305        let id = self.cx.create_metadata(b"branch_weights");
306
307        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
308        // This function handles switch instructions with more than 2 targets and it needs to
309        // emit branch weights metadata instead of using the intrinsic.
310        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
311        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
312        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
313        let weight =
314            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
315
316        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
317        md.push(id);
318        md.push(weight(else_is_cold));
319
320        let switch =
321            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
322        for (on_val, dest, is_cold) in cases {
323            let on_val = self.const_uint_big(self.val_ty(v), on_val);
324            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
325            md.push(weight(is_cold));
326        }
327
328        unsafe {
329            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
330            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
331        }
332    }
333
334    fn invoke(
335        &mut self,
336        llty: &'ll Type,
337        fn_attrs: Option<&CodegenFnAttrs>,
338        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
339        llfn: &'ll Value,
340        args: &[&'ll Value],
341        then: &'ll BasicBlock,
342        catch: &'ll BasicBlock,
343        funclet: Option<&Funclet<'ll>>,
344        instance: Option<Instance<'tcx>>,
345    ) -> &'ll Value {
346        debug!("invoke {:?} with args ({:?})", llfn, args);
347
348        let args = self.check_call("invoke", llty, llfn, args);
349        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
350        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
351        if let Some(funclet_bundle) = funclet_bundle {
352            bundles.push(funclet_bundle);
353        }
354
355        // Emit CFI pointer type membership test
356        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
357
358        // Emit KCFI operand bundle
359        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
360        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
361            bundles.push(kcfi_bundle);
362        }
363
364        let invoke = unsafe {
365            llvm::LLVMBuildInvokeWithOperandBundles(
366                self.llbuilder,
367                llty,
368                llfn,
369                args.as_ptr(),
370                args.len() as c_uint,
371                then,
372                catch,
373                bundles.as_ptr(),
374                bundles.len() as c_uint,
375                UNNAMED,
376            )
377        };
378        if let Some(fn_abi) = fn_abi {
379            fn_abi.apply_attrs_callsite(self, invoke);
380        }
381        invoke
382    }
383
384    fn unreachable(&mut self) {
385        unsafe {
386            llvm::LLVMBuildUnreachable(self.llbuilder);
387        }
388    }
389
390    math_builder_methods! {
391        add(a, b) => LLVMBuildAdd,
392        fadd(a, b) => LLVMBuildFAdd,
393        sub(a, b) => LLVMBuildSub,
394        fsub(a, b) => LLVMBuildFSub,
395        mul(a, b) => LLVMBuildMul,
396        fmul(a, b) => LLVMBuildFMul,
397        udiv(a, b) => LLVMBuildUDiv,
398        exactudiv(a, b) => LLVMBuildExactUDiv,
399        sdiv(a, b) => LLVMBuildSDiv,
400        exactsdiv(a, b) => LLVMBuildExactSDiv,
401        fdiv(a, b) => LLVMBuildFDiv,
402        urem(a, b) => LLVMBuildURem,
403        srem(a, b) => LLVMBuildSRem,
404        frem(a, b) => LLVMBuildFRem,
405        shl(a, b) => LLVMBuildShl,
406        lshr(a, b) => LLVMBuildLShr,
407        ashr(a, b) => LLVMBuildAShr,
408        and(a, b) => LLVMBuildAnd,
409        or(a, b) => LLVMBuildOr,
410        xor(a, b) => LLVMBuildXor,
411        neg(x) => LLVMBuildNeg,
412        fneg(x) => LLVMBuildFNeg,
413        not(x) => LLVMBuildNot,
414        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
415        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
416        unchecked_ssub(x, y) => LLVMBuildNSWSub,
417        unchecked_usub(x, y) => LLVMBuildNUWSub,
418        unchecked_smul(x, y) => LLVMBuildNSWMul,
419        unchecked_umul(x, y) => LLVMBuildNUWMul,
420    }
421
422    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
423        unsafe {
424            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
425            if llvm::LLVMIsAInstruction(add).is_some() {
426                llvm::LLVMSetNUW(add, True);
427                llvm::LLVMSetNSW(add, True);
428            }
429            add
430        }
431    }
432    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
433        unsafe {
434            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
435            if llvm::LLVMIsAInstruction(sub).is_some() {
436                llvm::LLVMSetNUW(sub, True);
437                llvm::LLVMSetNSW(sub, True);
438            }
439            sub
440        }
441    }
442    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
443        unsafe {
444            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
445            if llvm::LLVMIsAInstruction(mul).is_some() {
446                llvm::LLVMSetNUW(mul, True);
447                llvm::LLVMSetNSW(mul, True);
448            }
449            mul
450        }
451    }
452
453    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
454        unsafe {
455            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
456
457            // If a and b are both values, then `or` is a value, rather than
458            // an instruction, so we need to check before setting the flag.
459            // (See also `LLVMBuildNUWNeg` which also needs a check.)
460            if llvm::LLVMIsAInstruction(or).is_some() {
461                llvm::LLVMSetIsDisjoint(or, True);
462            }
463            or
464        }
465    }
466
467    set_math_builder_methods! {
468        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
469        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
470        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
471        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
472        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
473        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
474        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
475        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
476        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
477        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
478    }
479
480    fn checked_binop(
481        &mut self,
482        oop: OverflowOp,
483        ty: Ty<'tcx>,
484        lhs: Self::Value,
485        rhs: Self::Value,
486    ) -> (Self::Value, Self::Value) {
487        let (size, signed) = ty.int_size_and_signed(self.tcx);
488        let width = size.bits();
489
490        if oop == OverflowOp::Sub && !signed {
491            // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
492            // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
493            // in the backend if profitable.
494            let sub = self.sub(lhs, rhs);
495            let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
496            return (sub, cmp);
497        }
498
499        let oop_str = match oop {
500            OverflowOp::Add => "add",
501            OverflowOp::Sub => "sub",
502            OverflowOp::Mul => "mul",
503        };
504
505        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
506
507        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
508        (self.extract_value(res, 0), self.extract_value(res, 1))
509    }
510
511    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
512        if self.cx().val_ty(val) == self.cx().type_i1() {
513            self.zext(val, self.cx().type_i8())
514        } else {
515            val
516        }
517    }
518
519    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
520        if scalar.is_bool() {
521            return self.unchecked_utrunc(val, self.cx().type_i1());
522        }
523        val
524    }
525
526    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
527        let mut bx = Builder::with_cx(self.cx);
528        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
529        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
530        unsafe {
531            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
532            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
533            // Cast to default addrspace if necessary
534            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
535        }
536    }
537
538    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
539        unsafe {
540            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
541            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
542            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
543            load
544        }
545    }
546
547    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
548        unsafe {
549            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
550            llvm::LLVMSetVolatile(load, llvm::True);
551            load
552        }
553    }
554
555    fn atomic_load(
556        &mut self,
557        ty: &'ll Type,
558        ptr: &'ll Value,
559        order: rustc_middle::ty::AtomicOrdering,
560        size: Size,
561    ) -> &'ll Value {
562        unsafe {
563            let load = llvm::LLVMRustBuildAtomicLoad(
564                self.llbuilder,
565                ty,
566                ptr,
567                UNNAMED,
568                AtomicOrdering::from_generic(order),
569            );
570            // LLVM requires the alignment of atomic loads to be at least the size of the type.
571            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
572            load
573        }
574    }
575
576    #[instrument(level = "trace", skip(self))]
577    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
578        if place.layout.is_unsized() {
579            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
580            if matches!(tail.kind(), ty::Foreign(..)) {
581                // Unsized locals and, at least conceptually, even unsized arguments must be copied
582                // around, which requires dynamically determining their size. Therefore, we cannot
583                // allow `extern` types here. Consult t-opsem before removing this check.
584                panic!("unsized locals must not be `extern` types");
585            }
586        }
587        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
588
589        if place.layout.is_zst() {
590            return OperandRef::zero_sized(place.layout);
591        }
592
593        #[instrument(level = "trace", skip(bx))]
594        fn scalar_load_metadata<'a, 'll, 'tcx>(
595            bx: &mut Builder<'a, 'll, 'tcx>,
596            load: &'ll Value,
597            scalar: abi::Scalar,
598            layout: TyAndLayout<'tcx>,
599            offset: Size,
600        ) {
601            if bx.cx.sess().opts.optimize == OptLevel::No {
602                // Don't emit metadata we're not going to use
603                return;
604            }
605
606            if !scalar.is_uninit_valid() {
607                bx.noundef_metadata(load);
608            }
609
610            match scalar.primitive() {
611                abi::Primitive::Int(..) => {
612                    if !scalar.is_always_valid(bx) {
613                        bx.range_metadata(load, scalar.valid_range(bx));
614                    }
615                }
616                abi::Primitive::Pointer(_) => {
617                    if !scalar.valid_range(bx).contains(0) {
618                        bx.nonnull_metadata(load);
619                    }
620
621                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
622                        if let Some(_) = pointee.safe {
623                            bx.align_metadata(load, pointee.align);
624                        }
625                    }
626                }
627                abi::Primitive::Float(_) => {}
628            }
629        }
630
631        let val = if let Some(_) = place.val.llextra {
632            // FIXME: Merge with the `else` below?
633            OperandValue::Ref(place.val)
634        } else if place.layout.is_llvm_immediate() {
635            let mut const_llval = None;
636            let llty = place.layout.llvm_type(self);
637            if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
638                if llvm::LLVMIsGlobalConstant(global) == llvm::True {
639                    if let Some(init) = llvm::LLVMGetInitializer(global) {
640                        if self.val_ty(init) == llty {
641                            const_llval = Some(init);
642                        }
643                    }
644                }
645            }
646
647            let llval = const_llval.unwrap_or_else(|| {
648                let load = self.load(llty, place.val.llval, place.val.align);
649                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
650                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
651                    self.to_immediate_scalar(load, scalar)
652                } else {
653                    load
654                }
655            });
656            OperandValue::Immediate(llval)
657        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
658            let b_offset = a.size(self).align_to(b.align(self).abi);
659
660            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
661                let llptr = if i == 0 {
662                    place.val.llval
663                } else {
664                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
665                };
666                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
667                let load = self.load(llty, llptr, align);
668                scalar_load_metadata(self, load, scalar, layout, offset);
669                self.to_immediate_scalar(load, scalar)
670            };
671
672            OperandValue::Pair(
673                load(0, a, place.layout, place.val.align, Size::ZERO),
674                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
675            )
676        } else {
677            OperandValue::Ref(place.val)
678        };
679
680        OperandRef { val, layout: place.layout }
681    }
682
683    fn write_operand_repeatedly(
684        &mut self,
685        cg_elem: OperandRef<'tcx, &'ll Value>,
686        count: u64,
687        dest: PlaceRef<'tcx, &'ll Value>,
688    ) {
689        let zero = self.const_usize(0);
690        let count = self.const_usize(count);
691
692        let header_bb = self.append_sibling_block("repeat_loop_header");
693        let body_bb = self.append_sibling_block("repeat_loop_body");
694        let next_bb = self.append_sibling_block("repeat_loop_next");
695
696        self.br(header_bb);
697
698        let mut header_bx = Self::build(self.cx, header_bb);
699        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
700
701        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
702        header_bx.cond_br(keep_going, body_bb, next_bb);
703
704        let mut body_bx = Self::build(self.cx, body_bb);
705        let dest_elem = dest.project_index(&mut body_bx, i);
706        cg_elem.val.store(&mut body_bx, dest_elem);
707
708        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
709        body_bx.br(header_bb);
710        header_bx.add_incoming_to_phi(i, next, body_bb);
711
712        *self = Self::build(self.cx, next_bb);
713    }
714
715    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
716        if self.cx.sess().opts.optimize == OptLevel::No {
717            // Don't emit metadata we're not going to use
718            return;
719        }
720
721        unsafe {
722            let llty = self.cx.val_ty(load);
723            let md = [
724                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
725                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
726            ];
727            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
728            self.set_metadata(load, llvm::MD_range, md);
729        }
730    }
731
732    fn nonnull_metadata(&mut self, load: &'ll Value) {
733        unsafe {
734            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
735            self.set_metadata(load, llvm::MD_nonnull, md);
736        }
737    }
738
739    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
740        self.store_with_flags(val, ptr, align, MemFlags::empty())
741    }
742
743    fn store_with_flags(
744        &mut self,
745        val: &'ll Value,
746        ptr: &'ll Value,
747        align: Align,
748        flags: MemFlags,
749    ) -> &'ll Value {
750        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
751        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
752        unsafe {
753            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
754            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
755            let align =
756                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
757            llvm::LLVMSetAlignment(store, align);
758            if flags.contains(MemFlags::VOLATILE) {
759                llvm::LLVMSetVolatile(store, llvm::True);
760            }
761            if flags.contains(MemFlags::NONTEMPORAL) {
762                // Make sure that the current target architectures supports "sane" non-temporal
763                // stores, i.e., non-temporal stores that are equivalent to regular stores except
764                // for performance. LLVM doesn't seem to care about this, and will happily treat
765                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
766                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
767                // regular stores but require special fences. So we keep a list of architectures
768                // where `!nontemporal` is known to be truly just a hint, and use regular stores
769                // everywhere else. (In the future, we could alternatively ensure that an sfence
770                // gets emitted after a sequence of movnt before any kind of synchronizing
771                // operation. But it's not clear how to do that with LLVM.)
772                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
773                // <https://github.com/llvm/llvm-project/issues/64521>.
774                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
775                    &["aarch64", "arm", "riscv32", "riscv64"];
776
777                let use_nontemporal =
778                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
779                if use_nontemporal {
780                    // According to LLVM [1] building a nontemporal store must
781                    // *always* point to a metadata value of the integer 1.
782                    //
783                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
784                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
785                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
786                    self.set_metadata(store, llvm::MD_nontemporal, md);
787                }
788            }
789            store
790        }
791    }
792
793    fn atomic_store(
794        &mut self,
795        val: &'ll Value,
796        ptr: &'ll Value,
797        order: rustc_middle::ty::AtomicOrdering,
798        size: Size,
799    ) {
800        debug!("Store {:?} -> {:?}", val, ptr);
801        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
802        unsafe {
803            let store = llvm::LLVMRustBuildAtomicStore(
804                self.llbuilder,
805                val,
806                ptr,
807                AtomicOrdering::from_generic(order),
808            );
809            // LLVM requires the alignment of atomic stores to be at least the size of the type.
810            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
811        }
812    }
813
814    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
815        unsafe {
816            llvm::LLVMBuildGEPWithNoWrapFlags(
817                self.llbuilder,
818                ty,
819                ptr,
820                indices.as_ptr(),
821                indices.len() as c_uint,
822                UNNAMED,
823                GEPNoWrapFlags::default(),
824            )
825        }
826    }
827
828    fn inbounds_gep(
829        &mut self,
830        ty: &'ll Type,
831        ptr: &'ll Value,
832        indices: &[&'ll Value],
833    ) -> &'ll Value {
834        unsafe {
835            llvm::LLVMBuildGEPWithNoWrapFlags(
836                self.llbuilder,
837                ty,
838                ptr,
839                indices.as_ptr(),
840                indices.len() as c_uint,
841                UNNAMED,
842                GEPNoWrapFlags::InBounds,
843            )
844        }
845    }
846
847    fn inbounds_nuw_gep(
848        &mut self,
849        ty: &'ll Type,
850        ptr: &'ll Value,
851        indices: &[&'ll Value],
852    ) -> &'ll Value {
853        unsafe {
854            llvm::LLVMBuildGEPWithNoWrapFlags(
855                self.llbuilder,
856                ty,
857                ptr,
858                indices.as_ptr(),
859                indices.len() as c_uint,
860                UNNAMED,
861                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
862            )
863        }
864    }
865
866    /* Casts */
867    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
868        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
869    }
870
871    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
872        debug_assert_ne!(self.val_ty(val), dest_ty);
873
874        let trunc = self.trunc(val, dest_ty);
875        unsafe {
876            if llvm::LLVMIsAInstruction(trunc).is_some() {
877                llvm::LLVMSetNUW(trunc, True);
878            }
879        }
880        trunc
881    }
882
883    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
884        debug_assert_ne!(self.val_ty(val), dest_ty);
885
886        let trunc = self.trunc(val, dest_ty);
887        unsafe {
888            if llvm::LLVMIsAInstruction(trunc).is_some() {
889                llvm::LLVMSetNSW(trunc, True);
890            }
891        }
892        trunc
893    }
894
895    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
896        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
897    }
898
899    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
900        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
901    }
902
903    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
904        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
905    }
906
907    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
908        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
909        // poor codegen. The reason for this is that the corresponding wasm
910        // instructions, `i32.trunc_f32_s` for example, will trap when the float
911        // is out-of-bounds, infinity, or nan. This means that LLVM
912        // automatically inserts control flow around `fptoui` and `fptosi`
913        // because the LLVM instruction `fptoui` is defined as producing a
914        // poison value, not having UB on out-of-bounds values.
915        //
916        // This method, however, is only used with non-saturating casts that
917        // have UB on out-of-bounds values. This means that it's ok if we use
918        // the raw wasm instruction since out-of-bounds values can do whatever
919        // we like. To ensure that LLVM picks the right instruction we choose
920        // the raw wasm intrinsic functions which avoid LLVM inserting all the
921        // other control flow automatically.
922        if self.sess().target.is_like_wasm {
923            let src_ty = self.cx.val_ty(val);
924            if self.cx.type_kind(src_ty) != TypeKind::Vector {
925                let float_width = self.cx.float_width(src_ty);
926                let int_width = self.cx.int_width(dest_ty);
927                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
928                    return self.call_intrinsic(
929                        "llvm.wasm.trunc.unsigned",
930                        &[dest_ty, src_ty],
931                        &[val],
932                    );
933                }
934            }
935        }
936        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
937    }
938
939    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
940        // see `fptoui` above for why wasm is different here
941        if self.sess().target.is_like_wasm {
942            let src_ty = self.cx.val_ty(val);
943            if self.cx.type_kind(src_ty) != TypeKind::Vector {
944                let float_width = self.cx.float_width(src_ty);
945                let int_width = self.cx.int_width(dest_ty);
946                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
947                    return self.call_intrinsic(
948                        "llvm.wasm.trunc.signed",
949                        &[dest_ty, src_ty],
950                        &[val],
951                    );
952                }
953            }
954        }
955        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
956    }
957
958    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
959        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
960    }
961
962    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
963        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
964    }
965
966    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
967        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
968    }
969
970    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
971        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
972    }
973
974    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
975        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
976    }
977
978    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
979        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
980    }
981
982    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
983        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
984    }
985
986    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
987        unsafe {
988            llvm::LLVMBuildIntCast2(
989                self.llbuilder,
990                val,
991                dest_ty,
992                if is_signed { True } else { False },
993                UNNAMED,
994            )
995        }
996    }
997
998    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
999        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1000    }
1001
1002    /* Comparisons */
1003    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1004        let op = llvm::IntPredicate::from_generic(op);
1005        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1006    }
1007
1008    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1009        let op = llvm::RealPredicate::from_generic(op);
1010        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1011    }
1012
1013    fn three_way_compare(
1014        &mut self,
1015        ty: Ty<'tcx>,
1016        lhs: Self::Value,
1017        rhs: Self::Value,
1018    ) -> Option<Self::Value> {
1019        // FIXME: See comment on the definition of `three_way_compare`.
1020        if crate::llvm_util::get_version() < (20, 0, 0) {
1021            return None;
1022        }
1023
1024        let size = ty.primitive_size(self.tcx);
1025        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1026
1027        Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
1028    }
1029
1030    /* Miscellaneous instructions */
1031    fn memcpy(
1032        &mut self,
1033        dst: &'ll Value,
1034        dst_align: Align,
1035        src: &'ll Value,
1036        src_align: Align,
1037        size: &'ll Value,
1038        flags: MemFlags,
1039    ) {
1040        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1041        let size = self.intcast(size, self.type_isize(), false);
1042        let is_volatile = flags.contains(MemFlags::VOLATILE);
1043        unsafe {
1044            llvm::LLVMRustBuildMemCpy(
1045                self.llbuilder,
1046                dst,
1047                dst_align.bytes() as c_uint,
1048                src,
1049                src_align.bytes() as c_uint,
1050                size,
1051                is_volatile,
1052            );
1053        }
1054    }
1055
1056    fn memmove(
1057        &mut self,
1058        dst: &'ll Value,
1059        dst_align: Align,
1060        src: &'ll Value,
1061        src_align: Align,
1062        size: &'ll Value,
1063        flags: MemFlags,
1064    ) {
1065        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1066        let size = self.intcast(size, self.type_isize(), false);
1067        let is_volatile = flags.contains(MemFlags::VOLATILE);
1068        unsafe {
1069            llvm::LLVMRustBuildMemMove(
1070                self.llbuilder,
1071                dst,
1072                dst_align.bytes() as c_uint,
1073                src,
1074                src_align.bytes() as c_uint,
1075                size,
1076                is_volatile,
1077            );
1078        }
1079    }
1080
1081    fn memset(
1082        &mut self,
1083        ptr: &'ll Value,
1084        fill_byte: &'ll Value,
1085        size: &'ll Value,
1086        align: Align,
1087        flags: MemFlags,
1088    ) {
1089        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1090        let is_volatile = flags.contains(MemFlags::VOLATILE);
1091        unsafe {
1092            llvm::LLVMRustBuildMemSet(
1093                self.llbuilder,
1094                ptr,
1095                align.bytes() as c_uint,
1096                fill_byte,
1097                size,
1098                is_volatile,
1099            );
1100        }
1101    }
1102
1103    fn select(
1104        &mut self,
1105        cond: &'ll Value,
1106        then_val: &'ll Value,
1107        else_val: &'ll Value,
1108    ) -> &'ll Value {
1109        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1110    }
1111
1112    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1113        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1114    }
1115
1116    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1117        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1118    }
1119
1120    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1121        unsafe {
1122            let elt_ty = self.cx.val_ty(elt);
1123            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1124            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1125            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1126            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1127        }
1128    }
1129
1130    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1131        assert_eq!(idx as c_uint as u64, idx);
1132        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1133    }
1134
1135    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1136        assert_eq!(idx as c_uint as u64, idx);
1137        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1138    }
1139
1140    fn set_personality_fn(&mut self, personality: &'ll Value) {
1141        unsafe {
1142            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1143        }
1144    }
1145
1146    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1147        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1148        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1149        unsafe {
1150            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1151        }
1152        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1153    }
1154
1155    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1156        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1157        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1158        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1159    }
1160
1161    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1162        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1163        let mut exn = self.const_poison(ty);
1164        exn = self.insert_value(exn, exn0, 0);
1165        exn = self.insert_value(exn, exn1, 1);
1166        unsafe {
1167            llvm::LLVMBuildResume(self.llbuilder, exn);
1168        }
1169    }
1170
1171    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1172        let ret = unsafe {
1173            llvm::LLVMBuildCleanupPad(
1174                self.llbuilder,
1175                parent,
1176                args.as_ptr(),
1177                args.len() as c_uint,
1178                c"cleanuppad".as_ptr(),
1179            )
1180        };
1181        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1182    }
1183
1184    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1185        unsafe {
1186            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1187                .expect("LLVM does not have support for cleanupret");
1188        }
1189    }
1190
1191    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1192        let ret = unsafe {
1193            llvm::LLVMBuildCatchPad(
1194                self.llbuilder,
1195                parent,
1196                args.as_ptr(),
1197                args.len() as c_uint,
1198                c"catchpad".as_ptr(),
1199            )
1200        };
1201        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1202    }
1203
1204    fn catch_switch(
1205        &mut self,
1206        parent: Option<&'ll Value>,
1207        unwind: Option<&'ll BasicBlock>,
1208        handlers: &[&'ll BasicBlock],
1209    ) -> &'ll Value {
1210        let ret = unsafe {
1211            llvm::LLVMBuildCatchSwitch(
1212                self.llbuilder,
1213                parent,
1214                unwind,
1215                handlers.len() as c_uint,
1216                c"catchswitch".as_ptr(),
1217            )
1218        };
1219        let ret = ret.expect("LLVM does not have support for catchswitch");
1220        for handler in handlers {
1221            unsafe {
1222                llvm::LLVMAddHandler(ret, handler);
1223            }
1224        }
1225        ret
1226    }
1227
1228    // Atomic Operations
1229    fn atomic_cmpxchg(
1230        &mut self,
1231        dst: &'ll Value,
1232        cmp: &'ll Value,
1233        src: &'ll Value,
1234        order: rustc_middle::ty::AtomicOrdering,
1235        failure_order: rustc_middle::ty::AtomicOrdering,
1236        weak: bool,
1237    ) -> (&'ll Value, &'ll Value) {
1238        let weak = if weak { llvm::True } else { llvm::False };
1239        unsafe {
1240            let value = llvm::LLVMBuildAtomicCmpXchg(
1241                self.llbuilder,
1242                dst,
1243                cmp,
1244                src,
1245                AtomicOrdering::from_generic(order),
1246                AtomicOrdering::from_generic(failure_order),
1247                llvm::False, // SingleThreaded
1248            );
1249            llvm::LLVMSetWeak(value, weak);
1250            let val = self.extract_value(value, 0);
1251            let success = self.extract_value(value, 1);
1252            (val, success)
1253        }
1254    }
1255
1256    fn atomic_rmw(
1257        &mut self,
1258        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1259        dst: &'ll Value,
1260        mut src: &'ll Value,
1261        order: rustc_middle::ty::AtomicOrdering,
1262    ) -> &'ll Value {
1263        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1264        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1265            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1266        if requires_cast_to_int {
1267            src = self.ptrtoint(src, self.type_isize());
1268        }
1269        let mut res = unsafe {
1270            llvm::LLVMBuildAtomicRMW(
1271                self.llbuilder,
1272                AtomicRmwBinOp::from_generic(op),
1273                dst,
1274                src,
1275                AtomicOrdering::from_generic(order),
1276                llvm::False, // SingleThreaded
1277            )
1278        };
1279        if requires_cast_to_int {
1280            res = self.inttoptr(res, self.type_ptr());
1281        }
1282        res
1283    }
1284
1285    fn atomic_fence(
1286        &mut self,
1287        order: rustc_middle::ty::AtomicOrdering,
1288        scope: SynchronizationScope,
1289    ) {
1290        let single_threaded = match scope {
1291            SynchronizationScope::SingleThread => llvm::True,
1292            SynchronizationScope::CrossThread => llvm::False,
1293        };
1294        unsafe {
1295            llvm::LLVMBuildFence(
1296                self.llbuilder,
1297                AtomicOrdering::from_generic(order),
1298                single_threaded,
1299                UNNAMED,
1300            );
1301        }
1302    }
1303
1304    fn set_invariant_load(&mut self, load: &'ll Value) {
1305        unsafe {
1306            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1307            self.set_metadata(load, llvm::MD_invariant_load, md);
1308        }
1309    }
1310
1311    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1312        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1313    }
1314
1315    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1316        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1317    }
1318
1319    fn call(
1320        &mut self,
1321        llty: &'ll Type,
1322        fn_attrs: Option<&CodegenFnAttrs>,
1323        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1324        llfn: &'ll Value,
1325        args: &[&'ll Value],
1326        funclet: Option<&Funclet<'ll>>,
1327        instance: Option<Instance<'tcx>>,
1328    ) -> &'ll Value {
1329        debug!("call {:?} with args ({:?})", llfn, args);
1330
1331        let args = self.check_call("call", llty, llfn, args);
1332        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1333        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1334        if let Some(funclet_bundle) = funclet_bundle {
1335            bundles.push(funclet_bundle);
1336        }
1337
1338        // Emit CFI pointer type membership test
1339        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1340
1341        // Emit KCFI operand bundle
1342        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1343        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1344            bundles.push(kcfi_bundle);
1345        }
1346
1347        let call = unsafe {
1348            llvm::LLVMBuildCallWithOperandBundles(
1349                self.llbuilder,
1350                llty,
1351                llfn,
1352                args.as_ptr() as *const &llvm::Value,
1353                args.len() as c_uint,
1354                bundles.as_ptr(),
1355                bundles.len() as c_uint,
1356                c"".as_ptr(),
1357            )
1358        };
1359        if let Some(fn_abi) = fn_abi {
1360            fn_abi.apply_attrs_callsite(self, call);
1361        }
1362        call
1363    }
1364
1365    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1366        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1367    }
1368
1369    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1370        // Cleanup is always the cold path.
1371        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1372        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1373    }
1374}
1375
1376impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1377    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1378        // Forward to the `get_static` method of `CodegenCx`
1379        let global = self.cx().get_static(def_id);
1380        if self.cx().tcx.is_thread_local_static(def_id) {
1381            let pointer =
1382                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1383            // Cast to default address space if globals are in a different addrspace
1384            self.pointercast(pointer, self.type_ptr())
1385        } else {
1386            // Cast to default address space if globals are in a different addrspace
1387            self.cx().const_pointercast(global, self.type_ptr())
1388        }
1389    }
1390}
1391
1392impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1393    pub(crate) fn llfn(&self) -> &'ll Value {
1394        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1395    }
1396}
1397
1398impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1399    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1400        unsafe {
1401            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1402        }
1403    }
1404}
1405impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1406    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1407        unsafe {
1408            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1409            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1410            self.set_metadata(load, llvm::MD_align, md);
1411        }
1412    }
1413
1414    fn noundef_metadata(&mut self, load: &'ll Value) {
1415        unsafe {
1416            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1417            self.set_metadata(load, llvm::MD_noundef, md);
1418        }
1419    }
1420
1421    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1422        unsafe {
1423            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1424            self.set_metadata(inst, llvm::MD_unpredictable, md);
1425        }
1426    }
1427}
1428impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1429    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1430        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1431    }
1432
1433    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1434        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1435    }
1436
1437    pub(crate) fn insert_element(
1438        &mut self,
1439        vec: &'ll Value,
1440        elt: &'ll Value,
1441        idx: &'ll Value,
1442    ) -> &'ll Value {
1443        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1444    }
1445
1446    pub(crate) fn shuffle_vector(
1447        &mut self,
1448        v1: &'ll Value,
1449        v2: &'ll Value,
1450        mask: &'ll Value,
1451    ) -> &'ll Value {
1452        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1453    }
1454
1455    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1456        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1457    }
1458    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1459        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1460    }
1461    pub(crate) fn vector_reduce_fadd_reassoc(
1462        &mut self,
1463        acc: &'ll Value,
1464        src: &'ll Value,
1465    ) -> &'ll Value {
1466        unsafe {
1467            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1468            llvm::LLVMRustSetAllowReassoc(instr);
1469            instr
1470        }
1471    }
1472    pub(crate) fn vector_reduce_fmul_reassoc(
1473        &mut self,
1474        acc: &'ll Value,
1475        src: &'ll Value,
1476    ) -> &'ll Value {
1477        unsafe {
1478            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1479            llvm::LLVMRustSetAllowReassoc(instr);
1480            instr
1481        }
1482    }
1483    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1484        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1485    }
1486    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1487        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1488    }
1489    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1490        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1491    }
1492    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1493        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1494    }
1495    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1496        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1497    }
1498    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1499        unsafe {
1500            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1501        }
1502    }
1503    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1504        unsafe {
1505            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1506        }
1507    }
1508    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1509        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1510    }
1511    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1512        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1513    }
1514
1515    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1516        unsafe {
1517            llvm::LLVMAddClause(landing_pad, clause);
1518        }
1519    }
1520
1521    pub(crate) fn catch_ret(
1522        &mut self,
1523        funclet: &Funclet<'ll>,
1524        unwind: &'ll BasicBlock,
1525    ) -> &'ll Value {
1526        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1527        ret.expect("LLVM does not have support for catchret")
1528    }
1529
1530    fn check_call<'b>(
1531        &mut self,
1532        typ: &str,
1533        fn_ty: &'ll Type,
1534        llfn: &'ll Value,
1535        args: &'b [&'ll Value],
1536    ) -> Cow<'b, [&'ll Value]> {
1537        assert!(
1538            self.cx.type_kind(fn_ty) == TypeKind::Function,
1539            "builder::{typ} not passed a function, but {fn_ty:?}"
1540        );
1541
1542        let param_tys = self.cx.func_params_types(fn_ty);
1543
1544        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1545            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1546
1547        if all_args_match {
1548            return Cow::Borrowed(args);
1549        }
1550
1551        let casted_args: Vec<_> = iter::zip(param_tys, args)
1552            .enumerate()
1553            .map(|(i, (expected_ty, &actual_val))| {
1554                let actual_ty = self.cx.val_ty(actual_val);
1555                if expected_ty != actual_ty {
1556                    debug!(
1557                        "type mismatch in function call of {:?}. \
1558                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1559                        llfn, expected_ty, i, actual_ty
1560                    );
1561                    self.bitcast(actual_val, expected_ty)
1562                } else {
1563                    actual_val
1564                }
1565            })
1566            .collect();
1567
1568        Cow::Owned(casted_args)
1569    }
1570
1571    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1572        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1573    }
1574}
1575
1576impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1577    pub(crate) fn call_intrinsic(
1578        &mut self,
1579        base_name: impl Into<Cow<'static, str>>,
1580        type_params: &[&'ll Type],
1581        args: &[&'ll Value],
1582    ) -> &'ll Value {
1583        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1584        self.call(ty, None, None, f, args, None, None)
1585    }
1586
1587    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1588        let size = size.bytes();
1589        if size == 0 {
1590            return;
1591        }
1592
1593        if !self.cx().sess().emit_lifetime_markers() {
1594            return;
1595        }
1596
1597        self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1598    }
1599}
1600impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1601    pub(crate) fn phi(
1602        &mut self,
1603        ty: &'ll Type,
1604        vals: &[&'ll Value],
1605        bbs: &[&'ll BasicBlock],
1606    ) -> &'ll Value {
1607        assert_eq!(vals.len(), bbs.len());
1608        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1609        unsafe {
1610            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1611            phi
1612        }
1613    }
1614
1615    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1616        unsafe {
1617            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1618        }
1619    }
1620}
1621impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1622    pub(crate) fn landing_pad(
1623        &mut self,
1624        ty: &'ll Type,
1625        pers_fn: &'ll Value,
1626        num_clauses: usize,
1627    ) -> &'ll Value {
1628        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1629        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1630        // personality lives on the parent function anyway.
1631        self.set_personality_fn(pers_fn);
1632        unsafe {
1633            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1634        }
1635    }
1636
1637    pub(crate) fn callbr(
1638        &mut self,
1639        llty: &'ll Type,
1640        fn_attrs: Option<&CodegenFnAttrs>,
1641        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1642        llfn: &'ll Value,
1643        args: &[&'ll Value],
1644        default_dest: &'ll BasicBlock,
1645        indirect_dest: &[&'ll BasicBlock],
1646        funclet: Option<&Funclet<'ll>>,
1647        instance: Option<Instance<'tcx>>,
1648    ) -> &'ll Value {
1649        debug!("invoke {:?} with args ({:?})", llfn, args);
1650
1651        let args = self.check_call("callbr", llty, llfn, args);
1652        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1653        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1654        if let Some(funclet_bundle) = funclet_bundle {
1655            bundles.push(funclet_bundle);
1656        }
1657
1658        // Emit CFI pointer type membership test
1659        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1660
1661        // Emit KCFI operand bundle
1662        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1663        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1664            bundles.push(kcfi_bundle);
1665        }
1666
1667        let callbr = unsafe {
1668            llvm::LLVMBuildCallBr(
1669                self.llbuilder,
1670                llty,
1671                llfn,
1672                default_dest,
1673                indirect_dest.as_ptr(),
1674                indirect_dest.len() as c_uint,
1675                args.as_ptr(),
1676                args.len() as c_uint,
1677                bundles.as_ptr(),
1678                bundles.len() as c_uint,
1679                UNNAMED,
1680            )
1681        };
1682        if let Some(fn_abi) = fn_abi {
1683            fn_abi.apply_attrs_callsite(self, callbr);
1684        }
1685        callbr
1686    }
1687
1688    // Emits CFI pointer type membership tests.
1689    fn cfi_type_test(
1690        &mut self,
1691        fn_attrs: Option<&CodegenFnAttrs>,
1692        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1693        instance: Option<Instance<'tcx>>,
1694        llfn: &'ll Value,
1695    ) {
1696        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1697        if self.tcx.sess.is_sanitizer_cfi_enabled()
1698            && let Some(fn_abi) = fn_abi
1699            && is_indirect_call
1700        {
1701            if let Some(fn_attrs) = fn_attrs
1702                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1703            {
1704                return;
1705            }
1706
1707            let mut options = cfi::TypeIdOptions::empty();
1708            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1709                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1710            }
1711            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1712                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1713            }
1714
1715            let typeid = if let Some(instance) = instance {
1716                cfi::typeid_for_instance(self.tcx, instance, options)
1717            } else {
1718                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1719            };
1720            let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
1721            let dbg_loc = self.get_dbg_loc();
1722
1723            // Test whether the function pointer is associated with the type identifier using the
1724            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1725            // calls to this intrinsic with code to test type membership.
1726            let typeid = self.get_metadata_value(typeid_metadata);
1727            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1728            let bb_pass = self.append_sibling_block("type_test.pass");
1729            let bb_fail = self.append_sibling_block("type_test.fail");
1730            self.cond_br(cond, bb_pass, bb_fail);
1731
1732            self.switch_to_block(bb_fail);
1733            if let Some(dbg_loc) = dbg_loc {
1734                self.set_dbg_loc(dbg_loc);
1735            }
1736            self.abort();
1737            self.unreachable();
1738
1739            self.switch_to_block(bb_pass);
1740            if let Some(dbg_loc) = dbg_loc {
1741                self.set_dbg_loc(dbg_loc);
1742            }
1743        }
1744    }
1745
1746    // Emits KCFI operand bundles.
1747    fn kcfi_operand_bundle(
1748        &mut self,
1749        fn_attrs: Option<&CodegenFnAttrs>,
1750        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1751        instance: Option<Instance<'tcx>>,
1752        llfn: &'ll Value,
1753    ) -> Option<llvm::OperandBundleBox<'ll>> {
1754        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1755        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1756            && let Some(fn_abi) = fn_abi
1757            && is_indirect_call
1758        {
1759            if let Some(fn_attrs) = fn_attrs
1760                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1761            {
1762                return None;
1763            }
1764
1765            let mut options = kcfi::TypeIdOptions::empty();
1766            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1767                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1768            }
1769            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1770                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1771            }
1772
1773            let kcfi_typeid = if let Some(instance) = instance {
1774                kcfi::typeid_for_instance(self.tcx, instance, options)
1775            } else {
1776                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1777            };
1778
1779            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1780        } else {
1781            None
1782        };
1783        kcfi_bundle
1784    }
1785
1786    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1787    #[instrument(level = "debug", skip(self))]
1788    pub(crate) fn instrprof_increment(
1789        &mut self,
1790        fn_name: &'ll Value,
1791        hash: &'ll Value,
1792        num_counters: &'ll Value,
1793        index: &'ll Value,
1794    ) {
1795        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1796    }
1797
1798    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1799    ///
1800    /// This doesn't produce any code directly, but is used as input by
1801    /// the LLVM pass that handles coverage instrumentation.
1802    ///
1803    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1804    ///
1805    /// [`CodeGenPGO::emitMCDCParameters`]:
1806    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1807    #[instrument(level = "debug", skip(self))]
1808    pub(crate) fn mcdc_parameters(
1809        &mut self,
1810        fn_name: &'ll Value,
1811        hash: &'ll Value,
1812        bitmap_bits: &'ll Value,
1813    ) {
1814        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]);
1815    }
1816
1817    #[instrument(level = "debug", skip(self))]
1818    pub(crate) fn mcdc_tvbitmap_update(
1819        &mut self,
1820        fn_name: &'ll Value,
1821        hash: &'ll Value,
1822        bitmap_index: &'ll Value,
1823        mcdc_temp: &'ll Value,
1824    ) {
1825        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1826        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args);
1827    }
1828
1829    #[instrument(level = "debug", skip(self))]
1830    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1831        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1832    }
1833
1834    #[instrument(level = "debug", skip(self))]
1835    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1836        let align = self.tcx.data_layout.i32_align.abi;
1837        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1838        let new_tv_index = self.add(current_tv_index, cond_index);
1839        self.store(new_tv_index, mcdc_temp, align);
1840    }
1841}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
