rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::bug;
18use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
19use rustc_middle::ty::layout::{
20    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
21    TyAndLayout,
22};
23use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
24use rustc_sanitizers::{cfi, kcfi};
25use rustc_session::config::OptLevel;
26use rustc_span::Span;
27use rustc_target::callconv::FnAbi;
28use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
29use smallvec::SmallVec;
30use tracing::{debug, instrument};
31
32use crate::abi::FnAbiLlvmExt;
33use crate::common::Funclet;
34use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
35use crate::llvm::{
36    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
37};
38use crate::type_::Type;
39use crate::type_of::LayoutLlvmExt;
40use crate::value::Value;
41use crate::{attributes, llvm_util};
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45    pub llbuilder: &'ll mut llvm::Builder<'ll>,
46    pub cx: &'a GenericCx<'ll, CX>,
47}
48
49pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
50pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
51
52impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
53    fn drop(&mut self) {
54        unsafe {
55            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
56        }
57    }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61    pub(crate) fn call(
62        &mut self,
63        llty: &'ll Type,
64        llfn: &'ll Value,
65        args: &[&'ll Value],
66        funclet: Option<&Funclet<'ll>>,
67    ) -> &'ll Value {
68        debug!("call {:?} with args ({:?})", llfn, args);
69
70        let args = self.check_call("call", llty, llfn, args);
71        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73        if let Some(funclet_bundle) = funclet_bundle {
74            bundles.push(funclet_bundle);
75        }
76
77        let call = unsafe {
78            llvm::LLVMBuildCallWithOperandBundles(
79                self.llbuilder,
80                llty,
81                llfn,
82                args.as_ptr() as *const &llvm::Value,
83                args.len() as c_uint,
84                bundles.as_ptr(),
85                bundles.len() as c_uint,
86                c"".as_ptr(),
87            )
88        };
89        call
90    }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95        // Create a fresh builder from the simple context.
96        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
97        GenericBuilder { llbuilder, cx: scx }
98    }
99
100    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
101        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
102    }
103
104    pub(crate) fn ret_void(&mut self) {
105        llvm::LLVMBuildRetVoid(self.llbuilder);
106    }
107
108    pub(crate) fn ret(&mut self, v: &'ll Value) {
109        unsafe {
110            llvm::LLVMBuildRet(self.llbuilder, v);
111        }
112    }
113
114    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
115        let bx = Self::with_cx(cx);
116        unsafe {
117            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
118        }
119        bx
120    }
121}
122
123/// Empty string, to be used where LLVM expects an instruction name, indicating
124/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
125// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
126const UNNAMED: *const c_char = c"".as_ptr();
127
128impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
129    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
130    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
131    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
132    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
133    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
134    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
135
136    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
137    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
138    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
139}
140
141impl abi::HasDataLayout for Builder<'_, '_, '_> {
142    fn data_layout(&self) -> &abi::TargetDataLayout {
143        self.cx.data_layout()
144    }
145}
146
147impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
148    #[inline]
149    fn tcx(&self) -> TyCtxt<'tcx> {
150        self.cx.tcx
151    }
152}
153
154impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
155    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
156        self.cx.typing_env()
157    }
158}
159
160impl HasTargetSpec for Builder<'_, '_, '_> {
161    #[inline]
162    fn target_spec(&self) -> &Target {
163        self.cx.target_spec()
164    }
165}
166
167impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
168    #[inline]
169    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
170        self.cx.handle_layout_err(err, span, ty)
171    }
172}
173
174impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
175    #[inline]
176    fn handle_fn_abi_err(
177        &self,
178        err: FnAbiError<'tcx>,
179        span: Span,
180        fn_abi_request: FnAbiRequest<'tcx>,
181    ) -> ! {
182        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
183    }
184}
185
186impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
187    type Target = CodegenCx<'ll, 'tcx>;
188
189    #[inline]
190    fn deref(&self) -> &Self::Target {
191        self.cx
192    }
193}
194
195macro_rules! math_builder_methods {
196    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
197        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
198            unsafe {
199                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
200            }
201        })+
202    }
203}
204
205macro_rules! set_math_builder_methods {
206    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
207        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
208            unsafe {
209                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
210                llvm::$llvm_set_math(instr);
211                instr
212            }
213        })+
214    }
215}
216
217impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
218    type CodegenCx = CodegenCx<'ll, 'tcx>;
219
220    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
221        let bx = Builder::with_cx(cx);
222        unsafe {
223            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
224        }
225        bx
226    }
227
228    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
229        self.cx
230    }
231
232    fn llbb(&self) -> &'ll BasicBlock {
233        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
234    }
235
236    fn set_span(&mut self, _span: Span) {}
237
238    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
239        unsafe {
240            let name = SmallCStr::new(name);
241            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
242        }
243    }
244
245    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
246        Self::append_block(self.cx, self.llfn(), name)
247    }
248
249    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
250        *self = Self::build(self.cx, llbb)
251    }
252
253    fn ret_void(&mut self) {
254        llvm::LLVMBuildRetVoid(self.llbuilder);
255    }
256
257    fn ret(&mut self, v: &'ll Value) {
258        unsafe {
259            llvm::LLVMBuildRet(self.llbuilder, v);
260        }
261    }
262
263    fn br(&mut self, dest: &'ll BasicBlock) {
264        unsafe {
265            llvm::LLVMBuildBr(self.llbuilder, dest);
266        }
267    }
268
269    fn cond_br(
270        &mut self,
271        cond: &'ll Value,
272        then_llbb: &'ll BasicBlock,
273        else_llbb: &'ll BasicBlock,
274    ) {
275        unsafe {
276            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
277        }
278    }
279
280    fn switch(
281        &mut self,
282        v: &'ll Value,
283        else_llbb: &'ll BasicBlock,
284        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
285    ) {
286        let switch =
287            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
288        for (on_val, dest) in cases {
289            let on_val = self.const_uint_big(self.val_ty(v), on_val);
290            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
291        }
292    }
293
294    fn switch_with_weights(
295        &mut self,
296        v: Self::Value,
297        else_llbb: Self::BasicBlock,
298        else_is_cold: bool,
299        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
300    ) {
301        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
302            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
303            return;
304        }
305
306        let id_str = "branch_weights";
307        let id = unsafe {
308            llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
309        };
310
311        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
312        // This function handles switch instructions with more than 2 targets and it needs to
313        // emit branch weights metadata instead of using the intrinsic.
314        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
315        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
316        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
317        let weight =
318            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
319
320        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
321        md.push(id);
322        md.push(weight(else_is_cold));
323
324        let switch =
325            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
326        for (on_val, dest, is_cold) in cases {
327            let on_val = self.const_uint_big(self.val_ty(v), on_val);
328            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
329            md.push(weight(is_cold));
330        }
331
332        unsafe {
333            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
334            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
335        }
336    }
337
338    fn invoke(
339        &mut self,
340        llty: &'ll Type,
341        fn_attrs: Option<&CodegenFnAttrs>,
342        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
343        llfn: &'ll Value,
344        args: &[&'ll Value],
345        then: &'ll BasicBlock,
346        catch: &'ll BasicBlock,
347        funclet: Option<&Funclet<'ll>>,
348        instance: Option<Instance<'tcx>>,
349    ) -> &'ll Value {
350        debug!("invoke {:?} with args ({:?})", llfn, args);
351
352        let args = self.check_call("invoke", llty, llfn, args);
353        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
354        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
355        if let Some(funclet_bundle) = funclet_bundle {
356            bundles.push(funclet_bundle);
357        }
358
359        // Emit CFI pointer type membership test
360        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
361
362        // Emit KCFI operand bundle
363        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
364        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) {
365            bundles.push(kcfi_bundle);
366        }
367
368        let invoke = unsafe {
369            llvm::LLVMBuildInvokeWithOperandBundles(
370                self.llbuilder,
371                llty,
372                llfn,
373                args.as_ptr(),
374                args.len() as c_uint,
375                then,
376                catch,
377                bundles.as_ptr(),
378                bundles.len() as c_uint,
379                UNNAMED,
380            )
381        };
382        if let Some(fn_abi) = fn_abi {
383            fn_abi.apply_attrs_callsite(self, invoke);
384        }
385        invoke
386    }
387
388    fn unreachable(&mut self) {
389        unsafe {
390            llvm::LLVMBuildUnreachable(self.llbuilder);
391        }
392    }
393
394    math_builder_methods! {
395        add(a, b) => LLVMBuildAdd,
396        fadd(a, b) => LLVMBuildFAdd,
397        sub(a, b) => LLVMBuildSub,
398        fsub(a, b) => LLVMBuildFSub,
399        mul(a, b) => LLVMBuildMul,
400        fmul(a, b) => LLVMBuildFMul,
401        udiv(a, b) => LLVMBuildUDiv,
402        exactudiv(a, b) => LLVMBuildExactUDiv,
403        sdiv(a, b) => LLVMBuildSDiv,
404        exactsdiv(a, b) => LLVMBuildExactSDiv,
405        fdiv(a, b) => LLVMBuildFDiv,
406        urem(a, b) => LLVMBuildURem,
407        srem(a, b) => LLVMBuildSRem,
408        frem(a, b) => LLVMBuildFRem,
409        shl(a, b) => LLVMBuildShl,
410        lshr(a, b) => LLVMBuildLShr,
411        ashr(a, b) => LLVMBuildAShr,
412        and(a, b) => LLVMBuildAnd,
413        or(a, b) => LLVMBuildOr,
414        xor(a, b) => LLVMBuildXor,
415        neg(x) => LLVMBuildNeg,
416        fneg(x) => LLVMBuildFNeg,
417        not(x) => LLVMBuildNot,
418        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
419        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
420        unchecked_ssub(x, y) => LLVMBuildNSWSub,
421        unchecked_usub(x, y) => LLVMBuildNUWSub,
422        unchecked_smul(x, y) => LLVMBuildNSWMul,
423        unchecked_umul(x, y) => LLVMBuildNUWMul,
424    }
425
426    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
427        unsafe {
428            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
429            if llvm::LLVMIsAInstruction(add).is_some() {
430                llvm::LLVMSetNUW(add, True);
431                llvm::LLVMSetNSW(add, True);
432            }
433            add
434        }
435    }
436    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
437        unsafe {
438            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
439            if llvm::LLVMIsAInstruction(sub).is_some() {
440                llvm::LLVMSetNUW(sub, True);
441                llvm::LLVMSetNSW(sub, True);
442            }
443            sub
444        }
445    }
446    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
447        unsafe {
448            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
449            if llvm::LLVMIsAInstruction(mul).is_some() {
450                llvm::LLVMSetNUW(mul, True);
451                llvm::LLVMSetNSW(mul, True);
452            }
453            mul
454        }
455    }
456
457    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
458        unsafe {
459            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
460
461            // If a and b are both values, then `or` is a value, rather than
462            // an instruction, so we need to check before setting the flag.
463            // (See also `LLVMBuildNUWNeg` which also needs a check.)
464            if llvm::LLVMIsAInstruction(or).is_some() {
465                llvm::LLVMSetIsDisjoint(or, True);
466            }
467            or
468        }
469    }
470
471    set_math_builder_methods! {
472        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
473        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
474        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
475        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
476        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
477        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
478        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
479        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
480        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
481        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
482    }
483
484    fn checked_binop(
485        &mut self,
486        oop: OverflowOp,
487        ty: Ty<'_>,
488        lhs: Self::Value,
489        rhs: Self::Value,
490    ) -> (Self::Value, Self::Value) {
491        use rustc_middle::ty::IntTy::*;
492        use rustc_middle::ty::UintTy::*;
493        use rustc_middle::ty::{Int, Uint};
494
495        let new_kind = match ty.kind() {
496            Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
497            Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
498            t @ (Uint(_) | Int(_)) => *t,
499            _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
500        };
501
502        let name = match oop {
503            OverflowOp::Add => match new_kind {
504                Int(I8) => "llvm.sadd.with.overflow.i8",
505                Int(I16) => "llvm.sadd.with.overflow.i16",
506                Int(I32) => "llvm.sadd.with.overflow.i32",
507                Int(I64) => "llvm.sadd.with.overflow.i64",
508                Int(I128) => "llvm.sadd.with.overflow.i128",
509
510                Uint(U8) => "llvm.uadd.with.overflow.i8",
511                Uint(U16) => "llvm.uadd.with.overflow.i16",
512                Uint(U32) => "llvm.uadd.with.overflow.i32",
513                Uint(U64) => "llvm.uadd.with.overflow.i64",
514                Uint(U128) => "llvm.uadd.with.overflow.i128",
515
516                _ => unreachable!(),
517            },
518            OverflowOp::Sub => match new_kind {
519                Int(I8) => "llvm.ssub.with.overflow.i8",
520                Int(I16) => "llvm.ssub.with.overflow.i16",
521                Int(I32) => "llvm.ssub.with.overflow.i32",
522                Int(I64) => "llvm.ssub.with.overflow.i64",
523                Int(I128) => "llvm.ssub.with.overflow.i128",
524
525                Uint(_) => {
526                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
527                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
528                    // in the backend if profitable.
529                    let sub = self.sub(lhs, rhs);
530                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
531                    return (sub, cmp);
532                }
533
534                _ => unreachable!(),
535            },
536            OverflowOp::Mul => match new_kind {
537                Int(I8) => "llvm.smul.with.overflow.i8",
538                Int(I16) => "llvm.smul.with.overflow.i16",
539                Int(I32) => "llvm.smul.with.overflow.i32",
540                Int(I64) => "llvm.smul.with.overflow.i64",
541                Int(I128) => "llvm.smul.with.overflow.i128",
542
543                Uint(U8) => "llvm.umul.with.overflow.i8",
544                Uint(U16) => "llvm.umul.with.overflow.i16",
545                Uint(U32) => "llvm.umul.with.overflow.i32",
546                Uint(U64) => "llvm.umul.with.overflow.i64",
547                Uint(U128) => "llvm.umul.with.overflow.i128",
548
549                _ => unreachable!(),
550            },
551        };
552
553        let res = self.call_intrinsic(name, &[lhs, rhs]);
554        (self.extract_value(res, 0), self.extract_value(res, 1))
555    }
556
557    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
558        if self.cx().val_ty(val) == self.cx().type_i1() {
559            self.zext(val, self.cx().type_i8())
560        } else {
561            val
562        }
563    }
564
565    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
566        if scalar.is_bool() {
567            return self.unchecked_utrunc(val, self.cx().type_i1());
568        }
569        val
570    }
571
572    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
573        let mut bx = Builder::with_cx(self.cx);
574        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
575        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
576        unsafe {
577            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
578            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
579            // Cast to default addrspace if necessary
580            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
581        }
582    }
583
584    fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
585        unsafe {
586            let alloca =
587                llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
588            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
589            // Cast to default addrspace if necessary
590            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
591        }
592    }
593
594    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
595        unsafe {
596            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
597            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
598            load
599        }
600    }
601
602    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
603        unsafe {
604            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
605            llvm::LLVMSetVolatile(load, llvm::True);
606            load
607        }
608    }
609
610    fn atomic_load(
611        &mut self,
612        ty: &'ll Type,
613        ptr: &'ll Value,
614        order: rustc_codegen_ssa::common::AtomicOrdering,
615        size: Size,
616    ) -> &'ll Value {
617        unsafe {
618            let load = llvm::LLVMRustBuildAtomicLoad(
619                self.llbuilder,
620                ty,
621                ptr,
622                UNNAMED,
623                AtomicOrdering::from_generic(order),
624            );
625            // LLVM requires the alignment of atomic loads to be at least the size of the type.
626            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
627            load
628        }
629    }
630
631    #[instrument(level = "trace", skip(self))]
632    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
633        if place.layout.is_unsized() {
634            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
635            if matches!(tail.kind(), ty::Foreign(..)) {
636                // Unsized locals and, at least conceptually, even unsized arguments must be copied
637                // around, which requires dynamically determining their size. Therefore, we cannot
638                // allow `extern` types here. Consult t-opsem before removing this check.
639                panic!("unsized locals must not be `extern` types");
640            }
641        }
642        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
643
644        if place.layout.is_zst() {
645            return OperandRef::zero_sized(place.layout);
646        }
647
648        #[instrument(level = "trace", skip(bx))]
649        fn scalar_load_metadata<'a, 'll, 'tcx>(
650            bx: &mut Builder<'a, 'll, 'tcx>,
651            load: &'ll Value,
652            scalar: abi::Scalar,
653            layout: TyAndLayout<'tcx>,
654            offset: Size,
655        ) {
656            if bx.cx.sess().opts.optimize == OptLevel::No {
657                // Don't emit metadata we're not going to use
658                return;
659            }
660
661            if !scalar.is_uninit_valid() {
662                bx.noundef_metadata(load);
663            }
664
665            match scalar.primitive() {
666                abi::Primitive::Int(..) => {
667                    if !scalar.is_always_valid(bx) {
668                        bx.range_metadata(load, scalar.valid_range(bx));
669                    }
670                }
671                abi::Primitive::Pointer(_) => {
672                    if !scalar.valid_range(bx).contains(0) {
673                        bx.nonnull_metadata(load);
674                    }
675
676                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
677                        if let Some(_) = pointee.safe {
678                            bx.align_metadata(load, pointee.align);
679                        }
680                    }
681                }
682                abi::Primitive::Float(_) => {}
683            }
684        }
685
686        let val = if let Some(_) = place.val.llextra {
687            // FIXME: Merge with the `else` below?
688            OperandValue::Ref(place.val)
689        } else if place.layout.is_llvm_immediate() {
690            let mut const_llval = None;
691            let llty = place.layout.llvm_type(self);
692            unsafe {
693                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
694                    if llvm::LLVMIsGlobalConstant(global) == llvm::True {
695                        if let Some(init) = llvm::LLVMGetInitializer(global) {
696                            if self.val_ty(init) == llty {
697                                const_llval = Some(init);
698                            }
699                        }
700                    }
701                }
702            }
703            let llval = const_llval.unwrap_or_else(|| {
704                let load = self.load(llty, place.val.llval, place.val.align);
705                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
706                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
707                    self.to_immediate_scalar(load, scalar)
708                } else {
709                    load
710                }
711            });
712            OperandValue::Immediate(llval)
713        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
714            let b_offset = a.size(self).align_to(b.align(self).abi);
715
716            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
717                let llptr = if i == 0 {
718                    place.val.llval
719                } else {
720                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
721                };
722                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
723                let load = self.load(llty, llptr, align);
724                scalar_load_metadata(self, load, scalar, layout, offset);
725                self.to_immediate_scalar(load, scalar)
726            };
727
728            OperandValue::Pair(
729                load(0, a, place.layout, place.val.align, Size::ZERO),
730                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
731            )
732        } else {
733            OperandValue::Ref(place.val)
734        };
735
736        OperandRef { val, layout: place.layout }
737    }
738
739    fn write_operand_repeatedly(
740        &mut self,
741        cg_elem: OperandRef<'tcx, &'ll Value>,
742        count: u64,
743        dest: PlaceRef<'tcx, &'ll Value>,
744    ) {
745        let zero = self.const_usize(0);
746        let count = self.const_usize(count);
747
748        let header_bb = self.append_sibling_block("repeat_loop_header");
749        let body_bb = self.append_sibling_block("repeat_loop_body");
750        let next_bb = self.append_sibling_block("repeat_loop_next");
751
752        self.br(header_bb);
753
754        let mut header_bx = Self::build(self.cx, header_bb);
755        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
756
757        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
758        header_bx.cond_br(keep_going, body_bb, next_bb);
759
760        let mut body_bx = Self::build(self.cx, body_bb);
761        let dest_elem = dest.project_index(&mut body_bx, i);
762        cg_elem.val.store(&mut body_bx, dest_elem);
763
764        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
765        body_bx.br(header_bb);
766        header_bx.add_incoming_to_phi(i, next, body_bb);
767
768        *self = Self::build(self.cx, next_bb);
769    }
770
771    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
772        if self.cx.sess().opts.optimize == OptLevel::No {
773            // Don't emit metadata we're not going to use
774            return;
775        }
776
777        unsafe {
778            let llty = self.cx.val_ty(load);
779            let md = [
780                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
781                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
782            ];
783            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
784            self.set_metadata(load, llvm::MD_range, md);
785        }
786    }
787
788    fn nonnull_metadata(&mut self, load: &'ll Value) {
789        unsafe {
790            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
791            self.set_metadata(load, llvm::MD_nonnull, md);
792        }
793    }
794
795    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
796        self.store_with_flags(val, ptr, align, MemFlags::empty())
797    }
798
799    fn store_with_flags(
800        &mut self,
801        val: &'ll Value,
802        ptr: &'ll Value,
803        align: Align,
804        flags: MemFlags,
805    ) -> &'ll Value {
806        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
807        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
808        unsafe {
809            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
810            let align =
811                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
812            llvm::LLVMSetAlignment(store, align);
813            if flags.contains(MemFlags::VOLATILE) {
814                llvm::LLVMSetVolatile(store, llvm::True);
815            }
816            if flags.contains(MemFlags::NONTEMPORAL) {
817                // Make sure that the current target architectures supports "sane" non-temporal
818                // stores, i.e., non-temporal stores that are equivalent to regular stores except
819                // for performance. LLVM doesn't seem to care about this, and will happily treat
820                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
821                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
822                // regular stores but require special fences. So we keep a list of architectures
823                // where `!nontemporal` is known to be truly just a hint, and use regular stores
824                // everywhere else. (In the future, we could alternatively ensure that an sfence
825                // gets emitted after a sequence of movnt before any kind of synchronizing
826                // operation. But it's not clear how to do that with LLVM.)
827                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
828                // <https://github.com/llvm/llvm-project/issues/64521>.
829                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
830                    &["aarch64", "arm", "riscv32", "riscv64"];
831
832                let use_nontemporal =
833                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
834                if use_nontemporal {
835                    // According to LLVM [1] building a nontemporal store must
836                    // *always* point to a metadata value of the integer 1.
837                    //
838                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
839                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
840                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
841                    self.set_metadata(store, llvm::MD_nontemporal, md);
842                }
843            }
844            store
845        }
846    }
847
848    fn atomic_store(
849        &mut self,
850        val: &'ll Value,
851        ptr: &'ll Value,
852        order: rustc_codegen_ssa::common::AtomicOrdering,
853        size: Size,
854    ) {
855        debug!("Store {:?} -> {:?}", val, ptr);
856        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
857        unsafe {
858            let store = llvm::LLVMRustBuildAtomicStore(
859                self.llbuilder,
860                val,
861                ptr,
862                AtomicOrdering::from_generic(order),
863            );
864            // LLVM requires the alignment of atomic stores to be at least the size of the type.
865            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
866        }
867    }
868
869    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
870        unsafe {
871            llvm::LLVMBuildGEPWithNoWrapFlags(
872                self.llbuilder,
873                ty,
874                ptr,
875                indices.as_ptr(),
876                indices.len() as c_uint,
877                UNNAMED,
878                GEPNoWrapFlags::default(),
879            )
880        }
881    }
882
883    fn inbounds_gep(
884        &mut self,
885        ty: &'ll Type,
886        ptr: &'ll Value,
887        indices: &[&'ll Value],
888    ) -> &'ll Value {
889        unsafe {
890            llvm::LLVMBuildGEPWithNoWrapFlags(
891                self.llbuilder,
892                ty,
893                ptr,
894                indices.as_ptr(),
895                indices.len() as c_uint,
896                UNNAMED,
897                GEPNoWrapFlags::InBounds,
898            )
899        }
900    }
901
902    fn inbounds_nuw_gep(
903        &mut self,
904        ty: &'ll Type,
905        ptr: &'ll Value,
906        indices: &[&'ll Value],
907    ) -> &'ll Value {
908        unsafe {
909            llvm::LLVMBuildGEPWithNoWrapFlags(
910                self.llbuilder,
911                ty,
912                ptr,
913                indices.as_ptr(),
914                indices.len() as c_uint,
915                UNNAMED,
916                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
917            )
918        }
919    }
920
921    /* Casts */
922    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
923        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
924    }
925
926    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
927        debug_assert_ne!(self.val_ty(val), dest_ty);
928
929        let trunc = self.trunc(val, dest_ty);
930        if llvm_util::get_version() >= (19, 0, 0) {
931            unsafe {
932                if llvm::LLVMIsAInstruction(trunc).is_some() {
933                    llvm::LLVMSetNUW(trunc, True);
934                }
935            }
936        }
937        trunc
938    }
939
940    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
941        debug_assert_ne!(self.val_ty(val), dest_ty);
942
943        let trunc = self.trunc(val, dest_ty);
944        if llvm_util::get_version() >= (19, 0, 0) {
945            unsafe {
946                if llvm::LLVMIsAInstruction(trunc).is_some() {
947                    llvm::LLVMSetNSW(trunc, True);
948                }
949            }
950        }
951        trunc
952    }
953
954    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
955        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
956    }
957
958    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
959        self.fptoint_sat(false, val, dest_ty)
960    }
961
962    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
963        self.fptoint_sat(true, val, dest_ty)
964    }
965
966    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
967        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
968        // poor codegen. The reason for this is that the corresponding wasm
969        // instructions, `i32.trunc_f32_s` for example, will trap when the float
970        // is out-of-bounds, infinity, or nan. This means that LLVM
971        // automatically inserts control flow around `fptoui` and `fptosi`
972        // because the LLVM instruction `fptoui` is defined as producing a
973        // poison value, not having UB on out-of-bounds values.
974        //
975        // This method, however, is only used with non-saturating casts that
976        // have UB on out-of-bounds values. This means that it's ok if we use
977        // the raw wasm instruction since out-of-bounds values can do whatever
978        // we like. To ensure that LLVM picks the right instruction we choose
979        // the raw wasm intrinsic functions which avoid LLVM inserting all the
980        // other control flow automatically.
981        if self.sess().target.is_like_wasm {
982            let src_ty = self.cx.val_ty(val);
983            if self.cx.type_kind(src_ty) != TypeKind::Vector {
984                let float_width = self.cx.float_width(src_ty);
985                let int_width = self.cx.int_width(dest_ty);
986                let name = match (int_width, float_width) {
987                    (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
988                    (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
989                    (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
990                    (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
991                    _ => None,
992                };
993                if let Some(name) = name {
994                    return self.call_intrinsic(name, &[val]);
995                }
996            }
997        }
998        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
999    }
1000
1001    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1002        // see `fptoui` above for why wasm is different here
1003        if self.sess().target.is_like_wasm {
1004            let src_ty = self.cx.val_ty(val);
1005            if self.cx.type_kind(src_ty) != TypeKind::Vector {
1006                let float_width = self.cx.float_width(src_ty);
1007                let int_width = self.cx.int_width(dest_ty);
1008                let name = match (int_width, float_width) {
1009                    (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
1010                    (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
1011                    (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
1012                    (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
1013                    _ => None,
1014                };
1015                if let Some(name) = name {
1016                    return self.call_intrinsic(name, &[val]);
1017                }
1018            }
1019        }
1020        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1021    }
1022
1023    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1024        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1025    }
1026
1027    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1028        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1029    }
1030
1031    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1032        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1033    }
1034
1035    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1036        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1037    }
1038
1039    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1040        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1041    }
1042
1043    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1044        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1045    }
1046
1047    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1048        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1049    }
1050
1051    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1052        unsafe {
1053            llvm::LLVMBuildIntCast2(
1054                self.llbuilder,
1055                val,
1056                dest_ty,
1057                if is_signed { True } else { False },
1058                UNNAMED,
1059            )
1060        }
1061    }
1062
1063    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1064        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1065    }
1066
1067    /* Comparisons */
1068    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1069        let op = llvm::IntPredicate::from_generic(op);
1070        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1071    }
1072
1073    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1074        let op = llvm::RealPredicate::from_generic(op);
1075        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1076    }
1077
1078    fn three_way_compare(
1079        &mut self,
1080        ty: Ty<'tcx>,
1081        lhs: Self::Value,
1082        rhs: Self::Value,
1083    ) -> Option<Self::Value> {
1084        // FIXME: See comment on the definition of `three_way_compare`.
1085        if crate::llvm_util::get_version() < (20, 0, 0) {
1086            return None;
1087        }
1088
1089        let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) {
1090            (true, 8) => "llvm.scmp.i8.i8",
1091            (true, 16) => "llvm.scmp.i8.i16",
1092            (true, 32) => "llvm.scmp.i8.i32",
1093            (true, 64) => "llvm.scmp.i8.i64",
1094            (true, 128) => "llvm.scmp.i8.i128",
1095
1096            (false, 8) => "llvm.ucmp.i8.i8",
1097            (false, 16) => "llvm.ucmp.i8.i16",
1098            (false, 32) => "llvm.ucmp.i8.i32",
1099            (false, 64) => "llvm.ucmp.i8.i64",
1100            (false, 128) => "llvm.ucmp.i8.i128",
1101
1102            _ => bug!("three-way compare unsupported for type {ty:?}"),
1103        };
1104        Some(self.call_intrinsic(name, &[lhs, rhs]))
1105    }
1106
1107    /* Miscellaneous instructions */
1108    fn memcpy(
1109        &mut self,
1110        dst: &'ll Value,
1111        dst_align: Align,
1112        src: &'ll Value,
1113        src_align: Align,
1114        size: &'ll Value,
1115        flags: MemFlags,
1116    ) {
1117        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1118        let size = self.intcast(size, self.type_isize(), false);
1119        let is_volatile = flags.contains(MemFlags::VOLATILE);
1120        unsafe {
1121            llvm::LLVMRustBuildMemCpy(
1122                self.llbuilder,
1123                dst,
1124                dst_align.bytes() as c_uint,
1125                src,
1126                src_align.bytes() as c_uint,
1127                size,
1128                is_volatile,
1129            );
1130        }
1131    }
1132
1133    fn memmove(
1134        &mut self,
1135        dst: &'ll Value,
1136        dst_align: Align,
1137        src: &'ll Value,
1138        src_align: Align,
1139        size: &'ll Value,
1140        flags: MemFlags,
1141    ) {
1142        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1143        let size = self.intcast(size, self.type_isize(), false);
1144        let is_volatile = flags.contains(MemFlags::VOLATILE);
1145        unsafe {
1146            llvm::LLVMRustBuildMemMove(
1147                self.llbuilder,
1148                dst,
1149                dst_align.bytes() as c_uint,
1150                src,
1151                src_align.bytes() as c_uint,
1152                size,
1153                is_volatile,
1154            );
1155        }
1156    }
1157
1158    fn memset(
1159        &mut self,
1160        ptr: &'ll Value,
1161        fill_byte: &'ll Value,
1162        size: &'ll Value,
1163        align: Align,
1164        flags: MemFlags,
1165    ) {
1166        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1167        let is_volatile = flags.contains(MemFlags::VOLATILE);
1168        unsafe {
1169            llvm::LLVMRustBuildMemSet(
1170                self.llbuilder,
1171                ptr,
1172                align.bytes() as c_uint,
1173                fill_byte,
1174                size,
1175                is_volatile,
1176            );
1177        }
1178    }
1179
1180    fn select(
1181        &mut self,
1182        cond: &'ll Value,
1183        then_val: &'ll Value,
1184        else_val: &'ll Value,
1185    ) -> &'ll Value {
1186        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1187    }
1188
1189    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1190        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1191    }
1192
1193    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1194        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1195    }
1196
1197    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1198        unsafe {
1199            let elt_ty = self.cx.val_ty(elt);
1200            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1201            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1202            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1203            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1204        }
1205    }
1206
1207    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1208        assert_eq!(idx as c_uint as u64, idx);
1209        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1210    }
1211
1212    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1213        assert_eq!(idx as c_uint as u64, idx);
1214        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1215    }
1216
1217    fn set_personality_fn(&mut self, personality: &'ll Value) {
1218        unsafe {
1219            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1220        }
1221    }
1222
1223    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1224        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1225        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1226        unsafe {
1227            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1228        }
1229        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1230    }
1231
1232    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1233        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1234        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1235        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1236        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1237    }
1238
1239    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1240        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1241        let mut exn = self.const_poison(ty);
1242        exn = self.insert_value(exn, exn0, 0);
1243        exn = self.insert_value(exn, exn1, 1);
1244        unsafe {
1245            llvm::LLVMBuildResume(self.llbuilder, exn);
1246        }
1247    }
1248
1249    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1250        let ret = unsafe {
1251            llvm::LLVMBuildCleanupPad(
1252                self.llbuilder,
1253                parent,
1254                args.as_ptr(),
1255                args.len() as c_uint,
1256                c"cleanuppad".as_ptr(),
1257            )
1258        };
1259        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1260    }
1261
1262    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1263        unsafe {
1264            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1265                .expect("LLVM does not have support for cleanupret");
1266        }
1267    }
1268
1269    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1270        let ret = unsafe {
1271            llvm::LLVMBuildCatchPad(
1272                self.llbuilder,
1273                parent,
1274                args.as_ptr(),
1275                args.len() as c_uint,
1276                c"catchpad".as_ptr(),
1277            )
1278        };
1279        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1280    }
1281
1282    fn catch_switch(
1283        &mut self,
1284        parent: Option<&'ll Value>,
1285        unwind: Option<&'ll BasicBlock>,
1286        handlers: &[&'ll BasicBlock],
1287    ) -> &'ll Value {
1288        let ret = unsafe {
1289            llvm::LLVMBuildCatchSwitch(
1290                self.llbuilder,
1291                parent,
1292                unwind,
1293                handlers.len() as c_uint,
1294                c"catchswitch".as_ptr(),
1295            )
1296        };
1297        let ret = ret.expect("LLVM does not have support for catchswitch");
1298        for handler in handlers {
1299            unsafe {
1300                llvm::LLVMAddHandler(ret, handler);
1301            }
1302        }
1303        ret
1304    }
1305
1306    // Atomic Operations
1307    fn atomic_cmpxchg(
1308        &mut self,
1309        dst: &'ll Value,
1310        cmp: &'ll Value,
1311        src: &'ll Value,
1312        order: rustc_codegen_ssa::common::AtomicOrdering,
1313        failure_order: rustc_codegen_ssa::common::AtomicOrdering,
1314        weak: bool,
1315    ) -> (&'ll Value, &'ll Value) {
1316        let weak = if weak { llvm::True } else { llvm::False };
1317        unsafe {
1318            let value = llvm::LLVMBuildAtomicCmpXchg(
1319                self.llbuilder,
1320                dst,
1321                cmp,
1322                src,
1323                AtomicOrdering::from_generic(order),
1324                AtomicOrdering::from_generic(failure_order),
1325                llvm::False, // SingleThreaded
1326            );
1327            llvm::LLVMSetWeak(value, weak);
1328            let val = self.extract_value(value, 0);
1329            let success = self.extract_value(value, 1);
1330            (val, success)
1331        }
1332    }
1333
1334    fn atomic_rmw(
1335        &mut self,
1336        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1337        dst: &'ll Value,
1338        mut src: &'ll Value,
1339        order: rustc_codegen_ssa::common::AtomicOrdering,
1340    ) -> &'ll Value {
1341        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1342        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1343            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1344        if requires_cast_to_int {
1345            src = self.ptrtoint(src, self.type_isize());
1346        }
1347        let mut res = unsafe {
1348            llvm::LLVMBuildAtomicRMW(
1349                self.llbuilder,
1350                AtomicRmwBinOp::from_generic(op),
1351                dst,
1352                src,
1353                AtomicOrdering::from_generic(order),
1354                llvm::False, // SingleThreaded
1355            )
1356        };
1357        if requires_cast_to_int {
1358            res = self.inttoptr(res, self.type_ptr());
1359        }
1360        res
1361    }
1362
1363    fn atomic_fence(
1364        &mut self,
1365        order: rustc_codegen_ssa::common::AtomicOrdering,
1366        scope: SynchronizationScope,
1367    ) {
1368        let single_threaded = match scope {
1369            SynchronizationScope::SingleThread => llvm::True,
1370            SynchronizationScope::CrossThread => llvm::False,
1371        };
1372        unsafe {
1373            llvm::LLVMBuildFence(
1374                self.llbuilder,
1375                AtomicOrdering::from_generic(order),
1376                single_threaded,
1377                UNNAMED,
1378            );
1379        }
1380    }
1381
1382    fn set_invariant_load(&mut self, load: &'ll Value) {
1383        unsafe {
1384            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1385            self.set_metadata(load, llvm::MD_invariant_load, md);
1386        }
1387    }
1388
1389    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1390        self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
1391    }
1392
1393    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1394        self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
1395    }
1396
1397    fn call(
1398        &mut self,
1399        llty: &'ll Type,
1400        fn_attrs: Option<&CodegenFnAttrs>,
1401        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1402        llfn: &'ll Value,
1403        args: &[&'ll Value],
1404        funclet: Option<&Funclet<'ll>>,
1405        instance: Option<Instance<'tcx>>,
1406    ) -> &'ll Value {
1407        debug!("call {:?} with args ({:?})", llfn, args);
1408
1409        let args = self.check_call("call", llty, llfn, args);
1410        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1411        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1412        if let Some(funclet_bundle) = funclet_bundle {
1413            bundles.push(funclet_bundle);
1414        }
1415
1416        // Emit CFI pointer type membership test
1417        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1418
1419        // Emit KCFI operand bundle
1420        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1421        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) {
1422            bundles.push(kcfi_bundle);
1423        }
1424
1425        let call = unsafe {
1426            llvm::LLVMBuildCallWithOperandBundles(
1427                self.llbuilder,
1428                llty,
1429                llfn,
1430                args.as_ptr() as *const &llvm::Value,
1431                args.len() as c_uint,
1432                bundles.as_ptr(),
1433                bundles.len() as c_uint,
1434                c"".as_ptr(),
1435            )
1436        };
1437        if let Some(fn_abi) = fn_abi {
1438            fn_abi.apply_attrs_callsite(self, call);
1439        }
1440        call
1441    }
1442
1443    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1444        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1445    }
1446
1447    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1448        // Cleanup is always the cold path.
1449        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1450        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1451    }
1452}
1453
1454impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1455    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1456        // Forward to the `get_static` method of `CodegenCx`
1457        let s = self.cx().get_static(def_id);
1458        // Cast to default address space if globals are in a different addrspace
1459        self.cx().const_pointercast(s, self.type_ptr())
1460    }
1461}
1462
1463impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1464    pub(crate) fn llfn(&self) -> &'ll Value {
1465        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1466    }
1467}
1468
1469impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1470    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1471        unsafe {
1472            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1473        }
1474    }
1475}
1476impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1477    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1478        unsafe {
1479            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1480            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1481            self.set_metadata(load, llvm::MD_align, md);
1482        }
1483    }
1484
1485    fn noundef_metadata(&mut self, load: &'ll Value) {
1486        unsafe {
1487            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1488            self.set_metadata(load, llvm::MD_noundef, md);
1489        }
1490    }
1491
1492    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1493        unsafe {
1494            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1495            self.set_metadata(inst, llvm::MD_unpredictable, md);
1496        }
1497    }
1498}
1499impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1500    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1501        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1502    }
1503
1504    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1505        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1506    }
1507
1508    pub(crate) fn insert_element(
1509        &mut self,
1510        vec: &'ll Value,
1511        elt: &'ll Value,
1512        idx: &'ll Value,
1513    ) -> &'ll Value {
1514        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1515    }
1516
1517    pub(crate) fn shuffle_vector(
1518        &mut self,
1519        v1: &'ll Value,
1520        v2: &'ll Value,
1521        mask: &'ll Value,
1522    ) -> &'ll Value {
1523        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1524    }
1525
1526    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1527        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1528    }
1529    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1530        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1531    }
1532    pub(crate) fn vector_reduce_fadd_reassoc(
1533        &mut self,
1534        acc: &'ll Value,
1535        src: &'ll Value,
1536    ) -> &'ll Value {
1537        unsafe {
1538            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1539            llvm::LLVMRustSetAllowReassoc(instr);
1540            instr
1541        }
1542    }
1543    pub(crate) fn vector_reduce_fmul_reassoc(
1544        &mut self,
1545        acc: &'ll Value,
1546        src: &'ll Value,
1547    ) -> &'ll Value {
1548        unsafe {
1549            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1550            llvm::LLVMRustSetAllowReassoc(instr);
1551            instr
1552        }
1553    }
1554    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1555        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1556    }
1557    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1558        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1559    }
1560    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1561        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1562    }
1563    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1564        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1565    }
1566    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1567        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1568    }
1569    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1570        unsafe {
1571            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1572        }
1573    }
1574    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1575        unsafe {
1576            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1577        }
1578    }
1579    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1580        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1581    }
1582    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1583        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1584    }
1585
1586    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1587        unsafe {
1588            llvm::LLVMAddClause(landing_pad, clause);
1589        }
1590    }
1591
1592    pub(crate) fn catch_ret(
1593        &mut self,
1594        funclet: &Funclet<'ll>,
1595        unwind: &'ll BasicBlock,
1596    ) -> &'ll Value {
1597        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1598        ret.expect("LLVM does not have support for catchret")
1599    }
1600
1601    fn check_call<'b>(
1602        &mut self,
1603        typ: &str,
1604        fn_ty: &'ll Type,
1605        llfn: &'ll Value,
1606        args: &'b [&'ll Value],
1607    ) -> Cow<'b, [&'ll Value]> {
1608        assert!(
1609            self.cx.type_kind(fn_ty) == TypeKind::Function,
1610            "builder::{typ} not passed a function, but {fn_ty:?}"
1611        );
1612
1613        let param_tys = self.cx.func_params_types(fn_ty);
1614
1615        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1616            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1617
1618        if all_args_match {
1619            return Cow::Borrowed(args);
1620        }
1621
1622        let casted_args: Vec<_> = iter::zip(param_tys, args)
1623            .enumerate()
1624            .map(|(i, (expected_ty, &actual_val))| {
1625                let actual_ty = self.cx.val_ty(actual_val);
1626                if expected_ty != actual_ty {
1627                    debug!(
1628                        "type mismatch in function call of {:?}. \
1629                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1630                        llfn, expected_ty, i, actual_ty
1631                    );
1632                    self.bitcast(actual_val, expected_ty)
1633                } else {
1634                    actual_val
1635                }
1636            })
1637            .collect();
1638
1639        Cow::Owned(casted_args)
1640    }
1641
1642    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1643        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1644    }
1645}
1646
1647impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1648    pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
1649        let (ty, f) = self.cx.get_intrinsic(intrinsic);
1650        self.call(ty, None, None, f, args, None, None)
1651    }
1652
1653    fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
1654        let size = size.bytes();
1655        if size == 0 {
1656            return;
1657        }
1658
1659        if !self.cx().sess().emit_lifetime_markers() {
1660            return;
1661        }
1662
1663        self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
1664    }
1665}
1666impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1667    pub(crate) fn phi(
1668        &mut self,
1669        ty: &'ll Type,
1670        vals: &[&'ll Value],
1671        bbs: &[&'ll BasicBlock],
1672    ) -> &'ll Value {
1673        assert_eq!(vals.len(), bbs.len());
1674        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1675        unsafe {
1676            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1677            phi
1678        }
1679    }
1680
1681    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1682        unsafe {
1683            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1684        }
1685    }
1686}
1687impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1688    fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1689        let src_ty = self.cx.val_ty(val);
1690        let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
1691            assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
1692            (
1693                self.cx.element_type(src_ty),
1694                self.cx.element_type(dest_ty),
1695                Some(self.cx.vector_length(src_ty)),
1696            )
1697        } else {
1698            (src_ty, dest_ty, None)
1699        };
1700        let float_width = self.cx.float_width(float_ty);
1701        let int_width = self.cx.int_width(int_ty);
1702
1703        let instr = if signed { "fptosi" } else { "fptoui" };
1704        let name = if let Some(vector_length) = vector_length {
1705            format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}")
1706        } else {
1707            format!("llvm.{instr}.sat.i{int_width}.f{float_width}")
1708        };
1709        let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
1710        self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None)
1711    }
1712
1713    pub(crate) fn landing_pad(
1714        &mut self,
1715        ty: &'ll Type,
1716        pers_fn: &'ll Value,
1717        num_clauses: usize,
1718    ) -> &'ll Value {
1719        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1720        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1721        // personality lives on the parent function anyway.
1722        self.set_personality_fn(pers_fn);
1723        unsafe {
1724            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1725        }
1726    }
1727
1728    pub(crate) fn callbr(
1729        &mut self,
1730        llty: &'ll Type,
1731        fn_attrs: Option<&CodegenFnAttrs>,
1732        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1733        llfn: &'ll Value,
1734        args: &[&'ll Value],
1735        default_dest: &'ll BasicBlock,
1736        indirect_dest: &[&'ll BasicBlock],
1737        funclet: Option<&Funclet<'ll>>,
1738        instance: Option<Instance<'tcx>>,
1739    ) -> &'ll Value {
1740        debug!("invoke {:?} with args ({:?})", llfn, args);
1741
1742        let args = self.check_call("callbr", llty, llfn, args);
1743        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1744        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1745        if let Some(funclet_bundle) = funclet_bundle {
1746            bundles.push(funclet_bundle);
1747        }
1748
1749        // Emit CFI pointer type membership test
1750        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1751
1752        // Emit KCFI operand bundle
1753        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1754        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) {
1755            bundles.push(kcfi_bundle);
1756        }
1757
1758        let callbr = unsafe {
1759            llvm::LLVMBuildCallBr(
1760                self.llbuilder,
1761                llty,
1762                llfn,
1763                default_dest,
1764                indirect_dest.as_ptr(),
1765                indirect_dest.len() as c_uint,
1766                args.as_ptr(),
1767                args.len() as c_uint,
1768                bundles.as_ptr(),
1769                bundles.len() as c_uint,
1770                UNNAMED,
1771            )
1772        };
1773        if let Some(fn_abi) = fn_abi {
1774            fn_abi.apply_attrs_callsite(self, callbr);
1775        }
1776        callbr
1777    }
1778
1779    // Emits CFI pointer type membership tests.
1780    fn cfi_type_test(
1781        &mut self,
1782        fn_attrs: Option<&CodegenFnAttrs>,
1783        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1784        instance: Option<Instance<'tcx>>,
1785        llfn: &'ll Value,
1786    ) {
1787        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1788        if self.tcx.sess.is_sanitizer_cfi_enabled()
1789            && let Some(fn_abi) = fn_abi
1790            && is_indirect_call
1791        {
1792            if let Some(fn_attrs) = fn_attrs
1793                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1794            {
1795                return;
1796            }
1797
1798            let mut options = cfi::TypeIdOptions::empty();
1799            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1800                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1801            }
1802            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1803                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1804            }
1805
1806            let typeid = if let Some(instance) = instance {
1807                cfi::typeid_for_instance(self.tcx, instance, options)
1808            } else {
1809                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1810            };
1811            let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
1812            let dbg_loc = self.get_dbg_loc();
1813
1814            // Test whether the function pointer is associated with the type identifier.
1815            let cond = self.type_test(llfn, typeid_metadata);
1816            let bb_pass = self.append_sibling_block("type_test.pass");
1817            let bb_fail = self.append_sibling_block("type_test.fail");
1818            self.cond_br(cond, bb_pass, bb_fail);
1819
1820            self.switch_to_block(bb_fail);
1821            if let Some(dbg_loc) = dbg_loc {
1822                self.set_dbg_loc(dbg_loc);
1823            }
1824            self.abort();
1825            self.unreachable();
1826
1827            self.switch_to_block(bb_pass);
1828            if let Some(dbg_loc) = dbg_loc {
1829                self.set_dbg_loc(dbg_loc);
1830            }
1831        }
1832    }
1833
1834    // Emits KCFI operand bundles.
1835    fn kcfi_operand_bundle(
1836        &mut self,
1837        fn_attrs: Option<&CodegenFnAttrs>,
1838        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1839        instance: Option<Instance<'tcx>>,
1840        llfn: &'ll Value,
1841    ) -> Option<llvm::OperandBundleOwned<'ll>> {
1842        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1843        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1844            && let Some(fn_abi) = fn_abi
1845            && is_indirect_call
1846        {
1847            if let Some(fn_attrs) = fn_attrs
1848                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1849            {
1850                return None;
1851            }
1852
1853            let mut options = kcfi::TypeIdOptions::empty();
1854            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1855                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1856            }
1857            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1858                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1859            }
1860
1861            let kcfi_typeid = if let Some(instance) = instance {
1862                kcfi::typeid_for_instance(self.tcx, instance, options)
1863            } else {
1864                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1865            };
1866
1867            Some(llvm::OperandBundleOwned::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1868        } else {
1869            None
1870        };
1871        kcfi_bundle
1872    }
1873
1874    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1875    #[instrument(level = "debug", skip(self))]
1876    pub(crate) fn instrprof_increment(
1877        &mut self,
1878        fn_name: &'ll Value,
1879        hash: &'ll Value,
1880        num_counters: &'ll Value,
1881        index: &'ll Value,
1882    ) {
1883        self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]);
1884    }
1885
1886    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1887    ///
1888    /// This doesn't produce any code directly, but is used as input by
1889    /// the LLVM pass that handles coverage instrumentation.
1890    ///
1891    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1892    ///
1893    /// [`CodeGenPGO::emitMCDCParameters`]:
1894    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1895    #[instrument(level = "debug", skip(self))]
1896    pub(crate) fn mcdc_parameters(
1897        &mut self,
1898        fn_name: &'ll Value,
1899        hash: &'ll Value,
1900        bitmap_bits: &'ll Value,
1901    ) {
1902        assert!(
1903            crate::llvm_util::get_version() >= (19, 0, 0),
1904            "MCDC intrinsics require LLVM 19 or later"
1905        );
1906        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]);
1907    }
1908
1909    #[instrument(level = "debug", skip(self))]
1910    pub(crate) fn mcdc_tvbitmap_update(
1911        &mut self,
1912        fn_name: &'ll Value,
1913        hash: &'ll Value,
1914        bitmap_index: &'ll Value,
1915        mcdc_temp: &'ll Value,
1916    ) {
1917        assert!(
1918            crate::llvm_util::get_version() >= (19, 0, 0),
1919            "MCDC intrinsics require LLVM 19 or later"
1920        );
1921        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1922        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args);
1923    }
1924
1925    #[instrument(level = "debug", skip(self))]
1926    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1927        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1928    }
1929
1930    #[instrument(level = "debug", skip(self))]
1931    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1932        assert!(
1933            crate::llvm_util::get_version() >= (19, 0, 0),
1934            "MCDC intrinsics require LLVM 19 or later"
1935        );
1936        let align = self.tcx.data_layout.i32_align.abi;
1937        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1938        let new_tv_index = self.add(current_tv_index, cond_index);
1939        self.store(new_tv_index, mcdc_temp, align);
1940    }
1941}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
