rustc_codegen_ssa/traits/
builder.rs

1use std::assert_matches;
2use std::ops::Deref;
3
4use rustc_abi::{Align, Scalar, Size, WrappingRange};
5use rustc_ast::expand::typetree::{FncTree, TypeTree};
6use rustc_hir::attrs::AttributeKind;
7use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
8use rustc_middle::mir;
9use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
10use rustc_middle::ty::typetree::typetree_from_ty;
11use rustc_middle::ty::{AtomicOrdering, Instance, Ty};
12use rustc_session::config::OptLevel;
13use rustc_span::Span;
14use rustc_target::callconv::FnAbi;
15
16use super::abi::AbiBuilderMethods;
17use super::asm::AsmBuilderMethods;
18use super::consts::ConstCodegenMethods;
19use super::coverageinfo::CoverageInfoBuilderMethods;
20use super::debuginfo::DebugInfoBuilderMethods;
21use super::intrinsic::IntrinsicCallBuilderMethods;
22use super::misc::MiscCodegenMethods;
23use super::type_::{ArgAbiBuilderMethods, BaseTypeCodegenMethods, LayoutTypeCodegenMethods};
24use super::{CodegenMethods, StaticBuilderMethods};
25use crate::MemFlags;
26use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
27use crate::mir::operand::{OperandRef, OperandValue};
28use crate::mir::place::{PlaceRef, PlaceValue};
29
30#[derive(#[automatically_derived]
impl ::core::marker::Copy for OverflowOp { }Copy, #[automatically_derived]
#[doc(hidden)]
unsafe impl ::core::clone::TrivialClone for OverflowOp { }
#[automatically_derived]
impl ::core::clone::Clone for OverflowOp {
    #[inline]
    fn clone(&self) -> OverflowOp { *self }
}Clone, #[automatically_derived]
impl ::core::fmt::Debug for OverflowOp {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                OverflowOp::Add => "Add",
                OverflowOp::Sub => "Sub",
                OverflowOp::Mul => "Mul",
            })
    }
}Debug, #[automatically_derived]
impl ::core::marker::StructuralPartialEq for OverflowOp { }
#[automatically_derived]
impl ::core::cmp::PartialEq for OverflowOp {
    #[inline]
    fn eq(&self, other: &OverflowOp) -> bool {
        let __self_discr = ::core::intrinsics::discriminant_value(self);
        let __arg1_discr = ::core::intrinsics::discriminant_value(other);
        __self_discr == __arg1_discr
    }
}PartialEq, #[automatically_derived]
impl ::core::cmp::Eq for OverflowOp { }Eq)]
31pub enum OverflowOp {
32    Add,
33    Sub,
34    Mul,
35}
36
37/// The location of the return value for the call.
38#[derive(#[automatically_derived]
impl<V: ::core::marker::Copy> ::core::marker::Copy for ReturnSlot<V> { }Copy, #[automatically_derived]
impl<V: ::core::clone::Clone> ::core::clone::Clone for ReturnSlot<V> {
    #[inline]
    fn clone(&self) -> ReturnSlot<V> {
        match self {
            ReturnSlot::Direct => ReturnSlot::Direct,
            ReturnSlot::Indirect(__self_0) =>
                ReturnSlot::Indirect(::core::clone::Clone::clone(__self_0)),
        }
    }
}Clone, #[automatically_derived]
impl<V: ::core::fmt::Debug> ::core::fmt::Debug for ReturnSlot<V> {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            ReturnSlot::Direct =>
                ::core::fmt::Formatter::write_str(f, "Direct"),
            ReturnSlot::Indirect(__self_0) =>
                ::core::fmt::Formatter::debug_tuple_field1_finish(f,
                    "Indirect", &__self_0),
        }
    }
}Debug)]
39pub enum ReturnSlot<V> {
40    Direct,
41    /// The return value will be passed via sret (e.g. `PassMode::Indirect`).
42    Indirect(V),
43}
44
45impl<V> ReturnSlot<V> {
46    pub fn is_indirect(&self) -> bool {
47        #[allow(non_exhaustive_omitted_patterns)] match self {
    ReturnSlot::Indirect(_) => true,
    _ => false,
}matches!(self, ReturnSlot::Indirect(_))
48    }
49}
50
51pub trait BuilderMethods<'a, 'tcx>:
52    Sized
53    + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
54    + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
55    + Deref<Target = Self::CodegenCx>
56    + CoverageInfoBuilderMethods<'tcx>
57    + DebugInfoBuilderMethods<'tcx>
58    + ArgAbiBuilderMethods<'tcx>
59    + AbiBuilderMethods
60    + IntrinsicCallBuilderMethods<'tcx>
61    + AsmBuilderMethods<'tcx>
62    + StaticBuilderMethods
63{
64    // `BackendTypes` is a supertrait of both `CodegenMethods` and
65    // `BuilderMethods`. This bound ensures all impls agree on the associated
66    // types within.
67    type CodegenCx: CodegenMethods<
68            'tcx,
69            Value = Self::Value,
70            Function = Self::Function,
71            BasicBlock = Self::BasicBlock,
72            Type = Self::Type,
73            FunctionSignature = Self::FunctionSignature,
74            Funclet = Self::Funclet,
75            DIScope = Self::DIScope,
76            DILocation = Self::DILocation,
77            DIVariable = Self::DIVariable,
78        >;
79
80    fn build(cx: &'a Self::CodegenCx, llbb: Self::BasicBlock) -> Self;
81
82    fn cx(&self) -> &Self::CodegenCx;
83    fn llbb(&self) -> Self::BasicBlock;
84
85    fn set_span(&mut self, span: Span);
86
87    // FIXME(eddyb) replace uses of this with `append_sibling_block`.
88    fn append_block(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &str) -> Self::BasicBlock;
89
90    fn append_sibling_block(&mut self, name: &str) -> Self::BasicBlock;
91
92    fn switch_to_block(&mut self, llbb: Self::BasicBlock);
93
94    fn ret_void(&mut self);
95    fn ret(&mut self, v: Self::Value);
96    fn br(&mut self, dest: Self::BasicBlock);
97    fn br_with_attrs(&mut self, dest: Self::BasicBlock, _attributes: &[AttributeKind]) {
98        self.br(dest)
99    }
100    fn cond_br(
101        &mut self,
102        cond: Self::Value,
103        then_llbb: Self::BasicBlock,
104        else_llbb: Self::BasicBlock,
105    );
106
107    // Conditional with expectation.
108    //
109    // This function is opt-in for back ends.
110    //
111    // The default implementation calls `self.expect()` before emitting the branch
112    // by calling `self.cond_br()`
113    fn cond_br_with_expect(
114        &mut self,
115        mut cond: Self::Value,
116        then_llbb: Self::BasicBlock,
117        else_llbb: Self::BasicBlock,
118        expect: Option<bool>,
119    ) {
120        if let Some(expect) = expect {
121            cond = self.expect(cond, expect);
122        }
123        self.cond_br(cond, then_llbb, else_llbb)
124    }
125
126    fn switch(
127        &mut self,
128        v: Self::Value,
129        else_llbb: Self::BasicBlock,
130        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock)>,
131    );
132
133    // This is like `switch()`, but every case has a bool flag indicating whether it's cold.
134    //
135    // Default implementation throws away the cold flags and calls `switch()`.
136    fn switch_with_weights(
137        &mut self,
138        v: Self::Value,
139        else_llbb: Self::BasicBlock,
140        _else_is_cold: bool,
141        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
142    ) {
143        self.switch(v, else_llbb, cases.map(|(val, bb, _)| (val, bb)))
144    }
145
146    fn invoke(
147        &mut self,
148        llty: Self::FunctionSignature,
149        fn_attrs: Option<&CodegenFnAttrs>,
150        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
151        llfn: Self::Value,
152        return_slot: ReturnSlot<Self::Value>,
153        args: &[Self::Value],
154        then: Self::BasicBlock,
155        catch: Self::BasicBlock,
156        funclet: Option<&Self::Funclet>,
157        instance: Option<Instance<'tcx>>,
158    ) -> Self::Value;
159    fn unreachable(&mut self);
160
161    /// Like [`Self::unreachable`], but for use in the middle of a basic block.
162    fn unreachable_nonterminator(&mut self) {
163        // This is the preferred LLVM incantation for this per
164        // https://llvm.org/docs/Frontend/PerformanceTips.html#other-things-to-consider
165        // Other backends may override if they have a better way.
166        let const_true = self.cx().const_bool(true);
167        let poison_ptr = self.const_poison(self.cx().type_ptr());
168        self.store(const_true, poison_ptr, Align::ONE);
169    }
170
171    fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
172    fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
173    fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
174    fn fadd_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
175    fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
176    fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
177    fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
178    fn fsub_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
179    fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
180    fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
181    fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
182    fn fmul_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
183    fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
184    fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
185    fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
186    fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
187    fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
188    fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
189    fn fdiv_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
190    fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
191    fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
192    fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
193    fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
194    fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
195    /// Generate a left-shift. Both operands must have the same size. The right operand must be
196    /// interpreted as unsigned and can be assumed to be less than the size of the left operand.
197    fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
198    /// Generate a logical right-shift. Both operands must have the same size. The right operand
199    /// must be interpreted as unsigned and can be assumed to be less than the size of the left
200    /// operand.
201    fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
202    /// Generate an arithmetic right-shift. Both operands must have the same size. The right operand
203    /// must be interpreted as unsigned and can be assumed to be less than the size of the left
204    /// operand.
205    fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
206    fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
207        self.add(lhs, rhs)
208    }
209    fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
210        self.add(lhs, rhs)
211    }
212    fn unchecked_suadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
213        self.unchecked_sadd(lhs, rhs)
214    }
215    fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
216        self.sub(lhs, rhs)
217    }
218    fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
219        self.sub(lhs, rhs)
220    }
221    fn unchecked_susub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
222        self.unchecked_ssub(lhs, rhs)
223    }
224    fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
225        self.mul(lhs, rhs)
226    }
227    fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
228        self.mul(lhs, rhs)
229    }
230    fn unchecked_sumul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
231        // Which to default to is a fairly arbitrary choice,
232        // but this is what slice layout was using before.
233        self.unchecked_smul(lhs, rhs)
234    }
235    fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
236    fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
237    /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends
238    /// can emit something more helpful for optimizations.
239    fn or_disjoint(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
240        self.or(lhs, rhs)
241    }
242    fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
243    fn neg(&mut self, v: Self::Value) -> Self::Value;
244    fn fneg(&mut self, v: Self::Value) -> Self::Value;
245    fn not(&mut self, v: Self::Value) -> Self::Value;
246
247    fn checked_binop(
248        &mut self,
249        oop: OverflowOp,
250        ty: Ty<'tcx>,
251        lhs: Self::Value,
252        rhs: Self::Value,
253    ) -> (Self::Value, Self::Value);
254
255    fn from_immediate(&mut self, val: Self::Value) -> Self::Value;
256    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value;
257
258    fn alloca(&mut self, size: Size, align: Align) -> Self::Value;
259    fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value;
260
261    fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
262    fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
263    fn atomic_load(
264        &mut self,
265        ty: Self::Type,
266        ptr: Self::Value,
267        order: AtomicOrdering,
268        volatile: bool,
269        size: Size,
270    ) -> Self::Value;
271    fn load_from_place(&mut self, ty: Self::Type, place: PlaceValue<Self::Value>) -> Self::Value {
272        {
    match (&place.llextra, &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(place.llextra, None);
273        self.load(ty, place.llval, place.align)
274    }
275    fn load_operand(&mut self, place: PlaceRef<'tcx, Self::Value>)
276    -> OperandRef<'tcx, Self::Value>;
277
278    /// Called for Rvalue::Repeat when the elem is neither a ZST nor optimizable using memset.
279    fn write_operand_repeatedly(
280        &mut self,
281        elem: OperandRef<'tcx, Self::Value>,
282        count: u64,
283        dest: PlaceRef<'tcx, Self::Value>,
284    );
285
286    /// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`.
287    ///
288    /// This *always* emits the assumption, so you probably want to check the
289    /// optimization level and `Scalar::is_always_valid` before calling it.
290    fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) {
291        let WrappingRange { start, end } = range;
292
293        // Perhaps one day we'll be able to use assume operand bundles for this,
294        // but for now this encoding with a single icmp+assume is best per
295        // <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158>
296        let shifted = if start == 0 {
297            imm
298        } else {
299            let low = self.const_uint_big(ty, start);
300            self.sub(imm, low)
301        };
302        let width = self.const_uint_big(ty, u128::wrapping_sub(end, start));
303        let cmp = self.icmp(IntPredicate::IntULE, shifted, width);
304        self.assume(cmp);
305    }
306
307    /// Emits an `assume` that the `val` of pointer type is non-null.
308    ///
309    /// You may want to check the optimization level before bothering calling this.
310    fn assume_nonnull(&mut self, val: Self::Value) {
311        // Arguably in LLVM it'd be better to emit an assume operand bundle instead
312        // <https://llvm.org/docs/LangRef.html#assume-operand-bundles>
313        // but this works fine for all backends.
314
315        let null = self.const_null(self.type_ptr());
316        let is_null = self.icmp(IntPredicate::IntNE, val, null);
317        self.assume(is_null);
318    }
319
320    fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
321    fn nonnull_metadata(&mut self, load: Self::Value);
322
323    fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) -> Self::Value;
324    fn store_to_place(&mut self, val: Self::Value, place: PlaceValue<Self::Value>) -> Self::Value {
325        {
    match (&place.llextra, &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(place.llextra, None);
326        self.store(val, place.llval, place.align)
327    }
328    fn store_with_flags(
329        &mut self,
330        val: Self::Value,
331        ptr: Self::Value,
332        align: Align,
333        flags: MemFlags,
334    ) -> Self::Value;
335    fn store_to_place_with_flags(
336        &mut self,
337        val: Self::Value,
338        place: PlaceValue<Self::Value>,
339        flags: MemFlags,
340    ) -> Self::Value {
341        {
    match (&place.llextra, &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(place.llextra, None);
342        self.store_with_flags(val, place.llval, place.align, flags)
343    }
344    fn atomic_store(
345        &mut self,
346        val: Self::Value,
347        ptr: Self::Value,
348        order: AtomicOrdering,
349        volatile: bool,
350        size: Size,
351    );
352
353    fn gep(&mut self, ty: Self::Type, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value;
354    fn inbounds_gep(
355        &mut self,
356        ty: Self::Type,
357        ptr: Self::Value,
358        indices: &[Self::Value],
359    ) -> Self::Value;
360    fn inbounds_nuw_gep(
361        &mut self,
362        ty: Self::Type,
363        ptr: Self::Value,
364        indices: &[Self::Value],
365    ) -> Self::Value {
366        self.inbounds_gep(ty, ptr, indices)
367    }
368    fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
369        self.gep(self.cx().type_i8(), ptr, &[offset])
370    }
371    fn inbounds_ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
372        self.inbounds_gep(self.cx().type_i8(), ptr, &[offset])
373    }
374
375    fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
376    /// Produces the same value as [`Self::trunc`] (and defaults to that),
377    /// but is UB unless the *zero*-extending the result can reproduce `val`.
378    fn unchecked_utrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
379        self.trunc(val, dest_ty)
380    }
381    /// Produces the same value as [`Self::trunc`] (and defaults to that),
382    /// but is UB unless the *sign*-extending the result can reproduce `val`.
383    fn unchecked_strunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
384        self.trunc(val, dest_ty)
385    }
386
387    fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
388    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
389    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
390    fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
391    fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
392    fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
393    fn sitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
394    fn fptrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
395    fn fpext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
396    fn ptrtoint(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
397    fn inttoptr(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
398    fn bitcast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
399    fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value;
400    fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
401
402    fn cast_float_to_int(
403        &mut self,
404        signed: bool,
405        x: Self::Value,
406        dest_ty: Self::Type,
407    ) -> Self::Value {
408        let in_ty = self.cx().val_ty(x);
409        let (float_ty, int_ty) = if self.cx().type_kind(dest_ty) == TypeKind::Vector
410            && self.cx().type_kind(in_ty) == TypeKind::Vector
411        {
412            (self.cx().element_type(in_ty), self.cx().element_type(dest_ty))
413        } else {
414            (in_ty, dest_ty)
415        };
416        {
    match self.cx().type_kind(float_ty) {
        TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128
            => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
417            self.cx().type_kind(float_ty),
418            TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128
419        );
420        {
    match (&self.cx().type_kind(int_ty), &TypeKind::Integer) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer);
421
422        if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }
423    }
424
425    fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
426    fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
427
428    /// Returns `-1` if `lhs < rhs`, `0` if `lhs == rhs`, and `1` if `lhs > rhs`.
429    fn three_way_compare(
430        &mut self,
431        ty: Ty<'tcx>,
432        lhs: Self::Value,
433        rhs: Self::Value,
434    ) -> Self::Value {
435        // FIXME: This implementation was designed around LLVM's ability to optimize, but `cg_llvm`
436        // overrides this to just use `@llvm.scmp`/`ucmp` since LLVM 20. This default impl should be
437        // reevaluated with respect to the remaining backends like cg_gcc, whether they might use
438        // specialized implementations as well, or continue to use a generic implementation here.
439        use std::cmp::Ordering;
440        let pred = |op| crate::base::bin_op_to_icmp_predicate(op, ty.is_signed());
441        if self.cx().sess().opts.optimize == OptLevel::No {
442            // This actually generates tighter assembly, and is a classic trick:
443            // <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>.
444            // However, as of 2023-11 it optimized worse in LLVM in things like derived
445            // `PartialOrd`, so we were only using it in debug. Since LLVM now uses its own
446            // intrinsics, it may be be worth trying it in optimized builds for other backends.
447            let is_gt = self.icmp(pred(mir::BinOp::Gt), lhs, rhs);
448            let gtext = self.zext(is_gt, self.type_i8());
449            let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs);
450            let ltext = self.zext(is_lt, self.type_i8());
451            self.unchecked_ssub(gtext, ltext)
452        } else {
453            // These operations were better optimized by LLVM, before `@llvm.scmp`/`ucmp` in 20.
454            // See <https://github.com/rust-lang/rust/pull/63767>.
455            let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs);
456            let is_ne = self.icmp(pred(mir::BinOp::Ne), lhs, rhs);
457            let ge = self.select(
458                is_ne,
459                self.cx().const_i8(Ordering::Greater as i8),
460                self.cx().const_i8(Ordering::Equal as i8),
461            );
462            self.select(is_lt, self.cx().const_i8(Ordering::Less as i8), ge)
463        }
464    }
465
466    fn memcpy(
467        &mut self,
468        dst: Self::Value,
469        dst_align: Align,
470        src: Self::Value,
471        src_align: Align,
472        size: Self::Value,
473        flags: MemFlags,
474        tt: Option<FncTree>,
475    );
476    fn memmove(
477        &mut self,
478        dst: Self::Value,
479        dst_align: Align,
480        src: Self::Value,
481        src_align: Align,
482        size: Self::Value,
483        flags: MemFlags,
484    );
485    fn memset(
486        &mut self,
487        ptr: Self::Value,
488        fill_byte: Self::Value,
489        size: Self::Value,
490        align: Align,
491        flags: MemFlags,
492    );
493
494    // Produce a value from calling the `vscale` intrinsic (containing the `vscale` multiplier that
495    // a scalable vector's element size and count can be multiplied by to get the real size of the
496    // vector)
497    fn vscale(&mut self, ty: Self::Type) -> Self::Value;
498
499    /// *Typed* copy for non-overlapping places.
500    ///
501    /// Has a default implementation in terms of `memcpy`, but specific backends
502    /// can override to do something smarter if possible.
503    ///
504    /// (For example, typed load-stores with alias metadata.)
505    fn typed_place_copy(
506        &mut self,
507        dst: PlaceValue<Self::Value>,
508        src: PlaceValue<Self::Value>,
509        layout: TyAndLayout<'tcx>,
510    ) {
511        self.typed_place_copy_with_flags(dst, src, layout, MemFlags::empty());
512    }
513
514    fn typed_place_copy_with_flags(
515        &mut self,
516        dst: PlaceValue<Self::Value>,
517        src: PlaceValue<Self::Value>,
518        layout: TyAndLayout<'tcx>,
519        flags: MemFlags,
520    ) {
521        if !layout.is_sized() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot typed-copy an unsigned type"));
    }
};assert!(layout.is_sized(), "cannot typed-copy an unsigned type");
522        if !src.llextra.is_none() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot directly copy from unsized values"));
    }
};assert!(src.llextra.is_none(), "cannot directly copy from unsized values");
523        if !dst.llextra.is_none() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot directly copy into unsized values"));
    }
};assert!(dst.llextra.is_none(), "cannot directly copy into unsized values");
524        if flags.contains(MemFlags::NONTEMPORAL) {
525            // HACK(nox): This is inefficient but there is no nontemporal memcpy.
526            let ty = self.backend_type(layout);
527            let val = self.load_from_place(ty, src);
528            self.store_to_place_with_flags(val, dst, flags);
529        } else if self.sess().opts.optimize == OptLevel::No
530            && layout.backend_repr.is_scalar_or_simd()
531        {
532            // If we're not optimizing, the aliasing information from `memcpy`
533            // isn't useful, so just load-store the value for smaller code.
534            let temp = self.load_operand(src.with_type(layout));
535            temp.val.store_with_flags(self, dst.with_type(layout), flags);
536        } else if !layout.is_zst() {
537            let tt = typetree_from_ty(self.tcx(), layout.ty);
538            // We seem to pass all values to memcpy with one more indirection.
539            let tt = tt.add_indirection();
540            let fnc_tree = FncTree { args: ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [tt.clone(), tt]))vec![tt.clone(), tt], ret: TypeTree::new() };
541            let bytes = self.const_usize(layout.size.bytes());
542            let bytes = if layout.peel_transparent_wrappers(self).ty.is_scalable_vector() {
543                let vscale = self.vscale(self.type_i64());
544                self.mul(vscale, bytes)
545            } else {
546                bytes
547            };
548            self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, Some(fnc_tree));
549        }
550    }
551
552    /// *Typed* swap for non-overlapping places.
553    ///
554    /// Avoids `alloca`s for Immediates and ScalarPairs.
555    ///
556    /// FIXME: Maybe do something smarter for Ref types too?
557    /// For now, the `typed_swap_nonoverlapping` intrinsic just doesn't call this for those
558    /// cases (in non-debug), preferring the fallback body instead.
559    fn typed_place_swap(
560        &mut self,
561        left: PlaceValue<Self::Value>,
562        right: PlaceValue<Self::Value>,
563        layout: TyAndLayout<'tcx>,
564    ) {
565        let mut temp = self.load_operand(left.with_type(layout));
566        if let OperandValue::Ref(..) = temp.val {
567            // The SSA value isn't stand-alone, so we need to copy it elsewhere
568            let alloca = PlaceRef::alloca(self, layout);
569            self.typed_place_copy(alloca.val, left, layout);
570            temp = self.load_operand(alloca);
571        }
572        self.typed_place_copy(left, right, layout);
573        temp.val.store(self, right.with_type(layout));
574    }
575
576    fn select(
577        &mut self,
578        cond: Self::Value,
579        then_val: Self::Value,
580        else_val: Self::Value,
581    ) -> Self::Value;
582
583    fn va_arg(&mut self, list: Self::Value, ty: Self::Type) -> Self::Value;
584    fn extract_element(&mut self, vec: Self::Value, idx: Self::Value) -> Self::Value;
585    fn vector_splat(&mut self, num_elts: usize, elt: Self::Value) -> Self::Value;
586    fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value;
587    fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value;
588
589    fn set_personality_fn(&mut self, personality: Self::Function);
590
591    // These are used by everyone except msvc and wasm EH
592    fn cleanup_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
593    fn filter_landing_pad(&mut self, pers_fn: Self::Function);
594    fn resume(&mut self, exn0: Self::Value, exn1: Self::Value);
595
596    // These are used by msvc and wasm EH
597    fn cleanup_pad(&mut self, parent: Option<Self::Value>, args: &[Self::Value]) -> Self::Funclet;
598    fn cleanup_ret(&mut self, funclet: &Self::Funclet, unwind: Option<Self::BasicBlock>);
599    fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet;
600    fn catch_switch(
601        &mut self,
602        parent: Option<Self::Value>,
603        unwind: Option<Self::BasicBlock>,
604        handlers: &[Self::BasicBlock],
605    ) -> Self::Value;
606    fn get_funclet_cleanuppad(&self, funclet: &Self::Funclet) -> Self::Value;
607
608    fn atomic_cmpxchg(
609        &mut self,
610        dst: Self::Value,
611        cmp: Self::Value,
612        src: Self::Value,
613        order: AtomicOrdering,
614        failure_order: AtomicOrdering,
615        weak: bool,
616    ) -> (Self::Value, Self::Value);
617    /// `ret_ptr` indicates whether the return type (which is also the type `dst` points to)
618    /// is a pointer or the same type as `src`.
619    fn atomic_rmw(
620        &mut self,
621        op: AtomicRmwBinOp,
622        dst: Self::Value,
623        src: Self::Value,
624        order: AtomicOrdering,
625        ret_ptr: bool,
626    ) -> Self::Value;
627    fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope);
628    fn set_invariant_load(&mut self, load: Self::Value);
629
630    /// Called for `StorageLive`
631    fn lifetime_start(&mut self, ptr: Self::Value, size: Size);
632
633    /// Called for `StorageDead`
634    fn lifetime_end(&mut self, ptr: Self::Value, size: Size);
635
636    /// "Finally codegen the call"
637    ///
638    /// ## Arguments
639    ///
640    /// `caller_attrs` are the attributes of the surrounding caller; they have nothing to do with
641    /// the callee.
642    ///
643    /// The `caller_attrs`, `fn_abi`, and `callee_instance` arguments are Options because they are
644    /// advisory. They relate to optional codegen enhancements like LLVM CFI, and do not affect ABI
645    /// per se. Any ABI-related transformations should be handled by different, earlier stages of
646    /// codegen. For instance, in the caller of `BuilderMethods::call`.
647    ///
648    /// This means that a codegen backend which disregards `fn_attrs`, `fn_abi`, and `instance`
649    /// should still do correct codegen, and code should not be miscompiled if they are omitted.
650    /// It is not a miscompilation in this sense if it fails to run under CFI, other sanitizers, or
651    /// in the context of other compiler-enhanced security features.
652    ///
653    /// The typical case that they are None is during the codegen of intrinsics and lang-items,
654    /// as those are "fake functions" with only a trivial ABI if any, et cetera.
655    ///
656    /// `return_slot` must be `ReturnSlot::Indirect` if an argument uses `PassMode::Indirect`.
657    ///
658    /// ## Return
659    ///
660    /// Must return the value the function will return so it can be written to the destination.
661    /// For calls with an indirect return, the returned value is meaningless and must not be
662    /// used: the return value lives in the return slot.
663    fn call(
664        &mut self,
665        llty: Self::FunctionSignature,
666        caller_attrs: Option<&CodegenFnAttrs>,
667        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
668        fn_val: Self::Value,
669        return_slot: ReturnSlot<Self::Value>,
670        args: &[Self::Value],
671        funclet: Option<&Self::Funclet>,
672        callee_instance: Option<Instance<'tcx>>,
673    ) -> Self::Value;
674
675    fn tail_call(
676        &mut self,
677        llty: Self::FunctionSignature,
678        caller_attrs: Option<&CodegenFnAttrs>,
679        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
680        llfn: Self::Value,
681        return_slot: ReturnSlot<Self::Value>,
682        args: &[Self::Value],
683        funclet: Option<&Self::Funclet>,
684        callee_instance: Option<Instance<'tcx>>,
685    );
686
687    fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
688
689    fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value);
690}
rustc_codegen_ssa/traits/builder.rs

rustc_codegen_ssa/traits/
builder.rs
