rustc_const_eval/interpret/
intrinsics.rs

1//! Intrinsics and other functions that the interpreter executes without
2//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
3//! and miri.
4
5use std::assert_matches::assert_matches;
6
7use rustc_abi::Size;
8use rustc_apfloat::ieee::{Double, Half, Quad, Single};
9use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
10use rustc_middle::ty::layout::TyAndLayout;
11use rustc_middle::ty::{Ty, TyCtxt};
12use rustc_middle::{bug, ty};
13use rustc_span::{Symbol, sym};
14use tracing::trace;
15
16use super::memory::MemoryKind;
17use super::util::ensure_monomorphic_enough;
18use super::{
19    Allocation, CheckInAllocMsg, ConstAllocation, ImmTy, InterpCx, InterpResult, Machine, OpTy,
20    PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format,
21    interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
22};
23use crate::fluent_generated as fluent;
24
25/// Directly returns an `Allocation` containing an absolute path representation of the given type.
26pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
27    let path = crate::util::type_name(tcx, ty);
28    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
29    tcx.mk_const_alloc(alloc)
30}
31
32impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
33    /// Returns `true` if emulation happened.
34    /// Here we implement the intrinsics that are common to all Miri instances; individual machines can add their own
35    /// intrinsic handling.
36    pub fn eval_intrinsic(
37        &mut self,
38        instance: ty::Instance<'tcx>,
39        args: &[OpTy<'tcx, M::Provenance>],
40        dest: &PlaceTy<'tcx, M::Provenance>,
41        ret: Option<mir::BasicBlock>,
42    ) -> InterpResult<'tcx, bool> {
43        let instance_args = instance.args;
44        let intrinsic_name = self.tcx.item_name(instance.def_id());
45        let tcx = self.tcx.tcx;
46
47        match intrinsic_name {
48            sym::type_name => {
49                let tp_ty = instance.args.type_at(0);
50                ensure_monomorphic_enough(tcx, tp_ty)?;
51                let alloc = alloc_type_name(tcx, tp_ty);
52                let val = ConstValue::Slice { data: alloc, meta: alloc.inner().size().bytes() };
53                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
54                self.copy_op(&val, dest)?;
55            }
56            sym::needs_drop => {
57                let tp_ty = instance.args.type_at(0);
58                ensure_monomorphic_enough(tcx, tp_ty)?;
59                let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
60                let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
61                self.copy_op(&val, dest)?;
62            }
63            sym::type_id => {
64                let tp_ty = instance.args.type_at(0);
65                ensure_monomorphic_enough(tcx, tp_ty)?;
66                let val = ConstValue::from_u128(tcx.type_id_hash(tp_ty).as_u128());
67                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
68                self.copy_op(&val, dest)?;
69            }
70            sym::variant_count => {
71                let tp_ty = instance.args.type_at(0);
72                let ty = match tp_ty.kind() {
73                    // Pattern types have the same number of variants as their base type.
74                    // Even if we restrict e.g. which variants are valid, the variants are essentially just uninhabited.
75                    // And `Result<(), !>` still has two variants according to `variant_count`.
76                    ty::Pat(base, _) => *base,
77                    _ => tp_ty,
78                };
79                let val = match ty.kind() {
80                    // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
81                    ty::Adt(adt, _) => {
82                        ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
83                    }
84                    ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
85                        throw_inval!(TooGeneric)
86                    }
87                    ty::Pat(..) => unreachable!(),
88                    ty::Bound(_, _) => bug!("bound ty during ctfe"),
89                    ty::Bool
90                    | ty::Char
91                    | ty::Int(_)
92                    | ty::Uint(_)
93                    | ty::Float(_)
94                    | ty::Foreign(_)
95                    | ty::Str
96                    | ty::Array(_, _)
97                    | ty::Slice(_)
98                    | ty::RawPtr(_, _)
99                    | ty::Ref(_, _, _)
100                    | ty::FnDef(_, _)
101                    | ty::FnPtr(..)
102                    | ty::Dynamic(_, _, _)
103                    | ty::Closure(_, _)
104                    | ty::CoroutineClosure(_, _)
105                    | ty::Coroutine(_, _)
106                    | ty::CoroutineWitness(..)
107                    | ty::UnsafeBinder(_)
108                    | ty::Never
109                    | ty::Tuple(_)
110                    | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
111                };
112                let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
113                self.copy_op(&val, dest)?;
114            }
115
116            sym::caller_location => {
117                let span = self.find_closest_untracked_caller_location();
118                let val = self.tcx.span_as_caller_location(span);
119                let val =
120                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
121                self.copy_op(&val, dest)?;
122            }
123
124            sym::align_of_val | sym::size_of_val => {
125                // Avoid `deref_pointer` -- this is not a deref, the ptr does not have to be
126                // dereferenceable!
127                let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
128                let (size, align) = self
129                    .size_and_align_of_val(&place)?
130                    .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
131
132                let result = match intrinsic_name {
133                    sym::align_of_val => align.bytes(),
134                    sym::size_of_val => size.bytes(),
135                    _ => bug!(),
136                };
137
138                self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
139            }
140
141            sym::fadd_algebraic
142            | sym::fsub_algebraic
143            | sym::fmul_algebraic
144            | sym::fdiv_algebraic
145            | sym::frem_algebraic => {
146                let a = self.read_immediate(&args[0])?;
147                let b = self.read_immediate(&args[1])?;
148
149                let op = match intrinsic_name {
150                    sym::fadd_algebraic => BinOp::Add,
151                    sym::fsub_algebraic => BinOp::Sub,
152                    sym::fmul_algebraic => BinOp::Mul,
153                    sym::fdiv_algebraic => BinOp::Div,
154                    sym::frem_algebraic => BinOp::Rem,
155
156                    _ => bug!(),
157                };
158
159                let res = self.binary_op(op, &a, &b)?;
160                // `binary_op` already called `generate_nan` if needed.
161                let res = M::apply_float_nondet(self, res)?;
162                self.write_immediate(*res, dest)?;
163            }
164
165            sym::ctpop
166            | sym::cttz
167            | sym::cttz_nonzero
168            | sym::ctlz
169            | sym::ctlz_nonzero
170            | sym::bswap
171            | sym::bitreverse => {
172                let ty = instance_args.type_at(0);
173                let layout = self.layout_of(ty)?;
174                let val = self.read_scalar(&args[0])?;
175
176                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
177                self.write_scalar(out_val, dest)?;
178            }
179            sym::saturating_add | sym::saturating_sub => {
180                let l = self.read_immediate(&args[0])?;
181                let r = self.read_immediate(&args[1])?;
182                let val = self.saturating_arith(
183                    if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
184                    &l,
185                    &r,
186                )?;
187                self.write_scalar(val, dest)?;
188            }
189            sym::discriminant_value => {
190                let place = self.deref_pointer(&args[0])?;
191                let variant = self.read_discriminant(&place)?;
192                let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
193                self.write_immediate(*discr, dest)?;
194            }
195            sym::exact_div => {
196                let l = self.read_immediate(&args[0])?;
197                let r = self.read_immediate(&args[1])?;
198                self.exact_div(&l, &r, dest)?;
199            }
200            sym::rotate_left | sym::rotate_right => {
201                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
202                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
203                let layout_val = self.layout_of(instance_args.type_at(0))?;
204                let val = self.read_scalar(&args[0])?;
205                let val_bits = val.to_bits(layout_val.size)?; // sign is ignored here
206
207                let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
208                let raw_shift = self.read_scalar(&args[1])?;
209                let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
210
211                let width_bits = u128::from(layout_val.size.bits());
212                let shift_bits = raw_shift_bits % width_bits;
213                let inv_shift_bits = (width_bits - shift_bits) % width_bits;
214                let result_bits = if intrinsic_name == sym::rotate_left {
215                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
216                } else {
217                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
218                };
219                let truncated_bits = layout_val.size.truncate(result_bits);
220                let result = Scalar::from_uint(truncated_bits, layout_val.size);
221                self.write_scalar(result, dest)?;
222            }
223            sym::copy => {
224                self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
225            }
226            sym::write_bytes => {
227                self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
228            }
229            sym::compare_bytes => {
230                let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
231                self.write_scalar(result, dest)?;
232            }
233            sym::arith_offset => {
234                let ptr = self.read_pointer(&args[0])?;
235                let offset_count = self.read_target_isize(&args[1])?;
236                let pointee_ty = instance_args.type_at(0);
237
238                let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
239                let offset_bytes = offset_count.wrapping_mul(pointee_size);
240                let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
241                self.write_pointer(offset_ptr, dest)?;
242            }
243            sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
244                let a = self.read_pointer(&args[0])?;
245                let b = self.read_pointer(&args[1])?;
246
247                let usize_layout = self.layout_of(self.tcx.types.usize)?;
248                let isize_layout = self.layout_of(self.tcx.types.isize)?;
249
250                // Get offsets for both that are at least relative to the same base.
251                // With `OFFSET_IS_ADDR` this is trivial; without it we need either
252                // two integers or two pointers into the same allocation.
253                let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
254                    (a.addr().bytes(), b.addr().bytes(), /*is_addr*/ true)
255                } else {
256                    match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
257                        (Err(a), Err(b)) => {
258                            // Neither pointer points to an allocation, so they are both absolute.
259                            (a, b, /*is_addr*/ true)
260                        }
261                        (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
262                            if a_alloc_id == b_alloc_id =>
263                        {
264                            // Found allocation for both, and it's the same.
265                            // Use these offsets for distance calculation.
266                            (a_offset.bytes(), b_offset.bytes(), /*is_addr*/ false)
267                        }
268                        _ => {
269                            // Not into the same allocation -- this is UB.
270                            throw_ub_custom!(
271                                fluent::const_eval_offset_from_different_allocations,
272                                name = intrinsic_name,
273                            );
274                        }
275                    }
276                };
277
278                // Compute distance: a - b.
279                let dist = {
280                    // Addresses are unsigned, so this is a `usize` computation. We have to do the
281                    // overflow check separately anyway.
282                    let (val, overflowed) = {
283                        let a_offset = ImmTy::from_uint(a_offset, usize_layout);
284                        let b_offset = ImmTy::from_uint(b_offset, usize_layout);
285                        self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
286                            .to_scalar_pair()
287                    };
288                    if overflowed.to_bool()? {
289                        // a < b
290                        if intrinsic_name == sym::ptr_offset_from_unsigned {
291                            throw_ub_custom!(
292                                fluent::const_eval_offset_from_unsigned_overflow,
293                                a_offset = a_offset,
294                                b_offset = b_offset,
295                                is_addr = is_addr,
296                            );
297                        }
298                        // The signed form of the intrinsic allows this. If we interpret the
299                        // difference as isize, we'll get the proper signed difference. If that
300                        // seems *positive* or equal to isize::MIN, they were more than isize::MAX apart.
301                        let dist = val.to_target_isize(self)?;
302                        if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
303                            throw_ub_custom!(
304                                fluent::const_eval_offset_from_underflow,
305                                name = intrinsic_name,
306                            );
307                        }
308                        dist
309                    } else {
310                        // b >= a
311                        let dist = val.to_target_isize(self)?;
312                        // If converting to isize produced a *negative* result, we had an overflow
313                        // because they were more than isize::MAX apart.
314                        if dist < 0 {
315                            throw_ub_custom!(
316                                fluent::const_eval_offset_from_overflow,
317                                name = intrinsic_name,
318                            );
319                        }
320                        dist
321                    }
322                };
323
324                // Check that the memory between them is dereferenceable at all, starting from the
325                // origin pointer: `dist` is `a - b`, so it is based on `b`.
326                self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
327                    .map_err_kind(|_| {
328                        // This could mean they point to different allocations, or they point to the same allocation
329                        // but not the entire range between the pointers is in-bounds.
330                        if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
331                            && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
332                            && a_alloc_id == b_alloc_id
333                        {
334                            err_ub_custom!(
335                                fluent::const_eval_offset_from_out_of_bounds,
336                                name = intrinsic_name,
337                            )
338                        } else {
339                            err_ub_custom!(
340                                fluent::const_eval_offset_from_different_allocations,
341                                name = intrinsic_name,
342                            )
343                        }
344                    })?;
345                // Then check that this is also dereferenceable from `a`. This ensures that they are
346                // derived from the same allocation.
347                self.check_ptr_access_signed(
348                    a,
349                    dist.checked_neg().unwrap(), // i64::MIN is impossible as no allocation can be that large
350                    CheckInAllocMsg::Dereferenceable,
351                )
352                .map_err_kind(|_| {
353                    // Make the error more specific.
354                    err_ub_custom!(
355                        fluent::const_eval_offset_from_different_allocations,
356                        name = intrinsic_name,
357                    )
358                })?;
359
360                // Perform division by size to compute return value.
361                let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
362                    assert!(0 <= dist && dist <= self.target_isize_max());
363                    usize_layout
364                } else {
365                    assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
366                    isize_layout
367                };
368                let pointee_layout = self.layout_of(instance_args.type_at(0))?;
369                // If ret_layout is unsigned, we checked that so is the distance, so we are good.
370                let val = ImmTy::from_int(dist, ret_layout);
371                let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
372                self.exact_div(&val, &size, dest)?;
373            }
374
375            sym::simd_insert => {
376                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
377                let elem = &args[2];
378                let (input, input_len) = self.project_to_simd(&args[0])?;
379                let (dest, dest_len) = self.project_to_simd(dest)?;
380                assert_eq!(input_len, dest_len, "Return vector length must match input length");
381                // Bounds are not checked by typeck so we have to do it ourselves.
382                if index >= input_len {
383                    throw_ub_format!(
384                        "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
385                    );
386                }
387
388                for i in 0..dest_len {
389                    let place = self.project_index(&dest, i)?;
390                    let value =
391                        if i == index { elem.clone() } else { self.project_index(&input, i)? };
392                    self.copy_op(&value, &place)?;
393                }
394            }
395            sym::simd_extract => {
396                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
397                let (input, input_len) = self.project_to_simd(&args[0])?;
398                // Bounds are not checked by typeck so we have to do it ourselves.
399                if index >= input_len {
400                    throw_ub_format!(
401                        "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
402                    );
403                }
404                self.copy_op(&self.project_index(&input, index)?, dest)?;
405            }
406            sym::black_box => {
407                // These just return their argument
408                self.copy_op(&args[0], dest)?;
409            }
410            sym::raw_eq => {
411                let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
412                self.write_scalar(result, dest)?;
413            }
414            sym::typed_swap_nonoverlapping => {
415                self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
416            }
417
418            sym::vtable_size => {
419                let ptr = self.read_pointer(&args[0])?;
420                // `None` because we don't know which trait to expect here; any vtable is okay.
421                let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
422                self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
423            }
424            sym::vtable_align => {
425                let ptr = self.read_pointer(&args[0])?;
426                // `None` because we don't know which trait to expect here; any vtable is okay.
427                let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
428                self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
429            }
430
431            sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
432            sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
433            sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
434            sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
435
436            sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
437            sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
438            sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
439            sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
440
441            sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
442            sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
443            sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
444            sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
445
446            sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
447            sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
448            sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
449            sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
450
451            sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
452            sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
453            sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
454            sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
455
456            sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
457            sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
458            sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
459            sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
460
461            sym::floorf16 => self.float_round_intrinsic::<Half>(
462                args,
463                dest,
464                rustc_apfloat::Round::TowardNegative,
465            )?,
466            sym::floorf32 => self.float_round_intrinsic::<Single>(
467                args,
468                dest,
469                rustc_apfloat::Round::TowardNegative,
470            )?,
471            sym::floorf64 => self.float_round_intrinsic::<Double>(
472                args,
473                dest,
474                rustc_apfloat::Round::TowardNegative,
475            )?,
476            sym::floorf128 => self.float_round_intrinsic::<Quad>(
477                args,
478                dest,
479                rustc_apfloat::Round::TowardNegative,
480            )?,
481
482            sym::ceilf16 => self.float_round_intrinsic::<Half>(
483                args,
484                dest,
485                rustc_apfloat::Round::TowardPositive,
486            )?,
487            sym::ceilf32 => self.float_round_intrinsic::<Single>(
488                args,
489                dest,
490                rustc_apfloat::Round::TowardPositive,
491            )?,
492            sym::ceilf64 => self.float_round_intrinsic::<Double>(
493                args,
494                dest,
495                rustc_apfloat::Round::TowardPositive,
496            )?,
497            sym::ceilf128 => self.float_round_intrinsic::<Quad>(
498                args,
499                dest,
500                rustc_apfloat::Round::TowardPositive,
501            )?,
502
503            sym::truncf16 => {
504                self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
505            }
506            sym::truncf32 => {
507                self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
508            }
509            sym::truncf64 => {
510                self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
511            }
512            sym::truncf128 => {
513                self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
514            }
515
516            sym::roundf16 => self.float_round_intrinsic::<Half>(
517                args,
518                dest,
519                rustc_apfloat::Round::NearestTiesToAway,
520            )?,
521            sym::roundf32 => self.float_round_intrinsic::<Single>(
522                args,
523                dest,
524                rustc_apfloat::Round::NearestTiesToAway,
525            )?,
526            sym::roundf64 => self.float_round_intrinsic::<Double>(
527                args,
528                dest,
529                rustc_apfloat::Round::NearestTiesToAway,
530            )?,
531            sym::roundf128 => self.float_round_intrinsic::<Quad>(
532                args,
533                dest,
534                rustc_apfloat::Round::NearestTiesToAway,
535            )?,
536
537            sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
538                args,
539                dest,
540                rustc_apfloat::Round::NearestTiesToEven,
541            )?,
542            sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
543                args,
544                dest,
545                rustc_apfloat::Round::NearestTiesToEven,
546            )?,
547            sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
548                args,
549                dest,
550                rustc_apfloat::Round::NearestTiesToEven,
551            )?,
552            sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
553                args,
554                dest,
555                rustc_apfloat::Round::NearestTiesToEven,
556            )?,
557
558            // Unsupported intrinsic: skip the return_to_block below.
559            _ => return interp_ok(false),
560        }
561
562        trace!("{:?}", self.dump_place(&dest.clone().into()));
563        self.return_to_block(ret)?;
564        interp_ok(true)
565    }
566
567    pub(super) fn eval_nondiverging_intrinsic(
568        &mut self,
569        intrinsic: &NonDivergingIntrinsic<'tcx>,
570    ) -> InterpResult<'tcx> {
571        match intrinsic {
572            NonDivergingIntrinsic::Assume(op) => {
573                let op = self.eval_operand(op, None)?;
574                let cond = self.read_scalar(&op)?.to_bool()?;
575                if !cond {
576                    throw_ub_custom!(fluent::const_eval_assume_false);
577                }
578                interp_ok(())
579            }
580            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
581                count,
582                src,
583                dst,
584            }) => {
585                let src = self.eval_operand(src, None)?;
586                let dst = self.eval_operand(dst, None)?;
587                let count = self.eval_operand(count, None)?;
588                self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)
589            }
590        }
591    }
592
593    pub fn numeric_intrinsic(
594        &self,
595        name: Symbol,
596        val: Scalar<M::Provenance>,
597        layout: TyAndLayout<'tcx>,
598        ret_layout: TyAndLayout<'tcx>,
599    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
600        assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
601        let bits = val.to_bits(layout.size)?; // these operations all ignore the sign
602        let extra = 128 - u128::from(layout.size.bits());
603        let bits_out = match name {
604            sym::ctpop => u128::from(bits.count_ones()),
605            sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
606                throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
607            }
608            sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
609            sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
610            sym::bswap => {
611                assert_eq!(layout, ret_layout);
612                (bits << extra).swap_bytes()
613            }
614            sym::bitreverse => {
615                assert_eq!(layout, ret_layout);
616                (bits << extra).reverse_bits()
617            }
618            _ => bug!("not a numeric intrinsic: {}", name),
619        };
620        interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
621    }
622
623    pub fn exact_div(
624        &mut self,
625        a: &ImmTy<'tcx, M::Provenance>,
626        b: &ImmTy<'tcx, M::Provenance>,
627        dest: &PlaceTy<'tcx, M::Provenance>,
628    ) -> InterpResult<'tcx> {
629        assert_eq!(a.layout.ty, b.layout.ty);
630        assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
631
632        // Performs an exact division, resulting in undefined behavior where
633        // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`.
634        // First, check x % y != 0 (or if that computation overflows).
635        let rem = self.binary_op(BinOp::Rem, a, b)?;
636        // sign does not matter for 0 test, so `to_bits` is fine
637        if rem.to_scalar().to_bits(a.layout.size)? != 0 {
638            throw_ub_custom!(
639                fluent::const_eval_exact_div_has_remainder,
640                a = format!("{a}"),
641                b = format!("{b}")
642            )
643        }
644        // `Rem` says this is all right, so we can let `Div` do its job.
645        let res = self.binary_op(BinOp::Div, a, b)?;
646        self.write_immediate(*res, dest)
647    }
648
649    pub fn saturating_arith(
650        &self,
651        mir_op: BinOp,
652        l: &ImmTy<'tcx, M::Provenance>,
653        r: &ImmTy<'tcx, M::Provenance>,
654    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
655        assert_eq!(l.layout.ty, r.layout.ty);
656        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
657        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
658
659        let (val, overflowed) =
660            self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
661        interp_ok(if overflowed.to_bool()? {
662            let size = l.layout.size;
663            if l.layout.backend_repr.is_signed() {
664                // For signed ints the saturated value depends on the sign of the first
665                // term since the sign of the second term can be inferred from this and
666                // the fact that the operation has overflowed (if either is 0 no
667                // overflow can occur)
668                let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
669                if first_term >= 0 {
670                    // Negative overflow not possible since the positive first term
671                    // can only increase an (in range) negative term for addition
672                    // or corresponding negated positive term for subtraction.
673                    Scalar::from_int(size.signed_int_max(), size)
674                } else {
675                    // Positive overflow not possible for similar reason.
676                    Scalar::from_int(size.signed_int_min(), size)
677                }
678            } else {
679                // unsigned
680                if matches!(mir_op, BinOp::Add) {
681                    // max unsigned
682                    Scalar::from_uint(size.unsigned_int_max(), size)
683                } else {
684                    // underflow to 0
685                    Scalar::from_uint(0u128, size)
686                }
687            }
688        } else {
689            val
690        })
691    }
692
693    /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
694    /// allocation.
695    pub fn ptr_offset_inbounds(
696        &self,
697        ptr: Pointer<Option<M::Provenance>>,
698        offset_bytes: i64,
699    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
700        // The offset must be in bounds starting from `ptr`.
701        self.check_ptr_access_signed(
702            ptr,
703            offset_bytes,
704            CheckInAllocMsg::InboundsPointerArithmetic,
705        )?;
706        // This also implies that there is no overflow, so we are done.
707        interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
708    }
709
710    /// Copy `count*size_of::<T>()` many bytes from `*src` to `*dst`.
711    pub(crate) fn copy_intrinsic(
712        &mut self,
713        src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
714        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
715        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
716        nonoverlapping: bool,
717    ) -> InterpResult<'tcx> {
718        let count = self.read_target_usize(count)?;
719        let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
720        let (size, align) = (layout.size, layout.align.abi);
721
722        let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
723            err_ub_custom!(
724                fluent::const_eval_size_overflow,
725                name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
726            )
727        })?;
728
729        let src = self.read_pointer(src)?;
730        let dst = self.read_pointer(dst)?;
731
732        self.check_ptr_align(src, align)?;
733        self.check_ptr_align(dst, align)?;
734
735        self.mem_copy(src, dst, size, nonoverlapping)
736    }
737
738    /// Does a *typed* swap of `*left` and `*right`.
739    fn typed_swap_nonoverlapping_intrinsic(
740        &mut self,
741        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
742        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
743    ) -> InterpResult<'tcx> {
744        let left = self.deref_pointer(left)?;
745        let right = self.deref_pointer(right)?;
746        assert_eq!(left.layout, right.layout);
747        assert!(left.layout.is_sized());
748        let kind = MemoryKind::Stack;
749        let temp = self.allocate(left.layout, kind)?;
750        self.copy_op(&left, &temp)?; // checks alignment of `left`
751
752        // We want to always enforce non-overlapping, even if this is a scalar type.
753        // Therefore we directly use the underlying `mem_copy` here.
754        self.mem_copy(right.ptr(), left.ptr(), left.layout.size, /*nonoverlapping*/ true)?;
755        // This means we also need to do the validation of the value that used to be in `right`
756        // ourselves. This value is now in `left.` The one that started out in `left` already got
757        // validated by the copy above.
758        if M::enforce_validity(self, left.layout) {
759            self.validate_operand(
760                &left.clone().into(),
761                M::enforce_validity_recursively(self, left.layout),
762                /*reset_provenance_and_padding*/ true,
763            )?;
764        }
765
766        self.copy_op(&temp, &right)?; // checks alignment of `right`
767
768        self.deallocate_ptr(temp.ptr(), None, kind)?;
769        interp_ok(())
770    }
771
772    pub fn write_bytes_intrinsic(
773        &mut self,
774        dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
775        byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
776        count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
777        name: &'static str,
778    ) -> InterpResult<'tcx> {
779        let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
780
781        let dst = self.read_pointer(dst)?;
782        let byte = self.read_scalar(byte)?.to_u8()?;
783        let count = self.read_target_usize(count)?;
784
785        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
786        // but no actual allocation can be big enough for the difference to be noticeable.
787        let len = self
788            .compute_size_in_bytes(layout.size, count)
789            .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
790
791        let bytes = std::iter::repeat(byte).take(len.bytes_usize());
792        self.write_bytes_ptr(dst, bytes)
793    }
794
795    pub(crate) fn compare_bytes_intrinsic(
796        &mut self,
797        left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
798        right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
799        byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
800    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
801        let left = self.read_pointer(left)?;
802        let right = self.read_pointer(right)?;
803        let n = Size::from_bytes(self.read_target_usize(byte_count)?);
804
805        let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
806        let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
807
808        // `Ordering`'s discriminants are -1/0/+1, so casting does the right thing.
809        let result = Ord::cmp(left_bytes, right_bytes) as i32;
810        interp_ok(Scalar::from_i32(result))
811    }
812
813    pub(crate) fn raw_eq_intrinsic(
814        &mut self,
815        lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
816        rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
817    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
818        let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
819        assert!(layout.is_sized());
820
821        let get_bytes = |this: &InterpCx<'tcx, M>,
822                         op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
823         -> InterpResult<'tcx, &[u8]> {
824            let ptr = this.read_pointer(op)?;
825            this.check_ptr_align(ptr, layout.align.abi)?;
826            let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
827                // zero-sized access
828                return interp_ok(&[]);
829            };
830            alloc_ref.get_bytes_strip_provenance()
831        };
832
833        let lhs_bytes = get_bytes(self, lhs)?;
834        let rhs_bytes = get_bytes(self, rhs)?;
835        interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
836    }
837
838    fn float_min_intrinsic<F>(
839        &mut self,
840        args: &[OpTy<'tcx, M::Provenance>],
841        dest: &PlaceTy<'tcx, M::Provenance>,
842    ) -> InterpResult<'tcx, ()>
843    where
844        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
845    {
846        let a: F = self.read_scalar(&args[0])?.to_float()?;
847        let b: F = self.read_scalar(&args[1])?.to_float()?;
848        let res = if a == b {
849            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
850            // Let the machine decide which one to return.
851            M::equal_float_min_max(self, a, b)
852        } else {
853            self.adjust_nan(a.min(b), &[a, b])
854        };
855        self.write_scalar(res, dest)?;
856        interp_ok(())
857    }
858
859    fn float_max_intrinsic<F>(
860        &mut self,
861        args: &[OpTy<'tcx, M::Provenance>],
862        dest: &PlaceTy<'tcx, M::Provenance>,
863    ) -> InterpResult<'tcx, ()>
864    where
865        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
866    {
867        let a: F = self.read_scalar(&args[0])?.to_float()?;
868        let b: F = self.read_scalar(&args[1])?.to_float()?;
869        let res = if a == b {
870            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
871            // Let the machine decide which one to return.
872            M::equal_float_min_max(self, a, b)
873        } else {
874            self.adjust_nan(a.max(b), &[a, b])
875        };
876        self.write_scalar(res, dest)?;
877        interp_ok(())
878    }
879
880    fn float_minimum_intrinsic<F>(
881        &mut self,
882        args: &[OpTy<'tcx, M::Provenance>],
883        dest: &PlaceTy<'tcx, M::Provenance>,
884    ) -> InterpResult<'tcx, ()>
885    where
886        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
887    {
888        let a: F = self.read_scalar(&args[0])?.to_float()?;
889        let b: F = self.read_scalar(&args[1])?.to_float()?;
890        let res = a.minimum(b);
891        let res = self.adjust_nan(res, &[a, b]);
892        self.write_scalar(res, dest)?;
893        interp_ok(())
894    }
895
896    fn float_maximum_intrinsic<F>(
897        &mut self,
898        args: &[OpTy<'tcx, M::Provenance>],
899        dest: &PlaceTy<'tcx, M::Provenance>,
900    ) -> InterpResult<'tcx, ()>
901    where
902        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
903    {
904        let a: F = self.read_scalar(&args[0])?.to_float()?;
905        let b: F = self.read_scalar(&args[1])?.to_float()?;
906        let res = a.maximum(b);
907        let res = self.adjust_nan(res, &[a, b]);
908        self.write_scalar(res, dest)?;
909        interp_ok(())
910    }
911
912    fn float_copysign_intrinsic<F>(
913        &mut self,
914        args: &[OpTy<'tcx, M::Provenance>],
915        dest: &PlaceTy<'tcx, M::Provenance>,
916    ) -> InterpResult<'tcx, ()>
917    where
918        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
919    {
920        let a: F = self.read_scalar(&args[0])?.to_float()?;
921        let b: F = self.read_scalar(&args[1])?.to_float()?;
922        // bitwise, no NaN adjustments
923        self.write_scalar(a.copy_sign(b), dest)?;
924        interp_ok(())
925    }
926
927    fn float_abs_intrinsic<F>(
928        &mut self,
929        args: &[OpTy<'tcx, M::Provenance>],
930        dest: &PlaceTy<'tcx, M::Provenance>,
931    ) -> InterpResult<'tcx, ()>
932    where
933        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
934    {
935        let x: F = self.read_scalar(&args[0])?.to_float()?;
936        // bitwise, no NaN adjustments
937        self.write_scalar(x.abs(), dest)?;
938        interp_ok(())
939    }
940
941    fn float_round_intrinsic<F>(
942        &mut self,
943        args: &[OpTy<'tcx, M::Provenance>],
944        dest: &PlaceTy<'tcx, M::Provenance>,
945        mode: rustc_apfloat::Round,
946    ) -> InterpResult<'tcx, ()>
947    where
948        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
949    {
950        let x: F = self.read_scalar(&args[0])?.to_float()?;
951        let res = x.round_to_integral(mode).value;
952        let res = self.adjust_nan(res, &[x]);
953        self.write_scalar(res, dest)?;
954        interp_ok(())
955    }
956}