1use std::assert_matches::assert_matches;
6
7use rustc_abi::Size;
8use rustc_apfloat::ieee::{Double, Half, Quad, Single};
9use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
10use rustc_middle::ty::layout::TyAndLayout;
11use rustc_middle::ty::{Ty, TyCtxt};
12use rustc_middle::{bug, ty};
13use rustc_span::{Symbol, sym};
14use tracing::trace;
15
16use super::memory::MemoryKind;
17use super::util::ensure_monomorphic_enough;
18use super::{
19 Allocation, CheckInAllocMsg, ConstAllocation, ImmTy, InterpCx, InterpResult, Machine, OpTy,
20 PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format,
21 interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
22};
23use crate::fluent_generated as fluent;
24
25pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
27 let path = crate::util::type_name(tcx, ty);
28 let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
29 tcx.mk_const_alloc(alloc)
30}
31
32impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
33 pub fn eval_intrinsic(
37 &mut self,
38 instance: ty::Instance<'tcx>,
39 args: &[OpTy<'tcx, M::Provenance>],
40 dest: &PlaceTy<'tcx, M::Provenance>,
41 ret: Option<mir::BasicBlock>,
42 ) -> InterpResult<'tcx, bool> {
43 let instance_args = instance.args;
44 let intrinsic_name = self.tcx.item_name(instance.def_id());
45 let tcx = self.tcx.tcx;
46
47 match intrinsic_name {
48 sym::type_name => {
49 let tp_ty = instance.args.type_at(0);
50 ensure_monomorphic_enough(tcx, tp_ty)?;
51 let alloc = alloc_type_name(tcx, tp_ty);
52 let val = ConstValue::Slice { data: alloc, meta: alloc.inner().size().bytes() };
53 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
54 self.copy_op(&val, dest)?;
55 }
56 sym::needs_drop => {
57 let tp_ty = instance.args.type_at(0);
58 ensure_monomorphic_enough(tcx, tp_ty)?;
59 let val = ConstValue::from_bool(tp_ty.needs_drop(tcx, self.typing_env));
60 let val = self.const_val_to_op(val, tcx.types.bool, Some(dest.layout))?;
61 self.copy_op(&val, dest)?;
62 }
63 sym::type_id => {
64 let tp_ty = instance.args.type_at(0);
65 ensure_monomorphic_enough(tcx, tp_ty)?;
66 let val = ConstValue::from_u128(tcx.type_id_hash(tp_ty).as_u128());
67 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
68 self.copy_op(&val, dest)?;
69 }
70 sym::variant_count => {
71 let tp_ty = instance.args.type_at(0);
72 let ty = match tp_ty.kind() {
73 ty::Pat(base, _) => *base,
77 _ => tp_ty,
78 };
79 let val = match ty.kind() {
80 ty::Adt(adt, _) => {
82 ConstValue::from_target_usize(adt.variants().len() as u64, &tcx)
83 }
84 ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
85 throw_inval!(TooGeneric)
86 }
87 ty::Pat(..) => unreachable!(),
88 ty::Bound(_, _) => bug!("bound ty during ctfe"),
89 ty::Bool
90 | ty::Char
91 | ty::Int(_)
92 | ty::Uint(_)
93 | ty::Float(_)
94 | ty::Foreign(_)
95 | ty::Str
96 | ty::Array(_, _)
97 | ty::Slice(_)
98 | ty::RawPtr(_, _)
99 | ty::Ref(_, _, _)
100 | ty::FnDef(_, _)
101 | ty::FnPtr(..)
102 | ty::Dynamic(_, _, _)
103 | ty::Closure(_, _)
104 | ty::CoroutineClosure(_, _)
105 | ty::Coroutine(_, _)
106 | ty::CoroutineWitness(..)
107 | ty::UnsafeBinder(_)
108 | ty::Never
109 | ty::Tuple(_)
110 | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
111 };
112 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
113 self.copy_op(&val, dest)?;
114 }
115
116 sym::caller_location => {
117 let span = self.find_closest_untracked_caller_location();
118 let val = self.tcx.span_as_caller_location(span);
119 let val =
120 self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
121 self.copy_op(&val, dest)?;
122 }
123
124 sym::align_of_val | sym::size_of_val => {
125 let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?;
128 let (size, align) = self
129 .size_and_align_of_val(&place)?
130 .ok_or_else(|| err_unsup_format!("`extern type` does not have known layout"))?;
131
132 let result = match intrinsic_name {
133 sym::align_of_val => align.bytes(),
134 sym::size_of_val => size.bytes(),
135 _ => bug!(),
136 };
137
138 self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
139 }
140
141 sym::fadd_algebraic
142 | sym::fsub_algebraic
143 | sym::fmul_algebraic
144 | sym::fdiv_algebraic
145 | sym::frem_algebraic => {
146 let a = self.read_immediate(&args[0])?;
147 let b = self.read_immediate(&args[1])?;
148
149 let op = match intrinsic_name {
150 sym::fadd_algebraic => BinOp::Add,
151 sym::fsub_algebraic => BinOp::Sub,
152 sym::fmul_algebraic => BinOp::Mul,
153 sym::fdiv_algebraic => BinOp::Div,
154 sym::frem_algebraic => BinOp::Rem,
155
156 _ => bug!(),
157 };
158
159 let res = self.binary_op(op, &a, &b)?;
160 let res = M::apply_float_nondet(self, res)?;
162 self.write_immediate(*res, dest)?;
163 }
164
165 sym::ctpop
166 | sym::cttz
167 | sym::cttz_nonzero
168 | sym::ctlz
169 | sym::ctlz_nonzero
170 | sym::bswap
171 | sym::bitreverse => {
172 let ty = instance_args.type_at(0);
173 let layout = self.layout_of(ty)?;
174 let val = self.read_scalar(&args[0])?;
175
176 let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?;
177 self.write_scalar(out_val, dest)?;
178 }
179 sym::saturating_add | sym::saturating_sub => {
180 let l = self.read_immediate(&args[0])?;
181 let r = self.read_immediate(&args[1])?;
182 let val = self.saturating_arith(
183 if intrinsic_name == sym::saturating_add { BinOp::Add } else { BinOp::Sub },
184 &l,
185 &r,
186 )?;
187 self.write_scalar(val, dest)?;
188 }
189 sym::discriminant_value => {
190 let place = self.deref_pointer(&args[0])?;
191 let variant = self.read_discriminant(&place)?;
192 let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
193 self.write_immediate(*discr, dest)?;
194 }
195 sym::exact_div => {
196 let l = self.read_immediate(&args[0])?;
197 let r = self.read_immediate(&args[1])?;
198 self.exact_div(&l, &r, dest)?;
199 }
200 sym::rotate_left | sym::rotate_right => {
201 let layout_val = self.layout_of(instance_args.type_at(0))?;
204 let val = self.read_scalar(&args[0])?;
205 let val_bits = val.to_bits(layout_val.size)?; let layout_raw_shift = self.layout_of(self.tcx.types.u32)?;
208 let raw_shift = self.read_scalar(&args[1])?;
209 let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?;
210
211 let width_bits = u128::from(layout_val.size.bits());
212 let shift_bits = raw_shift_bits % width_bits;
213 let inv_shift_bits = (width_bits - shift_bits) % width_bits;
214 let result_bits = if intrinsic_name == sym::rotate_left {
215 (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
216 } else {
217 (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
218 };
219 let truncated_bits = layout_val.size.truncate(result_bits);
220 let result = Scalar::from_uint(truncated_bits, layout_val.size);
221 self.write_scalar(result, dest)?;
222 }
223 sym::copy => {
224 self.copy_intrinsic(&args[0], &args[1], &args[2], false)?;
225 }
226 sym::write_bytes => {
227 self.write_bytes_intrinsic(&args[0], &args[1], &args[2], "write_bytes")?;
228 }
229 sym::compare_bytes => {
230 let result = self.compare_bytes_intrinsic(&args[0], &args[1], &args[2])?;
231 self.write_scalar(result, dest)?;
232 }
233 sym::arith_offset => {
234 let ptr = self.read_pointer(&args[0])?;
235 let offset_count = self.read_target_isize(&args[1])?;
236 let pointee_ty = instance_args.type_at(0);
237
238 let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
239 let offset_bytes = offset_count.wrapping_mul(pointee_size);
240 let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
241 self.write_pointer(offset_ptr, dest)?;
242 }
243 sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
244 let a = self.read_pointer(&args[0])?;
245 let b = self.read_pointer(&args[1])?;
246
247 let usize_layout = self.layout_of(self.tcx.types.usize)?;
248 let isize_layout = self.layout_of(self.tcx.types.isize)?;
249
250 let (a_offset, b_offset, is_addr) = if M::Provenance::OFFSET_IS_ADDR {
254 (a.addr().bytes(), b.addr().bytes(), true)
255 } else {
256 match (self.ptr_try_get_alloc_id(a, 0), self.ptr_try_get_alloc_id(b, 0)) {
257 (Err(a), Err(b)) => {
258 (a, b, true)
260 }
261 (Ok((a_alloc_id, a_offset, _)), Ok((b_alloc_id, b_offset, _)))
262 if a_alloc_id == b_alloc_id =>
263 {
264 (a_offset.bytes(), b_offset.bytes(), false)
267 }
268 _ => {
269 throw_ub_custom!(
271 fluent::const_eval_offset_from_different_allocations,
272 name = intrinsic_name,
273 );
274 }
275 }
276 };
277
278 let dist = {
280 let (val, overflowed) = {
283 let a_offset = ImmTy::from_uint(a_offset, usize_layout);
284 let b_offset = ImmTy::from_uint(b_offset, usize_layout);
285 self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)?
286 .to_scalar_pair()
287 };
288 if overflowed.to_bool()? {
289 if intrinsic_name == sym::ptr_offset_from_unsigned {
291 throw_ub_custom!(
292 fluent::const_eval_offset_from_unsigned_overflow,
293 a_offset = a_offset,
294 b_offset = b_offset,
295 is_addr = is_addr,
296 );
297 }
298 let dist = val.to_target_isize(self)?;
302 if dist >= 0 || i128::from(dist) == self.pointer_size().signed_int_min() {
303 throw_ub_custom!(
304 fluent::const_eval_offset_from_underflow,
305 name = intrinsic_name,
306 );
307 }
308 dist
309 } else {
310 let dist = val.to_target_isize(self)?;
312 if dist < 0 {
315 throw_ub_custom!(
316 fluent::const_eval_offset_from_overflow,
317 name = intrinsic_name,
318 );
319 }
320 dist
321 }
322 };
323
324 self.check_ptr_access_signed(b, dist, CheckInAllocMsg::Dereferenceable)
327 .map_err_kind(|_| {
328 if let Ok((a_alloc_id, ..)) = self.ptr_try_get_alloc_id(a, 0)
331 && let Ok((b_alloc_id, ..)) = self.ptr_try_get_alloc_id(b, 0)
332 && a_alloc_id == b_alloc_id
333 {
334 err_ub_custom!(
335 fluent::const_eval_offset_from_out_of_bounds,
336 name = intrinsic_name,
337 )
338 } else {
339 err_ub_custom!(
340 fluent::const_eval_offset_from_different_allocations,
341 name = intrinsic_name,
342 )
343 }
344 })?;
345 self.check_ptr_access_signed(
348 a,
349 dist.checked_neg().unwrap(), CheckInAllocMsg::Dereferenceable,
351 )
352 .map_err_kind(|_| {
353 err_ub_custom!(
355 fluent::const_eval_offset_from_different_allocations,
356 name = intrinsic_name,
357 )
358 })?;
359
360 let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
362 assert!(0 <= dist && dist <= self.target_isize_max());
363 usize_layout
364 } else {
365 assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
366 isize_layout
367 };
368 let pointee_layout = self.layout_of(instance_args.type_at(0))?;
369 let val = ImmTy::from_int(dist, ret_layout);
371 let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
372 self.exact_div(&val, &size, dest)?;
373 }
374
375 sym::simd_insert => {
376 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
377 let elem = &args[2];
378 let (input, input_len) = self.project_to_simd(&args[0])?;
379 let (dest, dest_len) = self.project_to_simd(dest)?;
380 assert_eq!(input_len, dest_len, "Return vector length must match input length");
381 if index >= input_len {
383 throw_ub_format!(
384 "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
385 );
386 }
387
388 for i in 0..dest_len {
389 let place = self.project_index(&dest, i)?;
390 let value =
391 if i == index { elem.clone() } else { self.project_index(&input, i)? };
392 self.copy_op(&value, &place)?;
393 }
394 }
395 sym::simd_extract => {
396 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
397 let (input, input_len) = self.project_to_simd(&args[0])?;
398 if index >= input_len {
400 throw_ub_format!(
401 "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
402 );
403 }
404 self.copy_op(&self.project_index(&input, index)?, dest)?;
405 }
406 sym::black_box => {
407 self.copy_op(&args[0], dest)?;
409 }
410 sym::raw_eq => {
411 let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
412 self.write_scalar(result, dest)?;
413 }
414 sym::typed_swap_nonoverlapping => {
415 self.typed_swap_nonoverlapping_intrinsic(&args[0], &args[1])?;
416 }
417
418 sym::vtable_size => {
419 let ptr = self.read_pointer(&args[0])?;
420 let (size, _align) = self.get_vtable_size_and_align(ptr, None)?;
422 self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
423 }
424 sym::vtable_align => {
425 let ptr = self.read_pointer(&args[0])?;
426 let (_size, align) = self.get_vtable_size_and_align(ptr, None)?;
428 self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
429 }
430
431 sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
432 sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
433 sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
434 sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
435
436 sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
437 sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
438 sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
439 sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
440
441 sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
442 sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
443 sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
444 sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
445
446 sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
447 sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
448 sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
449 sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
450
451 sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
452 sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
453 sym::copysignf64 => self.float_copysign_intrinsic::<Double>(args, dest)?,
454 sym::copysignf128 => self.float_copysign_intrinsic::<Quad>(args, dest)?,
455
456 sym::fabsf16 => self.float_abs_intrinsic::<Half>(args, dest)?,
457 sym::fabsf32 => self.float_abs_intrinsic::<Single>(args, dest)?,
458 sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
459 sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
460
461 sym::floorf16 => self.float_round_intrinsic::<Half>(
462 args,
463 dest,
464 rustc_apfloat::Round::TowardNegative,
465 )?,
466 sym::floorf32 => self.float_round_intrinsic::<Single>(
467 args,
468 dest,
469 rustc_apfloat::Round::TowardNegative,
470 )?,
471 sym::floorf64 => self.float_round_intrinsic::<Double>(
472 args,
473 dest,
474 rustc_apfloat::Round::TowardNegative,
475 )?,
476 sym::floorf128 => self.float_round_intrinsic::<Quad>(
477 args,
478 dest,
479 rustc_apfloat::Round::TowardNegative,
480 )?,
481
482 sym::ceilf16 => self.float_round_intrinsic::<Half>(
483 args,
484 dest,
485 rustc_apfloat::Round::TowardPositive,
486 )?,
487 sym::ceilf32 => self.float_round_intrinsic::<Single>(
488 args,
489 dest,
490 rustc_apfloat::Round::TowardPositive,
491 )?,
492 sym::ceilf64 => self.float_round_intrinsic::<Double>(
493 args,
494 dest,
495 rustc_apfloat::Round::TowardPositive,
496 )?,
497 sym::ceilf128 => self.float_round_intrinsic::<Quad>(
498 args,
499 dest,
500 rustc_apfloat::Round::TowardPositive,
501 )?,
502
503 sym::truncf16 => {
504 self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
505 }
506 sym::truncf32 => {
507 self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
508 }
509 sym::truncf64 => {
510 self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
511 }
512 sym::truncf128 => {
513 self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
514 }
515
516 sym::roundf16 => self.float_round_intrinsic::<Half>(
517 args,
518 dest,
519 rustc_apfloat::Round::NearestTiesToAway,
520 )?,
521 sym::roundf32 => self.float_round_intrinsic::<Single>(
522 args,
523 dest,
524 rustc_apfloat::Round::NearestTiesToAway,
525 )?,
526 sym::roundf64 => self.float_round_intrinsic::<Double>(
527 args,
528 dest,
529 rustc_apfloat::Round::NearestTiesToAway,
530 )?,
531 sym::roundf128 => self.float_round_intrinsic::<Quad>(
532 args,
533 dest,
534 rustc_apfloat::Round::NearestTiesToAway,
535 )?,
536
537 sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
538 args,
539 dest,
540 rustc_apfloat::Round::NearestTiesToEven,
541 )?,
542 sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
543 args,
544 dest,
545 rustc_apfloat::Round::NearestTiesToEven,
546 )?,
547 sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
548 args,
549 dest,
550 rustc_apfloat::Round::NearestTiesToEven,
551 )?,
552 sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
553 args,
554 dest,
555 rustc_apfloat::Round::NearestTiesToEven,
556 )?,
557
558 _ => return interp_ok(false),
560 }
561
562 trace!("{:?}", self.dump_place(&dest.clone().into()));
563 self.return_to_block(ret)?;
564 interp_ok(true)
565 }
566
567 pub(super) fn eval_nondiverging_intrinsic(
568 &mut self,
569 intrinsic: &NonDivergingIntrinsic<'tcx>,
570 ) -> InterpResult<'tcx> {
571 match intrinsic {
572 NonDivergingIntrinsic::Assume(op) => {
573 let op = self.eval_operand(op, None)?;
574 let cond = self.read_scalar(&op)?.to_bool()?;
575 if !cond {
576 throw_ub_custom!(fluent::const_eval_assume_false);
577 }
578 interp_ok(())
579 }
580 NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
581 count,
582 src,
583 dst,
584 }) => {
585 let src = self.eval_operand(src, None)?;
586 let dst = self.eval_operand(dst, None)?;
587 let count = self.eval_operand(count, None)?;
588 self.copy_intrinsic(&src, &dst, &count, true)
589 }
590 }
591 }
592
593 pub fn numeric_intrinsic(
594 &self,
595 name: Symbol,
596 val: Scalar<M::Provenance>,
597 layout: TyAndLayout<'tcx>,
598 ret_layout: TyAndLayout<'tcx>,
599 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
600 assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
601 let bits = val.to_bits(layout.size)?; let extra = 128 - u128::from(layout.size.bits());
603 let bits_out = match name {
604 sym::ctpop => u128::from(bits.count_ones()),
605 sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
606 throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
607 }
608 sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
609 sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
610 sym::bswap => {
611 assert_eq!(layout, ret_layout);
612 (bits << extra).swap_bytes()
613 }
614 sym::bitreverse => {
615 assert_eq!(layout, ret_layout);
616 (bits << extra).reverse_bits()
617 }
618 _ => bug!("not a numeric intrinsic: {}", name),
619 };
620 interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
621 }
622
623 pub fn exact_div(
624 &mut self,
625 a: &ImmTy<'tcx, M::Provenance>,
626 b: &ImmTy<'tcx, M::Provenance>,
627 dest: &PlaceTy<'tcx, M::Provenance>,
628 ) -> InterpResult<'tcx> {
629 assert_eq!(a.layout.ty, b.layout.ty);
630 assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
631
632 let rem = self.binary_op(BinOp::Rem, a, b)?;
636 if rem.to_scalar().to_bits(a.layout.size)? != 0 {
638 throw_ub_custom!(
639 fluent::const_eval_exact_div_has_remainder,
640 a = format!("{a}"),
641 b = format!("{b}")
642 )
643 }
644 let res = self.binary_op(BinOp::Div, a, b)?;
646 self.write_immediate(*res, dest)
647 }
648
649 pub fn saturating_arith(
650 &self,
651 mir_op: BinOp,
652 l: &ImmTy<'tcx, M::Provenance>,
653 r: &ImmTy<'tcx, M::Provenance>,
654 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
655 assert_eq!(l.layout.ty, r.layout.ty);
656 assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
657 assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
658
659 let (val, overflowed) =
660 self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
661 interp_ok(if overflowed.to_bool()? {
662 let size = l.layout.size;
663 if l.layout.backend_repr.is_signed() {
664 let first_term: i128 = l.to_scalar().to_int(l.layout.size)?;
669 if first_term >= 0 {
670 Scalar::from_int(size.signed_int_max(), size)
674 } else {
675 Scalar::from_int(size.signed_int_min(), size)
677 }
678 } else {
679 if matches!(mir_op, BinOp::Add) {
681 Scalar::from_uint(size.unsigned_int_max(), size)
683 } else {
684 Scalar::from_uint(0u128, size)
686 }
687 }
688 } else {
689 val
690 })
691 }
692
693 pub fn ptr_offset_inbounds(
696 &self,
697 ptr: Pointer<Option<M::Provenance>>,
698 offset_bytes: i64,
699 ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
700 self.check_ptr_access_signed(
702 ptr,
703 offset_bytes,
704 CheckInAllocMsg::InboundsPointerArithmetic,
705 )?;
706 interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
708 }
709
710 pub(crate) fn copy_intrinsic(
712 &mut self,
713 src: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
714 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
715 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
716 nonoverlapping: bool,
717 ) -> InterpResult<'tcx> {
718 let count = self.read_target_usize(count)?;
719 let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?;
720 let (size, align) = (layout.size, layout.align.abi);
721
722 let size = self.compute_size_in_bytes(size, count).ok_or_else(|| {
723 err_ub_custom!(
724 fluent::const_eval_size_overflow,
725 name = if nonoverlapping { "copy_nonoverlapping" } else { "copy" }
726 )
727 })?;
728
729 let src = self.read_pointer(src)?;
730 let dst = self.read_pointer(dst)?;
731
732 self.check_ptr_align(src, align)?;
733 self.check_ptr_align(dst, align)?;
734
735 self.mem_copy(src, dst, size, nonoverlapping)
736 }
737
738 fn typed_swap_nonoverlapping_intrinsic(
740 &mut self,
741 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
742 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
743 ) -> InterpResult<'tcx> {
744 let left = self.deref_pointer(left)?;
745 let right = self.deref_pointer(right)?;
746 assert_eq!(left.layout, right.layout);
747 assert!(left.layout.is_sized());
748 let kind = MemoryKind::Stack;
749 let temp = self.allocate(left.layout, kind)?;
750 self.copy_op(&left, &temp)?; self.mem_copy(right.ptr(), left.ptr(), left.layout.size, true)?;
755 if M::enforce_validity(self, left.layout) {
759 self.validate_operand(
760 &left.clone().into(),
761 M::enforce_validity_recursively(self, left.layout),
762 true,
763 )?;
764 }
765
766 self.copy_op(&temp, &right)?; self.deallocate_ptr(temp.ptr(), None, kind)?;
769 interp_ok(())
770 }
771
772 pub fn write_bytes_intrinsic(
773 &mut self,
774 dst: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
775 byte: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
776 count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
777 name: &'static str,
778 ) -> InterpResult<'tcx> {
779 let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?;
780
781 let dst = self.read_pointer(dst)?;
782 let byte = self.read_scalar(byte)?.to_u8()?;
783 let count = self.read_target_usize(count)?;
784
785 let len = self
788 .compute_size_in_bytes(layout.size, count)
789 .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?;
790
791 let bytes = std::iter::repeat(byte).take(len.bytes_usize());
792 self.write_bytes_ptr(dst, bytes)
793 }
794
795 pub(crate) fn compare_bytes_intrinsic(
796 &mut self,
797 left: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
798 right: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
799 byte_count: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
800 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
801 let left = self.read_pointer(left)?;
802 let right = self.read_pointer(right)?;
803 let n = Size::from_bytes(self.read_target_usize(byte_count)?);
804
805 let left_bytes = self.read_bytes_ptr_strip_provenance(left, n)?;
806 let right_bytes = self.read_bytes_ptr_strip_provenance(right, n)?;
807
808 let result = Ord::cmp(left_bytes, right_bytes) as i32;
810 interp_ok(Scalar::from_i32(result))
811 }
812
813 pub(crate) fn raw_eq_intrinsic(
814 &mut self,
815 lhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
816 rhs: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>,
817 ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
818 let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?;
819 assert!(layout.is_sized());
820
821 let get_bytes = |this: &InterpCx<'tcx, M>,
822 op: &OpTy<'tcx, <M as Machine<'tcx>>::Provenance>|
823 -> InterpResult<'tcx, &[u8]> {
824 let ptr = this.read_pointer(op)?;
825 this.check_ptr_align(ptr, layout.align.abi)?;
826 let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
827 return interp_ok(&[]);
829 };
830 alloc_ref.get_bytes_strip_provenance()
831 };
832
833 let lhs_bytes = get_bytes(self, lhs)?;
834 let rhs_bytes = get_bytes(self, rhs)?;
835 interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
836 }
837
838 fn float_min_intrinsic<F>(
839 &mut self,
840 args: &[OpTy<'tcx, M::Provenance>],
841 dest: &PlaceTy<'tcx, M::Provenance>,
842 ) -> InterpResult<'tcx, ()>
843 where
844 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
845 {
846 let a: F = self.read_scalar(&args[0])?.to_float()?;
847 let b: F = self.read_scalar(&args[1])?.to_float()?;
848 let res = if a == b {
849 M::equal_float_min_max(self, a, b)
852 } else {
853 self.adjust_nan(a.min(b), &[a, b])
854 };
855 self.write_scalar(res, dest)?;
856 interp_ok(())
857 }
858
859 fn float_max_intrinsic<F>(
860 &mut self,
861 args: &[OpTy<'tcx, M::Provenance>],
862 dest: &PlaceTy<'tcx, M::Provenance>,
863 ) -> InterpResult<'tcx, ()>
864 where
865 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
866 {
867 let a: F = self.read_scalar(&args[0])?.to_float()?;
868 let b: F = self.read_scalar(&args[1])?.to_float()?;
869 let res = if a == b {
870 M::equal_float_min_max(self, a, b)
873 } else {
874 self.adjust_nan(a.max(b), &[a, b])
875 };
876 self.write_scalar(res, dest)?;
877 interp_ok(())
878 }
879
880 fn float_minimum_intrinsic<F>(
881 &mut self,
882 args: &[OpTy<'tcx, M::Provenance>],
883 dest: &PlaceTy<'tcx, M::Provenance>,
884 ) -> InterpResult<'tcx, ()>
885 where
886 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
887 {
888 let a: F = self.read_scalar(&args[0])?.to_float()?;
889 let b: F = self.read_scalar(&args[1])?.to_float()?;
890 let res = a.minimum(b);
891 let res = self.adjust_nan(res, &[a, b]);
892 self.write_scalar(res, dest)?;
893 interp_ok(())
894 }
895
896 fn float_maximum_intrinsic<F>(
897 &mut self,
898 args: &[OpTy<'tcx, M::Provenance>],
899 dest: &PlaceTy<'tcx, M::Provenance>,
900 ) -> InterpResult<'tcx, ()>
901 where
902 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
903 {
904 let a: F = self.read_scalar(&args[0])?.to_float()?;
905 let b: F = self.read_scalar(&args[1])?.to_float()?;
906 let res = a.maximum(b);
907 let res = self.adjust_nan(res, &[a, b]);
908 self.write_scalar(res, dest)?;
909 interp_ok(())
910 }
911
912 fn float_copysign_intrinsic<F>(
913 &mut self,
914 args: &[OpTy<'tcx, M::Provenance>],
915 dest: &PlaceTy<'tcx, M::Provenance>,
916 ) -> InterpResult<'tcx, ()>
917 where
918 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
919 {
920 let a: F = self.read_scalar(&args[0])?.to_float()?;
921 let b: F = self.read_scalar(&args[1])?.to_float()?;
922 self.write_scalar(a.copy_sign(b), dest)?;
924 interp_ok(())
925 }
926
927 fn float_abs_intrinsic<F>(
928 &mut self,
929 args: &[OpTy<'tcx, M::Provenance>],
930 dest: &PlaceTy<'tcx, M::Provenance>,
931 ) -> InterpResult<'tcx, ()>
932 where
933 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
934 {
935 let x: F = self.read_scalar(&args[0])?.to_float()?;
936 self.write_scalar(x.abs(), dest)?;
938 interp_ok(())
939 }
940
941 fn float_round_intrinsic<F>(
942 &mut self,
943 args: &[OpTy<'tcx, M::Provenance>],
944 dest: &PlaceTy<'tcx, M::Provenance>,
945 mode: rustc_apfloat::Round,
946 ) -> InterpResult<'tcx, ()>
947 where
948 F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
949 {
950 let x: F = self.read_scalar(&args[0])?.to_float()?;
951 let res = x.round_to_integral(mode).value;
952 let res = self.adjust_nan(res, &[x]);
953 self.write_scalar(res, dest)?;
954 interp_ok(())
955 }
956}