1#![warn(clippy::arithmetic_side_effects)]
2
3mod atomic;
4mod simd;
5
6use std::ops::Neg;
7
8use rand::Rng;
9use rustc_abi::Size;
10use rustc_apfloat::ieee::{IeeeFloat, Semantics};
11use rustc_apfloat::{self, Float, Round};
12use rustc_middle::mir;
13use rustc_middle::ty::{self, FloatTy, ScalarInt};
14use rustc_span::{Symbol, sym};
15
16use self::atomic::EvalContextExt as _;
17use self::helpers::{ToHost, ToSoft, check_intrinsic_arg_count};
18use self::simd::EvalContextExt as _;
19use crate::math::{IeeeExt, apply_random_float_error_ulp};
20use crate::*;
21
22impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
23pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
24 fn call_intrinsic(
25 &mut self,
26 instance: ty::Instance<'tcx>,
27 args: &[OpTy<'tcx>],
28 dest: &PlaceTy<'tcx>,
29 ret: Option<mir::BasicBlock>,
30 unwind: mir::UnwindAction,
31 ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
32 let this = self.eval_context_mut();
33
34 if this.machine.force_intrinsic_fallback
36 && !this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden
37 {
38 return interp_ok(Some(ty::Instance {
39 def: ty::InstanceKind::Item(instance.def_id()),
40 args: instance.args,
41 }));
42 }
43
44 if this.eval_intrinsic(instance, args, dest, ret)? {
46 return interp_ok(None);
47 }
48 let intrinsic_name = this.tcx.item_name(instance.def_id());
49 let intrinsic_name = intrinsic_name.as_str();
50
51 let dest = this.force_allocation(dest)?;
53
54 match this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, &dest, ret)? {
55 EmulateItemResult::NotSupported => {
56 if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
58 throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
59 }
60 let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
61 if this
62 .tcx
63 .get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
64 .next()
65 .is_none()
66 {
67 throw_unsup_format!(
68 "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
69 );
70 }
71 interp_ok(Some(ty::Instance {
72 def: ty::InstanceKind::Item(instance.def_id()),
73 args: instance.args,
74 }))
75 }
76 EmulateItemResult::NeedsReturn => {
77 trace!("{:?}", this.dump_place(&dest.clone().into()));
78 this.return_to_block(ret)?;
79 interp_ok(None)
80 }
81 EmulateItemResult::NeedsUnwind => {
82 this.unwind_to_block(unwind)?;
84 interp_ok(None)
85 }
86 EmulateItemResult::AlreadyJumped => interp_ok(None),
87 }
88 }
89
90 fn emulate_intrinsic_by_name(
93 &mut self,
94 intrinsic_name: &str,
95 generic_args: ty::GenericArgsRef<'tcx>,
96 args: &[OpTy<'tcx>],
97 dest: &MPlaceTy<'tcx>,
98 ret: Option<mir::BasicBlock>,
99 ) -> InterpResult<'tcx, EmulateItemResult> {
100 let this = self.eval_context_mut();
101
102 if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
103 return this.emulate_atomic_intrinsic(name, generic_args, args, dest);
104 }
105 if let Some(name) = intrinsic_name.strip_prefix("simd_") {
106 return this.emulate_simd_intrinsic(name, generic_args, args, dest);
107 }
108
109 match intrinsic_name {
110 "abort" => {
112 throw_machine_stop!(TerminationInfo::Abort(
113 "the program aborted execution".to_owned()
114 ));
115 }
116 "catch_unwind" => {
117 this.handle_catch_unwind(args, dest, ret)?;
118 return interp_ok(EmulateItemResult::AlreadyJumped);
120 }
121
122 "volatile_load" => {
124 let [place] = check_intrinsic_arg_count(args)?;
125 let place = this.deref_pointer(place)?;
126 this.copy_op(&place, dest)?;
127 }
128 "volatile_store" => {
129 let [place, dest] = check_intrinsic_arg_count(args)?;
130 let place = this.deref_pointer(place)?;
131 this.copy_op(dest, &place)?;
132 }
133
134 "volatile_set_memory" => {
135 let [ptr, val_byte, count] = check_intrinsic_arg_count(args)?;
136 this.write_bytes_intrinsic(ptr, val_byte, count, "volatile_set_memory")?;
137 }
138
139 "ptr_mask" => {
141 let [ptr, mask] = check_intrinsic_arg_count(args)?;
142
143 let ptr = this.read_pointer(ptr)?;
144 let mask = this.read_target_usize(mask)?;
145
146 let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
147
148 this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
149 }
150
151 "is_val_statically_known" => {
157 let [_arg] = check_intrinsic_arg_count(args)?;
158 let branch: bool = this.machine.rng.get_mut().random();
162 this.write_scalar(Scalar::from_bool(branch), dest)?;
163 }
164
165 "sqrtf32" => {
166 let [f] = check_intrinsic_arg_count(args)?;
167 let f = this.read_scalar(f)?.to_f32()?;
168 let res = math::sqrt(f);
170 let res = this.adjust_nan(res, &[f]);
171 this.write_scalar(res, dest)?;
172 }
173 "sqrtf64" => {
174 let [f] = check_intrinsic_arg_count(args)?;
175 let f = this.read_scalar(f)?.to_f64()?;
176 let res = math::sqrt(f);
178 let res = this.adjust_nan(res, &[f]);
179 this.write_scalar(res, dest)?;
180 }
181
182 #[rustfmt::skip]
183 | "sinf32"
184 | "cosf32"
185 | "expf32"
186 | "exp2f32"
187 | "logf32"
188 | "log10f32"
189 | "log2f32"
190 => {
191 let [f] = check_intrinsic_arg_count(args)?;
192 let f = this.read_scalar(f)?.to_f32()?;
193
194 let res = fixed_float_value(intrinsic_name, &[f]).unwrap_or_else(||{
195 let host = f.to_host();
198 let res = match intrinsic_name {
199 "sinf32" => host.sin(),
200 "cosf32" => host.cos(),
201 "expf32" => host.exp(),
202 "exp2f32" => host.exp2(),
203 "logf32" => host.ln(),
204 "log10f32" => host.log10(),
205 "log2f32" => host.log2(),
206 _ => bug!(),
207 };
208 let res = res.to_soft();
209
210 let res = apply_random_float_error_ulp(
213 this,
214 res,
215 2, );
217
218 clamp_float_value(intrinsic_name, res)
221 });
222 let res = this.adjust_nan(res, &[f]);
223 this.write_scalar(res, dest)?;
224 }
225
226 #[rustfmt::skip]
227 | "sinf64"
228 | "cosf64"
229 | "expf64"
230 | "exp2f64"
231 | "logf64"
232 | "log10f64"
233 | "log2f64"
234 => {
235 let [f] = check_intrinsic_arg_count(args)?;
236 let f = this.read_scalar(f)?.to_f64()?;
237
238 let res = fixed_float_value(intrinsic_name, &[f]).unwrap_or_else(||{
239 let host = f.to_host();
242 let res = match intrinsic_name {
243 "sinf64" => host.sin(),
244 "cosf64" => host.cos(),
245 "expf64" => host.exp(),
246 "exp2f64" => host.exp2(),
247 "logf64" => host.ln(),
248 "log10f64" => host.log10(),
249 "log2f64" => host.log2(),
250 _ => bug!(),
251 };
252 let res = res.to_soft();
253
254 let res = apply_random_float_error_ulp(
257 this,
258 res,
259 2, );
261
262 clamp_float_value(intrinsic_name, res)
265 });
266 let res = this.adjust_nan(res, &[f]);
267 this.write_scalar(res, dest)?;
268 }
269
270 "fmaf32" => {
271 let [a, b, c] = check_intrinsic_arg_count(args)?;
272 let a = this.read_scalar(a)?.to_f32()?;
273 let b = this.read_scalar(b)?.to_f32()?;
274 let c = this.read_scalar(c)?.to_f32()?;
275 let res = a.mul_add(b, c).value;
276 let res = this.adjust_nan(res, &[a, b, c]);
277 this.write_scalar(res, dest)?;
278 }
279 "fmaf64" => {
280 let [a, b, c] = check_intrinsic_arg_count(args)?;
281 let a = this.read_scalar(a)?.to_f64()?;
282 let b = this.read_scalar(b)?.to_f64()?;
283 let c = this.read_scalar(c)?.to_f64()?;
284 let res = a.mul_add(b, c).value;
285 let res = this.adjust_nan(res, &[a, b, c]);
286 this.write_scalar(res, dest)?;
287 }
288
289 "fmuladdf32" => {
290 let [a, b, c] = check_intrinsic_arg_count(args)?;
291 let a = this.read_scalar(a)?.to_f32()?;
292 let b = this.read_scalar(b)?.to_f32()?;
293 let c = this.read_scalar(c)?.to_f32()?;
294 let fuse: bool = this.machine.float_nondet && this.machine.rng.get_mut().random();
295 let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
296 let res = this.adjust_nan(res, &[a, b, c]);
297 this.write_scalar(res, dest)?;
298 }
299 "fmuladdf64" => {
300 let [a, b, c] = check_intrinsic_arg_count(args)?;
301 let a = this.read_scalar(a)?.to_f64()?;
302 let b = this.read_scalar(b)?.to_f64()?;
303 let c = this.read_scalar(c)?.to_f64()?;
304 let fuse: bool = this.machine.float_nondet && this.machine.rng.get_mut().random();
305 let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
306 let res = this.adjust_nan(res, &[a, b, c]);
307 this.write_scalar(res, dest)?;
308 }
309
310 "powf32" => {
311 let [f1, f2] = check_intrinsic_arg_count(args)?;
312 let f1 = this.read_scalar(f1)?.to_f32()?;
313 let f2 = this.read_scalar(f2)?.to_f32()?;
314
315 let res = fixed_float_value(intrinsic_name, &[f1, f2]).unwrap_or_else(|| {
316 let res = f1.to_host().powf(f2.to_host()).to_soft();
318
319 apply_random_float_error_ulp(
322 this, res, 2, )
324 });
325 let res = this.adjust_nan(res, &[f1, f2]);
326 this.write_scalar(res, dest)?;
327 }
328 "powf64" => {
329 let [f1, f2] = check_intrinsic_arg_count(args)?;
330 let f1 = this.read_scalar(f1)?.to_f64()?;
331 let f2 = this.read_scalar(f2)?.to_f64()?;
332
333 let res = fixed_float_value(intrinsic_name, &[f1, f2]).unwrap_or_else(|| {
334 let res = f1.to_host().powf(f2.to_host()).to_soft();
336
337 apply_random_float_error_ulp(
340 this, res, 2, )
342 });
343 let res = this.adjust_nan(res, &[f1, f2]);
344 this.write_scalar(res, dest)?;
345 }
346
347 "powif32" => {
348 let [f, i] = check_intrinsic_arg_count(args)?;
349 let f = this.read_scalar(f)?.to_f32()?;
350 let i = this.read_scalar(i)?.to_i32()?;
351
352 let res = fixed_powi_float_value(f, i).unwrap_or_else(|| {
353 let res = f.to_host().powi(i).to_soft();
355
356 apply_random_float_error_ulp(
359 this, res, 2, )
361 });
362 let res = this.adjust_nan(res, &[f]);
363 this.write_scalar(res, dest)?;
364 }
365 "powif64" => {
366 let [f, i] = check_intrinsic_arg_count(args)?;
367 let f = this.read_scalar(f)?.to_f64()?;
368 let i = this.read_scalar(i)?.to_i32()?;
369
370 let res = fixed_powi_float_value(f, i).unwrap_or_else(|| {
371 let res = f.to_host().powi(i).to_soft();
373
374 apply_random_float_error_ulp(
377 this, res, 2, )
379 });
380 let res = this.adjust_nan(res, &[f]);
381 this.write_scalar(res, dest)?;
382 }
383
384 #[rustfmt::skip]
385 | "fadd_fast"
386 | "fsub_fast"
387 | "fmul_fast"
388 | "fdiv_fast"
389 | "frem_fast"
390 => {
391 let [a, b] = check_intrinsic_arg_count(args)?;
392 let a = this.read_immediate(a)?;
393 let b = this.read_immediate(b)?;
394 let op = match intrinsic_name {
395 "fadd_fast" => mir::BinOp::Add,
396 "fsub_fast" => mir::BinOp::Sub,
397 "fmul_fast" => mir::BinOp::Mul,
398 "fdiv_fast" => mir::BinOp::Div,
399 "frem_fast" => mir::BinOp::Rem,
400 _ => bug!(),
401 };
402 let float_finite = |x: &ImmTy<'tcx>| -> InterpResult<'tcx, bool> {
403 let ty::Float(fty) = x.layout.ty.kind() else {
404 bug!("float_finite: non-float input type {}", x.layout.ty)
405 };
406 interp_ok(match fty {
407 FloatTy::F16 => x.to_scalar().to_f16()?.is_finite(),
408 FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(),
409 FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(),
410 FloatTy::F128 => x.to_scalar().to_f128()?.is_finite(),
411 })
412 };
413 match (float_finite(&a)?, float_finite(&b)?) {
414 (false, false) => throw_ub_format!(
415 "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
416 ),
417 (false, _) => throw_ub_format!(
418 "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
419 ),
420 (_, false) => throw_ub_format!(
421 "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
422 ),
423 _ => {}
424 }
425 let res = this.binary_op(op, &a, &b)?;
426 if !float_finite(&res)? {
429 throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
430 }
431 let res = apply_random_float_error_to_imm(this, res, 2 )?;
434 this.write_immediate(*res, dest)?;
435 }
436
437 "float_to_int_unchecked" => {
438 let [val] = check_intrinsic_arg_count(args)?;
439 let val = this.read_immediate(val)?;
440
441 let res = this
442 .float_to_int_checked(&val, dest.layout, Round::TowardZero)?
443 .ok_or_else(|| {
444 err_ub_format!(
445 "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`",
446 dest.layout.ty
447 )
448 })?;
449
450 this.write_immediate(*res, dest)?;
451 }
452
453 "breakpoint" => {
455 let [] = check_intrinsic_arg_count(args)?;
456 throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
458 }
459
460 _ => return interp_ok(EmulateItemResult::NotSupported),
461 }
462
463 interp_ok(EmulateItemResult::NeedsReturn)
464 }
465}
466
467fn apply_random_float_error_to_imm<'tcx>(
472 ecx: &mut MiriInterpCx<'tcx>,
473 val: ImmTy<'tcx>,
474 ulp_exponent: u32,
475) -> InterpResult<'tcx, ImmTy<'tcx>> {
476 let scalar = val.to_scalar_int()?;
477 let res: ScalarInt = match val.layout.ty.kind() {
478 ty::Float(FloatTy::F16) =>
479 apply_random_float_error_ulp(ecx, scalar.to_f16(), ulp_exponent).into(),
480 ty::Float(FloatTy::F32) =>
481 apply_random_float_error_ulp(ecx, scalar.to_f32(), ulp_exponent).into(),
482 ty::Float(FloatTy::F64) =>
483 apply_random_float_error_ulp(ecx, scalar.to_f64(), ulp_exponent).into(),
484 ty::Float(FloatTy::F128) =>
485 apply_random_float_error_ulp(ecx, scalar.to_f128(), ulp_exponent).into(),
486 _ => bug!("intrinsic called with non-float input type"),
487 };
488
489 interp_ok(ImmTy::from_scalar_int(res, val.layout))
490}
491
492fn fixed_float_value<S: Semantics>(
504 intrinsic_name: &str,
505 args: &[IeeeFloat<S>],
506) -> Option<IeeeFloat<S>> {
507 let one = IeeeFloat::<S>::one();
508 match (intrinsic_name, args) {
509 ("cosf32" | "cosf64", [input]) if input.is_zero() => Some(one),
511
512 ("expf32" | "expf64" | "exp2f32" | "exp2f64", [input]) if input.is_zero() => Some(one),
514
515 ("powf32" | "powf64", [base, _]) if *base == one => Some(one),
517
518 ("powf32" | "powf64", [base, exp]) if *base == -one && exp.is_infinite() => Some(one),
520
521 ("powf32" | "powf64", [_, exp]) if exp.is_zero() => Some(one),
526
527 _ => None,
530 }
531}
532
533fn fixed_powi_float_value<S: Semantics>(base: IeeeFloat<S>, exp: i32) -> Option<IeeeFloat<S>> {
536 match (base.category(), exp) {
537 (_, 0) => Some(IeeeFloat::<S>::one()),
542
543 _ => None,
544 }
545}
546
547fn clamp_float_value<S: Semantics>(intrinsic_name: &str, val: IeeeFloat<S>) -> IeeeFloat<S> {
550 match intrinsic_name {
551 "sinf32" | "cosf32" | "sinf64" | "cosf64" =>
553 val.clamp(IeeeFloat::<S>::one().neg(), IeeeFloat::<S>::one()),
554 "expf32" | "exp2f32" | "expf64" | "exp2f64" =>
556 IeeeFloat::<S>::maximum(val, IeeeFloat::<S>::ZERO),
557 _ => val,
558 }
559}