miri/intrinsics/
mod.rs

1#![warn(clippy::arithmetic_side_effects)]
2
3mod atomic;
4mod simd;
5
6use std::ops::Neg;
7
8use rand::Rng;
9use rustc_abi::Size;
10use rustc_apfloat::ieee::{IeeeFloat, Semantics};
11use rustc_apfloat::{self, Float, Round};
12use rustc_middle::mir;
13use rustc_middle::ty::{self, FloatTy, ScalarInt};
14use rustc_span::{Symbol, sym};
15
16use self::atomic::EvalContextExt as _;
17use self::helpers::{ToHost, ToSoft, check_intrinsic_arg_count};
18use self::simd::EvalContextExt as _;
19use crate::math::{IeeeExt, apply_random_float_error_ulp};
20use crate::*;
21
22impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
23pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
24    fn call_intrinsic(
25        &mut self,
26        instance: ty::Instance<'tcx>,
27        args: &[OpTy<'tcx>],
28        dest: &PlaceTy<'tcx>,
29        ret: Option<mir::BasicBlock>,
30        unwind: mir::UnwindAction,
31    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
32        let this = self.eval_context_mut();
33
34        // Force use of fallback body, if available.
35        if this.machine.force_intrinsic_fallback
36            && !this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden
37        {
38            return interp_ok(Some(ty::Instance {
39                def: ty::InstanceKind::Item(instance.def_id()),
40                args: instance.args,
41            }));
42        }
43
44        // See if the core engine can handle this intrinsic.
45        if this.eval_intrinsic(instance, args, dest, ret)? {
46            return interp_ok(None);
47        }
48        let intrinsic_name = this.tcx.item_name(instance.def_id());
49        let intrinsic_name = intrinsic_name.as_str();
50
51        // FIXME: avoid allocating memory
52        let dest = this.force_allocation(dest)?;
53
54        match this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, &dest, ret)? {
55            EmulateItemResult::NotSupported => {
56                // We haven't handled the intrinsic, let's see if we can use a fallback body.
57                if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
58                    throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
59                }
60                let intrinsic_fallback_is_spec = Symbol::intern("intrinsic_fallback_is_spec");
61                if this
62                    .tcx
63                    .get_attrs_by_path(instance.def_id(), &[sym::miri, intrinsic_fallback_is_spec])
64                    .next()
65                    .is_none()
66                {
67                    throw_unsup_format!(
68                        "Miri can only use intrinsic fallback bodies that exactly reflect the specification: they fully check for UB and are as non-deterministic as possible. After verifying that `{intrinsic_name}` does so, add the `#[miri::intrinsic_fallback_is_spec]` attribute to it; also ping @rust-lang/miri when you do that"
69                    );
70                }
71                interp_ok(Some(ty::Instance {
72                    def: ty::InstanceKind::Item(instance.def_id()),
73                    args: instance.args,
74                }))
75            }
76            EmulateItemResult::NeedsReturn => {
77                trace!("{:?}", this.dump_place(&dest.clone().into()));
78                this.return_to_block(ret)?;
79                interp_ok(None)
80            }
81            EmulateItemResult::NeedsUnwind => {
82                // Jump to the unwind block to begin unwinding.
83                this.unwind_to_block(unwind)?;
84                interp_ok(None)
85            }
86            EmulateItemResult::AlreadyJumped => interp_ok(None),
87        }
88    }
89
90    /// Emulates a Miri-supported intrinsic (not supported by the core engine).
91    /// Returns `Ok(true)` if the intrinsic was handled.
92    fn emulate_intrinsic_by_name(
93        &mut self,
94        intrinsic_name: &str,
95        generic_args: ty::GenericArgsRef<'tcx>,
96        args: &[OpTy<'tcx>],
97        dest: &MPlaceTy<'tcx>,
98        ret: Option<mir::BasicBlock>,
99    ) -> InterpResult<'tcx, EmulateItemResult> {
100        let this = self.eval_context_mut();
101
102        if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
103            return this.emulate_atomic_intrinsic(name, generic_args, args, dest);
104        }
105        if let Some(name) = intrinsic_name.strip_prefix("simd_") {
106            return this.emulate_simd_intrinsic(name, generic_args, args, dest);
107        }
108
109        match intrinsic_name {
110            // Basic control flow
111            "abort" => {
112                throw_machine_stop!(TerminationInfo::Abort(
113                    "the program aborted execution".to_owned()
114                ));
115            }
116            "catch_unwind" => {
117                this.handle_catch_unwind(args, dest, ret)?;
118                // This pushed a stack frame, don't jump to `ret`.
119                return interp_ok(EmulateItemResult::AlreadyJumped);
120            }
121
122            // Raw memory accesses
123            "volatile_load" => {
124                let [place] = check_intrinsic_arg_count(args)?;
125                let place = this.deref_pointer(place)?;
126                this.copy_op(&place, dest)?;
127            }
128            "volatile_store" => {
129                let [place, dest] = check_intrinsic_arg_count(args)?;
130                let place = this.deref_pointer(place)?;
131                this.copy_op(dest, &place)?;
132            }
133
134            "volatile_set_memory" => {
135                let [ptr, val_byte, count] = check_intrinsic_arg_count(args)?;
136                this.write_bytes_intrinsic(ptr, val_byte, count, "volatile_set_memory")?;
137            }
138
139            // Memory model / provenance manipulation
140            "ptr_mask" => {
141                let [ptr, mask] = check_intrinsic_arg_count(args)?;
142
143                let ptr = this.read_pointer(ptr)?;
144                let mask = this.read_target_usize(mask)?;
145
146                let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask);
147
148                this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
149            }
150
151            // We want to return either `true` or `false` at random, or else something like
152            // ```
153            // if !is_val_statically_known(0) { unreachable_unchecked(); }
154            // ```
155            // Would not be considered UB, or the other way around (`is_val_statically_known(0)`).
156            "is_val_statically_known" => {
157                let [_arg] = check_intrinsic_arg_count(args)?;
158                // FIXME: should we check for validity here? It's tricky because we do not have a
159                // place. Codegen does not seem to set any attributes like `noundef` for intrinsic
160                // calls, so we don't *have* to do anything.
161                let branch: bool = this.machine.rng.get_mut().random();
162                this.write_scalar(Scalar::from_bool(branch), dest)?;
163            }
164
165            "sqrtf32" => {
166                let [f] = check_intrinsic_arg_count(args)?;
167                let f = this.read_scalar(f)?.to_f32()?;
168                // Sqrt is specified to be fully precise.
169                let res = math::sqrt(f);
170                let res = this.adjust_nan(res, &[f]);
171                this.write_scalar(res, dest)?;
172            }
173            "sqrtf64" => {
174                let [f] = check_intrinsic_arg_count(args)?;
175                let f = this.read_scalar(f)?.to_f64()?;
176                // Sqrt is specified to be fully precise.
177                let res = math::sqrt(f);
178                let res = this.adjust_nan(res, &[f]);
179                this.write_scalar(res, dest)?;
180            }
181
182            #[rustfmt::skip]
183            | "sinf32"
184            | "cosf32"
185            | "expf32"
186            | "exp2f32"
187            | "logf32"
188            | "log10f32"
189            | "log2f32"
190            => {
191                let [f] = check_intrinsic_arg_count(args)?;
192                let f = this.read_scalar(f)?.to_f32()?;
193
194                let res = fixed_float_value(intrinsic_name, &[f]).unwrap_or_else(||{
195                    // Using host floats (but it's fine, these operations do not have
196                    // guaranteed precision).
197                    let host = f.to_host();
198                    let res = match intrinsic_name {
199                        "sinf32" => host.sin(),
200                        "cosf32" => host.cos(),
201                        "expf32" => host.exp(),
202                        "exp2f32" => host.exp2(),
203                        "logf32" => host.ln(),
204                        "log10f32" => host.log10(),
205                        "log2f32" => host.log2(),
206                        _ => bug!(),
207                    };
208                    let res = res.to_soft();
209
210                    // Apply a relative error of 4ULP to introduce some non-determinism
211                    // simulating imprecise implementations and optimizations.
212                    let res = apply_random_float_error_ulp(
213                        this,
214                        res,
215                        2, // log2(4)
216                    );
217
218                    // Clamp the result to the guaranteed range of this function according to the C standard,
219                    // if any.
220                    clamp_float_value(intrinsic_name, res)
221                });
222                let res = this.adjust_nan(res, &[f]);
223                this.write_scalar(res, dest)?;
224            }
225
226            #[rustfmt::skip]
227            | "sinf64"
228            | "cosf64"
229            | "expf64"
230            | "exp2f64"
231            | "logf64"
232            | "log10f64"
233            | "log2f64"
234            => {
235                let [f] = check_intrinsic_arg_count(args)?;
236                let f = this.read_scalar(f)?.to_f64()?;
237
238                let res = fixed_float_value(intrinsic_name, &[f]).unwrap_or_else(||{
239                    // Using host floats (but it's fine, these operations do not have
240                    // guaranteed precision).
241                    let host = f.to_host();
242                    let res = match intrinsic_name {
243                        "sinf64" => host.sin(),
244                        "cosf64" => host.cos(),
245                        "expf64" => host.exp(),
246                        "exp2f64" => host.exp2(),
247                        "logf64" => host.ln(),
248                        "log10f64" => host.log10(),
249                        "log2f64" => host.log2(),
250                        _ => bug!(),
251                    };
252                    let res = res.to_soft();
253
254                    // Apply a relative error of 4ULP to introduce some non-determinism
255                    // simulating imprecise implementations and optimizations.
256                    let res = apply_random_float_error_ulp(
257                        this,
258                        res,
259                        2, // log2(4)
260                    );
261
262                    // Clamp the result to the guaranteed range of this function according to the C standard,
263                    // if any.
264                    clamp_float_value(intrinsic_name, res)
265                });
266                let res = this.adjust_nan(res, &[f]);
267                this.write_scalar(res, dest)?;
268            }
269
270            "fmaf32" => {
271                let [a, b, c] = check_intrinsic_arg_count(args)?;
272                let a = this.read_scalar(a)?.to_f32()?;
273                let b = this.read_scalar(b)?.to_f32()?;
274                let c = this.read_scalar(c)?.to_f32()?;
275                let res = a.mul_add(b, c).value;
276                let res = this.adjust_nan(res, &[a, b, c]);
277                this.write_scalar(res, dest)?;
278            }
279            "fmaf64" => {
280                let [a, b, c] = check_intrinsic_arg_count(args)?;
281                let a = this.read_scalar(a)?.to_f64()?;
282                let b = this.read_scalar(b)?.to_f64()?;
283                let c = this.read_scalar(c)?.to_f64()?;
284                let res = a.mul_add(b, c).value;
285                let res = this.adjust_nan(res, &[a, b, c]);
286                this.write_scalar(res, dest)?;
287            }
288
289            "fmuladdf32" => {
290                let [a, b, c] = check_intrinsic_arg_count(args)?;
291                let a = this.read_scalar(a)?.to_f32()?;
292                let b = this.read_scalar(b)?.to_f32()?;
293                let c = this.read_scalar(c)?.to_f32()?;
294                let fuse: bool = this.machine.float_nondet && this.machine.rng.get_mut().random();
295                let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
296                let res = this.adjust_nan(res, &[a, b, c]);
297                this.write_scalar(res, dest)?;
298            }
299            "fmuladdf64" => {
300                let [a, b, c] = check_intrinsic_arg_count(args)?;
301                let a = this.read_scalar(a)?.to_f64()?;
302                let b = this.read_scalar(b)?.to_f64()?;
303                let c = this.read_scalar(c)?.to_f64()?;
304                let fuse: bool = this.machine.float_nondet && this.machine.rng.get_mut().random();
305                let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
306                let res = this.adjust_nan(res, &[a, b, c]);
307                this.write_scalar(res, dest)?;
308            }
309
310            "powf32" => {
311                let [f1, f2] = check_intrinsic_arg_count(args)?;
312                let f1 = this.read_scalar(f1)?.to_f32()?;
313                let f2 = this.read_scalar(f2)?.to_f32()?;
314
315                let res = fixed_float_value(intrinsic_name, &[f1, f2]).unwrap_or_else(|| {
316                    // Using host floats (but it's fine, this operation does not have guaranteed precision).
317                    let res = f1.to_host().powf(f2.to_host()).to_soft();
318
319                    // Apply a relative error of 4ULP to introduce some non-determinism
320                    // simulating imprecise implementations and optimizations.
321                    apply_random_float_error_ulp(
322                        this, res, 2, // log2(4)
323                    )
324                });
325                let res = this.adjust_nan(res, &[f1, f2]);
326                this.write_scalar(res, dest)?;
327            }
328            "powf64" => {
329                let [f1, f2] = check_intrinsic_arg_count(args)?;
330                let f1 = this.read_scalar(f1)?.to_f64()?;
331                let f2 = this.read_scalar(f2)?.to_f64()?;
332
333                let res = fixed_float_value(intrinsic_name, &[f1, f2]).unwrap_or_else(|| {
334                    // Using host floats (but it's fine, this operation does not have guaranteed precision).
335                    let res = f1.to_host().powf(f2.to_host()).to_soft();
336
337                    // Apply a relative error of 4ULP to introduce some non-determinism
338                    // simulating imprecise implementations and optimizations.
339                    apply_random_float_error_ulp(
340                        this, res, 2, // log2(4)
341                    )
342                });
343                let res = this.adjust_nan(res, &[f1, f2]);
344                this.write_scalar(res, dest)?;
345            }
346
347            "powif32" => {
348                let [f, i] = check_intrinsic_arg_count(args)?;
349                let f = this.read_scalar(f)?.to_f32()?;
350                let i = this.read_scalar(i)?.to_i32()?;
351
352                let res = fixed_powi_float_value(f, i).unwrap_or_else(|| {
353                    // Using host floats (but it's fine, this operation does not have guaranteed precision).
354                    let res = f.to_host().powi(i).to_soft();
355
356                    // Apply a relative error of 4ULP to introduce some non-determinism
357                    // simulating imprecise implementations and optimizations.
358                    apply_random_float_error_ulp(
359                        this, res, 2, // log2(4)
360                    )
361                });
362                let res = this.adjust_nan(res, &[f]);
363                this.write_scalar(res, dest)?;
364            }
365            "powif64" => {
366                let [f, i] = check_intrinsic_arg_count(args)?;
367                let f = this.read_scalar(f)?.to_f64()?;
368                let i = this.read_scalar(i)?.to_i32()?;
369
370                let res = fixed_powi_float_value(f, i).unwrap_or_else(|| {
371                    // Using host floats (but it's fine, this operation does not have guaranteed precision).
372                    let res = f.to_host().powi(i).to_soft();
373
374                    // Apply a relative error of 4ULP to introduce some non-determinism
375                    // simulating imprecise implementations and optimizations.
376                    apply_random_float_error_ulp(
377                        this, res, 2, // log2(4)
378                    )
379                });
380                let res = this.adjust_nan(res, &[f]);
381                this.write_scalar(res, dest)?;
382            }
383
384            #[rustfmt::skip]
385            | "fadd_fast"
386            | "fsub_fast"
387            | "fmul_fast"
388            | "fdiv_fast"
389            | "frem_fast"
390            => {
391                let [a, b] = check_intrinsic_arg_count(args)?;
392                let a = this.read_immediate(a)?;
393                let b = this.read_immediate(b)?;
394                let op = match intrinsic_name {
395                    "fadd_fast" => mir::BinOp::Add,
396                    "fsub_fast" => mir::BinOp::Sub,
397                    "fmul_fast" => mir::BinOp::Mul,
398                    "fdiv_fast" => mir::BinOp::Div,
399                    "frem_fast" => mir::BinOp::Rem,
400                    _ => bug!(),
401                };
402                let float_finite = |x: &ImmTy<'tcx>| -> InterpResult<'tcx, bool> {
403                    let ty::Float(fty) = x.layout.ty.kind() else {
404                        bug!("float_finite: non-float input type {}", x.layout.ty)
405                    };
406                    interp_ok(match fty {
407                        FloatTy::F16 => x.to_scalar().to_f16()?.is_finite(),
408                        FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(),
409                        FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(),
410                        FloatTy::F128 => x.to_scalar().to_f128()?.is_finite(),
411                    })
412                };
413                match (float_finite(&a)?, float_finite(&b)?) {
414                    (false, false) => throw_ub_format!(
415                        "`{intrinsic_name}` intrinsic called with non-finite value as both parameters",
416                    ),
417                    (false, _) => throw_ub_format!(
418                        "`{intrinsic_name}` intrinsic called with non-finite value as first parameter",
419                    ),
420                    (_, false) => throw_ub_format!(
421                        "`{intrinsic_name}` intrinsic called with non-finite value as second parameter",
422                    ),
423                    _ => {}
424                }
425                let res = this.binary_op(op, &a, &b)?;
426                // This cannot be a NaN so we also don't have to apply any non-determinism.
427                // (Also, `binary_op` already called `generate_nan` if needed.)
428                if !float_finite(&res)? {
429                    throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
430                }
431                // Apply a relative error of 4ULP to simulate non-deterministic precision loss
432                // due to optimizations.
433                let res = apply_random_float_error_to_imm(this, res, 2 /* log2(4) */)?;
434                this.write_immediate(*res, dest)?;
435            }
436
437            "float_to_int_unchecked" => {
438                let [val] = check_intrinsic_arg_count(args)?;
439                let val = this.read_immediate(val)?;
440
441                let res = this
442                    .float_to_int_checked(&val, dest.layout, Round::TowardZero)?
443                    .ok_or_else(|| {
444                        err_ub_format!(
445                            "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`",
446                            dest.layout.ty
447                        )
448                    })?;
449
450                this.write_immediate(*res, dest)?;
451            }
452
453            // Other
454            "breakpoint" => {
455                let [] = check_intrinsic_arg_count(args)?;
456                // normally this would raise a SIGTRAP, which aborts if no debugger is connected
457                throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
458            }
459
460            _ => return interp_ok(EmulateItemResult::NotSupported),
461        }
462
463        interp_ok(EmulateItemResult::NeedsReturn)
464    }
465}
466
467/// Applies a random ULP floating point error to `val` and returns the new value.
468/// So if you want an X ULP error, `ulp_exponent` should be log2(X).
469///
470/// Will fail if `val` is not a floating point number.
471fn apply_random_float_error_to_imm<'tcx>(
472    ecx: &mut MiriInterpCx<'tcx>,
473    val: ImmTy<'tcx>,
474    ulp_exponent: u32,
475) -> InterpResult<'tcx, ImmTy<'tcx>> {
476    let scalar = val.to_scalar_int()?;
477    let res: ScalarInt = match val.layout.ty.kind() {
478        ty::Float(FloatTy::F16) =>
479            apply_random_float_error_ulp(ecx, scalar.to_f16(), ulp_exponent).into(),
480        ty::Float(FloatTy::F32) =>
481            apply_random_float_error_ulp(ecx, scalar.to_f32(), ulp_exponent).into(),
482        ty::Float(FloatTy::F64) =>
483            apply_random_float_error_ulp(ecx, scalar.to_f64(), ulp_exponent).into(),
484        ty::Float(FloatTy::F128) =>
485            apply_random_float_error_ulp(ecx, scalar.to_f128(), ulp_exponent).into(),
486        _ => bug!("intrinsic called with non-float input type"),
487    };
488
489    interp_ok(ImmTy::from_scalar_int(res, val.layout))
490}
491
492/// For the intrinsics:
493/// - sinf32, sinf64
494/// - cosf32, cosf64
495/// - expf32, expf64, exp2f32, exp2f64
496/// - logf32, logf64, log2f32, log2f64, log10f32, log10f64
497/// - powf32, powf64
498///
499/// Returns `Some(output)` if the `intrinsic` results in a defined fixed `output` specified in the C standard
500/// (specifically, C23 annex F.10)  when given `args` as arguments. Outputs that are unaffected by a relative error
501/// (such as INF and zero) are not handled here, they are assumed to be handled by the underlying
502/// implementation. Returns `None` if no specific value is guaranteed.
503fn fixed_float_value<S: Semantics>(
504    intrinsic_name: &str,
505    args: &[IeeeFloat<S>],
506) -> Option<IeeeFloat<S>> {
507    let one = IeeeFloat::<S>::one();
508    match (intrinsic_name, args) {
509        // cos(+- 0) = 1
510        ("cosf32" | "cosf64", [input]) if input.is_zero() => Some(one),
511
512        // e^0 = 1
513        ("expf32" | "expf64" | "exp2f32" | "exp2f64", [input]) if input.is_zero() => Some(one),
514
515        // 1^y = 1 for any y, even a NaN.
516        ("powf32" | "powf64", [base, _]) if *base == one => Some(one),
517
518        // (-1)^(±INF) = 1
519        ("powf32" | "powf64", [base, exp]) if *base == -one && exp.is_infinite() => Some(one),
520
521        // FIXME(#4286): The C ecosystem is inconsistent with handling sNaN's, some return 1 others propogate
522        // the NaN. We should return either 1 or the NaN non-deterministically here.
523        // But for now, just handle them all the same.
524        // x^(±0) = 1 for any x, even a NaN
525        ("powf32" | "powf64", [_, exp]) if exp.is_zero() => Some(one),
526
527        // There are a lot of cases for fixed outputs according to the C Standard, but these are mainly INF or zero
528        // which are not affected by the applied error.
529        _ => None,
530    }
531}
532
533/// Returns `Some(output)` if `powi` (called `pown` in C) results in a fixed value specified in the C standard
534/// (specifically, C23 annex F.10.4.6) when doing `base^exp`. Otherwise, returns `None`.
535fn fixed_powi_float_value<S: Semantics>(base: IeeeFloat<S>, exp: i32) -> Option<IeeeFloat<S>> {
536    match (base.category(), exp) {
537        // x^0 = 1, if x is not a Signaling NaN
538        // FIXME(#4286): The C ecosystem is inconsistent with handling sNaN's, some return 1 others propogate
539        // the NaN. We should return either 1 or the NaN non-deterministically here.
540        // But for now, just handle them all the same.
541        (_, 0) => Some(IeeeFloat::<S>::one()),
542
543        _ => None,
544    }
545}
546
547/// Given an floating-point operation and a floating-point value, clamps the result to the output
548/// range of the given operation.
549fn clamp_float_value<S: Semantics>(intrinsic_name: &str, val: IeeeFloat<S>) -> IeeeFloat<S> {
550    match intrinsic_name {
551        // sin and cos: [-1, 1]
552        "sinf32" | "cosf32" | "sinf64" | "cosf64" =>
553            val.clamp(IeeeFloat::<S>::one().neg(), IeeeFloat::<S>::one()),
554        // exp: [0, +INF]
555        "expf32" | "exp2f32" | "expf64" | "exp2f64" =>
556            IeeeFloat::<S>::maximum(val, IeeeFloat::<S>::ZERO),
557        _ => val,
558    }
559}