miri/intrinsics/
simd.rs

1use either::Either;
2use rand::Rng;
3use rustc_abi::{Endian, HasDataLayout};
4use rustc_apfloat::{Float, Round};
5use rustc_middle::ty::FloatTy;
6use rustc_middle::{mir, ty};
7use rustc_span::{Symbol, sym};
8
9use crate::helpers::{
10    ToHost, ToSoft, bool_to_simd_element, check_intrinsic_arg_count, simd_element_to_bool,
11};
12use crate::*;
13
14#[derive(Copy, Clone)]
15pub(crate) enum MinMax {
16    Min,
17    Max,
18}
19
20impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
21pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
22    /// Calls the simd intrinsic `intrinsic`; the `simd_` prefix has already been removed.
23    /// Returns `Ok(true)` if the intrinsic was handled.
24    fn emulate_simd_intrinsic(
25        &mut self,
26        intrinsic_name: &str,
27        generic_args: ty::GenericArgsRef<'tcx>,
28        args: &[OpTy<'tcx>],
29        dest: &MPlaceTy<'tcx>,
30    ) -> InterpResult<'tcx, EmulateItemResult> {
31        let this = self.eval_context_mut();
32        match intrinsic_name {
33            #[rustfmt::skip]
34            | "neg"
35            | "fabs"
36            | "ceil"
37            | "floor"
38            | "round"
39            | "round_ties_even"
40            | "trunc"
41            | "fsqrt"
42            | "fsin"
43            | "fcos"
44            | "fexp"
45            | "fexp2"
46            | "flog"
47            | "flog2"
48            | "flog10"
49            | "ctlz"
50            | "ctpop"
51            | "cttz"
52            | "bswap"
53            | "bitreverse"
54            => {
55                let [op] = check_intrinsic_arg_count(args)?;
56                let (op, op_len) = this.project_to_simd(op)?;
57                let (dest, dest_len) = this.project_to_simd(dest)?;
58
59                assert_eq!(dest_len, op_len);
60
61                #[derive(Copy, Clone)]
62                enum Op<'a> {
63                    MirOp(mir::UnOp),
64                    Abs,
65                    Round(rustc_apfloat::Round),
66                    Numeric(Symbol),
67                    HostOp(&'a str),
68                }
69                let which = match intrinsic_name {
70                    "neg" => Op::MirOp(mir::UnOp::Neg),
71                    "fabs" => Op::Abs,
72                    "ceil" => Op::Round(rustc_apfloat::Round::TowardPositive),
73                    "floor" => Op::Round(rustc_apfloat::Round::TowardNegative),
74                    "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway),
75                    "round_ties_even" => Op::Round(rustc_apfloat::Round::NearestTiesToEven),
76                    "trunc" => Op::Round(rustc_apfloat::Round::TowardZero),
77                    "ctlz" => Op::Numeric(sym::ctlz),
78                    "ctpop" => Op::Numeric(sym::ctpop),
79                    "cttz" => Op::Numeric(sym::cttz),
80                    "bswap" => Op::Numeric(sym::bswap),
81                    "bitreverse" => Op::Numeric(sym::bitreverse),
82                    _ => Op::HostOp(intrinsic_name),
83                };
84
85                for i in 0..dest_len {
86                    let op = this.read_immediate(&this.project_index(&op, i)?)?;
87                    let dest = this.project_index(&dest, i)?;
88                    let val = match which {
89                        Op::MirOp(mir_op) => {
90                            // This already does NaN adjustments
91                            this.unary_op(mir_op, &op)?.to_scalar()
92                        }
93                        Op::Abs => {
94                            // Works for f32 and f64.
95                            let ty::Float(float_ty) = op.layout.ty.kind() else {
96                                span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
97                            };
98                            let op = op.to_scalar();
99                            // "Bitwise" operation, no NaN adjustments
100                            match float_ty {
101                                FloatTy::F16 => unimplemented!("f16_f128"),
102                                FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
103                                FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
104                                FloatTy::F128 => unimplemented!("f16_f128"),
105                            }
106                        }
107                        Op::HostOp(host_op) => {
108                            let ty::Float(float_ty) = op.layout.ty.kind() else {
109                                span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
110                            };
111                            // Using host floats except for sqrt (but it's fine, these operations do not
112                            // have guaranteed precision).
113                            match float_ty {
114                                FloatTy::F16 => unimplemented!("f16_f128"),
115                                FloatTy::F32 => {
116                                    let f = op.to_scalar().to_f32()?;
117                                    let res = match host_op {
118                                        "fsqrt" => math::sqrt(f),
119                                        "fsin" => f.to_host().sin().to_soft(),
120                                        "fcos" => f.to_host().cos().to_soft(),
121                                        "fexp" => f.to_host().exp().to_soft(),
122                                        "fexp2" => f.to_host().exp2().to_soft(),
123                                        "flog" => f.to_host().ln().to_soft(),
124                                        "flog2" => f.to_host().log2().to_soft(),
125                                        "flog10" => f.to_host().log10().to_soft(),
126                                        _ => bug!(),
127                                    };
128                                    let res = this.adjust_nan(res, &[f]);
129                                    Scalar::from(res)
130                                }
131                                FloatTy::F64 => {
132                                    let f = op.to_scalar().to_f64()?;
133                                    let res = match host_op {
134                                        "fsqrt" => math::sqrt(f),
135                                        "fsin" => f.to_host().sin().to_soft(),
136                                        "fcos" => f.to_host().cos().to_soft(),
137                                        "fexp" => f.to_host().exp().to_soft(),
138                                        "fexp2" => f.to_host().exp2().to_soft(),
139                                        "flog" => f.to_host().ln().to_soft(),
140                                        "flog2" => f.to_host().log2().to_soft(),
141                                        "flog10" => f.to_host().log10().to_soft(),
142                                        _ => bug!(),
143                                    };
144                                    let res = this.adjust_nan(res, &[f]);
145                                    Scalar::from(res)
146                                }
147                                FloatTy::F128 => unimplemented!("f16_f128"),
148                            }
149                        }
150                        Op::Round(rounding) => {
151                            let ty::Float(float_ty) = op.layout.ty.kind() else {
152                                span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
153                            };
154                            match float_ty {
155                                FloatTy::F16 => unimplemented!("f16_f128"),
156                                FloatTy::F32 => {
157                                    let f = op.to_scalar().to_f32()?;
158                                    let res = f.round_to_integral(rounding).value;
159                                    let res = this.adjust_nan(res, &[f]);
160                                    Scalar::from_f32(res)
161                                }
162                                FloatTy::F64 => {
163                                    let f = op.to_scalar().to_f64()?;
164                                    let res = f.round_to_integral(rounding).value;
165                                    let res = this.adjust_nan(res, &[f]);
166                                    Scalar::from_f64(res)
167                                }
168                                FloatTy::F128 => unimplemented!("f16_f128"),
169                            }
170                        }
171                        Op::Numeric(name) => {
172                            this.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)?
173                        }
174                    };
175                    this.write_scalar(val, &dest)?;
176                }
177            }
178            #[rustfmt::skip]
179            | "add"
180            | "sub"
181            | "mul"
182            | "div"
183            | "rem"
184            | "shl"
185            | "shr"
186            | "and"
187            | "or"
188            | "xor"
189            | "eq"
190            | "ne"
191            | "lt"
192            | "le"
193            | "gt"
194            | "ge"
195            | "fmax"
196            | "fmin"
197            | "saturating_add"
198            | "saturating_sub"
199            | "arith_offset"
200            => {
201                use mir::BinOp;
202
203                let [left, right] = check_intrinsic_arg_count(args)?;
204                let (left, left_len) = this.project_to_simd(left)?;
205                let (right, right_len) = this.project_to_simd(right)?;
206                let (dest, dest_len) = this.project_to_simd(dest)?;
207
208                assert_eq!(dest_len, left_len);
209                assert_eq!(dest_len, right_len);
210
211                enum Op {
212                    MirOp(BinOp),
213                    SaturatingOp(BinOp),
214                    FMinMax(MinMax),
215                    WrappingOffset,
216                }
217                let which = match intrinsic_name {
218                    "add" => Op::MirOp(BinOp::Add),
219                    "sub" => Op::MirOp(BinOp::Sub),
220                    "mul" => Op::MirOp(BinOp::Mul),
221                    "div" => Op::MirOp(BinOp::Div),
222                    "rem" => Op::MirOp(BinOp::Rem),
223                    "shl" => Op::MirOp(BinOp::ShlUnchecked),
224                    "shr" => Op::MirOp(BinOp::ShrUnchecked),
225                    "and" => Op::MirOp(BinOp::BitAnd),
226                    "or" => Op::MirOp(BinOp::BitOr),
227                    "xor" => Op::MirOp(BinOp::BitXor),
228                    "eq" => Op::MirOp(BinOp::Eq),
229                    "ne" => Op::MirOp(BinOp::Ne),
230                    "lt" => Op::MirOp(BinOp::Lt),
231                    "le" => Op::MirOp(BinOp::Le),
232                    "gt" => Op::MirOp(BinOp::Gt),
233                    "ge" => Op::MirOp(BinOp::Ge),
234                    "fmax" => Op::FMinMax(MinMax::Max),
235                    "fmin" => Op::FMinMax(MinMax::Min),
236                    "saturating_add" => Op::SaturatingOp(BinOp::Add),
237                    "saturating_sub" => Op::SaturatingOp(BinOp::Sub),
238                    "arith_offset" => Op::WrappingOffset,
239                    _ => unreachable!(),
240                };
241
242                for i in 0..dest_len {
243                    let left = this.read_immediate(&this.project_index(&left, i)?)?;
244                    let right = this.read_immediate(&this.project_index(&right, i)?)?;
245                    let dest = this.project_index(&dest, i)?;
246                    let val = match which {
247                        Op::MirOp(mir_op) => {
248                            // This does NaN adjustments.
249                            let val = this.binary_op(mir_op, &left, &right).map_err_kind(|kind| {
250                                match kind {
251                                    InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ShiftOverflow { shift_amount, .. }) => {
252                                        // This resets the interpreter backtrace, but it's not worth avoiding that.
253                                        let shift_amount = match shift_amount {
254                                            Either::Left(v) => v.to_string(),
255                                            Either::Right(v) => v.to_string(),
256                                        };
257                                        err_ub_format!("overflowing shift by {shift_amount} in `simd_{intrinsic_name}` in lane {i}")
258                                    }
259                                    kind => kind
260                                }
261                            })?;
262                            if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) {
263                                // Special handling for boolean-returning operations
264                                assert_eq!(val.layout.ty, this.tcx.types.bool);
265                                let val = val.to_scalar().to_bool().unwrap();
266                                bool_to_simd_element(val, dest.layout.size)
267                            } else {
268                                assert_ne!(val.layout.ty, this.tcx.types.bool);
269                                assert_eq!(val.layout.ty, dest.layout.ty);
270                                val.to_scalar()
271                            }
272                        }
273                        Op::SaturatingOp(mir_op) => {
274                            this.saturating_arith(mir_op, &left, &right)?
275                        }
276                        Op::WrappingOffset => {
277                            let ptr = left.to_scalar().to_pointer(this)?;
278                            let offset_count = right.to_scalar().to_target_isize(this)?;
279                            let pointee_ty = left.layout.ty.builtin_deref(true).unwrap();
280
281                            let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap();
282                            let offset_bytes = offset_count.wrapping_mul(pointee_size);
283                            let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this);
284                            Scalar::from_maybe_pointer(offset_ptr, this)
285                        }
286                        Op::FMinMax(op) => {
287                            this.fminmax_op(op, &left, &right)?
288                        }
289                    };
290                    this.write_scalar(val, &dest)?;
291                }
292            }
293            "fma" | "relaxed_fma" => {
294                let [a, b, c] = check_intrinsic_arg_count(args)?;
295                let (a, a_len) = this.project_to_simd(a)?;
296                let (b, b_len) = this.project_to_simd(b)?;
297                let (c, c_len) = this.project_to_simd(c)?;
298                let (dest, dest_len) = this.project_to_simd(dest)?;
299
300                assert_eq!(dest_len, a_len);
301                assert_eq!(dest_len, b_len);
302                assert_eq!(dest_len, c_len);
303
304                for i in 0..dest_len {
305                    let a = this.read_scalar(&this.project_index(&a, i)?)?;
306                    let b = this.read_scalar(&this.project_index(&b, i)?)?;
307                    let c = this.read_scalar(&this.project_index(&c, i)?)?;
308                    let dest = this.project_index(&dest, i)?;
309
310                    let fuse: bool = intrinsic_name == "fma"
311                        || (this.machine.float_nondet && this.machine.rng.get_mut().random());
312
313                    // Works for f32 and f64.
314                    // FIXME: using host floats to work around https://github.com/rust-lang/miri/issues/2468.
315                    let ty::Float(float_ty) = dest.layout.ty.kind() else {
316                        span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name)
317                    };
318                    let val = match float_ty {
319                        FloatTy::F16 => unimplemented!("f16_f128"),
320                        FloatTy::F32 => {
321                            let a = a.to_f32()?;
322                            let b = b.to_f32()?;
323                            let c = c.to_f32()?;
324                            let res = if fuse {
325                                a.mul_add(b, c).value
326                            } else {
327                                ((a * b).value + c).value
328                            };
329                            let res = this.adjust_nan(res, &[a, b, c]);
330                            Scalar::from(res)
331                        }
332                        FloatTy::F64 => {
333                            let a = a.to_f64()?;
334                            let b = b.to_f64()?;
335                            let c = c.to_f64()?;
336                            let res = if fuse {
337                                a.mul_add(b, c).value
338                            } else {
339                                ((a * b).value + c).value
340                            };
341                            let res = this.adjust_nan(res, &[a, b, c]);
342                            Scalar::from(res)
343                        }
344                        FloatTy::F128 => unimplemented!("f16_f128"),
345                    };
346                    this.write_scalar(val, &dest)?;
347                }
348            }
349            #[rustfmt::skip]
350            | "reduce_and"
351            | "reduce_or"
352            | "reduce_xor"
353            | "reduce_any"
354            | "reduce_all"
355            | "reduce_max"
356            | "reduce_min" => {
357                use mir::BinOp;
358
359                let [op] = check_intrinsic_arg_count(args)?;
360                let (op, op_len) = this.project_to_simd(op)?;
361
362                let imm_from_bool =
363                    |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
364
365                enum Op {
366                    MirOp(BinOp),
367                    MirOpBool(BinOp),
368                    MinMax(MinMax),
369                }
370                let which = match intrinsic_name {
371                    "reduce_and" => Op::MirOp(BinOp::BitAnd),
372                    "reduce_or" => Op::MirOp(BinOp::BitOr),
373                    "reduce_xor" => Op::MirOp(BinOp::BitXor),
374                    "reduce_any" => Op::MirOpBool(BinOp::BitOr),
375                    "reduce_all" => Op::MirOpBool(BinOp::BitAnd),
376                    "reduce_max" => Op::MinMax(MinMax::Max),
377                    "reduce_min" => Op::MinMax(MinMax::Min),
378                    _ => unreachable!(),
379                };
380
381                // Initialize with first lane, then proceed with the rest.
382                let mut res = this.read_immediate(&this.project_index(&op, 0)?)?;
383                if matches!(which, Op::MirOpBool(_)) {
384                    // Convert to `bool` scalar.
385                    res = imm_from_bool(simd_element_to_bool(res)?);
386                }
387                for i in 1..op_len {
388                    let op = this.read_immediate(&this.project_index(&op, i)?)?;
389                    res = match which {
390                        Op::MirOp(mir_op) => {
391                            this.binary_op(mir_op, &res, &op)?
392                        }
393                        Op::MirOpBool(mir_op) => {
394                            let op = imm_from_bool(simd_element_to_bool(op)?);
395                            this.binary_op(mir_op, &res, &op)?
396                        }
397                        Op::MinMax(mmop) => {
398                            if matches!(res.layout.ty.kind(), ty::Float(_)) {
399                                ImmTy::from_scalar(this.fminmax_op(mmop, &res, &op)?, res.layout)
400                            } else {
401                                // Just boring integers, so NaNs to worry about
402                                let mirop = match mmop {
403                                    MinMax::Min => BinOp::Le,
404                                    MinMax::Max => BinOp::Ge,
405                                };
406                                if this.binary_op(mirop, &res, &op)?.to_scalar().to_bool()? {
407                                    res
408                                } else {
409                                    op
410                                }
411                            }
412                        }
413                    };
414                }
415                this.write_immediate(*res, dest)?;
416            }
417            #[rustfmt::skip]
418            | "reduce_add_ordered"
419            | "reduce_mul_ordered" => {
420                use mir::BinOp;
421
422                let [op, init] = check_intrinsic_arg_count(args)?;
423                let (op, op_len) = this.project_to_simd(op)?;
424                let init = this.read_immediate(init)?;
425
426                let mir_op = match intrinsic_name {
427                    "reduce_add_ordered" => BinOp::Add,
428                    "reduce_mul_ordered" => BinOp::Mul,
429                    _ => unreachable!(),
430                };
431
432                let mut res = init;
433                for i in 0..op_len {
434                    let op = this.read_immediate(&this.project_index(&op, i)?)?;
435                    res = this.binary_op(mir_op, &res, &op)?;
436                }
437                this.write_immediate(*res, dest)?;
438            }
439            "select" => {
440                let [mask, yes, no] = check_intrinsic_arg_count(args)?;
441                let (mask, mask_len) = this.project_to_simd(mask)?;
442                let (yes, yes_len) = this.project_to_simd(yes)?;
443                let (no, no_len) = this.project_to_simd(no)?;
444                let (dest, dest_len) = this.project_to_simd(dest)?;
445
446                assert_eq!(dest_len, mask_len);
447                assert_eq!(dest_len, yes_len);
448                assert_eq!(dest_len, no_len);
449
450                for i in 0..dest_len {
451                    let mask = this.read_immediate(&this.project_index(&mask, i)?)?;
452                    let yes = this.read_immediate(&this.project_index(&yes, i)?)?;
453                    let no = this.read_immediate(&this.project_index(&no, i)?)?;
454                    let dest = this.project_index(&dest, i)?;
455
456                    let val = if simd_element_to_bool(mask)? { yes } else { no };
457                    this.write_immediate(*val, &dest)?;
458                }
459            }
460            // Variant of `select` that takes a bitmask rather than a "vector of bool".
461            "select_bitmask" => {
462                let [mask, yes, no] = check_intrinsic_arg_count(args)?;
463                let (yes, yes_len) = this.project_to_simd(yes)?;
464                let (no, no_len) = this.project_to_simd(no)?;
465                let (dest, dest_len) = this.project_to_simd(dest)?;
466                let bitmask_len = dest_len.next_multiple_of(8);
467                if bitmask_len > 64 {
468                    throw_unsup_format!(
469                        "simd_select_bitmask: vectors larger than 64 elements are currently not supported"
470                    );
471                }
472
473                assert_eq!(dest_len, yes_len);
474                assert_eq!(dest_len, no_len);
475
476                // Read the mask, either as an integer or as an array.
477                let mask: u64 = match mask.layout.ty.kind() {
478                    ty::Uint(_) => {
479                        // Any larger integer type is fine.
480                        assert!(mask.layout.size.bits() >= bitmask_len);
481                        this.read_scalar(mask)?.to_bits(mask.layout.size)?.try_into().unwrap()
482                    }
483                    ty::Array(elem, _len) if elem == &this.tcx.types.u8 => {
484                        // The array must have exactly the right size.
485                        assert_eq!(mask.layout.size.bits(), bitmask_len);
486                        // Read the raw bytes.
487                        let mask = mask.assert_mem_place(); // arrays cannot be immediate
488                        let mask_bytes =
489                            this.read_bytes_ptr_strip_provenance(mask.ptr(), mask.layout.size)?;
490                        // Turn them into a `u64` in the right way.
491                        let mask_size = mask.layout.size.bytes_usize();
492                        let mut mask_arr = [0u8; 8];
493                        match this.data_layout().endian {
494                            Endian::Little => {
495                                // Fill the first N bytes.
496                                mask_arr[..mask_size].copy_from_slice(mask_bytes);
497                                u64::from_le_bytes(mask_arr)
498                            }
499                            Endian::Big => {
500                                // Fill the last N bytes.
501                                let i = mask_arr.len().strict_sub(mask_size);
502                                mask_arr[i..].copy_from_slice(mask_bytes);
503                                u64::from_be_bytes(mask_arr)
504                            }
505                        }
506                    }
507                    _ => bug!("simd_select_bitmask: invalid mask type {}", mask.layout.ty),
508                };
509
510                let dest_len = u32::try_from(dest_len).unwrap();
511                for i in 0..dest_len {
512                    let bit_i = simd_bitmask_index(i, dest_len, this.data_layout().endian);
513                    let mask = mask & 1u64.strict_shl(bit_i);
514                    let yes = this.read_immediate(&this.project_index(&yes, i.into())?)?;
515                    let no = this.read_immediate(&this.project_index(&no, i.into())?)?;
516                    let dest = this.project_index(&dest, i.into())?;
517
518                    let val = if mask != 0 { yes } else { no };
519                    this.write_immediate(*val, &dest)?;
520                }
521                // The remaining bits of the mask are ignored.
522            }
523            // Converts a "vector of bool" into a bitmask.
524            "bitmask" => {
525                let [op] = check_intrinsic_arg_count(args)?;
526                let (op, op_len) = this.project_to_simd(op)?;
527                let bitmask_len = op_len.next_multiple_of(8);
528                if bitmask_len > 64 {
529                    throw_unsup_format!(
530                        "simd_bitmask: vectors larger than 64 elements are currently not supported"
531                    );
532                }
533
534                let op_len = u32::try_from(op_len).unwrap();
535                let mut res = 0u64;
536                for i in 0..op_len {
537                    let op = this.read_immediate(&this.project_index(&op, i.into())?)?;
538                    if simd_element_to_bool(op)? {
539                        let bit_i = simd_bitmask_index(i, op_len, this.data_layout().endian);
540                        res |= 1u64.strict_shl(bit_i);
541                    }
542                }
543                // Write the result, depending on the `dest` type.
544                // Returns either an unsigned integer or array of `u8`.
545                match dest.layout.ty.kind() {
546                    ty::Uint(_) => {
547                        // Any larger integer type is fine, it will be zero-extended.
548                        assert!(dest.layout.size.bits() >= bitmask_len);
549                        this.write_int(res, dest)?;
550                    }
551                    ty::Array(elem, _len) if elem == &this.tcx.types.u8 => {
552                        // The array must have exactly the right size.
553                        assert_eq!(dest.layout.size.bits(), bitmask_len);
554                        // We have to write the result byte-for-byte.
555                        let res_size = dest.layout.size.bytes_usize();
556                        let res_bytes;
557                        let res_bytes_slice = match this.data_layout().endian {
558                            Endian::Little => {
559                                res_bytes = res.to_le_bytes();
560                                &res_bytes[..res_size] // take the first N bytes
561                            }
562                            Endian::Big => {
563                                res_bytes = res.to_be_bytes();
564                                &res_bytes[res_bytes.len().strict_sub(res_size)..] // take the last N bytes
565                            }
566                        };
567                        this.write_bytes_ptr(dest.ptr(), res_bytes_slice.iter().cloned())?;
568                    }
569                    _ => bug!("simd_bitmask: invalid return type {}", dest.layout.ty),
570                }
571            }
572            "cast" | "as" | "cast_ptr" | "expose_provenance" | "with_exposed_provenance" => {
573                let [op] = check_intrinsic_arg_count(args)?;
574                let (op, op_len) = this.project_to_simd(op)?;
575                let (dest, dest_len) = this.project_to_simd(dest)?;
576
577                assert_eq!(dest_len, op_len);
578
579                let unsafe_cast = intrinsic_name == "cast";
580                let safe_cast = intrinsic_name == "as";
581                let ptr_cast = intrinsic_name == "cast_ptr";
582                let expose_cast = intrinsic_name == "expose_provenance";
583                let from_exposed_cast = intrinsic_name == "with_exposed_provenance";
584
585                for i in 0..dest_len {
586                    let op = this.read_immediate(&this.project_index(&op, i)?)?;
587                    let dest = this.project_index(&dest, i)?;
588
589                    let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
590                        // Int-to-(int|float): always safe
591                        (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_))
592                            if safe_cast || unsafe_cast =>
593                            this.int_to_int_or_float(&op, dest.layout)?,
594                        // Float-to-float: always safe
595                        (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast =>
596                            this.float_to_float_or_int(&op, dest.layout)?,
597                        // Float-to-int in safe mode
598                        (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
599                            this.float_to_float_or_int(&op, dest.layout)?,
600                        // Float-to-int in unchecked mode
601                        (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => {
602                            this.float_to_int_checked(&op, dest.layout, Round::TowardZero)?
603                                .ok_or_else(|| {
604                                    err_ub_format!(
605                                        "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`",
606                                        dest.layout.ty
607                                    )
608                                })?
609                        }
610                        // Ptr-to-ptr cast
611                        (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast =>
612                            this.ptr_to_ptr(&op, dest.layout)?,
613                        // Ptr/Int casts
614                        (ty::RawPtr(..), ty::Int(_) | ty::Uint(_)) if expose_cast =>
615                            this.pointer_expose_provenance_cast(&op, dest.layout)?,
616                        (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast =>
617                            this.pointer_with_exposed_provenance_cast(&op, dest.layout)?,
618                        // Error otherwise
619                        _ =>
620                            throw_unsup_format!(
621                                "Unsupported SIMD cast from element type {from_ty} to {to_ty}",
622                                from_ty = op.layout.ty,
623                                to_ty = dest.layout.ty,
624                            ),
625                    };
626                    this.write_immediate(*val, &dest)?;
627                }
628            }
629            "shuffle_const_generic" => {
630                let [left, right] = check_intrinsic_arg_count(args)?;
631                let (left, left_len) = this.project_to_simd(left)?;
632                let (right, right_len) = this.project_to_simd(right)?;
633                let (dest, dest_len) = this.project_to_simd(dest)?;
634
635                let index = generic_args[2].expect_const().to_value().valtree.unwrap_branch();
636                let index_len = index.len();
637
638                assert_eq!(left_len, right_len);
639                assert_eq!(u64::try_from(index_len).unwrap(), dest_len);
640
641                for i in 0..dest_len {
642                    let src_index: u64 =
643                        index[usize::try_from(i).unwrap()].unwrap_leaf().to_u32().into();
644                    let dest = this.project_index(&dest, i)?;
645
646                    let val = if src_index < left_len {
647                        this.read_immediate(&this.project_index(&left, src_index)?)?
648                    } else if src_index < left_len.strict_add(right_len) {
649                        let right_idx = src_index.strict_sub(left_len);
650                        this.read_immediate(&this.project_index(&right, right_idx)?)?
651                    } else {
652                        throw_ub_format!(
653                            "`simd_shuffle_const_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}"
654                        );
655                    };
656                    this.write_immediate(*val, &dest)?;
657                }
658            }
659            "shuffle" => {
660                let [left, right, index] = check_intrinsic_arg_count(args)?;
661                let (left, left_len) = this.project_to_simd(left)?;
662                let (right, right_len) = this.project_to_simd(right)?;
663                let (index, index_len) = this.project_to_simd(index)?;
664                let (dest, dest_len) = this.project_to_simd(dest)?;
665
666                assert_eq!(left_len, right_len);
667                assert_eq!(index_len, dest_len);
668
669                for i in 0..dest_len {
670                    let src_index: u64 = this
671                        .read_immediate(&this.project_index(&index, i)?)?
672                        .to_scalar()
673                        .to_u32()?
674                        .into();
675                    let dest = this.project_index(&dest, i)?;
676
677                    let val = if src_index < left_len {
678                        this.read_immediate(&this.project_index(&left, src_index)?)?
679                    } else if src_index < left_len.strict_add(right_len) {
680                        let right_idx = src_index.strict_sub(left_len);
681                        this.read_immediate(&this.project_index(&right, right_idx)?)?
682                    } else {
683                        throw_ub_format!(
684                            "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}"
685                        );
686                    };
687                    this.write_immediate(*val, &dest)?;
688                }
689            }
690            "gather" => {
691                let [passthru, ptrs, mask] = check_intrinsic_arg_count(args)?;
692                let (passthru, passthru_len) = this.project_to_simd(passthru)?;
693                let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?;
694                let (mask, mask_len) = this.project_to_simd(mask)?;
695                let (dest, dest_len) = this.project_to_simd(dest)?;
696
697                assert_eq!(dest_len, passthru_len);
698                assert_eq!(dest_len, ptrs_len);
699                assert_eq!(dest_len, mask_len);
700
701                for i in 0..dest_len {
702                    let passthru = this.read_immediate(&this.project_index(&passthru, i)?)?;
703                    let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?;
704                    let mask = this.read_immediate(&this.project_index(&mask, i)?)?;
705                    let dest = this.project_index(&dest, i)?;
706
707                    let val = if simd_element_to_bool(mask)? {
708                        let place = this.deref_pointer(&ptr)?;
709                        this.read_immediate(&place)?
710                    } else {
711                        passthru
712                    };
713                    this.write_immediate(*val, &dest)?;
714                }
715            }
716            "scatter" => {
717                let [value, ptrs, mask] = check_intrinsic_arg_count(args)?;
718                let (value, value_len) = this.project_to_simd(value)?;
719                let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?;
720                let (mask, mask_len) = this.project_to_simd(mask)?;
721
722                assert_eq!(ptrs_len, value_len);
723                assert_eq!(ptrs_len, mask_len);
724
725                for i in 0..ptrs_len {
726                    let value = this.read_immediate(&this.project_index(&value, i)?)?;
727                    let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?;
728                    let mask = this.read_immediate(&this.project_index(&mask, i)?)?;
729
730                    if simd_element_to_bool(mask)? {
731                        let place = this.deref_pointer(&ptr)?;
732                        this.write_immediate(*value, &place)?;
733                    }
734                }
735            }
736            "masked_load" => {
737                let [mask, ptr, default] = check_intrinsic_arg_count(args)?;
738                let (mask, mask_len) = this.project_to_simd(mask)?;
739                let ptr = this.read_pointer(ptr)?;
740                let (default, default_len) = this.project_to_simd(default)?;
741                let (dest, dest_len) = this.project_to_simd(dest)?;
742
743                assert_eq!(dest_len, mask_len);
744                assert_eq!(dest_len, default_len);
745
746                for i in 0..dest_len {
747                    let mask = this.read_immediate(&this.project_index(&mask, i)?)?;
748                    let default = this.read_immediate(&this.project_index(&default, i)?)?;
749                    let dest = this.project_index(&dest, i)?;
750
751                    let val = if simd_element_to_bool(mask)? {
752                        // Size * u64 is implemented as always checked
753                        let ptr = ptr.wrapping_offset(dest.layout.size * i, this);
754                        let place = this.ptr_to_mplace(ptr, dest.layout);
755                        this.read_immediate(&place)?
756                    } else {
757                        default
758                    };
759                    this.write_immediate(*val, &dest)?;
760                }
761            }
762            "masked_store" => {
763                let [mask, ptr, vals] = check_intrinsic_arg_count(args)?;
764                let (mask, mask_len) = this.project_to_simd(mask)?;
765                let ptr = this.read_pointer(ptr)?;
766                let (vals, vals_len) = this.project_to_simd(vals)?;
767
768                assert_eq!(mask_len, vals_len);
769
770                for i in 0..vals_len {
771                    let mask = this.read_immediate(&this.project_index(&mask, i)?)?;
772                    let val = this.read_immediate(&this.project_index(&vals, i)?)?;
773
774                    if simd_element_to_bool(mask)? {
775                        // Size * u64 is implemented as always checked
776                        let ptr = ptr.wrapping_offset(val.layout.size * i, this);
777                        let place = this.ptr_to_mplace(ptr, val.layout);
778                        this.write_immediate(*val, &place)?
779                    };
780                }
781            }
782
783            _ => return interp_ok(EmulateItemResult::NotSupported),
784        }
785        interp_ok(EmulateItemResult::NeedsReturn)
786    }
787
788    fn fminmax_op(
789        &self,
790        op: MinMax,
791        left: &ImmTy<'tcx>,
792        right: &ImmTy<'tcx>,
793    ) -> InterpResult<'tcx, Scalar> {
794        let this = self.eval_context_ref();
795        assert_eq!(left.layout.ty, right.layout.ty);
796        let ty::Float(float_ty) = left.layout.ty.kind() else {
797            bug!("fmax operand is not a float")
798        };
799        let left = left.to_scalar();
800        let right = right.to_scalar();
801        interp_ok(match float_ty {
802            FloatTy::F16 => unimplemented!("f16_f128"),
803            FloatTy::F32 => {
804                let left = left.to_f32()?;
805                let right = right.to_f32()?;
806                let res = match op {
807                    MinMax::Min => left.min(right),
808                    MinMax::Max => left.max(right),
809                };
810                let res = this.adjust_nan(res, &[left, right]);
811                Scalar::from_f32(res)
812            }
813            FloatTy::F64 => {
814                let left = left.to_f64()?;
815                let right = right.to_f64()?;
816                let res = match op {
817                    MinMax::Min => left.min(right),
818                    MinMax::Max => left.max(right),
819                };
820                let res = this.adjust_nan(res, &[left, right]);
821                Scalar::from_f64(res)
822            }
823            FloatTy::F128 => unimplemented!("f16_f128"),
824        })
825    }
826}
827
828fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 {
829    assert!(idx < vec_len);
830    match endianness {
831        Endian::Little => idx,
832        #[expect(clippy::arithmetic_side_effects)] // idx < vec_len
833        Endian::Big => vec_len - 1 - idx, // reverse order of bits
834    }
835}
miri/intrinsics/simd.rs

miri/intrinsics/
simd.rs
