miri/shims/x86/
sse.rs

1use rustc_abi::CanonAbi;
2use rustc_apfloat::ieee::Single;
3use rustc_middle::ty::Ty;
4use rustc_span::Symbol;
5use rustc_target::callconv::FnAbi;
6
7use super::{
8    FloatBinOp, FloatUnaryOp, bin_op_simd_float_all, bin_op_simd_float_first, unary_op_ps,
9    unary_op_ss,
10};
11use crate::*;
12
13impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
14pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
15    fn emulate_x86_sse_intrinsic(
16        &mut self,
17        link_name: Symbol,
18        abi: &FnAbi<'tcx, Ty<'tcx>>,
19        args: &[OpTy<'tcx>],
20        dest: &MPlaceTy<'tcx>,
21    ) -> InterpResult<'tcx, EmulateItemResult> {
22        let this = self.eval_context_mut();
23        this.expect_target_feature_for_intrinsic(link_name, "sse")?;
24        // Prefix should have already been checked.
25        let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.sse.").unwrap();
26        // All these intrinsics operate on 128-bit (f32x4) SIMD vectors unless stated otherwise.
27        // Many intrinsic names are sufixed with "ps" (packed single) or "ss" (scalar single),
28        // where single means single precision floating point (f32). "ps" means thet the operation
29        // is performed on each element of the vector, while "ss" means that the operation is
30        // performed only on the first element, copying the remaining elements from the input
31        // vector (for binary operations, from the left-hand side).
32        match unprefixed_name {
33            // Used to implement _mm_{min,max}_ss functions.
34            // Performs the operations on the first component of `left` and
35            // `right` and copies the remaining components from `left`.
36            "min.ss" | "max.ss" => {
37                let [left, right] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
38
39                let which = match unprefixed_name {
40                    "min.ss" => FloatBinOp::Min,
41                    "max.ss" => FloatBinOp::Max,
42                    _ => unreachable!(),
43                };
44
45                bin_op_simd_float_first::<Single>(this, which, left, right, dest)?;
46            }
47            // Used to implement _mm_min_ps and _mm_max_ps functions.
48            // Note that the semantics are a bit different from Rust simd_min
49            // and simd_max intrinsics regarding handling of NaN and -0.0: Rust
50            // matches the IEEE min/max operations, while x86 has different
51            // semantics.
52            "min.ps" | "max.ps" => {
53                let [left, right] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
54
55                let which = match unprefixed_name {
56                    "min.ps" => FloatBinOp::Min,
57                    "max.ps" => FloatBinOp::Max,
58                    _ => unreachable!(),
59                };
60
61                bin_op_simd_float_all::<Single>(this, which, left, right, dest)?;
62            }
63            // Used to implement _mm_{rcp,rsqrt}_ss functions.
64            // Performs the operations on the first component of `op` and
65            // copies the remaining components from `op`.
66            "rcp.ss" | "rsqrt.ss" => {
67                let [op] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
68
69                let which = match unprefixed_name {
70                    "rcp.ss" => FloatUnaryOp::Rcp,
71                    "rsqrt.ss" => FloatUnaryOp::Rsqrt,
72                    _ => unreachable!(),
73                };
74
75                unary_op_ss(this, which, op, dest)?;
76            }
77            // Used to implement _mm_{sqrt,rcp,rsqrt}_ps functions.
78            // Performs the operations on all components of `op`.
79            "rcp.ps" | "rsqrt.ps" => {
80                let [op] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
81
82                let which = match unprefixed_name {
83                    "rcp.ps" => FloatUnaryOp::Rcp,
84                    "rsqrt.ps" => FloatUnaryOp::Rsqrt,
85                    _ => unreachable!(),
86                };
87
88                unary_op_ps(this, which, op, dest)?;
89            }
90            // Used to implement the _mm_cmp*_ss functions.
91            // Performs a comparison operation on the first component of `left`
92            // and `right`, returning 0 if false or `u32::MAX` if true. The remaining
93            // components are copied from `left`.
94            // _mm_cmp_ss is actually an AVX function where the operation is specified
95            // by a const parameter.
96            // _mm_cmp{eq,lt,le,gt,ge,neq,nlt,nle,ngt,nge,ord,unord}_ss are SSE functions
97            // with hard-coded operations.
98            "cmp.ss" => {
99                let [left, right, imm] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
100
101                let which =
102                    FloatBinOp::cmp_from_imm(this, this.read_scalar(imm)?.to_i8()?, link_name)?;
103
104                bin_op_simd_float_first::<Single>(this, which, left, right, dest)?;
105            }
106            // Used to implement the _mm_cmp*_ps functions.
107            // Performs a comparison operation on each component of `left`
108            // and `right`. For each component, returns 0 if false or u32::MAX
109            // if true.
110            // _mm_cmp_ps is actually an AVX function where the operation is specified
111            // by a const parameter.
112            // _mm_cmp{eq,lt,le,gt,ge,neq,nlt,nle,ngt,nge,ord,unord}_ps are SSE functions
113            // with hard-coded operations.
114            "cmp.ps" => {
115                let [left, right, imm] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
116
117                let which =
118                    FloatBinOp::cmp_from_imm(this, this.read_scalar(imm)?.to_i8()?, link_name)?;
119
120                bin_op_simd_float_all::<Single>(this, which, left, right, dest)?;
121            }
122            // Used to implement _mm_{,u}comi{eq,lt,le,gt,ge,neq}_ss functions.
123            // Compares the first component of `left` and `right` and returns
124            // a scalar value (0 or 1).
125            "comieq.ss" | "comilt.ss" | "comile.ss" | "comigt.ss" | "comige.ss" | "comineq.ss"
126            | "ucomieq.ss" | "ucomilt.ss" | "ucomile.ss" | "ucomigt.ss" | "ucomige.ss"
127            | "ucomineq.ss" => {
128                let [left, right] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
129
130                let (left, left_len) = this.project_to_simd(left)?;
131                let (right, right_len) = this.project_to_simd(right)?;
132
133                assert_eq!(left_len, right_len);
134
135                let left = this.read_scalar(&this.project_index(&left, 0)?)?.to_f32()?;
136                let right = this.read_scalar(&this.project_index(&right, 0)?)?.to_f32()?;
137                // The difference between the com* and ucom* variants is signaling
138                // of exceptions when either argument is a quiet NaN. We do not
139                // support accessing the SSE status register from miri (or from Rust,
140                // for that matter), so we treat both variants equally.
141                let res = match unprefixed_name {
142                    "comieq.ss" | "ucomieq.ss" => left == right,
143                    "comilt.ss" | "ucomilt.ss" => left < right,
144                    "comile.ss" | "ucomile.ss" => left <= right,
145                    "comigt.ss" | "ucomigt.ss" => left > right,
146                    "comige.ss" | "ucomige.ss" => left >= right,
147                    "comineq.ss" | "ucomineq.ss" => left != right,
148                    _ => unreachable!(),
149                };
150                this.write_scalar(Scalar::from_i32(i32::from(res)), dest)?;
151            }
152            // Use to implement the _mm_cvtss_si32, _mm_cvttss_si32,
153            // _mm_cvtss_si64 and _mm_cvttss_si64 functions.
154            // Converts the first component of `op` from f32 to i32/i64.
155            "cvtss2si" | "cvttss2si" | "cvtss2si64" | "cvttss2si64" => {
156                let [op] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
157                let (op, _) = this.project_to_simd(op)?;
158
159                let op = this.read_immediate(&this.project_index(&op, 0)?)?;
160
161                let rnd = match unprefixed_name {
162                    // "current SSE rounding mode", assume nearest
163                    // https://www.felixcloutier.com/x86/cvtss2si
164                    "cvtss2si" | "cvtss2si64" => rustc_apfloat::Round::NearestTiesToEven,
165                    // always truncate
166                    // https://www.felixcloutier.com/x86/cvttss2si
167                    "cvttss2si" | "cvttss2si64" => rustc_apfloat::Round::TowardZero,
168                    _ => unreachable!(),
169                };
170
171                let res = this.float_to_int_checked(&op, dest.layout, rnd)?.unwrap_or_else(|| {
172                    // Fallback to minimum according to SSE semantics.
173                    ImmTy::from_int(dest.layout.size.signed_int_min(), dest.layout)
174                });
175
176                this.write_immediate(*res, dest)?;
177            }
178            // Used to implement the _mm_cvtsi32_ss and _mm_cvtsi64_ss functions.
179            // Converts `right` from i32/i64 to f32. Returns a SIMD vector with
180            // the result in the first component and the remaining components
181            // are copied from `left`.
182            // https://www.felixcloutier.com/x86/cvtsi2ss
183            "cvtsi2ss" | "cvtsi642ss" => {
184                let [left, right] = this.check_shim(abi, CanonAbi::C, link_name, args)?;
185
186                let (left, left_len) = this.project_to_simd(left)?;
187                let (dest, dest_len) = this.project_to_simd(dest)?;
188
189                assert_eq!(dest_len, left_len);
190
191                let right = this.read_immediate(right)?;
192                let dest0 = this.project_index(&dest, 0)?;
193                let res0 = this.int_to_int_or_float(&right, dest0.layout)?;
194                this.write_immediate(*res0, &dest0)?;
195
196                for i in 1..dest_len {
197                    this.copy_op(&this.project_index(&left, i)?, &this.project_index(&dest, i)?)?;
198                }
199            }
200            _ => return interp_ok(EmulateItemResult::NotSupported),
201        }
202        interp_ok(EmulateItemResult::NeedsReturn)
203    }
204}