miri/intrinsics/
atomic.rs

1use rustc_middle::mir::BinOp;
2use rustc_middle::ty::AtomicOrdering;
3use rustc_middle::{mir, ty};
4
5use self::helpers::check_intrinsic_arg_count;
6use crate::*;
7
8pub enum AtomicOp {
9    /// The `bool` indicates whether the result of the operation should be negated (`UnOp::Not`,
10    /// must be a boolean-typed operation).
11    MirOp(mir::BinOp, bool),
12    Max,
13    Min,
14}
15
16impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
17pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
18    /// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed.
19    /// Returns `Ok(true)` if the intrinsic was handled.
20    fn emulate_atomic_intrinsic(
21        &mut self,
22        intrinsic_name: &str,
23        generic_args: ty::GenericArgsRef<'tcx>,
24        args: &[OpTy<'tcx>],
25        dest: &MPlaceTy<'tcx>,
26    ) -> InterpResult<'tcx, EmulateItemResult> {
27        let this = self.eval_context_mut();
28
29        let get_ord_at = |i: usize| {
30            let ordering = generic_args.const_at(i).to_value();
31            ordering.valtree.unwrap_branch()[0].unwrap_leaf().to_atomic_ordering()
32        };
33
34        fn read_ord(ord: AtomicOrdering) -> AtomicReadOrd {
35            match ord {
36                AtomicOrdering::SeqCst => AtomicReadOrd::SeqCst,
37                AtomicOrdering::Acquire => AtomicReadOrd::Acquire,
38                AtomicOrdering::Relaxed => AtomicReadOrd::Relaxed,
39                _ => panic!("invalid read ordering `{ord:?}`"),
40            }
41        }
42
43        fn write_ord(ord: AtomicOrdering) -> AtomicWriteOrd {
44            match ord {
45                AtomicOrdering::SeqCst => AtomicWriteOrd::SeqCst,
46                AtomicOrdering::Release => AtomicWriteOrd::Release,
47                AtomicOrdering::Relaxed => AtomicWriteOrd::Relaxed,
48                _ => panic!("invalid write ordering `{ord:?}`"),
49            }
50        }
51
52        fn rw_ord(ord: AtomicOrdering) -> AtomicRwOrd {
53            match ord {
54                AtomicOrdering::SeqCst => AtomicRwOrd::SeqCst,
55                AtomicOrdering::AcqRel => AtomicRwOrd::AcqRel,
56                AtomicOrdering::Acquire => AtomicRwOrd::Acquire,
57                AtomicOrdering::Release => AtomicRwOrd::Release,
58                AtomicOrdering::Relaxed => AtomicRwOrd::Relaxed,
59            }
60        }
61
62        fn fence_ord(ord: AtomicOrdering) -> AtomicFenceOrd {
63            match ord {
64                AtomicOrdering::SeqCst => AtomicFenceOrd::SeqCst,
65                AtomicOrdering::AcqRel => AtomicFenceOrd::AcqRel,
66                AtomicOrdering::Acquire => AtomicFenceOrd::Acquire,
67                AtomicOrdering::Release => AtomicFenceOrd::Release,
68                _ => panic!("invalid fence ordering `{ord:?}`"),
69            }
70        }
71
72        match intrinsic_name {
73            "load" => {
74                let ord = get_ord_at(1);
75                this.atomic_load(args, dest, read_ord(ord))?;
76            }
77
78            "store" => {
79                let ord = get_ord_at(1);
80                this.atomic_store(args, write_ord(ord))?
81            }
82
83            "fence" => {
84                let ord = get_ord_at(0);
85                this.atomic_fence_intrinsic(args, fence_ord(ord))?
86            }
87            "singlethreadfence" => {
88                let ord = get_ord_at(0);
89                this.compiler_fence_intrinsic(args, fence_ord(ord))?;
90            }
91
92            "xchg" => {
93                let ord = get_ord_at(1);
94                this.atomic_exchange(args, dest, rw_ord(ord))?;
95            }
96            "cxchg" => {
97                let ord1 = get_ord_at(1);
98                let ord2 = get_ord_at(2);
99                this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?;
100            }
101            "cxchgweak" => {
102                let ord1 = get_ord_at(1);
103                let ord2 = get_ord_at(2);
104                this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?;
105            }
106
107            "or" => {
108                let ord = get_ord_at(1);
109                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?;
110            }
111            "xor" => {
112                let ord = get_ord_at(1);
113                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?;
114            }
115            "and" => {
116                let ord = get_ord_at(1);
117                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?;
118            }
119            "nand" => {
120                let ord = get_ord_at(1);
121                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?;
122            }
123            "xadd" => {
124                let ord = get_ord_at(1);
125                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?;
126            }
127            "xsub" => {
128                let ord = get_ord_at(1);
129                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?;
130            }
131            "min" => {
132                let ord = get_ord_at(1);
133                // Later we will use the type to indicate signed vs unsigned,
134                // so make sure it matches the intrinsic name.
135                assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
136                this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
137            }
138            "umin" => {
139                let ord = get_ord_at(1);
140                // Later we will use the type to indicate signed vs unsigned,
141                // so make sure it matches the intrinsic name.
142                assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
143                this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
144            }
145            "max" => {
146                let ord = get_ord_at(1);
147                // Later we will use the type to indicate signed vs unsigned,
148                // so make sure it matches the intrinsic name.
149                assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
150                this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord))?;
151            }
152            "umax" => {
153                let ord = get_ord_at(1);
154                // Later we will use the type to indicate signed vs unsigned,
155                // so make sure it matches the intrinsic name.
156                assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
157                this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord))?;
158            }
159
160            _ => return interp_ok(EmulateItemResult::NotSupported),
161        }
162        interp_ok(EmulateItemResult::NeedsReturn)
163    }
164}
165
166impl<'tcx> EvalContextPrivExt<'tcx> for MiriInterpCx<'tcx> {}
167trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
168    fn atomic_load(
169        &mut self,
170        args: &[OpTy<'tcx>],
171        dest: &MPlaceTy<'tcx>,
172        atomic: AtomicReadOrd,
173    ) -> InterpResult<'tcx> {
174        let this = self.eval_context_mut();
175
176        let [place] = check_intrinsic_arg_count(args)?;
177        let place = this.deref_pointer(place)?;
178
179        // Perform atomic load.
180        let val = this.read_scalar_atomic(&place, atomic)?;
181        // Perform regular store.
182        this.write_scalar(val, dest)?;
183        interp_ok(())
184    }
185
186    fn atomic_store(&mut self, args: &[OpTy<'tcx>], atomic: AtomicWriteOrd) -> InterpResult<'tcx> {
187        let this = self.eval_context_mut();
188
189        let [place, val] = check_intrinsic_arg_count(args)?;
190        let place = this.deref_pointer(place)?;
191
192        // Perform regular load.
193        let val = this.read_scalar(val)?;
194        // Perform atomic store.
195        this.write_scalar_atomic(val, &place, atomic)?;
196        interp_ok(())
197    }
198
199    fn compiler_fence_intrinsic(
200        &mut self,
201        args: &[OpTy<'tcx>],
202        atomic: AtomicFenceOrd,
203    ) -> InterpResult<'tcx> {
204        let [] = check_intrinsic_arg_count(args)?;
205        let _ = atomic;
206        // FIXME, FIXME(GenMC): compiler fences are currently ignored (also ignored in GenMC mode)
207        interp_ok(())
208    }
209
210    fn atomic_fence_intrinsic(
211        &mut self,
212        args: &[OpTy<'tcx>],
213        atomic: AtomicFenceOrd,
214    ) -> InterpResult<'tcx> {
215        let this = self.eval_context_mut();
216        let [] = check_intrinsic_arg_count(args)?;
217        this.atomic_fence(atomic)?;
218        interp_ok(())
219    }
220
221    fn atomic_rmw_op(
222        &mut self,
223        args: &[OpTy<'tcx>],
224        dest: &MPlaceTy<'tcx>,
225        atomic_op: AtomicOp,
226        atomic: AtomicRwOrd,
227    ) -> InterpResult<'tcx> {
228        let this = self.eval_context_mut();
229
230        let [place, rhs] = check_intrinsic_arg_count(args)?;
231        let place = this.deref_pointer(place)?;
232        let rhs = this.read_immediate(rhs)?;
233
234        if !place.layout.ty.is_integral() && !place.layout.ty.is_raw_ptr() {
235            span_bug!(
236                this.cur_span(),
237                "atomic arithmetic operations only work on integer and raw pointer types",
238            );
239        }
240        if rhs.layout.ty != place.layout.ty {
241            span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch");
242        }
243
244        let old = match atomic_op {
245            AtomicOp::Min =>
246                this.atomic_min_max_scalar(&place, rhs, /* min */ true, atomic)?,
247            AtomicOp::Max =>
248                this.atomic_min_max_scalar(&place, rhs, /* min */ false, atomic)?,
249            AtomicOp::MirOp(op, not) =>
250                this.atomic_rmw_op_immediate(&place, &rhs, op, not, atomic)?,
251        };
252        this.write_immediate(*old, dest)?; // old value is returned
253        interp_ok(())
254    }
255
256    fn atomic_exchange(
257        &mut self,
258        args: &[OpTy<'tcx>],
259        dest: &MPlaceTy<'tcx>,
260        atomic: AtomicRwOrd,
261    ) -> InterpResult<'tcx> {
262        let this = self.eval_context_mut();
263
264        let [place, new] = check_intrinsic_arg_count(args)?;
265        let place = this.deref_pointer(place)?;
266        let new = this.read_scalar(new)?;
267
268        let old = this.atomic_exchange_scalar(&place, new, atomic)?;
269        this.write_scalar(old, dest)?; // old value is returned
270        interp_ok(())
271    }
272
273    fn atomic_compare_exchange_impl(
274        &mut self,
275        args: &[OpTy<'tcx>],
276        dest: &MPlaceTy<'tcx>,
277        success: AtomicRwOrd,
278        fail: AtomicReadOrd,
279        can_fail_spuriously: bool,
280    ) -> InterpResult<'tcx> {
281        let this = self.eval_context_mut();
282
283        let [place, expect_old, new] = check_intrinsic_arg_count(args)?;
284        let place = this.deref_pointer(place)?;
285        let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()`
286        let new = this.read_scalar(new)?;
287
288        let old = this.atomic_compare_exchange_scalar(
289            &place,
290            &expect_old,
291            new,
292            success,
293            fail,
294            can_fail_spuriously,
295        )?;
296
297        // Return old value.
298        this.write_immediate(old, dest)?;
299        interp_ok(())
300    }
301
302    fn atomic_compare_exchange(
303        &mut self,
304        args: &[OpTy<'tcx>],
305        dest: &MPlaceTy<'tcx>,
306        success: AtomicRwOrd,
307        fail: AtomicReadOrd,
308    ) -> InterpResult<'tcx> {
309        self.atomic_compare_exchange_impl(args, dest, success, fail, false)
310    }
311
312    fn atomic_compare_exchange_weak(
313        &mut self,
314        args: &[OpTy<'tcx>],
315        dest: &MPlaceTy<'tcx>,
316        success: AtomicRwOrd,
317        fail: AtomicReadOrd,
318    ) -> InterpResult<'tcx> {
319        self.atomic_compare_exchange_impl(args, dest, success, fail, true)
320    }
321}
miri/intrinsics/atomic.rs

miri/intrinsics/
atomic.rs
