rustc_const_eval/interpret/
operand.rs

1//! Functions concerning immediate values and operands, and reading from operands.
2//! All high-level functions to read from memory work on operands as sources.
3
4use std::assert_matches::assert_matches;
5
6use either::{Either, Left, Right};
7use rustc_abi as abi;
8use rustc_abi::{BackendRepr, HasDataLayout, Size};
9use rustc_hir::def::Namespace;
10use rustc_middle::mir::interpret::ScalarSizeMismatch;
11use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, TyAndLayout};
12use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
13use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt};
14use rustc_middle::{bug, mir, span_bug, ty};
15use rustc_span::DUMMY_SP;
16use tracing::trace;
17
18use super::{
19    CtfeProvenance, Frame, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta,
20    OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub,
21    from_known_layout, interp_ok, mir_assign_valid_types, throw_ub,
22};
23
24/// An `Immediate` represents a single immediate self-contained Rust value.
25///
26/// For optimization of a few very common cases, there is also a representation for a pair of
27/// primitive values (`ScalarPair`). It allows Miri to avoid making allocations for checked binary
28/// operations and wide pointers. This idea was taken from rustc's codegen.
29/// In particular, thanks to `ScalarPair`, arithmetic operations and casts can be entirely
30/// defined on `Immediate`, and do not have to work with a `Place`.
31#[derive(Copy, Clone, Debug)]
32pub enum Immediate<Prov: Provenance = CtfeProvenance> {
33    /// A single scalar value (must have *initialized* `Scalar` ABI).
34    Scalar(Scalar<Prov>),
35    /// A pair of two scalar value (must have `ScalarPair` ABI where both fields are
36    /// `Scalar::Initialized`).
37    ScalarPair(Scalar<Prov>, Scalar<Prov>),
38    /// A value of fully uninitialized memory. Can have arbitrary size and layout, but must be sized.
39    Uninit,
40}
41
42impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> {
43    #[inline(always)]
44    fn from(val: Scalar<Prov>) -> Self {
45        Immediate::Scalar(val)
46    }
47}
48
49impl<Prov: Provenance> Immediate<Prov> {
50    pub fn new_pointer_with_meta(
51        ptr: Pointer<Option<Prov>>,
52        meta: MemPlaceMeta<Prov>,
53        cx: &impl HasDataLayout,
54    ) -> Self {
55        let ptr = Scalar::from_maybe_pointer(ptr, cx);
56        match meta {
57            MemPlaceMeta::None => Immediate::from(ptr),
58            MemPlaceMeta::Meta(meta) => Immediate::ScalarPair(ptr, meta),
59        }
60    }
61
62    pub fn new_slice(ptr: Pointer<Option<Prov>>, len: u64, cx: &impl HasDataLayout) -> Self {
63        Immediate::ScalarPair(
64            Scalar::from_maybe_pointer(ptr, cx),
65            Scalar::from_target_usize(len, cx),
66        )
67    }
68
69    pub fn new_dyn_trait(
70        val: Pointer<Option<Prov>>,
71        vtable: Pointer<Option<Prov>>,
72        cx: &impl HasDataLayout,
73    ) -> Self {
74        Immediate::ScalarPair(
75            Scalar::from_maybe_pointer(val, cx),
76            Scalar::from_maybe_pointer(vtable, cx),
77        )
78    }
79
80    #[inline]
81    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
82    pub fn to_scalar(self) -> Scalar<Prov> {
83        match self {
84            Immediate::Scalar(val) => val,
85            Immediate::ScalarPair(..) => bug!("Got a scalar pair where a scalar was expected"),
86            Immediate::Uninit => bug!("Got uninit where a scalar was expected"),
87        }
88    }
89
90    #[inline]
91    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
92    pub fn to_scalar_int(self) -> ScalarInt {
93        self.to_scalar().try_to_scalar_int().unwrap()
94    }
95
96    #[inline]
97    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
98    pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) {
99        match self {
100            Immediate::ScalarPair(val1, val2) => (val1, val2),
101            Immediate::Scalar(..) => bug!("Got a scalar where a scalar pair was expected"),
102            Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"),
103        }
104    }
105
106    /// Returns the scalar from the first component and optionally the 2nd component as metadata.
107    #[inline]
108    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
109    pub fn to_scalar_and_meta(self) -> (Scalar<Prov>, MemPlaceMeta<Prov>) {
110        match self {
111            Immediate::ScalarPair(val1, val2) => (val1, MemPlaceMeta::Meta(val2)),
112            Immediate::Scalar(val) => (val, MemPlaceMeta::None),
113            Immediate::Uninit => bug!("Got uninit where a scalar or scalar pair was expected"),
114        }
115    }
116
117    /// Assert that this immediate is a valid value for the given ABI.
118    pub fn assert_matches_abi(self, abi: BackendRepr, msg: &str, cx: &impl HasDataLayout) {
119        match (self, abi) {
120            (Immediate::Scalar(scalar), BackendRepr::Scalar(s)) => {
121                assert_eq!(scalar.size(), s.size(cx), "{msg}: scalar value has wrong size");
122                if !matches!(s.primitive(), abi::Primitive::Pointer(..)) {
123                    // This is not a pointer, it should not carry provenance.
124                    assert!(
125                        matches!(scalar, Scalar::Int(..)),
126                        "{msg}: scalar value should be an integer, but has provenance"
127                    );
128                }
129            }
130            (Immediate::ScalarPair(a_val, b_val), BackendRepr::ScalarPair(a, b)) => {
131                assert_eq!(
132                    a_val.size(),
133                    a.size(cx),
134                    "{msg}: first component of scalar pair has wrong size"
135                );
136                if !matches!(a.primitive(), abi::Primitive::Pointer(..)) {
137                    assert!(
138                        matches!(a_val, Scalar::Int(..)),
139                        "{msg}: first component of scalar pair should be an integer, but has provenance"
140                    );
141                }
142                assert_eq!(
143                    b_val.size(),
144                    b.size(cx),
145                    "{msg}: second component of scalar pair has wrong size"
146                );
147                if !matches!(b.primitive(), abi::Primitive::Pointer(..)) {
148                    assert!(
149                        matches!(b_val, Scalar::Int(..)),
150                        "{msg}: second component of scalar pair should be an integer, but has provenance"
151                    );
152                }
153            }
154            (Immediate::Uninit, _) => {
155                assert!(abi.is_sized(), "{msg}: unsized immediates are not a thing");
156            }
157            _ => {
158                bug!("{msg}: value {self:?} does not match ABI {abi:?})",)
159            }
160        }
161    }
162
163    pub fn clear_provenance<'tcx>(&mut self) -> InterpResult<'tcx> {
164        match self {
165            Immediate::Scalar(s) => {
166                s.clear_provenance()?;
167            }
168            Immediate::ScalarPair(a, b) => {
169                a.clear_provenance()?;
170                b.clear_provenance()?;
171            }
172            Immediate::Uninit => {}
173        }
174        interp_ok(())
175    }
176}
177
178// ScalarPair needs a type to interpret, so we often have an immediate and a type together
179// as input for binary and cast operations.
180#[derive(Clone)]
181pub struct ImmTy<'tcx, Prov: Provenance = CtfeProvenance> {
182    imm: Immediate<Prov>,
183    pub layout: TyAndLayout<'tcx>,
184}
185
186impl<Prov: Provenance> std::fmt::Display for ImmTy<'_, Prov> {
187    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
188        /// Helper function for printing a scalar to a FmtPrinter
189        fn p<'a, 'tcx, Prov: Provenance>(
190            cx: &mut FmtPrinter<'a, 'tcx>,
191            s: Scalar<Prov>,
192            ty: Ty<'tcx>,
193        ) -> Result<(), std::fmt::Error> {
194            match s {
195                Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true),
196                Scalar::Ptr(ptr, _sz) => {
197                    // Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to
198                    // print what is points to, which would fail since it has no access to the local
199                    // memory.
200                    cx.pretty_print_const_pointer(ptr, ty)
201                }
202            }
203        }
204        ty::tls::with(|tcx| {
205            match self.imm {
206                Immediate::Scalar(s) => {
207                    if let Some(ty) = tcx.lift(self.layout.ty) {
208                        let s =
209                            FmtPrinter::print_string(tcx, Namespace::ValueNS, |cx| p(cx, s, ty))?;
210                        f.write_str(&s)?;
211                        return Ok(());
212                    }
213                    write!(f, "{:x}: {}", s, self.layout.ty)
214                }
215                Immediate::ScalarPair(a, b) => {
216                    // FIXME(oli-obk): at least print tuples and slices nicely
217                    write!(f, "({:x}, {:x}): {}", a, b, self.layout.ty)
218                }
219                Immediate::Uninit => {
220                    write!(f, "uninit: {}", self.layout.ty)
221                }
222            }
223        })
224    }
225}
226
227impl<Prov: Provenance> std::fmt::Debug for ImmTy<'_, Prov> {
228    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229        // Printing `layout` results in too much noise; just print a nice version of the type.
230        f.debug_struct("ImmTy")
231            .field("imm", &self.imm)
232            .field("ty", &format_args!("{}", self.layout.ty))
233            .finish()
234    }
235}
236
237impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> {
238    type Target = Immediate<Prov>;
239    #[inline(always)]
240    fn deref(&self) -> &Immediate<Prov> {
241        &self.imm
242    }
243}
244
245impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
246    #[inline]
247    pub fn from_scalar(val: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
248        debug_assert!(layout.backend_repr.is_scalar(), "`ImmTy::from_scalar` on non-scalar layout");
249        debug_assert_eq!(val.size(), layout.size);
250        ImmTy { imm: val.into(), layout }
251    }
252
253    #[inline]
254    pub fn from_scalar_pair(a: Scalar<Prov>, b: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
255        debug_assert!(
256            matches!(layout.backend_repr, BackendRepr::ScalarPair(..)),
257            "`ImmTy::from_scalar_pair` on non-scalar-pair layout"
258        );
259        let imm = Immediate::ScalarPair(a, b);
260        ImmTy { imm, layout }
261    }
262
263    #[inline(always)]
264    pub fn from_immediate(imm: Immediate<Prov>, layout: TyAndLayout<'tcx>) -> Self {
265        // Without a `cx` we cannot call `assert_matches_abi`.
266        debug_assert!(
267            match (imm, layout.backend_repr) {
268                (Immediate::Scalar(..), BackendRepr::Scalar(..)) => true,
269                (Immediate::ScalarPair(..), BackendRepr::ScalarPair(..)) => true,
270                (Immediate::Uninit, _) if layout.is_sized() => true,
271                _ => false,
272            },
273            "immediate {imm:?} does not fit to layout {layout:?}",
274        );
275        ImmTy { imm, layout }
276    }
277
278    #[inline]
279    pub fn uninit(layout: TyAndLayout<'tcx>) -> Self {
280        debug_assert!(layout.is_sized(), "immediates must be sized");
281        ImmTy { imm: Immediate::Uninit, layout }
282    }
283
284    #[inline]
285    pub fn from_scalar_int(s: ScalarInt, layout: TyAndLayout<'tcx>) -> Self {
286        Self::from_scalar(Scalar::from(s), layout)
287    }
288
289    #[inline]
290    pub fn from_uint(i: impl Into<u128>, layout: TyAndLayout<'tcx>) -> Self {
291        Self::from_scalar(Scalar::from_uint(i, layout.size), layout)
292    }
293
294    #[inline]
295    pub fn from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Self {
296        Self::from_scalar(Scalar::from_int(i, layout.size), layout)
297    }
298
299    #[inline]
300    pub fn from_bool(b: bool, tcx: TyCtxt<'tcx>) -> Self {
301        // Can use any typing env, since `bool` is always monomorphic.
302        let layout = tcx
303            .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(tcx.types.bool))
304            .unwrap();
305        Self::from_scalar(Scalar::from_bool(b), layout)
306    }
307
308    #[inline]
309    pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
310        // Can use any typing env, since `Ordering` is always monomorphic.
311        let ty = tcx.ty_ordering_enum(DUMMY_SP);
312        let layout =
313            tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)).unwrap();
314        Self::from_scalar(Scalar::Int(c.into()), layout)
315    }
316
317    pub fn from_pair(a: Self, b: Self, cx: &(impl HasTypingEnv<'tcx> + HasTyCtxt<'tcx>)) -> Self {
318        let layout = cx
319            .tcx()
320            .layout_of(
321                cx.typing_env().as_query_input(Ty::new_tup(cx.tcx(), &[a.layout.ty, b.layout.ty])),
322            )
323            .unwrap();
324        Self::from_scalar_pair(a.to_scalar(), b.to_scalar(), layout)
325    }
326
327    /// Return the immediate as a `ScalarInt`. Ensures that it has the size that the layout of the
328    /// immediate indicates.
329    #[inline]
330    pub fn to_scalar_int(&self) -> InterpResult<'tcx, ScalarInt> {
331        let s = self.to_scalar().to_scalar_int()?;
332        if s.size() != self.layout.size {
333            throw_ub!(ScalarSizeMismatch(ScalarSizeMismatch {
334                target_size: self.layout.size.bytes(),
335                data_size: s.size().bytes(),
336            }));
337        }
338        interp_ok(s)
339    }
340
341    #[inline]
342    pub fn to_const_int(self) -> ConstInt {
343        assert!(self.layout.ty.is_integral());
344        let int = self.imm.to_scalar_int();
345        assert_eq!(int.size(), self.layout.size);
346        ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral())
347    }
348
349    #[inline]
350    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
351    pub fn to_pair(self, cx: &(impl HasTyCtxt<'tcx> + HasTypingEnv<'tcx>)) -> (Self, Self) {
352        let layout = self.layout;
353        let (val0, val1) = self.to_scalar_pair();
354        (
355            ImmTy::from_scalar(val0, layout.field(cx, 0)),
356            ImmTy::from_scalar(val1, layout.field(cx, 1)),
357        )
358    }
359
360    /// Compute the "sub-immediate" that is located within the `base` at the given offset with the
361    /// given layout.
362    // Not called `offset` to avoid confusion with the trait method.
363    fn offset_(&self, offset: Size, layout: TyAndLayout<'tcx>, cx: &impl HasDataLayout) -> Self {
364        // Verify that the input matches its type.
365        if cfg!(debug_assertions) {
366            self.assert_matches_abi(
367                self.layout.backend_repr,
368                "invalid input to Immediate::offset",
369                cx,
370            );
371        }
372        // `ImmTy` have already been checked to be in-bounds, so we can just check directly if this
373        // remains in-bounds. This cannot actually be violated since projections are type-checked
374        // and bounds-checked.
375        assert!(
376            offset + layout.size <= self.layout.size,
377            "attempting to project to field at offset {} with size {} into immediate with layout {:#?}",
378            offset.bytes(),
379            layout.size.bytes(),
380            self.layout,
381        );
382        // This makes several assumptions about what layouts we will encounter; we match what
383        // codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
384        let inner_val: Immediate<_> = match (**self, self.layout.backend_repr) {
385            // If the entire value is uninit, then so is the field (can happen in ConstProp).
386            (Immediate::Uninit, _) => Immediate::Uninit,
387            // If the field is uninhabited, we can forget the data (can happen in ConstProp).
388            // `enum S { A(!), B, C }` is an example of an enum with Scalar layout that
389            // has an uninhabited variant, which means this case is possible.
390            _ if layout.is_uninhabited() => Immediate::Uninit,
391            // the field contains no information, can be left uninit
392            // (Scalar/ScalarPair can contain even aligned ZST, not just 1-ZST)
393            _ if layout.is_zst() => Immediate::Uninit,
394            // some fieldless enum variants can have non-zero size but still `Aggregate` ABI... try
395            // to detect those here and also give them no data
396            _ if matches!(layout.backend_repr, BackendRepr::Memory { .. })
397                && matches!(layout.variants, abi::Variants::Single { .. })
398                && matches!(&layout.fields, abi::FieldsShape::Arbitrary { offsets, .. } if offsets.len() == 0) =>
399            {
400                Immediate::Uninit
401            }
402            // the field covers the entire type
403            _ if layout.size == self.layout.size => {
404                assert_eq!(offset.bytes(), 0);
405                **self
406            }
407            // extract fields from types with `ScalarPair` ABI
408            (Immediate::ScalarPair(a_val, b_val), BackendRepr::ScalarPair(a, b)) => {
409                Immediate::from(if offset.bytes() == 0 {
410                    a_val
411                } else {
412                    assert_eq!(offset, a.size(cx).align_to(b.align(cx).abi));
413                    b_val
414                })
415            }
416            // everything else is a bug
417            _ => bug!(
418                "invalid field access on immediate {} at offset {}, original layout {:#?}",
419                self,
420                offset.bytes(),
421                self.layout
422            ),
423        };
424        // Ensure the new layout matches the new value.
425        inner_val.assert_matches_abi(
426            layout.backend_repr,
427            "invalid field type in Immediate::offset",
428            cx,
429        );
430
431        ImmTy::from_immediate(inner_val, layout)
432    }
433}
434
435impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
436    #[inline(always)]
437    fn layout(&self) -> TyAndLayout<'tcx> {
438        self.layout
439    }
440
441    #[inline(always)]
442    fn meta(&self) -> MemPlaceMeta<Prov> {
443        debug_assert!(self.layout.is_sized()); // unsized ImmTy can only exist temporarily and should never reach this here
444        MemPlaceMeta::None
445    }
446
447    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
448        &self,
449        offset: Size,
450        _mode: OffsetMode,
451        meta: MemPlaceMeta<Prov>,
452        layout: TyAndLayout<'tcx>,
453        ecx: &InterpCx<'tcx, M>,
454    ) -> InterpResult<'tcx, Self> {
455        assert_matches!(meta, MemPlaceMeta::None); // we can't store this anywhere anyway
456        interp_ok(self.offset_(offset, layout, ecx))
457    }
458
459    #[inline(always)]
460    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
461        &self,
462        _ecx: &InterpCx<'tcx, M>,
463    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
464        interp_ok(self.clone().into())
465    }
466}
467
468/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate,
469/// or still in memory. The latter is an optimization, to delay reading that chunk of
470/// memory and to avoid having to store arbitrary-sized data here.
471#[derive(Copy, Clone, Debug)]
472pub(super) enum Operand<Prov: Provenance = CtfeProvenance> {
473    Immediate(Immediate<Prov>),
474    Indirect(MemPlace<Prov>),
475}
476
477#[derive(Clone)]
478pub struct OpTy<'tcx, Prov: Provenance = CtfeProvenance> {
479    op: Operand<Prov>, // Keep this private; it helps enforce invariants.
480    pub layout: TyAndLayout<'tcx>,
481}
482
483impl<Prov: Provenance> std::fmt::Debug for OpTy<'_, Prov> {
484    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
485        // Printing `layout` results in too much noise; just print a nice version of the type.
486        f.debug_struct("OpTy")
487            .field("op", &self.op)
488            .field("ty", &format_args!("{}", self.layout.ty))
489            .finish()
490    }
491}
492
493impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
494    #[inline(always)]
495    fn from(val: ImmTy<'tcx, Prov>) -> Self {
496        OpTy { op: Operand::Immediate(val.imm), layout: val.layout }
497    }
498}
499
500impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
501    #[inline(always)]
502    fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
503        OpTy { op: Operand::Indirect(*mplace.mplace()), layout: mplace.layout }
504    }
505}
506
507impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
508    #[inline(always)]
509    pub(super) fn op(&self) -> &Operand<Prov> {
510        &self.op
511    }
512}
513
514impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
515    #[inline(always)]
516    fn layout(&self) -> TyAndLayout<'tcx> {
517        self.layout
518    }
519
520    #[inline]
521    fn meta(&self) -> MemPlaceMeta<Prov> {
522        match self.as_mplace_or_imm() {
523            Left(mplace) => mplace.meta(),
524            Right(_) => {
525                debug_assert!(self.layout.is_sized(), "unsized immediates are not a thing");
526                MemPlaceMeta::None
527            }
528        }
529    }
530
531    fn offset_with_meta<M: Machine<'tcx, Provenance = Prov>>(
532        &self,
533        offset: Size,
534        mode: OffsetMode,
535        meta: MemPlaceMeta<Prov>,
536        layout: TyAndLayout<'tcx>,
537        ecx: &InterpCx<'tcx, M>,
538    ) -> InterpResult<'tcx, Self> {
539        match self.as_mplace_or_imm() {
540            Left(mplace) => {
541                interp_ok(mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into())
542            }
543            Right(imm) => {
544                assert_matches!(meta, MemPlaceMeta::None); // no place to store metadata here
545                // Every part of an uninit is uninit.
546                interp_ok(imm.offset_(offset, layout, ecx).into())
547            }
548        }
549    }
550
551    #[inline(always)]
552    fn to_op<M: Machine<'tcx, Provenance = Prov>>(
553        &self,
554        _ecx: &InterpCx<'tcx, M>,
555    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
556        interp_ok(self.clone())
557    }
558}
559
560impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
561    /// Try reading an immediate in memory; this is interesting particularly for `ScalarPair`.
562    /// Returns `None` if the layout does not permit loading this as a value.
563    ///
564    /// This is an internal function; call `read_immediate` instead.
565    fn read_immediate_from_mplace_raw(
566        &self,
567        mplace: &MPlaceTy<'tcx, M::Provenance>,
568    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
569        if mplace.layout.is_unsized() {
570            // Don't touch unsized
571            return interp_ok(None);
572        }
573
574        let Some(alloc) = self.get_place_alloc(mplace)? else {
575            // zero-sized type can be left uninit
576            return interp_ok(Some(ImmTy::uninit(mplace.layout)));
577        };
578
579        // It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point.
580        // However, `MaybeUninit<u64>` is considered a `Scalar` as far as its layout is concerned --
581        // and yet cannot be represented by an interpreter `Scalar`, since we have to handle the
582        // case where some of the bytes are initialized and others are not. So, we need an extra
583        // check that walks over the type of `mplace` to make sure it is truly correct to treat this
584        // like a `Scalar` (or `ScalarPair`).
585        interp_ok(match mplace.layout.backend_repr {
586            BackendRepr::Scalar(abi::Scalar::Initialized { value: s, .. }) => {
587                let size = s.size(self);
588                assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
589                let scalar = alloc.read_scalar(
590                    alloc_range(Size::ZERO, size),
591                    /*read_provenance*/ matches!(s, abi::Primitive::Pointer(_)),
592                )?;
593                Some(ImmTy::from_scalar(scalar, mplace.layout))
594            }
595            BackendRepr::ScalarPair(
596                abi::Scalar::Initialized { value: a, .. },
597                abi::Scalar::Initialized { value: b, .. },
598            ) => {
599                // We checked `ptr_align` above, so all fields will have the alignment they need.
600                // We would anyway check against `ptr_align.restrict_for_offset(b_offset)`,
601                // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
602                let (a_size, b_size) = (a.size(self), b.size(self));
603                let b_offset = a_size.align_to(b.align(self).abi);
604                assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
605                let a_val = alloc.read_scalar(
606                    alloc_range(Size::ZERO, a_size),
607                    /*read_provenance*/ matches!(a, abi::Primitive::Pointer(_)),
608                )?;
609                let b_val = alloc.read_scalar(
610                    alloc_range(b_offset, b_size),
611                    /*read_provenance*/ matches!(b, abi::Primitive::Pointer(_)),
612                )?;
613                Some(ImmTy::from_immediate(Immediate::ScalarPair(a_val, b_val), mplace.layout))
614            }
615            _ => {
616                // Neither a scalar nor scalar pair.
617                None
618            }
619        })
620    }
621
622    /// Try returning an immediate for the operand. If the layout does not permit loading this as an
623    /// immediate, return where in memory we can find the data.
624    /// Note that for a given layout, this operation will either always return Left or Right!
625    /// succeed!  Whether it returns Left depends on whether the layout can be represented
626    /// in an `Immediate`, not on which data is stored there currently.
627    ///
628    /// This is an internal function that should not usually be used; call `read_immediate` instead.
629    /// ConstProp needs it, though.
630    pub fn read_immediate_raw(
631        &self,
632        src: &impl Projectable<'tcx, M::Provenance>,
633    ) -> InterpResult<'tcx, Either<MPlaceTy<'tcx, M::Provenance>, ImmTy<'tcx, M::Provenance>>> {
634        interp_ok(match src.to_op(self)?.as_mplace_or_imm() {
635            Left(ref mplace) => {
636                if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
637                    Right(val)
638                } else {
639                    Left(mplace.clone())
640                }
641            }
642            Right(val) => Right(val),
643        })
644    }
645
646    /// Read an immediate from a place, asserting that that is possible with the given layout.
647    ///
648    /// If this succeeds, the `ImmTy` is never `Uninit`.
649    #[inline(always)]
650    pub fn read_immediate(
651        &self,
652        op: &impl Projectable<'tcx, M::Provenance>,
653    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
654        if !matches!(
655            op.layout().backend_repr,
656            BackendRepr::Scalar(abi::Scalar::Initialized { .. })
657                | BackendRepr::ScalarPair(
658                    abi::Scalar::Initialized { .. },
659                    abi::Scalar::Initialized { .. }
660                )
661        ) {
662            span_bug!(self.cur_span(), "primitive read not possible for type: {}", op.layout().ty);
663        }
664        let imm = self.read_immediate_raw(op)?.right().unwrap();
665        if matches!(*imm, Immediate::Uninit) {
666            throw_ub!(InvalidUninitBytes(None));
667        }
668        interp_ok(imm)
669    }
670
671    /// Read a scalar from a place
672    pub fn read_scalar(
673        &self,
674        op: &impl Projectable<'tcx, M::Provenance>,
675    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
676        interp_ok(self.read_immediate(op)?.to_scalar())
677    }
678
679    // Pointer-sized reads are fairly common and need target layout access, so we wrap them in
680    // convenience functions.
681
682    /// Read a pointer from a place.
683    pub fn read_pointer(
684        &self,
685        op: &impl Projectable<'tcx, M::Provenance>,
686    ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
687        self.read_scalar(op)?.to_pointer(self)
688    }
689    /// Read a pointer-sized unsigned integer from a place.
690    pub fn read_target_usize(
691        &self,
692        op: &impl Projectable<'tcx, M::Provenance>,
693    ) -> InterpResult<'tcx, u64> {
694        self.read_scalar(op)?.to_target_usize(self)
695    }
696    /// Read a pointer-sized signed integer from a place.
697    pub fn read_target_isize(
698        &self,
699        op: &impl Projectable<'tcx, M::Provenance>,
700    ) -> InterpResult<'tcx, i64> {
701        self.read_scalar(op)?.to_target_isize(self)
702    }
703
704    /// Turn the wide MPlace into a string (must already be dereferenced!)
705    pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
706        let len = mplace.len(self)?;
707        let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
708        let s = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
709        interp_ok(s)
710    }
711
712    /// Read from a local of the current frame. Convenience method for [`InterpCx::local_at_frame_to_op`].
713    pub fn local_to_op(
714        &self,
715        local: mir::Local,
716        layout: Option<TyAndLayout<'tcx>>,
717    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
718        self.local_at_frame_to_op(self.frame(), local, layout)
719    }
720
721    /// Read from a local of a given frame.
722    /// Will not access memory, instead an indirect `Operand` is returned.
723    ///
724    /// This is public because it is used by [Aquascope](https://github.com/cognitive-engineering-lab/aquascope/)
725    /// to get an OpTy from a local.
726    pub fn local_at_frame_to_op(
727        &self,
728        frame: &Frame<'tcx, M::Provenance, M::FrameExtra>,
729        local: mir::Local,
730        layout: Option<TyAndLayout<'tcx>>,
731    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
732        let layout = self.layout_of_local(frame, local, layout)?;
733        let op = *frame.locals[local].access()?;
734        if matches!(op, Operand::Immediate(_)) {
735            assert!(!layout.is_unsized());
736        }
737        M::after_local_read(self, frame, local)?;
738        interp_ok(OpTy { op, layout })
739    }
740
741    /// Every place can be read from, so we can turn them into an operand.
742    /// This will definitely return `Indirect` if the place is a `Ptr`, i.e., this
743    /// will never actually read from memory.
744    pub fn place_to_op(
745        &self,
746        place: &PlaceTy<'tcx, M::Provenance>,
747    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
748        match place.as_mplace_or_local() {
749            Left(mplace) => interp_ok(mplace.into()),
750            Right((local, offset, locals_addr, _)) => {
751                debug_assert!(place.layout.is_sized()); // only sized locals can ever be `Place::Local`.
752                debug_assert_eq!(locals_addr, self.frame().locals_addr());
753                let base = self.local_to_op(local, None)?;
754                interp_ok(match offset {
755                    Some(offset) => base.offset(offset, place.layout, self)?,
756                    None => {
757                        // In the common case this hasn't been projected.
758                        debug_assert_eq!(place.layout, base.layout);
759                        base
760                    }
761                })
762            }
763        }
764    }
765
766    /// Evaluate a place with the goal of reading from it. This lets us sometimes
767    /// avoid allocations.
768    pub fn eval_place_to_op(
769        &self,
770        mir_place: mir::Place<'tcx>,
771        layout: Option<TyAndLayout<'tcx>>,
772    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
773        // Do not use the layout passed in as argument if the base we are looking at
774        // here is not the entire place.
775        let layout = if mir_place.projection.is_empty() { layout } else { None };
776
777        let mut op = self.local_to_op(mir_place.local, layout)?;
778        // Using `try_fold` turned out to be bad for performance, hence the loop.
779        for elem in mir_place.projection.iter() {
780            op = self.project(&op, elem)?
781        }
782
783        trace!("eval_place_to_op: got {:?}", op);
784        // Sanity-check the type we ended up with.
785        if cfg!(debug_assertions) {
786            let normalized_place_ty = self
787                .instantiate_from_current_frame_and_normalize_erasing_regions(
788                    mir_place.ty(&self.frame().body.local_decls, *self.tcx).ty,
789                )?;
790            if !mir_assign_valid_types(
791                *self.tcx,
792                self.typing_env(),
793                self.layout_of(normalized_place_ty)?,
794                op.layout,
795            ) {
796                span_bug!(
797                    self.cur_span(),
798                    "eval_place of a MIR place with type {} produced an interpreter operand with type {}",
799                    normalized_place_ty,
800                    op.layout.ty,
801                )
802            }
803        }
804        interp_ok(op)
805    }
806
807    /// Evaluate the operand, returning a place where you can then find the data.
808    /// If you already know the layout, you can save two table lookups
809    /// by passing it in here.
810    #[inline]
811    pub fn eval_operand(
812        &self,
813        mir_op: &mir::Operand<'tcx>,
814        layout: Option<TyAndLayout<'tcx>>,
815    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
816        use rustc_middle::mir::Operand::*;
817        let op = match mir_op {
818            // FIXME: do some more logic on `move` to invalidate the old location
819            &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?,
820
821            Constant(constant) => {
822                let c = self.instantiate_from_current_frame_and_normalize_erasing_regions(
823                    constant.const_,
824                )?;
825
826                // This can still fail:
827                // * During ConstProp, with `TooGeneric` or since the `required_consts` were not all
828                //   checked yet.
829                // * During CTFE, since promoteds in `const`/`static` initializer bodies can fail.
830                self.eval_mir_constant(&c, constant.span, layout)?
831            }
832        };
833        trace!("{:?}: {:?}", mir_op, op);
834        interp_ok(op)
835    }
836
837    pub(crate) fn const_val_to_op(
838        &self,
839        val_val: mir::ConstValue<'tcx>,
840        ty: Ty<'tcx>,
841        layout: Option<TyAndLayout<'tcx>>,
842    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
843        // Other cases need layout.
844        let adjust_scalar = |scalar| -> InterpResult<'tcx, _> {
845            interp_ok(match scalar {
846                Scalar::Ptr(ptr, size) => Scalar::Ptr(self.global_root_pointer(ptr)?, size),
847                Scalar::Int(int) => Scalar::Int(int),
848            })
849        };
850        let layout =
851            from_known_layout(self.tcx, self.typing_env(), layout, || self.layout_of(ty).into())?;
852        let imm = match val_val {
853            mir::ConstValue::Indirect { alloc_id, offset } => {
854                // This is const data, no mutation allowed.
855                let ptr = self.global_root_pointer(Pointer::new(
856                    CtfeProvenance::from(alloc_id).as_immutable(),
857                    offset,
858                ))?;
859                return interp_ok(self.ptr_to_mplace(ptr.into(), layout).into());
860            }
861            mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
862            mir::ConstValue::ZeroSized => Immediate::Uninit,
863            mir::ConstValue::Slice { data, meta } => {
864                // This is const data, no mutation allowed.
865                let alloc_id = self.tcx.reserve_and_set_memory_alloc(data);
866                let ptr = Pointer::new(CtfeProvenance::from(alloc_id).as_immutable(), Size::ZERO);
867                Immediate::new_slice(self.global_root_pointer(ptr)?.into(), meta, self)
868            }
869        };
870        interp_ok(OpTy { op: Operand::Immediate(imm), layout })
871    }
872}
873
874// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
875#[cfg(target_pointer_width = "64")]
876mod size_asserts {
877    use rustc_data_structures::static_assert_size;
878
879    use super::*;
880    // tidy-alphabetical-start
881    static_assert_size!(ImmTy<'_>, 64);
882    static_assert_size!(Immediate, 48);
883    static_assert_size!(OpTy<'_>, 72);
884    static_assert_size!(Operand, 56);
885    // tidy-alphabetical-end
886}
rustc_const_eval/interpret/operand.rs

rustc_const_eval/interpret/
operand.rs
