rustc_const_eval/interpret/
memory.rs

1//! The memory subsystem.
2//!
3//! Generally, we use `Pointer` to denote memory addresses. However, some operations
4//! have a "size"-like parameter, and they take `Scalar` for the address because
5//! if the size is 0, then the pointer can also be a (properly aligned, non-null)
6//! integer. It is crucial that these operations call `check_align` *before*
7//! short-circuiting the empty case!
8
9use std::assert_matches::assert_matches;
10use std::borrow::{Borrow, Cow};
11use std::cell::Cell;
12use std::collections::VecDeque;
13use std::{fmt, ptr};
14
15use rustc_abi::{Align, HasDataLayout, Size};
16use rustc_ast::Mutability;
17use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
18use rustc_middle::bug;
19use rustc_middle::mir::display_allocation;
20use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
21use tracing::{debug, instrument, trace};
22
23use super::{
24    AllocBytes, AllocId, AllocInit, AllocMap, AllocRange, Allocation, CheckAlignMsg,
25    CheckInAllocMsg, CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak,
26    Misalignment, Pointer, PointerArithmetic, Provenance, Scalar, alloc_range, err_ub,
27    err_ub_custom, interp_ok, throw_ub, throw_ub_custom, throw_unsup, throw_unsup_format,
28};
29use crate::fluent_generated as fluent;
30
31#[derive(Debug, PartialEq, Copy, Clone)]
32pub enum MemoryKind<T> {
33    /// Stack memory. Error if deallocated except during a stack pop.
34    Stack,
35    /// Memory allocated by `caller_location` intrinsic. Error if ever deallocated.
36    CallerLocation,
37    /// Additional memory kinds a machine wishes to distinguish from the builtin ones.
38    Machine(T),
39}
40
41impl<T: MayLeak> MayLeak for MemoryKind<T> {
42    #[inline]
43    fn may_leak(self) -> bool {
44        match self {
45            MemoryKind::Stack => false,
46            MemoryKind::CallerLocation => true,
47            MemoryKind::Machine(k) => k.may_leak(),
48        }
49    }
50}
51
52impl<T: fmt::Display> fmt::Display for MemoryKind<T> {
53    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
54        match self {
55            MemoryKind::Stack => write!(f, "stack variable"),
56            MemoryKind::CallerLocation => write!(f, "caller location"),
57            MemoryKind::Machine(m) => write!(f, "{m}"),
58        }
59    }
60}
61
62/// The return value of `get_alloc_info` indicates the "kind" of the allocation.
63#[derive(Copy, Clone, PartialEq, Debug)]
64pub enum AllocKind {
65    /// A regular live data allocation.
66    LiveData,
67    /// A function allocation (that fn ptrs point to).
68    Function,
69    /// A (symbolic) vtable allocation.
70    VTable,
71    /// A dead allocation.
72    Dead,
73}
74
75/// Metadata about an `AllocId`.
76#[derive(Copy, Clone, PartialEq, Debug)]
77pub struct AllocInfo {
78    pub size: Size,
79    pub align: Align,
80    pub kind: AllocKind,
81    pub mutbl: Mutability,
82}
83
84impl AllocInfo {
85    fn new(size: Size, align: Align, kind: AllocKind, mutbl: Mutability) -> Self {
86        Self { size, align, kind, mutbl }
87    }
88}
89
90/// The value of a function pointer.
91#[derive(Debug, Copy, Clone)]
92pub enum FnVal<'tcx, Other> {
93    Instance(Instance<'tcx>),
94    Other(Other),
95}
96
97impl<'tcx, Other> FnVal<'tcx, Other> {
98    pub fn as_instance(self) -> InterpResult<'tcx, Instance<'tcx>> {
99        match self {
100            FnVal::Instance(instance) => interp_ok(instance),
101            FnVal::Other(_) => {
102                throw_unsup_format!("'foreign' function pointers are not supported in this context")
103            }
104        }
105    }
106}
107
108// `Memory` has to depend on the `Machine` because some of its operations
109// (e.g., `get`) call a `Machine` hook.
110pub struct Memory<'tcx, M: Machine<'tcx>> {
111    /// Allocations local to this instance of the interpreter. The kind
112    /// helps ensure that the same mechanism is used for allocation and
113    /// deallocation. When an allocation is not found here, it is a
114    /// global and looked up in the `tcx` for read access. Some machines may
115    /// have to mutate this map even on a read-only access to a global (because
116    /// they do pointer provenance tracking and the allocations in `tcx` have
117    /// the wrong type), so we let the machine override this type.
118    /// Either way, if the machine allows writing to a global, doing so will
119    /// create a copy of the global allocation here.
120    // FIXME: this should not be public, but interning currently needs access to it
121    pub(super) alloc_map: M::MemoryMap,
122
123    /// Map for "extra" function pointers.
124    extra_fn_ptr_map: FxIndexMap<AllocId, M::ExtraFnVal>,
125
126    /// To be able to compare pointers with null, and to check alignment for accesses
127    /// to ZSTs (where pointers may dangle), we keep track of the size even for allocations
128    /// that do not exist any more.
129    // FIXME: this should not be public, but interning currently needs access to it
130    pub(super) dead_alloc_map: FxIndexMap<AllocId, (Size, Align)>,
131
132    /// This stores whether we are currently doing reads purely for the purpose of validation.
133    /// Those reads do not trigger the machine's hooks for memory reads.
134    /// Needless to say, this must only be set with great care!
135    validation_in_progress: Cell<bool>,
136}
137
138/// A reference to some allocation that was already bounds-checked for the given region
139/// and had the on-access machine hooks run.
140#[derive(Copy, Clone)]
141pub struct AllocRef<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> {
142    alloc: &'a Allocation<Prov, Extra, Bytes>,
143    range: AllocRange,
144    tcx: TyCtxt<'tcx>,
145    alloc_id: AllocId,
146}
147/// A reference to some allocation that was already bounds-checked for the given region
148/// and had the on-access machine hooks run.
149pub struct AllocRefMut<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> {
150    alloc: &'a mut Allocation<Prov, Extra, Bytes>,
151    range: AllocRange,
152    tcx: TyCtxt<'tcx>,
153    alloc_id: AllocId,
154}
155
156impl<'tcx, M: Machine<'tcx>> Memory<'tcx, M> {
157    pub fn new() -> Self {
158        Memory {
159            alloc_map: M::MemoryMap::default(),
160            extra_fn_ptr_map: FxIndexMap::default(),
161            dead_alloc_map: FxIndexMap::default(),
162            validation_in_progress: Cell::new(false),
163        }
164    }
165
166    /// This is used by [priroda](https://github.com/oli-obk/priroda)
167    pub fn alloc_map(&self) -> &M::MemoryMap {
168        &self.alloc_map
169    }
170}
171
172impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
173    /// Call this to turn untagged "global" pointers (obtained via `tcx`) into
174    /// the machine pointer to the allocation. Must never be used
175    /// for any other pointers, nor for TLS statics.
176    ///
177    /// Using the resulting pointer represents a *direct* access to that memory
178    /// (e.g. by directly using a `static`),
179    /// as opposed to access through a pointer that was created by the program.
180    ///
181    /// This function can fail only if `ptr` points to an `extern static`.
182    #[inline]
183    pub fn global_root_pointer(
184        &self,
185        ptr: Pointer<CtfeProvenance>,
186    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
187        let alloc_id = ptr.provenance.alloc_id();
188        // We need to handle `extern static`.
189        match self.tcx.try_get_global_alloc(alloc_id) {
190            Some(GlobalAlloc::Static(def_id)) if self.tcx.is_thread_local_static(def_id) => {
191                // Thread-local statics do not have a constant address. They *must* be accessed via
192                // `ThreadLocalRef`; we can never have a pointer to them as a regular constant value.
193                bug!("global memory cannot point to thread-local static")
194            }
195            Some(GlobalAlloc::Static(def_id)) if self.tcx.is_foreign_item(def_id) => {
196                return M::extern_static_pointer(self, def_id);
197            }
198            None => {
199                assert!(
200                    self.memory.extra_fn_ptr_map.contains_key(&alloc_id),
201                    "{alloc_id:?} is neither global nor a function pointer"
202                );
203            }
204            _ => {}
205        }
206        // And we need to get the provenance.
207        M::adjust_alloc_root_pointer(self, ptr, M::GLOBAL_KIND.map(MemoryKind::Machine))
208    }
209
210    pub fn fn_ptr(&mut self, fn_val: FnVal<'tcx, M::ExtraFnVal>) -> Pointer<M::Provenance> {
211        let id = match fn_val {
212            FnVal::Instance(instance) => {
213                let salt = M::get_global_alloc_salt(self, Some(instance));
214                self.tcx.reserve_and_set_fn_alloc(instance, salt)
215            }
216            FnVal::Other(extra) => {
217                // FIXME(RalfJung): Should we have a cache here?
218                let id = self.tcx.reserve_alloc_id();
219                let old = self.memory.extra_fn_ptr_map.insert(id, extra);
220                assert!(old.is_none());
221                id
222            }
223        };
224        // Functions are global allocations, so make sure we get the right root pointer.
225        // We know this is not an `extern static` so this cannot fail.
226        self.global_root_pointer(Pointer::from(id)).unwrap()
227    }
228
229    pub fn allocate_ptr(
230        &mut self,
231        size: Size,
232        align: Align,
233        kind: MemoryKind<M::MemoryKind>,
234        init: AllocInit,
235    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
236        let params = self.machine.get_default_alloc_params();
237        let alloc = if M::PANIC_ON_ALLOC_FAIL {
238            Allocation::new(size, align, init, params)
239        } else {
240            Allocation::try_new(size, align, init, params)?
241        };
242        self.insert_allocation(alloc, kind)
243    }
244
245    pub fn allocate_bytes_ptr(
246        &mut self,
247        bytes: &[u8],
248        align: Align,
249        kind: MemoryKind<M::MemoryKind>,
250        mutability: Mutability,
251    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
252        let params = self.machine.get_default_alloc_params();
253        let alloc = Allocation::from_bytes(bytes, align, mutability, params);
254        self.insert_allocation(alloc, kind)
255    }
256
257    pub fn insert_allocation(
258        &mut self,
259        alloc: Allocation<M::Provenance, (), M::Bytes>,
260        kind: MemoryKind<M::MemoryKind>,
261    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
262        assert!(alloc.size() <= self.max_size_of_val());
263        let id = self.tcx.reserve_alloc_id();
264        debug_assert_ne!(
265            Some(kind),
266            M::GLOBAL_KIND.map(MemoryKind::Machine),
267            "dynamically allocating global memory"
268        );
269        // This cannot be merged with the `adjust_global_allocation` code path
270        // since here we have an allocation that already uses `M::Bytes`.
271        let extra = M::init_local_allocation(self, id, kind, alloc.size(), alloc.align)?;
272        let alloc = alloc.with_extra(extra);
273        self.memory.alloc_map.insert(id, (kind, alloc));
274        M::adjust_alloc_root_pointer(self, Pointer::from(id), Some(kind))
275    }
276
277    /// If this grows the allocation, `init_growth` determines
278    /// whether the additional space will be initialized.
279    pub fn reallocate_ptr(
280        &mut self,
281        ptr: Pointer<Option<M::Provenance>>,
282        old_size_and_align: Option<(Size, Align)>,
283        new_size: Size,
284        new_align: Align,
285        kind: MemoryKind<M::MemoryKind>,
286        init_growth: AllocInit,
287    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
288        let (alloc_id, offset, _prov) = self.ptr_get_alloc_id(ptr, 0)?;
289        if offset.bytes() != 0 {
290            throw_ub_custom!(
291                fluent::const_eval_realloc_or_alloc_with_offset,
292                ptr = format!("{ptr:?}"),
293                kind = "realloc"
294            );
295        }
296
297        // For simplicities' sake, we implement reallocate as "alloc, copy, dealloc".
298        // This happens so rarely, the perf advantage is outweighed by the maintenance cost.
299        // If requested, we zero-init the entire allocation, to ensure that a growing
300        // allocation has its new bytes properly set. For the part that is copied,
301        // `mem_copy` below will de-initialize things as necessary.
302        let new_ptr = self.allocate_ptr(new_size, new_align, kind, init_growth)?;
303        let old_size = match old_size_and_align {
304            Some((size, _align)) => size,
305            None => self.get_alloc_raw(alloc_id)?.size(),
306        };
307        // This will also call the access hooks.
308        self.mem_copy(ptr, new_ptr.into(), old_size.min(new_size), /*nonoverlapping*/ true)?;
309        self.deallocate_ptr(ptr, old_size_and_align, kind)?;
310
311        interp_ok(new_ptr)
312    }
313
314    #[instrument(skip(self), level = "debug")]
315    pub fn deallocate_ptr(
316        &mut self,
317        ptr: Pointer<Option<M::Provenance>>,
318        old_size_and_align: Option<(Size, Align)>,
319        kind: MemoryKind<M::MemoryKind>,
320    ) -> InterpResult<'tcx> {
321        let (alloc_id, offset, prov) = self.ptr_get_alloc_id(ptr, 0)?;
322        trace!("deallocating: {alloc_id:?}");
323
324        if offset.bytes() != 0 {
325            throw_ub_custom!(
326                fluent::const_eval_realloc_or_alloc_with_offset,
327                ptr = format!("{ptr:?}"),
328                kind = "dealloc",
329            );
330        }
331
332        let Some((alloc_kind, mut alloc)) = self.memory.alloc_map.remove(&alloc_id) else {
333            // Deallocating global memory -- always an error
334            return Err(match self.tcx.try_get_global_alloc(alloc_id) {
335                Some(GlobalAlloc::Function { .. }) => {
336                    err_ub_custom!(
337                        fluent::const_eval_invalid_dealloc,
338                        alloc_id = alloc_id,
339                        kind = "fn",
340                    )
341                }
342                Some(GlobalAlloc::VTable(..)) => {
343                    err_ub_custom!(
344                        fluent::const_eval_invalid_dealloc,
345                        alloc_id = alloc_id,
346                        kind = "vtable",
347                    )
348                }
349                Some(GlobalAlloc::Static(..) | GlobalAlloc::Memory(..)) => {
350                    err_ub_custom!(
351                        fluent::const_eval_invalid_dealloc,
352                        alloc_id = alloc_id,
353                        kind = "static_mem"
354                    )
355                }
356                None => err_ub!(PointerUseAfterFree(alloc_id, CheckInAllocMsg::MemoryAccess)),
357            })
358            .into();
359        };
360
361        if alloc.mutability.is_not() {
362            throw_ub_custom!(fluent::const_eval_dealloc_immutable, alloc = alloc_id,);
363        }
364        if alloc_kind != kind {
365            throw_ub_custom!(
366                fluent::const_eval_dealloc_kind_mismatch,
367                alloc = alloc_id,
368                alloc_kind = format!("{alloc_kind}"),
369                kind = format!("{kind}"),
370            );
371        }
372        if let Some((size, align)) = old_size_and_align {
373            if size != alloc.size() || align != alloc.align {
374                throw_ub_custom!(
375                    fluent::const_eval_dealloc_incorrect_layout,
376                    alloc = alloc_id,
377                    size = alloc.size().bytes(),
378                    align = alloc.align.bytes(),
379                    size_found = size.bytes(),
380                    align_found = align.bytes(),
381                )
382            }
383        }
384
385        // Let the machine take some extra action
386        let size = alloc.size();
387        M::before_memory_deallocation(
388            self.tcx,
389            &mut self.machine,
390            &mut alloc.extra,
391            ptr,
392            (alloc_id, prov),
393            size,
394            alloc.align,
395            kind,
396        )?;
397
398        // Don't forget to remember size and align of this now-dead allocation
399        let old = self.memory.dead_alloc_map.insert(alloc_id, (size, alloc.align));
400        if old.is_some() {
401            bug!("Nothing can be deallocated twice");
402        }
403
404        interp_ok(())
405    }
406
407    /// Internal helper function to determine the allocation and offset of a pointer (if any).
408    #[inline(always)]
409    fn get_ptr_access(
410        &self,
411        ptr: Pointer<Option<M::Provenance>>,
412        size: Size,
413    ) -> InterpResult<'tcx, Option<(AllocId, Size, M::ProvenanceExtra)>> {
414        let size = i64::try_from(size.bytes()).unwrap(); // it would be an error to even ask for more than isize::MAX bytes
415        Self::check_and_deref_ptr(
416            self,
417            ptr,
418            size,
419            CheckInAllocMsg::MemoryAccess,
420            |this, alloc_id, offset, prov| {
421                let (size, align) =
422                    this.get_live_alloc_size_and_align(alloc_id, CheckInAllocMsg::MemoryAccess)?;
423                interp_ok((size, align, (alloc_id, offset, prov)))
424            },
425        )
426    }
427
428    /// Check if the given pointer points to live memory of the given `size`.
429    /// The caller can control the error message for the out-of-bounds case.
430    #[inline(always)]
431    pub fn check_ptr_access(
432        &self,
433        ptr: Pointer<Option<M::Provenance>>,
434        size: Size,
435        msg: CheckInAllocMsg,
436    ) -> InterpResult<'tcx> {
437        let size = i64::try_from(size.bytes()).unwrap(); // it would be an error to even ask for more than isize::MAX bytes
438        Self::check_and_deref_ptr(self, ptr, size, msg, |this, alloc_id, _, _| {
439            let (size, align) = this.get_live_alloc_size_and_align(alloc_id, msg)?;
440            interp_ok((size, align, ()))
441        })?;
442        interp_ok(())
443    }
444
445    /// Check whether the given pointer points to live memory for a signed amount of bytes.
446    /// A negative amounts means that the given range of memory to the left of the pointer
447    /// needs to be dereferenceable.
448    pub fn check_ptr_access_signed(
449        &self,
450        ptr: Pointer<Option<M::Provenance>>,
451        size: i64,
452        msg: CheckInAllocMsg,
453    ) -> InterpResult<'tcx> {
454        Self::check_and_deref_ptr(self, ptr, size, msg, |this, alloc_id, _, _| {
455            let (size, align) = this.get_live_alloc_size_and_align(alloc_id, msg)?;
456            interp_ok((size, align, ()))
457        })?;
458        interp_ok(())
459    }
460
461    /// Low-level helper function to check if a ptr is in-bounds and potentially return a reference
462    /// to the allocation it points to. Supports both shared and mutable references, as the actual
463    /// checking is offloaded to a helper closure. Supports signed sizes for checks "to the left" of
464    /// a pointer.
465    ///
466    /// `alloc_size` will only get called for non-zero-sized accesses.
467    ///
468    /// Returns `None` if and only if the size is 0.
469    fn check_and_deref_ptr<T, R: Borrow<Self>>(
470        this: R,
471        ptr: Pointer<Option<M::Provenance>>,
472        size: i64,
473        msg: CheckInAllocMsg,
474        alloc_size: impl FnOnce(
475            R,
476            AllocId,
477            Size,
478            M::ProvenanceExtra,
479        ) -> InterpResult<'tcx, (Size, Align, T)>,
480    ) -> InterpResult<'tcx, Option<T>> {
481        // Everything is okay with size 0.
482        if size == 0 {
483            return interp_ok(None);
484        }
485
486        interp_ok(match this.borrow().ptr_try_get_alloc_id(ptr, size) {
487            Err(addr) => {
488                // We couldn't get a proper allocation.
489                throw_ub!(DanglingIntPointer { addr, inbounds_size: size, msg });
490            }
491            Ok((alloc_id, offset, prov)) => {
492                let tcx = this.borrow().tcx;
493                let (alloc_size, _alloc_align, ret_val) = alloc_size(this, alloc_id, offset, prov)?;
494                let offset = offset.bytes();
495                // Compute absolute begin and end of the range.
496                let (begin, end) = if size >= 0 {
497                    (Some(offset), offset.checked_add(size as u64))
498                } else {
499                    (offset.checked_sub(size.unsigned_abs()), Some(offset))
500                };
501                // Ensure both are within bounds.
502                let in_bounds = begin.is_some() && end.is_some_and(|e| e <= alloc_size.bytes());
503                if !in_bounds {
504                    throw_ub!(PointerOutOfBounds {
505                        alloc_id,
506                        alloc_size,
507                        ptr_offset: tcx.sign_extend_to_target_isize(offset),
508                        inbounds_size: size,
509                        msg,
510                    })
511                }
512
513                Some(ret_val)
514            }
515        })
516    }
517
518    pub(super) fn check_misalign(
519        &self,
520        misaligned: Option<Misalignment>,
521        msg: CheckAlignMsg,
522    ) -> InterpResult<'tcx> {
523        if let Some(misaligned) = misaligned {
524            throw_ub!(AlignmentCheckFailed(misaligned, msg))
525        }
526        interp_ok(())
527    }
528
529    pub(super) fn is_ptr_misaligned(
530        &self,
531        ptr: Pointer<Option<M::Provenance>>,
532        align: Align,
533    ) -> Option<Misalignment> {
534        if !M::enforce_alignment(self) || align.bytes() == 1 {
535            return None;
536        }
537
538        #[inline]
539        fn is_offset_misaligned(offset: u64, align: Align) -> Option<Misalignment> {
540            if offset.is_multiple_of(align.bytes()) {
541                None
542            } else {
543                // The biggest power of two through which `offset` is divisible.
544                let offset_pow2 = 1 << offset.trailing_zeros();
545                Some(Misalignment { has: Align::from_bytes(offset_pow2).unwrap(), required: align })
546            }
547        }
548
549        match self.ptr_try_get_alloc_id(ptr, 0) {
550            Err(addr) => is_offset_misaligned(addr, align),
551            Ok((alloc_id, offset, _prov)) => {
552                let alloc_info = self.get_alloc_info(alloc_id);
553                if let Some(misalign) = M::alignment_check(
554                    self,
555                    alloc_id,
556                    alloc_info.align,
557                    alloc_info.kind,
558                    offset,
559                    align,
560                ) {
561                    Some(misalign)
562                } else if M::Provenance::OFFSET_IS_ADDR {
563                    is_offset_misaligned(ptr.addr().bytes(), align)
564                } else {
565                    // Check allocation alignment and offset alignment.
566                    if alloc_info.align.bytes() < align.bytes() {
567                        Some(Misalignment { has: alloc_info.align, required: align })
568                    } else {
569                        is_offset_misaligned(offset.bytes(), align)
570                    }
571                }
572            }
573        }
574    }
575
576    /// Checks a pointer for misalignment.
577    ///
578    /// The error assumes this is checking the pointer used directly for an access.
579    pub fn check_ptr_align(
580        &self,
581        ptr: Pointer<Option<M::Provenance>>,
582        align: Align,
583    ) -> InterpResult<'tcx> {
584        self.check_misalign(self.is_ptr_misaligned(ptr, align), CheckAlignMsg::AccessedPtr)
585    }
586}
587
588impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
589    /// This function is used by Miri's provenance GC to remove unreachable entries from the dead_alloc_map.
590    pub fn remove_unreachable_allocs(&mut self, reachable_allocs: &FxHashSet<AllocId>) {
591        // Unlike all the other GC helpers where we check if an `AllocId` is found in the interpreter or
592        // is live, here all the IDs in the map are for dead allocations so we don't
593        // need to check for liveness.
594        #[allow(rustc::potential_query_instability)] // Only used from Miri, not queries.
595        self.memory.dead_alloc_map.retain(|id, _| reachable_allocs.contains(id));
596    }
597}
598
599/// Allocation accessors
600impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
601    /// Helper function to obtain a global (tcx) allocation.
602    /// This attempts to return a reference to an existing allocation if
603    /// one can be found in `tcx`. That, however, is only possible if `tcx` and
604    /// this machine use the same pointer provenance, so it is indirected through
605    /// `M::adjust_allocation`.
606    fn get_global_alloc(
607        &self,
608        id: AllocId,
609        is_write: bool,
610    ) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::Provenance, M::AllocExtra, M::Bytes>>> {
611        let (alloc, def_id) = match self.tcx.try_get_global_alloc(id) {
612            Some(GlobalAlloc::Memory(mem)) => {
613                // Memory of a constant or promoted or anonymous memory referenced by a static.
614                (mem, None)
615            }
616            Some(GlobalAlloc::Function { .. }) => throw_ub!(DerefFunctionPointer(id)),
617            Some(GlobalAlloc::VTable(..)) => throw_ub!(DerefVTablePointer(id)),
618            None => throw_ub!(PointerUseAfterFree(id, CheckInAllocMsg::MemoryAccess)),
619            Some(GlobalAlloc::Static(def_id)) => {
620                assert!(self.tcx.is_static(def_id));
621                // Thread-local statics do not have a constant address. They *must* be accessed via
622                // `ThreadLocalRef`; we can never have a pointer to them as a regular constant value.
623                assert!(!self.tcx.is_thread_local_static(def_id));
624                // Notice that every static has two `AllocId` that will resolve to the same
625                // thing here: one maps to `GlobalAlloc::Static`, this is the "lazy" ID,
626                // and the other one is maps to `GlobalAlloc::Memory`, this is returned by
627                // `eval_static_initializer` and it is the "resolved" ID.
628                // The resolved ID is never used by the interpreted program, it is hidden.
629                // This is relied upon for soundness of const-patterns; a pointer to the resolved
630                // ID would "sidestep" the checks that make sure consts do not point to statics!
631                // The `GlobalAlloc::Memory` branch here is still reachable though; when a static
632                // contains a reference to memory that was created during its evaluation (i.e., not
633                // to another static), those inner references only exist in "resolved" form.
634                if self.tcx.is_foreign_item(def_id) {
635                    // This is unreachable in Miri, but can happen in CTFE where we actually *do* support
636                    // referencing arbitrary (declared) extern statics.
637                    throw_unsup!(ExternStatic(def_id));
638                }
639
640                // We don't give a span -- statics don't need that, they cannot be generic or associated.
641                let val = self.ctfe_query(|tcx| tcx.eval_static_initializer(def_id))?;
642                (val, Some(def_id))
643            }
644        };
645        M::before_access_global(self.tcx, &self.machine, id, alloc, def_id, is_write)?;
646        // We got tcx memory. Let the machine initialize its "extra" stuff.
647        M::adjust_global_allocation(
648            self,
649            id, // always use the ID we got as input, not the "hidden" one.
650            alloc.inner(),
651        )
652    }
653
654    /// Gives raw access to the `Allocation`, without bounds or alignment checks.
655    /// The caller is responsible for calling the access hooks!
656    ///
657    /// You almost certainly want to use `get_ptr_alloc`/`get_ptr_alloc_mut` instead.
658    pub fn get_alloc_raw(
659        &self,
660        id: AllocId,
661    ) -> InterpResult<'tcx, &Allocation<M::Provenance, M::AllocExtra, M::Bytes>> {
662        // The error type of the inner closure here is somewhat funny. We have two
663        // ways of "erroring": An actual error, or because we got a reference from
664        // `get_global_alloc` that we can actually use directly without inserting anything anywhere.
665        // So the error type is `InterpResult<'tcx, &Allocation<M::Provenance>>`.
666        let a = self.memory.alloc_map.get_or(id, || {
667            // We have to funnel the `InterpErrorInfo` through a `Result` to match the `get_or` API,
668            // so we use `report_err` for that.
669            let alloc = self.get_global_alloc(id, /*is_write*/ false).report_err().map_err(Err)?;
670            match alloc {
671                Cow::Borrowed(alloc) => {
672                    // We got a ref, cheaply return that as an "error" so that the
673                    // map does not get mutated.
674                    Err(Ok(alloc))
675                }
676                Cow::Owned(alloc) => {
677                    // Need to put it into the map and return a ref to that
678                    let kind = M::GLOBAL_KIND.expect(
679                        "I got a global allocation that I have to copy but the machine does \
680                            not expect that to happen",
681                    );
682                    Ok((MemoryKind::Machine(kind), alloc))
683                }
684            }
685        });
686        // Now unpack that funny error type
687        match a {
688            Ok(a) => interp_ok(&a.1),
689            Err(a) => a.into(),
690        }
691    }
692
693    /// Gives raw, immutable access to the `Allocation` address, without bounds or alignment checks.
694    /// The caller is responsible for calling the access hooks!
695    pub fn get_alloc_bytes_unchecked_raw(&self, id: AllocId) -> InterpResult<'tcx, *const u8> {
696        let alloc = self.get_alloc_raw(id)?;
697        interp_ok(alloc.get_bytes_unchecked_raw())
698    }
699
700    /// Bounds-checked *but not align-checked* allocation access.
701    pub fn get_ptr_alloc<'a>(
702        &'a self,
703        ptr: Pointer<Option<M::Provenance>>,
704        size: Size,
705    ) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
706    {
707        let size_i64 = i64::try_from(size.bytes()).unwrap(); // it would be an error to even ask for more than isize::MAX bytes
708        let ptr_and_alloc = Self::check_and_deref_ptr(
709            self,
710            ptr,
711            size_i64,
712            CheckInAllocMsg::MemoryAccess,
713            |this, alloc_id, offset, prov| {
714                let alloc = this.get_alloc_raw(alloc_id)?;
715                interp_ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc)))
716            },
717        )?;
718        // We want to call the hook on *all* accesses that involve an AllocId, including zero-sized
719        // accesses. That means we cannot rely on the closure above or the `Some` branch below. We
720        // do this after `check_and_deref_ptr` to ensure some basic sanity has already been checked.
721        if !self.memory.validation_in_progress.get() {
722            if let Ok((alloc_id, ..)) = self.ptr_try_get_alloc_id(ptr, size_i64) {
723                M::before_alloc_access(self.tcx, &self.machine, alloc_id)?;
724            }
725        }
726
727        if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc {
728            let range = alloc_range(offset, size);
729            if !self.memory.validation_in_progress.get() {
730                M::before_memory_read(
731                    self.tcx,
732                    &self.machine,
733                    &alloc.extra,
734                    ptr,
735                    (alloc_id, prov),
736                    range,
737                )?;
738            }
739            interp_ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
740        } else {
741            interp_ok(None)
742        }
743    }
744
745    /// Return the `extra` field of the given allocation.
746    pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra> {
747        interp_ok(&self.get_alloc_raw(id)?.extra)
748    }
749
750    /// Return the `mutability` field of the given allocation.
751    pub fn get_alloc_mutability<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, Mutability> {
752        interp_ok(self.get_alloc_raw(id)?.mutability)
753    }
754
755    /// Gives raw mutable access to the `Allocation`, without bounds or alignment checks.
756    /// The caller is responsible for calling the access hooks!
757    ///
758    /// Also returns a ptr to `self.extra` so that the caller can use it in parallel with the
759    /// allocation.
760    ///
761    /// You almost certainly want to use `get_ptr_alloc`/`get_ptr_alloc_mut` instead.
762    pub fn get_alloc_raw_mut(
763        &mut self,
764        id: AllocId,
765    ) -> InterpResult<'tcx, (&mut Allocation<M::Provenance, M::AllocExtra, M::Bytes>, &mut M)> {
766        // We have "NLL problem case #3" here, which cannot be worked around without loss of
767        // efficiency even for the common case where the key is in the map.
768        // <https://rust-lang.github.io/rfcs/2094-nll.html#problem-case-3-conditional-control-flow-across-functions>
769        // (Cannot use `get_mut_or` since `get_global_alloc` needs `&self`, and that boils down to
770        // Miri's `adjust_alloc_root_pointer` needing to look up the size of the allocation.
771        // It could be avoided with a totally separate codepath in Miri for handling the absolute address
772        // of global allocations, but that's not worth it.)
773        if self.memory.alloc_map.get_mut(id).is_none() {
774            // Slow path.
775            // Allocation not found locally, go look global.
776            let alloc = self.get_global_alloc(id, /*is_write*/ true)?;
777            let kind = M::GLOBAL_KIND.expect(
778                "I got a global allocation that I have to copy but the machine does \
779                    not expect that to happen",
780            );
781            self.memory.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned()));
782        }
783
784        let (_kind, alloc) = self.memory.alloc_map.get_mut(id).unwrap();
785        if alloc.mutability.is_not() {
786            throw_ub!(WriteToReadOnly(id))
787        }
788        interp_ok((alloc, &mut self.machine))
789    }
790
791    /// Gives raw, mutable access to the `Allocation` address, without bounds or alignment checks.
792    /// The caller is responsible for calling the access hooks!
793    pub fn get_alloc_bytes_unchecked_raw_mut(
794        &mut self,
795        id: AllocId,
796    ) -> InterpResult<'tcx, *mut u8> {
797        let alloc = self.get_alloc_raw_mut(id)?.0;
798        interp_ok(alloc.get_bytes_unchecked_raw_mut())
799    }
800
801    /// Bounds-checked *but not align-checked* allocation access.
802    pub fn get_ptr_alloc_mut<'a>(
803        &'a mut self,
804        ptr: Pointer<Option<M::Provenance>>,
805        size: Size,
806    ) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
807    {
808        let tcx = self.tcx;
809        let validation_in_progress = self.memory.validation_in_progress.get();
810
811        let size_i64 = i64::try_from(size.bytes()).unwrap(); // it would be an error to even ask for more than isize::MAX bytes
812        let ptr_and_alloc = Self::check_and_deref_ptr(
813            self,
814            ptr,
815            size_i64,
816            CheckInAllocMsg::MemoryAccess,
817            |this, alloc_id, offset, prov| {
818                let (alloc, machine) = this.get_alloc_raw_mut(alloc_id)?;
819                interp_ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc, machine)))
820            },
821        )?;
822
823        if let Some((alloc_id, offset, prov, alloc, machine)) = ptr_and_alloc {
824            let range = alloc_range(offset, size);
825            if !validation_in_progress {
826                // For writes, it's okay to only call those when there actually is a non-zero
827                // amount of bytes to be written: a zero-sized write doesn't manifest anything.
828                M::before_alloc_access(tcx, machine, alloc_id)?;
829                M::before_memory_write(
830                    tcx,
831                    machine,
832                    &mut alloc.extra,
833                    ptr,
834                    (alloc_id, prov),
835                    range,
836                )?;
837            }
838            interp_ok(Some(AllocRefMut { alloc, range, tcx: *tcx, alloc_id }))
839        } else {
840            interp_ok(None)
841        }
842    }
843
844    /// Return the `extra` field of the given allocation.
845    pub fn get_alloc_extra_mut<'a>(
846        &'a mut self,
847        id: AllocId,
848    ) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M)> {
849        let (alloc, machine) = self.get_alloc_raw_mut(id)?;
850        interp_ok((&mut alloc.extra, machine))
851    }
852
853    /// Check whether an allocation is live. This is faster than calling
854    /// [`InterpCx::get_alloc_info`] if all you need to check is whether the kind is
855    /// [`AllocKind::Dead`] because it doesn't have to look up the type and layout of statics.
856    pub fn is_alloc_live(&self, id: AllocId) -> bool {
857        self.memory.alloc_map.contains_key_ref(&id)
858            || self.memory.extra_fn_ptr_map.contains_key(&id)
859            // We check `tcx` last as that has to acquire a lock in `many-seeds` mode.
860            // This also matches the order in `get_alloc_info`.
861            || self.tcx.try_get_global_alloc(id).is_some()
862    }
863
864    /// Obtain the size and alignment of an allocation, even if that allocation has
865    /// been deallocated.
866    pub fn get_alloc_info(&self, id: AllocId) -> AllocInfo {
867        // # Regular allocations
868        // Don't use `self.get_raw` here as that will
869        // a) cause cycles in case `id` refers to a static
870        // b) duplicate a global's allocation in miri
871        if let Some((_, alloc)) = self.memory.alloc_map.get(id) {
872            return AllocInfo::new(
873                alloc.size(),
874                alloc.align,
875                AllocKind::LiveData,
876                alloc.mutability,
877            );
878        }
879
880        // # Function pointers
881        // (both global from `alloc_map` and local from `extra_fn_ptr_map`)
882        if let Some(fn_val) = self.get_fn_alloc(id) {
883            let align = match fn_val {
884                FnVal::Instance(instance) => {
885                    self.tcx.codegen_fn_attrs(instance.def_id()).alignment.unwrap_or(Align::ONE)
886                }
887                // Machine-specific extra functions currently do not support alignment restrictions.
888                FnVal::Other(_) => Align::ONE,
889            };
890
891            return AllocInfo::new(Size::ZERO, align, AllocKind::Function, Mutability::Not);
892        }
893
894        // # Global allocations
895        if let Some(global_alloc) = self.tcx.try_get_global_alloc(id) {
896            let (size, align) = global_alloc.size_and_align(*self.tcx, self.typing_env);
897            let mutbl = global_alloc.mutability(*self.tcx, self.typing_env);
898            let kind = match global_alloc {
899                GlobalAlloc::Static { .. } | GlobalAlloc::Memory { .. } => AllocKind::LiveData,
900                GlobalAlloc::Function { .. } => bug!("We already checked function pointers above"),
901                GlobalAlloc::VTable { .. } => AllocKind::VTable,
902            };
903            return AllocInfo::new(size, align, kind, mutbl);
904        }
905
906        // # Dead pointers
907        let (size, align) = *self
908            .memory
909            .dead_alloc_map
910            .get(&id)
911            .expect("deallocated pointers should all be recorded in `dead_alloc_map`");
912        AllocInfo::new(size, align, AllocKind::Dead, Mutability::Not)
913    }
914
915    /// Obtain the size and alignment of a *live* allocation.
916    fn get_live_alloc_size_and_align(
917        &self,
918        id: AllocId,
919        msg: CheckInAllocMsg,
920    ) -> InterpResult<'tcx, (Size, Align)> {
921        let info = self.get_alloc_info(id);
922        if matches!(info.kind, AllocKind::Dead) {
923            throw_ub!(PointerUseAfterFree(id, msg))
924        }
925        interp_ok((info.size, info.align))
926    }
927
928    fn get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>> {
929        if let Some(extra) = self.memory.extra_fn_ptr_map.get(&id) {
930            Some(FnVal::Other(*extra))
931        } else {
932            match self.tcx.try_get_global_alloc(id) {
933                Some(GlobalAlloc::Function { instance, .. }) => Some(FnVal::Instance(instance)),
934                _ => None,
935            }
936        }
937    }
938
939    pub fn get_ptr_fn(
940        &self,
941        ptr: Pointer<Option<M::Provenance>>,
942    ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> {
943        trace!("get_ptr_fn({:?})", ptr);
944        let (alloc_id, offset, _prov) = self.ptr_get_alloc_id(ptr, 0)?;
945        if offset.bytes() != 0 {
946            throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset)))
947        }
948        self.get_fn_alloc(alloc_id)
949            .ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))))
950            .into()
951    }
952
953    /// Get the dynamic type of the given vtable pointer.
954    /// If `expected_trait` is `Some`, it must be a vtable for the given trait.
955    pub fn get_ptr_vtable_ty(
956        &self,
957        ptr: Pointer<Option<M::Provenance>>,
958        expected_trait: Option<&'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>>,
959    ) -> InterpResult<'tcx, Ty<'tcx>> {
960        trace!("get_ptr_vtable({:?})", ptr);
961        let (alloc_id, offset, _tag) = self.ptr_get_alloc_id(ptr, 0)?;
962        if offset.bytes() != 0 {
963            throw_ub!(InvalidVTablePointer(Pointer::new(alloc_id, offset)))
964        }
965        let Some(GlobalAlloc::VTable(ty, vtable_dyn_type)) =
966            self.tcx.try_get_global_alloc(alloc_id)
967        else {
968            throw_ub!(InvalidVTablePointer(Pointer::new(alloc_id, offset)))
969        };
970        if let Some(expected_dyn_type) = expected_trait {
971            self.check_vtable_for_type(vtable_dyn_type, expected_dyn_type)?;
972        }
973        interp_ok(ty)
974    }
975
976    pub fn alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> {
977        self.get_alloc_raw_mut(id)?.0.mutability = Mutability::Not;
978        interp_ok(())
979    }
980
981    /// Visit all allocations reachable from the given start set, by recursively traversing the
982    /// provenance information of those allocations.
983    pub fn visit_reachable_allocs(
984        &mut self,
985        start: Vec<AllocId>,
986        mut visit: impl FnMut(&mut Self, AllocId, &AllocInfo) -> InterpResult<'tcx>,
987    ) -> InterpResult<'tcx> {
988        let mut done = FxHashSet::default();
989        let mut todo = start;
990        while let Some(id) = todo.pop() {
991            if !done.insert(id) {
992                // We already saw this allocation before, don't process it again.
993                continue;
994            }
995            let info = self.get_alloc_info(id);
996
997            // Recurse, if there is data here.
998            // Do this *before* invoking the callback, as the callback might mutate the
999            // allocation and e.g. replace all provenance by wildcards!
1000            if matches!(info.kind, AllocKind::LiveData) {
1001                let alloc = self.get_alloc_raw(id)?;
1002                for prov in alloc.provenance().provenances() {
1003                    if let Some(id) = prov.get_alloc_id() {
1004                        todo.push(id);
1005                    }
1006                }
1007            }
1008
1009            // Call the callback.
1010            visit(self, id, &info)?;
1011        }
1012        interp_ok(())
1013    }
1014
1015    /// Create a lazy debug printer that prints the given allocation and all allocations it points
1016    /// to, recursively.
1017    #[must_use]
1018    pub fn dump_alloc<'a>(&'a self, id: AllocId) -> DumpAllocs<'a, 'tcx, M> {
1019        self.dump_allocs(vec![id])
1020    }
1021
1022    /// Create a lazy debug printer for a list of allocations and all allocations they point to,
1023    /// recursively.
1024    #[must_use]
1025    pub fn dump_allocs<'a>(&'a self, mut allocs: Vec<AllocId>) -> DumpAllocs<'a, 'tcx, M> {
1026        allocs.sort();
1027        allocs.dedup();
1028        DumpAllocs { ecx: self, allocs }
1029    }
1030
1031    /// Print the allocation's bytes, without any nested allocations.
1032    pub fn print_alloc_bytes_for_diagnostics(&self, id: AllocId) -> String {
1033        // Using the "raw" access to avoid the `before_alloc_read` hook, we specifically
1034        // want to be able to read all memory for diagnostics, even if that is cyclic.
1035        let alloc = self.get_alloc_raw(id).unwrap();
1036        let mut bytes = String::new();
1037        if alloc.size() != Size::ZERO {
1038            bytes = "\n".into();
1039            // FIXME(translation) there might be pieces that are translatable.
1040            rustc_middle::mir::pretty::write_allocation_bytes(*self.tcx, alloc, &mut bytes, "    ")
1041                .unwrap();
1042        }
1043        bytes
1044    }
1045
1046    /// Find leaked allocations, remove them from memory and return them. Allocations reachable from
1047    /// `static_roots` or a `Global` allocation are not considered leaked, as well as leaks whose
1048    /// kind's `may_leak()` returns true.
1049    ///
1050    /// This is highly destructive, no more execution can happen after this!
1051    pub fn take_leaked_allocations(
1052        &mut self,
1053        static_roots: impl FnOnce(&Self) -> &[AllocId],
1054    ) -> Vec<(AllocId, MemoryKind<M::MemoryKind>, Allocation<M::Provenance, M::AllocExtra, M::Bytes>)>
1055    {
1056        // Collect the set of allocations that are *reachable* from `Global` allocations.
1057        let reachable = {
1058            let mut reachable = FxHashSet::default();
1059            let global_kind = M::GLOBAL_KIND.map(MemoryKind::Machine);
1060            let mut todo: Vec<_> =
1061                self.memory.alloc_map.filter_map_collect(move |&id, &(kind, _)| {
1062                    if Some(kind) == global_kind { Some(id) } else { None }
1063                });
1064            todo.extend(static_roots(self));
1065            while let Some(id) = todo.pop() {
1066                if reachable.insert(id) {
1067                    // This is a new allocation, add the allocations it points to `todo`.
1068                    // We only need to care about `alloc_map` memory here, as entirely unchanged
1069                    // global memory cannot point to memory relevant for the leak check.
1070                    if let Some((_, alloc)) = self.memory.alloc_map.get(id) {
1071                        todo.extend(
1072                            alloc.provenance().provenances().filter_map(|prov| prov.get_alloc_id()),
1073                        );
1074                    }
1075                }
1076            }
1077            reachable
1078        };
1079
1080        // All allocations that are *not* `reachable` and *not* `may_leak` are considered leaking.
1081        let leaked: Vec<_> = self.memory.alloc_map.filter_map_collect(|&id, &(kind, _)| {
1082            if kind.may_leak() || reachable.contains(&id) { None } else { Some(id) }
1083        });
1084        let mut result = Vec::new();
1085        for &id in leaked.iter() {
1086            let (kind, alloc) = self.memory.alloc_map.remove(&id).unwrap();
1087            result.push((id, kind, alloc));
1088        }
1089        result
1090    }
1091
1092    /// Runs the closure in "validation" mode, which means the machine's memory read hooks will be
1093    /// suppressed. Needless to say, this must only be set with great care! Cannot be nested.
1094    ///
1095    /// We do this so Miri's allocation access tracking does not show the validation
1096    /// reads as spurious accesses.
1097    pub fn run_for_validation_mut<R>(&mut self, f: impl FnOnce(&mut Self) -> R) -> R {
1098        // This deliberately uses `==` on `bool` to follow the pattern
1099        // `assert!(val.replace(new) == old)`.
1100        assert!(
1101            self.memory.validation_in_progress.replace(true) == false,
1102            "`validation_in_progress` was already set"
1103        );
1104        let res = f(self);
1105        assert!(
1106            self.memory.validation_in_progress.replace(false) == true,
1107            "`validation_in_progress` was unset by someone else"
1108        );
1109        res
1110    }
1111
1112    /// Runs the closure in "validation" mode, which means the machine's memory read hooks will be
1113    /// suppressed. Needless to say, this must only be set with great care! Cannot be nested.
1114    ///
1115    /// We do this so Miri's allocation access tracking does not show the validation
1116    /// reads as spurious accesses.
1117    pub fn run_for_validation_ref<R>(&self, f: impl FnOnce(&Self) -> R) -> R {
1118        // This deliberately uses `==` on `bool` to follow the pattern
1119        // `assert!(val.replace(new) == old)`.
1120        assert!(
1121            self.memory.validation_in_progress.replace(true) == false,
1122            "`validation_in_progress` was already set"
1123        );
1124        let res = f(self);
1125        assert!(
1126            self.memory.validation_in_progress.replace(false) == true,
1127            "`validation_in_progress` was unset by someone else"
1128        );
1129        res
1130    }
1131
1132    pub(super) fn validation_in_progress(&self) -> bool {
1133        self.memory.validation_in_progress.get()
1134    }
1135}
1136
1137#[doc(hidden)]
1138/// There's no way to use this directly, it's just a helper struct for the `dump_alloc(s)` methods.
1139pub struct DumpAllocs<'a, 'tcx, M: Machine<'tcx>> {
1140    ecx: &'a InterpCx<'tcx, M>,
1141    allocs: Vec<AllocId>,
1142}
1143
1144impl<'a, 'tcx, M: Machine<'tcx>> std::fmt::Debug for DumpAllocs<'a, 'tcx, M> {
1145    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1146        // Cannot be a closure because it is generic in `Prov`, `Extra`.
1147        fn write_allocation_track_relocs<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(
1148            fmt: &mut std::fmt::Formatter<'_>,
1149            tcx: TyCtxt<'tcx>,
1150            allocs_to_print: &mut VecDeque<AllocId>,
1151            alloc: &Allocation<Prov, Extra, Bytes>,
1152        ) -> std::fmt::Result {
1153            for alloc_id in alloc.provenance().provenances().filter_map(|prov| prov.get_alloc_id())
1154            {
1155                allocs_to_print.push_back(alloc_id);
1156            }
1157            write!(fmt, "{}", display_allocation(tcx, alloc))
1158        }
1159
1160        let mut allocs_to_print: VecDeque<_> = self.allocs.iter().copied().collect();
1161        // `allocs_printed` contains all allocations that we have already printed.
1162        let mut allocs_printed = FxHashSet::default();
1163
1164        while let Some(id) = allocs_to_print.pop_front() {
1165            if !allocs_printed.insert(id) {
1166                // Already printed, so skip this.
1167                continue;
1168            }
1169
1170            write!(fmt, "{id:?}")?;
1171            match self.ecx.memory.alloc_map.get(id) {
1172                Some((kind, alloc)) => {
1173                    // normal alloc
1174                    write!(fmt, " ({kind}, ")?;
1175                    write_allocation_track_relocs(
1176                        &mut *fmt,
1177                        *self.ecx.tcx,
1178                        &mut allocs_to_print,
1179                        alloc,
1180                    )?;
1181                }
1182                None => {
1183                    // global alloc
1184                    match self.ecx.tcx.try_get_global_alloc(id) {
1185                        Some(GlobalAlloc::Memory(alloc)) => {
1186                            write!(fmt, " (unchanged global, ")?;
1187                            write_allocation_track_relocs(
1188                                &mut *fmt,
1189                                *self.ecx.tcx,
1190                                &mut allocs_to_print,
1191                                alloc.inner(),
1192                            )?;
1193                        }
1194                        Some(GlobalAlloc::Function { instance, .. }) => {
1195                            write!(fmt, " (fn: {instance})")?;
1196                        }
1197                        Some(GlobalAlloc::VTable(ty, dyn_ty)) => {
1198                            write!(fmt, " (vtable: impl {dyn_ty} for {ty})")?;
1199                        }
1200                        Some(GlobalAlloc::Static(did)) => {
1201                            write!(fmt, " (static: {})", self.ecx.tcx.def_path_str(did))?;
1202                        }
1203                        None => {
1204                            write!(fmt, " (deallocated)")?;
1205                        }
1206                    }
1207                }
1208            }
1209            writeln!(fmt)?;
1210        }
1211        Ok(())
1212    }
1213}
1214
1215/// Reading and writing.
1216impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>
1217    AllocRefMut<'a, 'tcx, Prov, Extra, Bytes>
1218{
1219    pub fn as_ref<'b>(&'b self) -> AllocRef<'b, 'tcx, Prov, Extra, Bytes> {
1220        AllocRef { alloc: self.alloc, range: self.range, tcx: self.tcx, alloc_id: self.alloc_id }
1221    }
1222
1223    /// `range` is relative to this allocation reference, not the base of the allocation.
1224    pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> {
1225        let range = self.range.subrange(range);
1226        debug!("write_scalar at {:?}{range:?}: {val:?}", self.alloc_id);
1227
1228        self.alloc
1229            .write_scalar(&self.tcx, range, val)
1230            .map_err(|e| e.to_interp_error(self.alloc_id))
1231            .into()
1232    }
1233
1234    /// `offset` is relative to this allocation reference, not the base of the allocation.
1235    pub fn write_ptr_sized(&mut self, offset: Size, val: Scalar<Prov>) -> InterpResult<'tcx> {
1236        self.write_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size()), val)
1237    }
1238
1239    /// Mark the given sub-range (relative to this allocation reference) as uninitialized.
1240    pub fn write_uninit(&mut self, range: AllocRange) -> InterpResult<'tcx> {
1241        let range = self.range.subrange(range);
1242
1243        self.alloc
1244            .write_uninit(&self.tcx, range)
1245            .map_err(|e| e.to_interp_error(self.alloc_id))
1246            .into()
1247    }
1248
1249    /// Mark the entire referenced range as uninitialized
1250    pub fn write_uninit_full(&mut self) -> InterpResult<'tcx> {
1251        self.alloc
1252            .write_uninit(&self.tcx, self.range)
1253            .map_err(|e| e.to_interp_error(self.alloc_id))
1254            .into()
1255    }
1256
1257    /// Remove all provenance in the reference range.
1258    pub fn clear_provenance(&mut self) -> InterpResult<'tcx> {
1259        self.alloc
1260            .clear_provenance(&self.tcx, self.range)
1261            .map_err(|e| e.to_interp_error(self.alloc_id))
1262            .into()
1263    }
1264}
1265
1266impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Prov, Extra, Bytes> {
1267    /// `range` is relative to this allocation reference, not the base of the allocation.
1268    pub fn read_scalar(
1269        &self,
1270        range: AllocRange,
1271        read_provenance: bool,
1272    ) -> InterpResult<'tcx, Scalar<Prov>> {
1273        let range = self.range.subrange(range);
1274        self.alloc
1275            .read_scalar(&self.tcx, range, read_provenance)
1276            .map_err(|e| e.to_interp_error(self.alloc_id))
1277            .into()
1278    }
1279
1280    /// `range` is relative to this allocation reference, not the base of the allocation.
1281    pub fn read_integer(&self, range: AllocRange) -> InterpResult<'tcx, Scalar<Prov>> {
1282        self.read_scalar(range, /*read_provenance*/ false)
1283    }
1284
1285    /// `offset` is relative to this allocation reference, not the base of the allocation.
1286    pub fn read_pointer(&self, offset: Size) -> InterpResult<'tcx, Scalar<Prov>> {
1287        self.read_scalar(
1288            alloc_range(offset, self.tcx.data_layout().pointer_size()),
1289            /*read_provenance*/ true,
1290        )
1291    }
1292
1293    /// `range` is relative to this allocation reference, not the base of the allocation.
1294    pub fn get_bytes_strip_provenance<'b>(&'b self) -> InterpResult<'tcx, &'a [u8]> {
1295        self.alloc
1296            .get_bytes_strip_provenance(&self.tcx, self.range)
1297            .map_err(|e| e.to_interp_error(self.alloc_id))
1298            .into()
1299    }
1300
1301    /// Returns whether the allocation has provenance anywhere in the range of the `AllocRef`.
1302    pub fn has_provenance(&self) -> bool {
1303        !self.alloc.provenance().range_empty(self.range, &self.tcx)
1304    }
1305}
1306
1307impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
1308    /// Reads the given number of bytes from memory, and strips their provenance if possible.
1309    /// Returns them as a slice.
1310    ///
1311    /// Performs appropriate bounds checks.
1312    pub fn read_bytes_ptr_strip_provenance(
1313        &self,
1314        ptr: Pointer<Option<M::Provenance>>,
1315        size: Size,
1316    ) -> InterpResult<'tcx, &[u8]> {
1317        let Some(alloc_ref) = self.get_ptr_alloc(ptr, size)? else {
1318            // zero-sized access
1319            return interp_ok(&[]);
1320        };
1321        // Side-step AllocRef and directly access the underlying bytes more efficiently.
1322        // (We are staying inside the bounds here so all is good.)
1323        interp_ok(
1324            alloc_ref
1325                .alloc
1326                .get_bytes_strip_provenance(&alloc_ref.tcx, alloc_ref.range)
1327                .map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?,
1328        )
1329    }
1330
1331    /// Writes the given stream of bytes into memory.
1332    ///
1333    /// Performs appropriate bounds checks.
1334    pub fn write_bytes_ptr(
1335        &mut self,
1336        ptr: Pointer<Option<M::Provenance>>,
1337        src: impl IntoIterator<Item = u8>,
1338    ) -> InterpResult<'tcx> {
1339        let mut src = src.into_iter();
1340        let (lower, upper) = src.size_hint();
1341        let len = upper.expect("can only write bounded iterators");
1342        assert_eq!(lower, len, "can only write iterators with a precise length");
1343
1344        let size = Size::from_bytes(len);
1345        let Some(alloc_ref) = self.get_ptr_alloc_mut(ptr, size)? else {
1346            // zero-sized access
1347            assert_matches!(src.next(), None, "iterator said it was empty but returned an element");
1348            return interp_ok(());
1349        };
1350
1351        // Side-step AllocRef and directly access the underlying bytes more efficiently.
1352        // (We are staying inside the bounds here and all bytes do get overwritten so all is good.)
1353        let alloc_id = alloc_ref.alloc_id;
1354        let bytes = alloc_ref
1355            .alloc
1356            .get_bytes_unchecked_for_overwrite(&alloc_ref.tcx, alloc_ref.range)
1357            .map_err(move |e| e.to_interp_error(alloc_id))?;
1358        // `zip` would stop when the first iterator ends; we want to definitely
1359        // cover all of `bytes`.
1360        for dest in bytes {
1361            *dest = src.next().expect("iterator was shorter than it said it would be");
1362        }
1363        assert_matches!(src.next(), None, "iterator was longer than it said it would be");
1364        interp_ok(())
1365    }
1366
1367    pub fn mem_copy(
1368        &mut self,
1369        src: Pointer<Option<M::Provenance>>,
1370        dest: Pointer<Option<M::Provenance>>,
1371        size: Size,
1372        nonoverlapping: bool,
1373    ) -> InterpResult<'tcx> {
1374        self.mem_copy_repeatedly(src, dest, size, 1, nonoverlapping)
1375    }
1376
1377    /// Performs `num_copies` many copies of `size` many bytes from `src` to `dest + i*size` (where
1378    /// `i` is the index of the copy).
1379    ///
1380    /// Either `nonoverlapping` must be true or `num_copies` must be 1; doing repeated copies that
1381    /// may overlap is not supported.
1382    pub fn mem_copy_repeatedly(
1383        &mut self,
1384        src: Pointer<Option<M::Provenance>>,
1385        dest: Pointer<Option<M::Provenance>>,
1386        size: Size,
1387        num_copies: u64,
1388        nonoverlapping: bool,
1389    ) -> InterpResult<'tcx> {
1390        let tcx = self.tcx;
1391        // We need to do our own bounds-checks.
1392        let src_parts = self.get_ptr_access(src, size)?;
1393        let dest_parts = self.get_ptr_access(dest, size * num_copies)?; // `Size` multiplication
1394
1395        // Similar to `get_ptr_alloc`, we need to call `before_alloc_access` even for zero-sized
1396        // reads. However, just like in `get_ptr_alloc_mut`, the write part is okay to skip for
1397        // zero-sized writes.
1398        if let Ok((alloc_id, ..)) = self.ptr_try_get_alloc_id(src, size.bytes().try_into().unwrap())
1399        {
1400            M::before_alloc_access(tcx, &self.machine, alloc_id)?;
1401        }
1402
1403        // FIXME: we look up both allocations twice here, once before for the `check_ptr_access`
1404        // and once below to get the underlying `&[mut] Allocation`.
1405
1406        // Source alloc preparations and access hooks.
1407        let Some((src_alloc_id, src_offset, src_prov)) = src_parts else {
1408            // Zero-sized *source*, that means dest is also zero-sized and we have nothing to do.
1409            return interp_ok(());
1410        };
1411        let src_alloc = self.get_alloc_raw(src_alloc_id)?;
1412        let src_range = alloc_range(src_offset, size);
1413        assert!(!self.memory.validation_in_progress.get(), "we can't be copying during validation");
1414
1415        // Trigger read hook.
1416        // For the overlapping case, it is crucial that we trigger the read hook
1417        // before the write hook -- the aliasing model cares about the order.
1418        M::before_memory_read(
1419            tcx,
1420            &self.machine,
1421            &src_alloc.extra,
1422            src,
1423            (src_alloc_id, src_prov),
1424            src_range,
1425        )?;
1426        // We need the `dest` ptr for the next operation, so we get it now.
1427        // We already did the source checks and called the hooks so we are good to return early.
1428        let Some((dest_alloc_id, dest_offset, dest_prov)) = dest_parts else {
1429            // Zero-sized *destination*.
1430            return interp_ok(());
1431        };
1432
1433        // Prepare getting source provenance.
1434        let src_bytes = src_alloc.get_bytes_unchecked(src_range).as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation
1435        // first copy the provenance to a temporary buffer, because
1436        // `get_bytes_mut` will clear the provenance, which is correct,
1437        // since we don't want to keep any provenance at the target.
1438        // This will also error if copying partial provenance is not supported.
1439        let provenance = src_alloc
1440            .provenance()
1441            .prepare_copy(src_range, dest_offset, num_copies, self)
1442            .map_err(|e| e.to_interp_error(src_alloc_id))?;
1443        // Prepare a copy of the initialization mask.
1444        let init = src_alloc.init_mask().prepare_copy(src_range);
1445
1446        // Destination alloc preparations...
1447        let (dest_alloc, machine) = self.get_alloc_raw_mut(dest_alloc_id)?;
1448        let dest_range = alloc_range(dest_offset, size * num_copies);
1449        // ...and access hooks.
1450        M::before_alloc_access(tcx, machine, dest_alloc_id)?;
1451        M::before_memory_write(
1452            tcx,
1453            machine,
1454            &mut dest_alloc.extra,
1455            dest,
1456            (dest_alloc_id, dest_prov),
1457            dest_range,
1458        )?;
1459        // Yes we do overwrite all bytes in `dest_bytes`.
1460        let dest_bytes = dest_alloc
1461            .get_bytes_unchecked_for_overwrite_ptr(&tcx, dest_range)
1462            .map_err(|e| e.to_interp_error(dest_alloc_id))?
1463            .as_mut_ptr();
1464
1465        if init.no_bytes_init() {
1466            // Fast path: If all bytes are `uninit` then there is nothing to copy. The target range
1467            // is marked as uninitialized but we otherwise omit changing the byte representation which may
1468            // be arbitrary for uninitialized bytes.
1469            // This also avoids writing to the target bytes so that the backing allocation is never
1470            // touched if the bytes stay uninitialized for the whole interpreter execution. On contemporary
1471            // operating system this can avoid physically allocating the page.
1472            dest_alloc
1473                .write_uninit(&tcx, dest_range)
1474                .map_err(|e| e.to_interp_error(dest_alloc_id))?;
1475            // We can forget about the provenance, this is all not initialized anyway.
1476            return interp_ok(());
1477        }
1478
1479        // SAFE: The above indexing would have panicked if there weren't at least `size` bytes
1480        // behind `src` and `dest`. Also, we use the overlapping-safe `ptr::copy` if `src` and
1481        // `dest` could possibly overlap.
1482        // The pointers above remain valid even if the `HashMap` table is moved around because they
1483        // point into the `Vec` storing the bytes.
1484        unsafe {
1485            if src_alloc_id == dest_alloc_id {
1486                if nonoverlapping {
1487                    // `Size` additions
1488                    if (src_offset <= dest_offset && src_offset + size > dest_offset)
1489                        || (dest_offset <= src_offset && dest_offset + size > src_offset)
1490                    {
1491                        throw_ub_custom!(fluent::const_eval_copy_nonoverlapping_overlapping);
1492                    }
1493                }
1494            }
1495            if num_copies > 1 {
1496                assert!(nonoverlapping, "multi-copy only supported in non-overlapping mode");
1497            }
1498
1499            let size_in_bytes = size.bytes_usize();
1500            // For particularly large arrays (where this is perf-sensitive) it's common that
1501            // we're writing a single byte repeatedly. So, optimize that case to a memset.
1502            if size_in_bytes == 1 {
1503                debug_assert!(num_copies >= 1); // we already handled the zero-sized cases above.
1504                // SAFETY: `src_bytes` would be read from anyway by `copy` below (num_copies >= 1).
1505                let value = *src_bytes;
1506                dest_bytes.write_bytes(value, (size * num_copies).bytes_usize());
1507            } else if src_alloc_id == dest_alloc_id {
1508                let mut dest_ptr = dest_bytes;
1509                for _ in 0..num_copies {
1510                    // Here we rely on `src` and `dest` being non-overlapping if there is more than
1511                    // one copy.
1512                    ptr::copy(src_bytes, dest_ptr, size_in_bytes);
1513                    dest_ptr = dest_ptr.add(size_in_bytes);
1514                }
1515            } else {
1516                let mut dest_ptr = dest_bytes;
1517                for _ in 0..num_copies {
1518                    ptr::copy_nonoverlapping(src_bytes, dest_ptr, size_in_bytes);
1519                    dest_ptr = dest_ptr.add(size_in_bytes);
1520                }
1521            }
1522        }
1523
1524        // now fill in all the "init" data
1525        dest_alloc.init_mask_apply_copy(
1526            init,
1527            alloc_range(dest_offset, size), // just a single copy (i.e., not full `dest_range`)
1528            num_copies,
1529        );
1530        // copy the provenance to the destination
1531        dest_alloc.provenance_apply_copy(provenance);
1532
1533        interp_ok(())
1534    }
1535}
1536
1537/// Machine pointer introspection.
1538impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
1539    /// Test if this value might be null.
1540    /// If the machine does not support ptr-to-int casts, this is conservative.
1541    pub fn scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool> {
1542        match scalar.try_to_scalar_int() {
1543            Ok(int) => interp_ok(int.is_null()),
1544            Err(_) => {
1545                // We can't cast this pointer to an integer. Can only happen during CTFE.
1546                let ptr = scalar.to_pointer(self)?;
1547                match self.ptr_try_get_alloc_id(ptr, 0) {
1548                    Ok((alloc_id, offset, _)) => {
1549                        let info = self.get_alloc_info(alloc_id);
1550                        // If the pointer is in-bounds (including "at the end"), it is definitely not null.
1551                        if offset <= info.size {
1552                            return interp_ok(false);
1553                        }
1554                        // If the allocation is N-aligned, and the offset is not divisible by N,
1555                        // then `base + offset` has a non-zero remainder after division by `N`,
1556                        // which means `base + offset` cannot be null.
1557                        if !offset.bytes().is_multiple_of(info.align.bytes()) {
1558                            return interp_ok(false);
1559                        }
1560                        // We don't know enough, this might be null.
1561                        interp_ok(true)
1562                    }
1563                    Err(_offset) => bug!("a non-int scalar is always a pointer"),
1564                }
1565            }
1566        }
1567    }
1568
1569    /// Turning a "maybe pointer" into a proper pointer (and some information
1570    /// about where it points), or an absolute address.
1571    ///
1572    /// `size` says how many bytes of memory are expected at that pointer. This is largely only used
1573    /// for error messages; however, the *sign* of `size` can be used to disambiguate situations
1574    /// where a wildcard pointer sits right in between two allocations.
1575    /// It is almost always okay to just set the size to 0; this will be treated like a positive size
1576    /// for handling wildcard pointers.
1577    ///
1578    /// The result must be used immediately; it is not allowed to convert
1579    /// the returned data back into a `Pointer` and store that in machine state.
1580    /// (In fact that's not even possible since `M::ProvenanceExtra` is generic and
1581    /// we don't have an operation to turn it back into `M::Provenance`.)
1582    pub fn ptr_try_get_alloc_id(
1583        &self,
1584        ptr: Pointer<Option<M::Provenance>>,
1585        size: i64,
1586    ) -> Result<(AllocId, Size, M::ProvenanceExtra), u64> {
1587        match ptr.into_pointer_or_addr() {
1588            Ok(ptr) => match M::ptr_get_alloc(self, ptr, size) {
1589                Some((alloc_id, offset, extra)) => Ok((alloc_id, offset, extra)),
1590                None => {
1591                    assert!(M::Provenance::OFFSET_IS_ADDR);
1592                    // Offset is absolute, as we just asserted.
1593                    let (_, addr) = ptr.into_raw_parts();
1594                    Err(addr.bytes())
1595                }
1596            },
1597            Err(addr) => Err(addr.bytes()),
1598        }
1599    }
1600
1601    /// Turning a "maybe pointer" into a proper pointer (and some information about where it points).
1602    ///
1603    /// `size` says how many bytes of memory are expected at that pointer. This is largely only used
1604    /// for error messages; however, the *sign* of `size` can be used to disambiguate situations
1605    /// where a wildcard pointer sits right in between two allocations.
1606    /// It is almost always okay to just set the size to 0; this will be treated like a positive size
1607    /// for handling wildcard pointers.
1608    ///
1609    /// The result must be used immediately; it is not allowed to convert
1610    /// the returned data back into a `Pointer` and store that in machine state.
1611    /// (In fact that's not even possible since `M::ProvenanceExtra` is generic and
1612    /// we don't have an operation to turn it back into `M::Provenance`.)
1613    #[inline(always)]
1614    pub fn ptr_get_alloc_id(
1615        &self,
1616        ptr: Pointer<Option<M::Provenance>>,
1617        size: i64,
1618    ) -> InterpResult<'tcx, (AllocId, Size, M::ProvenanceExtra)> {
1619        self.ptr_try_get_alloc_id(ptr, size)
1620            .map_err(|offset| {
1621                err_ub!(DanglingIntPointer {
1622                    addr: offset,
1623                    inbounds_size: size,
1624                    msg: CheckInAllocMsg::Dereferenceable
1625                })
1626            })
1627            .into()
1628    }
1629}