1use std::any::Any;
5use std::borrow::Cow;
6use std::cell::{Cell, RefCell};
7use std::collections::hash_map::Entry;
8use std::path::Path;
9use std::rc::Rc;
10use std::{fmt, process};
11
12use rand::rngs::StdRng;
13use rand::{Rng, SeedableRng};
14use rustc_abi::{Align, ExternAbi, Size};
15use rustc_apfloat::{Float, FloatConvert};
16use rustc_attr_data_structures::InlineAttr;
17use rustc_data_structures::fx::{FxHashMap, FxHashSet};
18#[allow(unused)]
19use rustc_data_structures::static_assert_size;
20use rustc_middle::mir;
21use rustc_middle::query::TyCtxtAt;
22use rustc_middle::ty::layout::{
23 HasTyCtxt, HasTypingEnv, LayoutCx, LayoutError, LayoutOf, TyAndLayout,
24};
25use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
26use rustc_session::config::InliningThreshold;
27use rustc_span::def_id::{CrateNum, DefId};
28use rustc_span::{Span, SpanData, Symbol};
29use rustc_target::callconv::FnAbi;
30
31use crate::alloc_addresses::EvalContextExt;
32use crate::concurrency::cpu_affinity::{self, CpuAffinityMask};
33use crate::concurrency::data_race::{self, NaReadType, NaWriteType};
34use crate::concurrency::{AllocDataRaceHandler, GenmcCtx, GlobalDataRaceHandler, weak_memory};
35use crate::*;
36
37pub const SIGRTMIN: i32 = 34;
41
42pub const SIGRTMAX: i32 = 42;
46
47const ADDRS_PER_ANON_GLOBAL: usize = 32;
51
52pub struct FrameExtra<'tcx> {
54 pub borrow_tracker: Option<borrow_tracker::FrameState>,
56
57 pub catch_unwind: Option<CatchUnwindData<'tcx>>,
61
62 pub timing: Option<measureme::DetachedTiming>,
66
67 pub is_user_relevant: bool,
72
73 salt: usize,
78
79 pub data_race: Option<data_race::FrameState>,
81}
82
83impl<'tcx> std::fmt::Debug for FrameExtra<'tcx> {
84 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85 let FrameExtra {
87 borrow_tracker,
88 catch_unwind,
89 timing: _,
90 is_user_relevant,
91 salt,
92 data_race,
93 } = self;
94 f.debug_struct("FrameData")
95 .field("borrow_tracker", borrow_tracker)
96 .field("catch_unwind", catch_unwind)
97 .field("is_user_relevant", is_user_relevant)
98 .field("salt", salt)
99 .field("data_race", data_race)
100 .finish()
101 }
102}
103
104impl VisitProvenance for FrameExtra<'_> {
105 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
106 let FrameExtra {
107 catch_unwind,
108 borrow_tracker,
109 timing: _,
110 is_user_relevant: _,
111 salt: _,
112 data_race: _,
113 } = self;
114
115 catch_unwind.visit_provenance(visit);
116 borrow_tracker.visit_provenance(visit);
117 }
118}
119
120#[derive(Debug, Copy, Clone, PartialEq, Eq)]
122pub enum MiriMemoryKind {
123 Rust,
125 Miri,
127 C,
129 WinHeap,
131 WinLocal,
133 Machine,
136 Runtime,
139 Global,
142 ExternStatic,
145 Tls,
148 Mmap,
150}
151
152impl From<MiriMemoryKind> for MemoryKind {
153 #[inline(always)]
154 fn from(kind: MiriMemoryKind) -> MemoryKind {
155 MemoryKind::Machine(kind)
156 }
157}
158
159impl MayLeak for MiriMemoryKind {
160 #[inline(always)]
161 fn may_leak(self) -> bool {
162 use self::MiriMemoryKind::*;
163 match self {
164 Rust | Miri | C | WinHeap | WinLocal | Runtime => false,
165 Machine | Global | ExternStatic | Tls | Mmap => true,
166 }
167 }
168}
169
170impl MiriMemoryKind {
171 fn should_save_allocation_span(self) -> bool {
173 use self::MiriMemoryKind::*;
174 match self {
175 Rust | Miri | C | WinHeap | WinLocal | Mmap => true,
177 Machine | Global | ExternStatic | Tls | Runtime => false,
179 }
180 }
181}
182
183impl fmt::Display for MiriMemoryKind {
184 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185 use self::MiriMemoryKind::*;
186 match self {
187 Rust => write!(f, "Rust heap"),
188 Miri => write!(f, "Miri bare-metal heap"),
189 C => write!(f, "C heap"),
190 WinHeap => write!(f, "Windows heap"),
191 WinLocal => write!(f, "Windows local memory"),
192 Machine => write!(f, "machine-managed memory"),
193 Runtime => write!(f, "language runtime memory"),
194 Global => write!(f, "global (static or const)"),
195 ExternStatic => write!(f, "extern static"),
196 Tls => write!(f, "thread-local static"),
197 Mmap => write!(f, "mmap"),
198 }
199 }
200}
201
202pub type MemoryKind = interpret::MemoryKind<MiriMemoryKind>;
203
204#[derive(Clone, Copy, PartialEq, Eq, Hash)]
210pub enum Provenance {
211 Concrete {
214 alloc_id: AllocId,
215 tag: BorTag,
217 },
218 Wildcard,
235}
236
237#[derive(Copy, Clone, PartialEq)]
239pub enum ProvenanceExtra {
240 Concrete(BorTag),
241 Wildcard,
242}
243
244#[cfg(target_pointer_width = "64")]
245static_assert_size!(StrictPointer, 24);
246#[cfg(target_pointer_width = "64")]
250static_assert_size!(Scalar, 32);
251
252impl fmt::Debug for Provenance {
253 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
254 match self {
255 Provenance::Concrete { alloc_id, tag } => {
256 if f.alternate() {
258 write!(f, "[{alloc_id:#?}]")?;
259 } else {
260 write!(f, "[{alloc_id:?}]")?;
261 }
262 write!(f, "{tag:?}")?;
264 }
265 Provenance::Wildcard => {
266 write!(f, "[wildcard]")?;
267 }
268 }
269 Ok(())
270 }
271}
272
273impl interpret::Provenance for Provenance {
274 const OFFSET_IS_ADDR: bool = true;
276
277 const WILDCARD: Option<Self> = Some(Provenance::Wildcard);
279
280 fn get_alloc_id(self) -> Option<AllocId> {
281 match self {
282 Provenance::Concrete { alloc_id, .. } => Some(alloc_id),
283 Provenance::Wildcard => None,
284 }
285 }
286
287 fn fmt(ptr: &interpret::Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
288 let (prov, addr) = ptr.into_parts(); write!(f, "{:#x}", addr.bytes())?;
290 if f.alternate() {
291 write!(f, "{prov:#?}")?;
292 } else {
293 write!(f, "{prov:?}")?;
294 }
295 Ok(())
296 }
297
298 fn join(left: Option<Self>, right: Option<Self>) -> Option<Self> {
299 match (left, right) {
300 (
302 Some(Provenance::Concrete { alloc_id: left_alloc, tag: left_tag }),
303 Some(Provenance::Concrete { alloc_id: right_alloc, tag: right_tag }),
304 ) if left_alloc == right_alloc && left_tag == right_tag => left,
305 (Some(Provenance::Wildcard), o) | (o, Some(Provenance::Wildcard)) => o,
308 _ => None,
310 }
311 }
312}
313
314impl fmt::Debug for ProvenanceExtra {
315 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
316 match self {
317 ProvenanceExtra::Concrete(pid) => write!(f, "{pid:?}"),
318 ProvenanceExtra::Wildcard => write!(f, "<wildcard>"),
319 }
320 }
321}
322
323impl ProvenanceExtra {
324 pub fn and_then<T>(self, f: impl FnOnce(BorTag) -> Option<T>) -> Option<T> {
325 match self {
326 ProvenanceExtra::Concrete(pid) => f(pid),
327 ProvenanceExtra::Wildcard => None,
328 }
329 }
330}
331
332#[derive(Debug)]
334pub struct AllocExtra<'tcx> {
335 pub borrow_tracker: Option<borrow_tracker::AllocState>,
337 pub data_race: AllocDataRaceHandler,
341 pub backtrace: Option<Vec<FrameInfo<'tcx>>>,
346 pub sync: FxHashMap<Size, Box<dyn Any>>,
351}
352
353impl<'tcx> Clone for AllocExtra<'tcx> {
356 fn clone(&self) -> Self {
357 panic!("our allocations should never be cloned");
358 }
359}
360
361impl VisitProvenance for AllocExtra<'_> {
362 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
363 let AllocExtra { borrow_tracker, data_race, backtrace: _, sync: _ } = self;
364
365 borrow_tracker.visit_provenance(visit);
366 data_race.visit_provenance(visit);
367 }
368}
369
370pub struct PrimitiveLayouts<'tcx> {
372 pub unit: TyAndLayout<'tcx>,
373 pub i8: TyAndLayout<'tcx>,
374 pub i16: TyAndLayout<'tcx>,
375 pub i32: TyAndLayout<'tcx>,
376 pub i64: TyAndLayout<'tcx>,
377 pub i128: TyAndLayout<'tcx>,
378 pub isize: TyAndLayout<'tcx>,
379 pub u8: TyAndLayout<'tcx>,
380 pub u16: TyAndLayout<'tcx>,
381 pub u32: TyAndLayout<'tcx>,
382 pub u64: TyAndLayout<'tcx>,
383 pub u128: TyAndLayout<'tcx>,
384 pub usize: TyAndLayout<'tcx>,
385 pub bool: TyAndLayout<'tcx>,
386 pub mut_raw_ptr: TyAndLayout<'tcx>, pub const_raw_ptr: TyAndLayout<'tcx>, }
389
390impl<'tcx> PrimitiveLayouts<'tcx> {
391 fn new(layout_cx: LayoutCx<'tcx>) -> Result<Self, &'tcx LayoutError<'tcx>> {
392 let tcx = layout_cx.tcx();
393 let mut_raw_ptr = Ty::new_mut_ptr(tcx, tcx.types.unit);
394 let const_raw_ptr = Ty::new_imm_ptr(tcx, tcx.types.unit);
395 Ok(Self {
396 unit: layout_cx.layout_of(tcx.types.unit)?,
397 i8: layout_cx.layout_of(tcx.types.i8)?,
398 i16: layout_cx.layout_of(tcx.types.i16)?,
399 i32: layout_cx.layout_of(tcx.types.i32)?,
400 i64: layout_cx.layout_of(tcx.types.i64)?,
401 i128: layout_cx.layout_of(tcx.types.i128)?,
402 isize: layout_cx.layout_of(tcx.types.isize)?,
403 u8: layout_cx.layout_of(tcx.types.u8)?,
404 u16: layout_cx.layout_of(tcx.types.u16)?,
405 u32: layout_cx.layout_of(tcx.types.u32)?,
406 u64: layout_cx.layout_of(tcx.types.u64)?,
407 u128: layout_cx.layout_of(tcx.types.u128)?,
408 usize: layout_cx.layout_of(tcx.types.usize)?,
409 bool: layout_cx.layout_of(tcx.types.bool)?,
410 mut_raw_ptr: layout_cx.layout_of(mut_raw_ptr)?,
411 const_raw_ptr: layout_cx.layout_of(const_raw_ptr)?,
412 })
413 }
414
415 pub fn uint(&self, size: Size) -> Option<TyAndLayout<'tcx>> {
416 match size.bits() {
417 8 => Some(self.u8),
418 16 => Some(self.u16),
419 32 => Some(self.u32),
420 64 => Some(self.u64),
421 128 => Some(self.u128),
422 _ => None,
423 }
424 }
425
426 pub fn int(&self, size: Size) -> Option<TyAndLayout<'tcx>> {
427 match size.bits() {
428 8 => Some(self.i8),
429 16 => Some(self.i16),
430 32 => Some(self.i32),
431 64 => Some(self.i64),
432 128 => Some(self.i128),
433 _ => None,
434 }
435 }
436}
437
438pub struct MiriMachine<'tcx> {
443 pub tcx: TyCtxt<'tcx>,
445
446 pub borrow_tracker: Option<borrow_tracker::GlobalState>,
448
449 pub data_race: GlobalDataRaceHandler,
455
456 pub alloc_addresses: alloc_addresses::GlobalState,
458
459 pub(crate) env_vars: EnvVars<'tcx>,
461
462 pub(crate) main_fn_ret_place: Option<MPlaceTy<'tcx>>,
464
465 pub(crate) argc: Option<Pointer>,
469 pub(crate) argv: Option<Pointer>,
470 pub(crate) cmd_line: Option<Pointer>,
471
472 pub(crate) tls: TlsData<'tcx>,
474
475 pub(crate) isolated_op: IsolatedOp,
479
480 pub(crate) validation: ValidationMode,
482
483 pub(crate) fds: shims::FdTable,
485 pub(crate) dirs: shims::DirTable,
487
488 pub(crate) epoll_interests: shims::EpollInterestTable,
490
491 pub(crate) monotonic_clock: MonotonicClock,
493
494 pub(crate) threads: ThreadManager<'tcx>,
496
497 pub(crate) thread_cpu_affinity: FxHashMap<ThreadId, CpuAffinityMask>,
501
502 pub(crate) sync: SynchronizationObjects,
504
505 pub(crate) layouts: PrimitiveLayouts<'tcx>,
507
508 pub(crate) static_roots: Vec<AllocId>,
510
511 profiler: Option<measureme::Profiler>,
514 string_cache: FxHashMap<String, measureme::StringId>,
517
518 pub(crate) exported_symbols_cache: FxHashMap<Symbol, Option<Instance<'tcx>>>,
521
522 pub(crate) backtrace_style: BacktraceStyle,
524
525 pub(crate) local_crates: Vec<CrateNum>,
527
528 extern_statics: FxHashMap<Symbol, StrictPointer>,
530
531 pub(crate) rng: RefCell<StdRng>,
534
535 tracked_alloc_ids: FxHashSet<AllocId>,
538 track_alloc_accesses: bool,
540
541 pub(crate) check_alignment: AlignmentCheck,
543
544 pub(crate) cmpxchg_weak_failure_rate: f64,
546
547 pub(crate) preemption_rate: f64,
549
550 pub(crate) report_progress: Option<u32>,
552 pub(crate) basic_block_count: u64,
554
555 #[cfg(unix)]
557 pub native_lib: Option<(libloading::Library, std::path::PathBuf)>,
558 #[cfg(not(unix))]
559 pub native_lib: Option<!>,
560
561 pub(crate) gc_interval: u32,
563 pub(crate) since_gc: u32,
565
566 pub(crate) num_cpus: u32,
568
569 pub(crate) page_size: u64,
571 pub(crate) stack_addr: u64,
572 pub(crate) stack_size: u64,
573
574 pub(crate) collect_leak_backtraces: bool,
576
577 pub(crate) allocation_spans: RefCell<FxHashMap<AllocId, (Span, Option<Span>)>>,
580
581 const_cache: RefCell<FxHashMap<(mir::Const<'tcx>, usize), OpTy<'tcx>>>,
585
586 pub(crate) symbolic_alignment: RefCell<FxHashMap<AllocId, (Size, Align)>>,
593
594 union_data_ranges: FxHashMap<Ty<'tcx>, RangeSet>,
596
597 pub(crate) pthread_mutex_sanity: Cell<bool>,
599 pub(crate) pthread_rwlock_sanity: Cell<bool>,
600 pub(crate) pthread_condvar_sanity: Cell<bool>,
601
602 pub(crate) sb_extern_type_warned: Cell<bool>,
604 #[cfg(unix)]
606 pub(crate) native_call_mem_warned: Cell<bool>,
607 pub(crate) reject_in_isolation_warned: RefCell<FxHashSet<String>>,
609 pub(crate) int2ptr_warned: RefCell<FxHashSet<Span>>,
611
612 pub(crate) mangle_internal_symbol_cache: FxHashMap<&'static str, String>,
614
615 pub force_intrinsic_fallback: bool,
617}
618
619impl<'tcx> MiriMachine<'tcx> {
620 pub(crate) fn new(
621 config: &MiriConfig,
622 layout_cx: LayoutCx<'tcx>,
623 genmc_ctx: Option<Rc<GenmcCtx>>,
624 ) -> Self {
625 let tcx = layout_cx.tcx();
626 let local_crates = helpers::get_local_crates(tcx);
627 let layouts =
628 PrimitiveLayouts::new(layout_cx).expect("Couldn't get layouts of primitive types");
629 let profiler = config.measureme_out.as_ref().map(|out| {
630 let crate_name =
631 tcx.sess.opts.crate_name.clone().unwrap_or_else(|| "unknown-crate".to_string());
632 let pid = process::id();
633 let filename = format!("{crate_name}-{pid:07}");
638 let path = Path::new(out).join(filename);
639 measureme::Profiler::new(path).expect("Couldn't create `measureme` profiler")
640 });
641 let rng = StdRng::seed_from_u64(config.seed.unwrap_or(0));
642 let borrow_tracker = config.borrow_tracker.map(|bt| bt.instantiate_global_state(config));
643 let data_race = if config.genmc_mode {
644 GlobalDataRaceHandler::Genmc(genmc_ctx.unwrap())
646 } else if config.data_race_detector {
647 GlobalDataRaceHandler::Vclocks(Box::new(data_race::GlobalState::new(config)))
648 } else {
649 GlobalDataRaceHandler::None
650 };
651 let page_size = if let Some(page_size) = config.page_size {
655 page_size
656 } else {
657 let target = &tcx.sess.target;
658 match target.arch.as_ref() {
659 "wasm32" | "wasm64" => 64 * 1024, "aarch64" => {
661 if target.options.vendor.as_ref() == "apple" {
662 16 * 1024
666 } else {
667 4 * 1024
668 }
669 }
670 _ => 4 * 1024,
671 }
672 };
673 let stack_addr = if tcx.pointer_size().bits() < 32 { page_size } else { page_size * 32 };
675 let stack_size =
676 if tcx.pointer_size().bits() < 32 { page_size * 4 } else { page_size * 16 };
677 assert!(
678 usize::try_from(config.num_cpus).unwrap() <= cpu_affinity::MAX_CPUS,
679 "miri only supports up to {} CPUs, but {} were configured",
680 cpu_affinity::MAX_CPUS,
681 config.num_cpus
682 );
683 let threads = ThreadManager::new(config);
684 let mut thread_cpu_affinity = FxHashMap::default();
685 if matches!(&*tcx.sess.target.os, "linux" | "freebsd" | "android") {
686 thread_cpu_affinity
687 .insert(threads.active_thread(), CpuAffinityMask::new(&layout_cx, config.num_cpus));
688 }
689 MiriMachine {
690 tcx,
691 borrow_tracker,
692 data_race,
693 alloc_addresses: RefCell::new(alloc_addresses::GlobalStateInner::new(config, stack_addr)),
694 env_vars: EnvVars::default(),
696 main_fn_ret_place: None,
697 argc: None,
698 argv: None,
699 cmd_line: None,
700 tls: TlsData::default(),
701 isolated_op: config.isolated_op,
702 validation: config.validation,
703 fds: shims::FdTable::init(config.mute_stdout_stderr),
704 epoll_interests: shims::EpollInterestTable::new(),
705 dirs: Default::default(),
706 layouts,
707 threads,
708 thread_cpu_affinity,
709 sync: SynchronizationObjects::default(),
710 static_roots: Vec::new(),
711 profiler,
712 string_cache: Default::default(),
713 exported_symbols_cache: FxHashMap::default(),
714 backtrace_style: config.backtrace_style,
715 local_crates,
716 extern_statics: FxHashMap::default(),
717 rng: RefCell::new(rng),
718 tracked_alloc_ids: config.tracked_alloc_ids.clone(),
719 track_alloc_accesses: config.track_alloc_accesses,
720 check_alignment: config.check_alignment,
721 cmpxchg_weak_failure_rate: config.cmpxchg_weak_failure_rate,
722 preemption_rate: config.preemption_rate,
723 report_progress: config.report_progress,
724 basic_block_count: 0,
725 monotonic_clock: MonotonicClock::new(config.isolated_op == IsolatedOp::Allow),
726 #[cfg(unix)]
727 native_lib: config.native_lib.as_ref().map(|lib_file_path| {
728 let host_triple = rustc_session::config::host_tuple();
729 let target_triple = tcx.sess.opts.target_triple.tuple();
730 if host_triple != target_triple {
732 panic!(
733 "calling native C functions in linked .so file requires host and target to be the same: \
734 host={host_triple}, target={target_triple}",
735 );
736 }
737 (
741 unsafe {
742 libloading::Library::new(lib_file_path)
743 .expect("failed to read specified extern shared object file")
744 },
745 lib_file_path.clone(),
746 )
747 }),
748 #[cfg(not(unix))]
749 native_lib: config.native_lib.as_ref().map(|_| {
750 panic!("calling functions from native libraries via FFI is only supported on Unix")
751 }),
752 gc_interval: config.gc_interval,
753 since_gc: 0,
754 num_cpus: config.num_cpus,
755 page_size,
756 stack_addr,
757 stack_size,
758 collect_leak_backtraces: config.collect_leak_backtraces,
759 allocation_spans: RefCell::new(FxHashMap::default()),
760 const_cache: RefCell::new(FxHashMap::default()),
761 symbolic_alignment: RefCell::new(FxHashMap::default()),
762 union_data_ranges: FxHashMap::default(),
763 pthread_mutex_sanity: Cell::new(false),
764 pthread_rwlock_sanity: Cell::new(false),
765 pthread_condvar_sanity: Cell::new(false),
766 sb_extern_type_warned: Cell::new(false),
767 #[cfg(unix)]
768 native_call_mem_warned: Cell::new(false),
769 reject_in_isolation_warned: Default::default(),
770 int2ptr_warned: Default::default(),
771 mangle_internal_symbol_cache: Default::default(),
772 force_intrinsic_fallback: config.force_intrinsic_fallback,
773 }
774 }
775
776 pub(crate) fn late_init(
777 ecx: &mut MiriInterpCx<'tcx>,
778 config: &MiriConfig,
779 on_main_stack_empty: StackEmptyCallback<'tcx>,
780 ) -> InterpResult<'tcx> {
781 EnvVars::init(ecx, config)?;
782 MiriMachine::init_extern_statics(ecx)?;
783 ThreadManager::init(ecx, on_main_stack_empty);
784 interp_ok(())
785 }
786
787 pub(crate) fn add_extern_static(ecx: &mut MiriInterpCx<'tcx>, name: &str, ptr: Pointer) {
788 let ptr = ptr.into_pointer_or_addr().unwrap();
790 ecx.machine.extern_statics.try_insert(Symbol::intern(name), ptr).unwrap();
791 }
792
793 pub(crate) fn communicate(&self) -> bool {
794 self.isolated_op == IsolatedOp::Allow
795 }
796
797 pub(crate) fn is_local(&self, frame: &FrameInfo<'_>) -> bool {
799 let def_id = frame.instance.def_id();
800 def_id.is_local() || self.local_crates.contains(&def_id.krate)
801 }
802
803 pub(crate) fn handle_abnormal_termination(&mut self) {
805 drop(self.profiler.take());
810 }
811
812 pub(crate) fn page_align(&self) -> Align {
813 Align::from_bytes(self.page_size).unwrap()
814 }
815
816 pub(crate) fn allocated_span(&self, alloc_id: AllocId) -> Option<SpanData> {
817 self.allocation_spans
818 .borrow()
819 .get(&alloc_id)
820 .map(|(allocated, _deallocated)| allocated.data())
821 }
822
823 pub(crate) fn deallocated_span(&self, alloc_id: AllocId) -> Option<SpanData> {
824 self.allocation_spans
825 .borrow()
826 .get(&alloc_id)
827 .and_then(|(_allocated, deallocated)| *deallocated)
828 .map(Span::data)
829 }
830
831 fn init_allocation(
832 ecx: &MiriInterpCx<'tcx>,
833 id: AllocId,
834 kind: MemoryKind,
835 size: Size,
836 align: Align,
837 ) -> InterpResult<'tcx, AllocExtra<'tcx>> {
838 if ecx.machine.tracked_alloc_ids.contains(&id) {
839 ecx.emit_diagnostic(NonHaltingDiagnostic::CreatedAlloc(id, size, align, kind));
840 }
841
842 let borrow_tracker = ecx
843 .machine
844 .borrow_tracker
845 .as_ref()
846 .map(|bt| bt.borrow_mut().new_allocation(id, size, kind, &ecx.machine));
847
848 let data_race = match &ecx.machine.data_race {
849 GlobalDataRaceHandler::None => AllocDataRaceHandler::None,
850 GlobalDataRaceHandler::Vclocks(data_race) =>
851 AllocDataRaceHandler::Vclocks(
852 data_race::AllocState::new_allocation(
853 data_race,
854 &ecx.machine.threads,
855 size,
856 kind,
857 ecx.machine.current_span(),
858 ),
859 data_race.weak_memory.then(weak_memory::AllocState::new_allocation),
860 ),
861 GlobalDataRaceHandler::Genmc(_genmc_ctx) => {
862 AllocDataRaceHandler::Genmc
865 }
866 };
867
868 let backtrace = if kind.may_leak() || !ecx.machine.collect_leak_backtraces {
872 None
873 } else {
874 Some(ecx.generate_stacktrace())
875 };
876
877 if matches!(kind, MemoryKind::Machine(kind) if kind.should_save_allocation_span()) {
878 ecx.machine
879 .allocation_spans
880 .borrow_mut()
881 .insert(id, (ecx.machine.current_span(), None));
882 }
883
884 interp_ok(AllocExtra { borrow_tracker, data_race, backtrace, sync: FxHashMap::default() })
885 }
886}
887
888impl VisitProvenance for MiriMachine<'_> {
889 fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
890 #[rustfmt::skip]
891 let MiriMachine {
892 threads,
893 thread_cpu_affinity: _,
894 sync: _,
895 tls,
896 env_vars,
897 main_fn_ret_place,
898 argc,
899 argv,
900 cmd_line,
901 extern_statics,
902 dirs,
903 borrow_tracker,
904 data_race,
905 alloc_addresses,
906 fds,
907 epoll_interests:_,
908 tcx: _,
909 isolated_op: _,
910 validation: _,
911 monotonic_clock: _,
912 layouts: _,
913 static_roots: _,
914 profiler: _,
915 string_cache: _,
916 exported_symbols_cache: _,
917 backtrace_style: _,
918 local_crates: _,
919 rng: _,
920 tracked_alloc_ids: _,
921 track_alloc_accesses: _,
922 check_alignment: _,
923 cmpxchg_weak_failure_rate: _,
924 preemption_rate: _,
925 report_progress: _,
926 basic_block_count: _,
927 native_lib: _,
928 gc_interval: _,
929 since_gc: _,
930 num_cpus: _,
931 page_size: _,
932 stack_addr: _,
933 stack_size: _,
934 collect_leak_backtraces: _,
935 allocation_spans: _,
936 const_cache: _,
937 symbolic_alignment: _,
938 union_data_ranges: _,
939 pthread_mutex_sanity: _,
940 pthread_rwlock_sanity: _,
941 pthread_condvar_sanity: _,
942 sb_extern_type_warned: _,
943 #[cfg(unix)]
944 native_call_mem_warned: _,
945 reject_in_isolation_warned: _,
946 int2ptr_warned: _,
947 mangle_internal_symbol_cache: _,
948 force_intrinsic_fallback: _,
949 } = self;
950
951 threads.visit_provenance(visit);
952 tls.visit_provenance(visit);
953 env_vars.visit_provenance(visit);
954 dirs.visit_provenance(visit);
955 fds.visit_provenance(visit);
956 data_race.visit_provenance(visit);
957 borrow_tracker.visit_provenance(visit);
958 alloc_addresses.visit_provenance(visit);
959 main_fn_ret_place.visit_provenance(visit);
960 argc.visit_provenance(visit);
961 argv.visit_provenance(visit);
962 cmd_line.visit_provenance(visit);
963 for ptr in extern_statics.values() {
964 ptr.visit_provenance(visit);
965 }
966 }
967}
968
969pub type MiriInterpCx<'tcx> = InterpCx<'tcx, MiriMachine<'tcx>>;
971
972pub trait MiriInterpCxExt<'tcx> {
974 fn eval_context_ref<'a>(&'a self) -> &'a MiriInterpCx<'tcx>;
975 fn eval_context_mut<'a>(&'a mut self) -> &'a mut MiriInterpCx<'tcx>;
976}
977impl<'tcx> MiriInterpCxExt<'tcx> for MiriInterpCx<'tcx> {
978 #[inline(always)]
979 fn eval_context_ref(&self) -> &MiriInterpCx<'tcx> {
980 self
981 }
982 #[inline(always)]
983 fn eval_context_mut(&mut self) -> &mut MiriInterpCx<'tcx> {
984 self
985 }
986}
987
988impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> {
990 type MemoryKind = MiriMemoryKind;
991 type ExtraFnVal = DynSym;
992
993 type FrameExtra = FrameExtra<'tcx>;
994 type AllocExtra = AllocExtra<'tcx>;
995
996 type Provenance = Provenance;
997 type ProvenanceExtra = ProvenanceExtra;
998 type Bytes = MiriAllocBytes;
999
1000 type MemoryMap =
1001 MonoHashMap<AllocId, (MemoryKind, Allocation<Provenance, Self::AllocExtra, Self::Bytes>)>;
1002
1003 const GLOBAL_KIND: Option<MiriMemoryKind> = Some(MiriMemoryKind::Global);
1004
1005 const PANIC_ON_ALLOC_FAIL: bool = false;
1006
1007 #[inline(always)]
1008 fn enforce_alignment(ecx: &MiriInterpCx<'tcx>) -> bool {
1009 ecx.machine.check_alignment != AlignmentCheck::None
1010 }
1011
1012 #[inline(always)]
1013 fn alignment_check(
1014 ecx: &MiriInterpCx<'tcx>,
1015 alloc_id: AllocId,
1016 alloc_align: Align,
1017 alloc_kind: AllocKind,
1018 offset: Size,
1019 align: Align,
1020 ) -> Option<Misalignment> {
1021 if ecx.machine.check_alignment != AlignmentCheck::Symbolic {
1022 return None;
1024 }
1025 if alloc_kind != AllocKind::LiveData {
1026 return None;
1028 }
1029 let (promised_offset, promised_align) = ecx
1031 .machine
1032 .symbolic_alignment
1033 .borrow()
1034 .get(&alloc_id)
1035 .copied()
1036 .unwrap_or((Size::ZERO, alloc_align));
1037 if promised_align < align {
1038 Some(Misalignment { has: promised_align, required: align })
1040 } else {
1041 let distance = offset.bytes().wrapping_sub(promised_offset.bytes());
1043 if distance % align.bytes() == 0 {
1045 None
1047 } else {
1048 let distance_pow2 = 1 << distance.trailing_zeros();
1050 Some(Misalignment {
1051 has: Align::from_bytes(distance_pow2).unwrap(),
1052 required: align,
1053 })
1054 }
1055 }
1056 }
1057
1058 #[inline(always)]
1059 fn enforce_validity(ecx: &MiriInterpCx<'tcx>, _layout: TyAndLayout<'tcx>) -> bool {
1060 ecx.machine.validation != ValidationMode::No
1061 }
1062 #[inline(always)]
1063 fn enforce_validity_recursively(
1064 ecx: &InterpCx<'tcx, Self>,
1065 _layout: TyAndLayout<'tcx>,
1066 ) -> bool {
1067 ecx.machine.validation == ValidationMode::Deep
1068 }
1069
1070 #[inline(always)]
1071 fn ignore_optional_overflow_checks(ecx: &MiriInterpCx<'tcx>) -> bool {
1072 !ecx.tcx.sess.overflow_checks()
1073 }
1074
1075 fn check_fn_target_features(
1076 ecx: &MiriInterpCx<'tcx>,
1077 instance: ty::Instance<'tcx>,
1078 ) -> InterpResult<'tcx> {
1079 let attrs = ecx.tcx.codegen_fn_attrs(instance.def_id());
1080 if attrs
1081 .target_features
1082 .iter()
1083 .any(|feature| !ecx.tcx.sess.target_features.contains(&feature.name))
1084 {
1085 let unavailable = attrs
1086 .target_features
1087 .iter()
1088 .filter(|&feature| {
1089 !feature.implied && !ecx.tcx.sess.target_features.contains(&feature.name)
1090 })
1091 .fold(String::new(), |mut s, feature| {
1092 if !s.is_empty() {
1093 s.push_str(", ");
1094 }
1095 s.push_str(feature.name.as_str());
1096 s
1097 });
1098 let msg = format!(
1099 "calling a function that requires unavailable target features: {unavailable}"
1100 );
1101 if ecx.tcx.sess.target.is_like_wasm {
1104 throw_machine_stop!(TerminationInfo::Abort(msg));
1105 } else {
1106 throw_ub_format!("{msg}");
1107 }
1108 }
1109 interp_ok(())
1110 }
1111
1112 #[inline(always)]
1113 fn find_mir_or_eval_fn(
1114 ecx: &mut MiriInterpCx<'tcx>,
1115 instance: ty::Instance<'tcx>,
1116 abi: &FnAbi<'tcx, Ty<'tcx>>,
1117 args: &[FnArg<'tcx, Provenance>],
1118 dest: &MPlaceTy<'tcx>,
1119 ret: Option<mir::BasicBlock>,
1120 unwind: mir::UnwindAction,
1121 ) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>> {
1122 if ecx.tcx.is_foreign_item(instance.def_id()) {
1124 let args = ecx.copy_fn_args(args); let link_name = Symbol::intern(ecx.tcx.symbol_name(instance).name);
1132 return ecx.emulate_foreign_item(link_name, abi, &args, dest, ret, unwind);
1133 }
1134
1135 interp_ok(Some((ecx.load_mir(instance.def, None)?, instance)))
1137 }
1138
1139 #[inline(always)]
1140 fn call_extra_fn(
1141 ecx: &mut MiriInterpCx<'tcx>,
1142 fn_val: DynSym,
1143 abi: &FnAbi<'tcx, Ty<'tcx>>,
1144 args: &[FnArg<'tcx, Provenance>],
1145 dest: &MPlaceTy<'tcx>,
1146 ret: Option<mir::BasicBlock>,
1147 unwind: mir::UnwindAction,
1148 ) -> InterpResult<'tcx> {
1149 let args = ecx.copy_fn_args(args); ecx.emulate_dyn_sym(fn_val, abi, &args, dest, ret, unwind)
1151 }
1152
1153 #[inline(always)]
1154 fn call_intrinsic(
1155 ecx: &mut MiriInterpCx<'tcx>,
1156 instance: ty::Instance<'tcx>,
1157 args: &[OpTy<'tcx>],
1158 dest: &MPlaceTy<'tcx>,
1159 ret: Option<mir::BasicBlock>,
1160 unwind: mir::UnwindAction,
1161 ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
1162 ecx.call_intrinsic(instance, args, dest, ret, unwind)
1163 }
1164
1165 #[inline(always)]
1166 fn assert_panic(
1167 ecx: &mut MiriInterpCx<'tcx>,
1168 msg: &mir::AssertMessage<'tcx>,
1169 unwind: mir::UnwindAction,
1170 ) -> InterpResult<'tcx> {
1171 ecx.assert_panic(msg, unwind)
1172 }
1173
1174 fn panic_nounwind(ecx: &mut InterpCx<'tcx, Self>, msg: &str) -> InterpResult<'tcx> {
1175 ecx.start_panic_nounwind(msg)
1176 }
1177
1178 fn unwind_terminate(
1179 ecx: &mut InterpCx<'tcx, Self>,
1180 reason: mir::UnwindTerminateReason,
1181 ) -> InterpResult<'tcx> {
1182 let panic = ecx.tcx.lang_items().get(reason.lang_item()).unwrap();
1184 let panic = ty::Instance::mono(ecx.tcx.tcx, panic);
1185 ecx.call_function(
1186 panic,
1187 ExternAbi::Rust,
1188 &[],
1189 None,
1190 StackPopCleanup::Goto { ret: None, unwind: mir::UnwindAction::Unreachable },
1191 )?;
1192 interp_ok(())
1193 }
1194
1195 #[inline(always)]
1196 fn binary_ptr_op(
1197 ecx: &MiriInterpCx<'tcx>,
1198 bin_op: mir::BinOp,
1199 left: &ImmTy<'tcx>,
1200 right: &ImmTy<'tcx>,
1201 ) -> InterpResult<'tcx, ImmTy<'tcx>> {
1202 ecx.binary_ptr_op(bin_op, left, right)
1203 }
1204
1205 #[inline(always)]
1206 fn generate_nan<F1: Float + FloatConvert<F2>, F2: Float>(
1207 ecx: &InterpCx<'tcx, Self>,
1208 inputs: &[F1],
1209 ) -> F2 {
1210 ecx.generate_nan(inputs)
1211 }
1212
1213 #[inline(always)]
1214 fn apply_float_nondet(
1215 ecx: &mut InterpCx<'tcx, Self>,
1216 val: ImmTy<'tcx>,
1217 ) -> InterpResult<'tcx, ImmTy<'tcx>> {
1218 crate::math::apply_random_float_error_to_imm(ecx, val, 2 )
1219 }
1220
1221 #[inline(always)]
1222 fn equal_float_min_max<F: Float>(ecx: &MiriInterpCx<'tcx>, a: F, b: F) -> F {
1223 ecx.equal_float_min_max(a, b)
1224 }
1225
1226 #[inline(always)]
1227 fn ub_checks(ecx: &InterpCx<'tcx, Self>) -> InterpResult<'tcx, bool> {
1228 interp_ok(ecx.tcx.sess.ub_checks())
1229 }
1230
1231 #[inline(always)]
1232 fn contract_checks(ecx: &InterpCx<'tcx, Self>) -> InterpResult<'tcx, bool> {
1233 interp_ok(ecx.tcx.sess.contract_checks())
1234 }
1235
1236 #[inline(always)]
1237 fn thread_local_static_pointer(
1238 ecx: &mut MiriInterpCx<'tcx>,
1239 def_id: DefId,
1240 ) -> InterpResult<'tcx, StrictPointer> {
1241 ecx.get_or_create_thread_local_alloc(def_id)
1242 }
1243
1244 fn extern_static_pointer(
1245 ecx: &MiriInterpCx<'tcx>,
1246 def_id: DefId,
1247 ) -> InterpResult<'tcx, StrictPointer> {
1248 let link_name = Symbol::intern(ecx.tcx.symbol_name(Instance::mono(*ecx.tcx, def_id)).name);
1249 if let Some(&ptr) = ecx.machine.extern_statics.get(&link_name) {
1250 let Provenance::Concrete { alloc_id, .. } = ptr.provenance else {
1254 panic!("extern_statics cannot contain wildcards")
1255 };
1256 let info = ecx.get_alloc_info(alloc_id);
1257 let def_ty = ecx.tcx.type_of(def_id).instantiate_identity();
1258 let extern_decl_layout =
1259 ecx.tcx.layout_of(ecx.typing_env().as_query_input(def_ty)).unwrap();
1260 if extern_decl_layout.size != info.size || extern_decl_layout.align.abi != info.align {
1261 throw_unsup_format!(
1262 "extern static `{link_name}` has been declared as `{krate}::{name}` \
1263 with a size of {decl_size} bytes and alignment of {decl_align} bytes, \
1264 but Miri emulates it via an extern static shim \
1265 with a size of {shim_size} bytes and alignment of {shim_align} bytes",
1266 name = ecx.tcx.def_path_str(def_id),
1267 krate = ecx.tcx.crate_name(def_id.krate),
1268 decl_size = extern_decl_layout.size.bytes(),
1269 decl_align = extern_decl_layout.align.abi.bytes(),
1270 shim_size = info.size.bytes(),
1271 shim_align = info.align.bytes(),
1272 )
1273 }
1274 interp_ok(ptr)
1275 } else {
1276 throw_unsup_format!("extern static `{link_name}` is not supported by Miri",)
1277 }
1278 }
1279
1280 fn init_local_allocation(
1281 ecx: &MiriInterpCx<'tcx>,
1282 id: AllocId,
1283 kind: MemoryKind,
1284 size: Size,
1285 align: Align,
1286 ) -> InterpResult<'tcx, Self::AllocExtra> {
1287 assert!(kind != MiriMemoryKind::Global.into());
1288 MiriMachine::init_allocation(ecx, id, kind, size, align)
1289 }
1290
1291 fn adjust_alloc_root_pointer(
1292 ecx: &MiriInterpCx<'tcx>,
1293 ptr: interpret::Pointer<CtfeProvenance>,
1294 kind: Option<MemoryKind>,
1295 ) -> InterpResult<'tcx, interpret::Pointer<Provenance>> {
1296 let kind = kind.expect("we set our GLOBAL_KIND so this cannot be None");
1297 let alloc_id = ptr.provenance.alloc_id();
1298 if cfg!(debug_assertions) {
1299 match ecx.tcx.try_get_global_alloc(alloc_id) {
1301 Some(GlobalAlloc::Static(def_id)) if ecx.tcx.is_thread_local_static(def_id) => {
1302 panic!("adjust_alloc_root_pointer called on thread-local static")
1303 }
1304 Some(GlobalAlloc::Static(def_id)) if ecx.tcx.is_foreign_item(def_id) => {
1305 panic!("adjust_alloc_root_pointer called on extern static")
1306 }
1307 _ => {}
1308 }
1309 }
1310 let tag = if let Some(borrow_tracker) = &ecx.machine.borrow_tracker {
1312 borrow_tracker.borrow_mut().root_ptr_tag(alloc_id, &ecx.machine)
1313 } else {
1314 BorTag::default()
1316 };
1317 ecx.adjust_alloc_root_pointer(ptr, tag, kind)
1318 }
1319
1320 #[inline(always)]
1322 fn ptr_from_addr_cast(ecx: &MiriInterpCx<'tcx>, addr: u64) -> InterpResult<'tcx, Pointer> {
1323 ecx.ptr_from_addr_cast(addr)
1324 }
1325
1326 #[inline(always)]
1330 fn expose_provenance(
1331 ecx: &InterpCx<'tcx, Self>,
1332 provenance: Self::Provenance,
1333 ) -> InterpResult<'tcx> {
1334 ecx.expose_provenance(provenance)
1335 }
1336
1337 fn ptr_get_alloc(
1349 ecx: &MiriInterpCx<'tcx>,
1350 ptr: StrictPointer,
1351 size: i64,
1352 ) -> Option<(AllocId, Size, Self::ProvenanceExtra)> {
1353 let rel = ecx.ptr_get_alloc(ptr, size);
1354
1355 rel.map(|(alloc_id, size)| {
1356 let tag = match ptr.provenance {
1357 Provenance::Concrete { tag, .. } => ProvenanceExtra::Concrete(tag),
1358 Provenance::Wildcard => ProvenanceExtra::Wildcard,
1359 };
1360 (alloc_id, size, tag)
1361 })
1362 }
1363
1364 fn adjust_global_allocation<'b>(
1373 ecx: &InterpCx<'tcx, Self>,
1374 id: AllocId,
1375 alloc: &'b Allocation,
1376 ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>
1377 {
1378 let alloc = alloc.adjust_from_tcx(
1379 &ecx.tcx,
1380 |bytes, align| ecx.get_global_alloc_bytes(id, bytes, align),
1381 |ptr| ecx.global_root_pointer(ptr),
1382 )?;
1383 let kind = MiriMemoryKind::Global.into();
1384 let extra = MiriMachine::init_allocation(ecx, id, kind, alloc.size(), alloc.align)?;
1385 interp_ok(Cow::Owned(alloc.with_extra(extra)))
1386 }
1387
1388 #[inline(always)]
1389 fn before_memory_read(
1390 _tcx: TyCtxtAt<'tcx>,
1391 machine: &Self,
1392 alloc_extra: &AllocExtra<'tcx>,
1393 ptr: Pointer,
1394 (alloc_id, prov_extra): (AllocId, Self::ProvenanceExtra),
1395 range: AllocRange,
1396 ) -> InterpResult<'tcx> {
1397 if machine.track_alloc_accesses && machine.tracked_alloc_ids.contains(&alloc_id) {
1398 machine
1399 .emit_diagnostic(NonHaltingDiagnostic::AccessedAlloc(alloc_id, AccessKind::Read));
1400 }
1401 match &machine.data_race {
1403 GlobalDataRaceHandler::None => {}
1404 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
1405 genmc_ctx.memory_load(machine, ptr.addr(), range.size)?,
1406 GlobalDataRaceHandler::Vclocks(_data_race) => {
1407 let AllocDataRaceHandler::Vclocks(data_race, weak_memory) = &alloc_extra.data_race
1408 else {
1409 unreachable!();
1410 };
1411 data_race.read(alloc_id, range, NaReadType::Read, None, machine)?;
1412 if let Some(weak_memory) = weak_memory {
1413 weak_memory.memory_accessed(range, machine.data_race.as_vclocks_ref().unwrap());
1414 }
1415 }
1416 }
1417 if let Some(borrow_tracker) = &alloc_extra.borrow_tracker {
1418 borrow_tracker.before_memory_read(alloc_id, prov_extra, range, machine)?;
1419 }
1420 interp_ok(())
1421 }
1422
1423 #[inline(always)]
1424 fn before_memory_write(
1425 _tcx: TyCtxtAt<'tcx>,
1426 machine: &mut Self,
1427 alloc_extra: &mut AllocExtra<'tcx>,
1428 ptr: Pointer,
1429 (alloc_id, prov_extra): (AllocId, Self::ProvenanceExtra),
1430 range: AllocRange,
1431 ) -> InterpResult<'tcx> {
1432 if machine.track_alloc_accesses && machine.tracked_alloc_ids.contains(&alloc_id) {
1433 machine
1434 .emit_diagnostic(NonHaltingDiagnostic::AccessedAlloc(alloc_id, AccessKind::Write));
1435 }
1436 match &machine.data_race {
1437 GlobalDataRaceHandler::None => {}
1438 GlobalDataRaceHandler::Genmc(genmc_ctx) => {
1439 genmc_ctx.memory_store(machine, ptr.addr(), range.size)?;
1440 }
1441 GlobalDataRaceHandler::Vclocks(_global_state) => {
1442 let AllocDataRaceHandler::Vclocks(data_race, weak_memory) =
1443 &mut alloc_extra.data_race
1444 else {
1445 unreachable!()
1446 };
1447 data_race.write(alloc_id, range, NaWriteType::Write, None, machine)?;
1448 if let Some(weak_memory) = weak_memory {
1449 weak_memory.memory_accessed(range, machine.data_race.as_vclocks_ref().unwrap());
1450 }
1451 }
1452 }
1453 if let Some(borrow_tracker) = &mut alloc_extra.borrow_tracker {
1454 borrow_tracker.before_memory_write(alloc_id, prov_extra, range, machine)?;
1455 }
1456 interp_ok(())
1457 }
1458
1459 #[inline(always)]
1460 fn before_memory_deallocation(
1461 _tcx: TyCtxtAt<'tcx>,
1462 machine: &mut Self,
1463 alloc_extra: &mut AllocExtra<'tcx>,
1464 ptr: Pointer,
1465 (alloc_id, prove_extra): (AllocId, Self::ProvenanceExtra),
1466 size: Size,
1467 align: Align,
1468 kind: MemoryKind,
1469 ) -> InterpResult<'tcx> {
1470 if machine.tracked_alloc_ids.contains(&alloc_id) {
1471 machine.emit_diagnostic(NonHaltingDiagnostic::FreedAlloc(alloc_id));
1472 }
1473 match &machine.data_race {
1474 GlobalDataRaceHandler::None => {}
1475 GlobalDataRaceHandler::Genmc(genmc_ctx) =>
1476 genmc_ctx.handle_dealloc(machine, ptr.addr(), size, align, kind)?,
1477 GlobalDataRaceHandler::Vclocks(_global_state) => {
1478 let data_race = alloc_extra.data_race.as_vclocks_mut().unwrap();
1479 data_race.write(
1480 alloc_id,
1481 alloc_range(Size::ZERO, size),
1482 NaWriteType::Deallocate,
1483 None,
1484 machine,
1485 )?;
1486 }
1487 }
1488 if let Some(borrow_tracker) = &mut alloc_extra.borrow_tracker {
1489 borrow_tracker.before_memory_deallocation(alloc_id, prove_extra, size, machine)?;
1490 }
1491 if let Some((_, deallocated_at)) = machine.allocation_spans.borrow_mut().get_mut(&alloc_id)
1492 {
1493 *deallocated_at = Some(machine.current_span());
1494 }
1495 machine.free_alloc_id(alloc_id, size, align, kind);
1496 interp_ok(())
1497 }
1498
1499 #[inline(always)]
1500 fn retag_ptr_value(
1501 ecx: &mut InterpCx<'tcx, Self>,
1502 kind: mir::RetagKind,
1503 val: &ImmTy<'tcx>,
1504 ) -> InterpResult<'tcx, ImmTy<'tcx>> {
1505 if ecx.machine.borrow_tracker.is_some() {
1506 ecx.retag_ptr_value(kind, val)
1507 } else {
1508 interp_ok(val.clone())
1509 }
1510 }
1511
1512 #[inline(always)]
1513 fn retag_place_contents(
1514 ecx: &mut InterpCx<'tcx, Self>,
1515 kind: mir::RetagKind,
1516 place: &PlaceTy<'tcx>,
1517 ) -> InterpResult<'tcx> {
1518 if ecx.machine.borrow_tracker.is_some() {
1519 ecx.retag_place_contents(kind, place)?;
1520 }
1521 interp_ok(())
1522 }
1523
1524 fn protect_in_place_function_argument(
1525 ecx: &mut InterpCx<'tcx, Self>,
1526 place: &MPlaceTy<'tcx>,
1527 ) -> InterpResult<'tcx> {
1528 let protected_place = if ecx.machine.borrow_tracker.is_some() {
1531 ecx.protect_place(place)?
1532 } else {
1533 place.clone()
1535 };
1536 ecx.write_uninit(&protected_place)?;
1541 interp_ok(())
1543 }
1544
1545 #[inline(always)]
1546 fn init_frame(
1547 ecx: &mut InterpCx<'tcx, Self>,
1548 frame: Frame<'tcx, Provenance>,
1549 ) -> InterpResult<'tcx, Frame<'tcx, Provenance, FrameExtra<'tcx>>> {
1550 let timing = if let Some(profiler) = ecx.machine.profiler.as_ref() {
1552 let fn_name = frame.instance().to_string();
1553 let entry = ecx.machine.string_cache.entry(fn_name.clone());
1554 let name = entry.or_insert_with(|| profiler.alloc_string(&*fn_name));
1555
1556 Some(profiler.start_recording_interval_event_detached(
1557 *name,
1558 measureme::EventId::from_label(*name),
1559 ecx.active_thread().to_u32(),
1560 ))
1561 } else {
1562 None
1563 };
1564
1565 let borrow_tracker = ecx.machine.borrow_tracker.as_ref();
1566
1567 let extra = FrameExtra {
1568 borrow_tracker: borrow_tracker.map(|bt| bt.borrow_mut().new_frame()),
1569 catch_unwind: None,
1570 timing,
1571 is_user_relevant: ecx.machine.is_user_relevant(&frame),
1572 salt: ecx.machine.rng.borrow_mut().random_range(0..ADDRS_PER_ANON_GLOBAL),
1573 data_race: ecx
1574 .machine
1575 .data_race
1576 .as_vclocks_ref()
1577 .map(|_| data_race::FrameState::default()),
1578 };
1579
1580 interp_ok(frame.with_extra(extra))
1581 }
1582
1583 fn stack<'a>(
1584 ecx: &'a InterpCx<'tcx, Self>,
1585 ) -> &'a [Frame<'tcx, Self::Provenance, Self::FrameExtra>] {
1586 ecx.active_thread_stack()
1587 }
1588
1589 fn stack_mut<'a>(
1590 ecx: &'a mut InterpCx<'tcx, Self>,
1591 ) -> &'a mut Vec<Frame<'tcx, Self::Provenance, Self::FrameExtra>> {
1592 ecx.active_thread_stack_mut()
1593 }
1594
1595 fn before_terminator(ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
1596 ecx.machine.basic_block_count += 1u64; ecx.machine.since_gc += 1;
1598 if let Some(report_progress) = ecx.machine.report_progress {
1600 if ecx.machine.basic_block_count % u64::from(report_progress) == 0 {
1601 ecx.emit_diagnostic(NonHaltingDiagnostic::ProgressReport {
1602 block_count: ecx.machine.basic_block_count,
1603 });
1604 }
1605 }
1606
1607 if ecx.machine.gc_interval > 0 && ecx.machine.since_gc >= ecx.machine.gc_interval {
1612 ecx.machine.since_gc = 0;
1613 ecx.run_provenance_gc();
1614 }
1615
1616 ecx.maybe_preempt_active_thread();
1619
1620 ecx.machine.monotonic_clock.tick();
1622
1623 interp_ok(())
1624 }
1625
1626 #[inline(always)]
1627 fn after_stack_push(ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
1628 if ecx.frame().extra.is_user_relevant {
1629 let stack_len = ecx.active_thread_stack().len();
1632 ecx.active_thread_mut().set_top_user_relevant_frame(stack_len - 1);
1633 }
1634 interp_ok(())
1635 }
1636
1637 fn before_stack_pop(
1638 ecx: &InterpCx<'tcx, Self>,
1639 frame: &Frame<'tcx, Self::Provenance, Self::FrameExtra>,
1640 ) -> InterpResult<'tcx> {
1641 if ecx.machine.borrow_tracker.is_some() {
1644 ecx.on_stack_pop(frame)?;
1645 }
1646 info!("Leaving {}", ecx.frame().instance());
1650 interp_ok(())
1651 }
1652
1653 #[inline(always)]
1654 fn after_stack_pop(
1655 ecx: &mut InterpCx<'tcx, Self>,
1656 frame: Frame<'tcx, Provenance, FrameExtra<'tcx>>,
1657 unwinding: bool,
1658 ) -> InterpResult<'tcx, ReturnAction> {
1659 if frame.extra.is_user_relevant {
1660 ecx.active_thread_mut().recompute_top_user_relevant_frame();
1665 }
1666 let res = {
1667 let mut frame = frame;
1669 let timing = frame.extra.timing.take();
1670 let res = ecx.handle_stack_pop_unwind(frame.extra, unwinding);
1671 if let Some(profiler) = ecx.machine.profiler.as_ref() {
1672 profiler.finish_recording_interval_event(timing.unwrap());
1673 }
1674 res
1675 };
1676 if !ecx.active_thread_stack().is_empty() {
1679 info!("Continuing in {}", ecx.frame().instance());
1680 }
1681 res
1682 }
1683
1684 fn after_local_read(
1685 ecx: &InterpCx<'tcx, Self>,
1686 frame: &Frame<'tcx, Provenance, FrameExtra<'tcx>>,
1687 local: mir::Local,
1688 ) -> InterpResult<'tcx> {
1689 if let Some(data_race) = &frame.extra.data_race {
1690 data_race.local_read(local, &ecx.machine);
1691 }
1692 interp_ok(())
1693 }
1694
1695 fn after_local_write(
1696 ecx: &mut InterpCx<'tcx, Self>,
1697 local: mir::Local,
1698 storage_live: bool,
1699 ) -> InterpResult<'tcx> {
1700 if let Some(data_race) = &ecx.frame().extra.data_race {
1701 data_race.local_write(local, storage_live, &ecx.machine);
1702 }
1703 interp_ok(())
1704 }
1705
1706 fn after_local_moved_to_memory(
1707 ecx: &mut InterpCx<'tcx, Self>,
1708 local: mir::Local,
1709 mplace: &MPlaceTy<'tcx>,
1710 ) -> InterpResult<'tcx> {
1711 let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr().provenance else {
1712 panic!("after_local_allocated should only be called on fresh allocations");
1713 };
1714 let local_decl = &ecx.frame().body().local_decls[local];
1716 let span = local_decl.source_info.span;
1717 ecx.machine.allocation_spans.borrow_mut().insert(alloc_id, (span, None));
1718 let (alloc_info, machine) = ecx.get_alloc_extra_mut(alloc_id)?;
1720 if let Some(data_race) =
1721 &machine.threads.active_thread_stack().last().unwrap().extra.data_race
1722 {
1723 data_race.local_moved_to_memory(
1724 local,
1725 alloc_info.data_race.as_vclocks_mut().unwrap(),
1726 machine,
1727 );
1728 }
1729 interp_ok(())
1730 }
1731
1732 fn eval_mir_constant<F>(
1733 ecx: &InterpCx<'tcx, Self>,
1734 val: mir::Const<'tcx>,
1735 span: Span,
1736 layout: Option<TyAndLayout<'tcx>>,
1737 eval: F,
1738 ) -> InterpResult<'tcx, OpTy<'tcx>>
1739 where
1740 F: Fn(
1741 &InterpCx<'tcx, Self>,
1742 mir::Const<'tcx>,
1743 Span,
1744 Option<TyAndLayout<'tcx>>,
1745 ) -> InterpResult<'tcx, OpTy<'tcx>>,
1746 {
1747 let frame = ecx.active_thread_stack().last().unwrap();
1748 let mut cache = ecx.machine.const_cache.borrow_mut();
1749 match cache.entry((val, frame.extra.salt)) {
1750 Entry::Vacant(ve) => {
1751 let op = eval(ecx, val, span, layout)?;
1752 ve.insert(op.clone());
1753 interp_ok(op)
1754 }
1755 Entry::Occupied(oe) => interp_ok(oe.get().clone()),
1756 }
1757 }
1758
1759 fn get_global_alloc_salt(
1760 ecx: &InterpCx<'tcx, Self>,
1761 instance: Option<ty::Instance<'tcx>>,
1762 ) -> usize {
1763 let unique = if let Some(instance) = instance {
1764 let is_generic = instance
1777 .args
1778 .into_iter()
1779 .any(|kind| !matches!(kind.unpack(), ty::GenericArgKind::Lifetime(_)));
1780 let can_be_inlined = matches!(
1781 ecx.tcx.sess.opts.unstable_opts.cross_crate_inline_threshold,
1782 InliningThreshold::Always
1783 ) || !matches!(
1784 ecx.tcx.codegen_fn_attrs(instance.def_id()).inline,
1785 InlineAttr::Never
1786 );
1787 !is_generic && !can_be_inlined
1788 } else {
1789 false
1791 };
1792 if unique {
1794 CTFE_ALLOC_SALT
1795 } else {
1796 ecx.machine.rng.borrow_mut().random_range(0..ADDRS_PER_ANON_GLOBAL)
1797 }
1798 }
1799
1800 fn cached_union_data_range<'e>(
1801 ecx: &'e mut InterpCx<'tcx, Self>,
1802 ty: Ty<'tcx>,
1803 compute_range: impl FnOnce() -> RangeSet,
1804 ) -> Cow<'e, RangeSet> {
1805 Cow::Borrowed(ecx.machine.union_data_ranges.entry(ty).or_insert_with(compute_range))
1806 }
1807}
1808
1809pub trait MachineCallback<'tcx, T>: VisitProvenance {
1811 fn call(
1813 self: Box<Self>,
1814 ecx: &mut InterpCx<'tcx, MiriMachine<'tcx>>,
1815 arg: T,
1816 ) -> InterpResult<'tcx>;
1817}
1818
1819pub type DynMachineCallback<'tcx, T> = Box<dyn MachineCallback<'tcx, T> + 'tcx>;
1821
1822#[macro_export]
1839macro_rules! callback {
1840 (@capture<$tcx:lifetime $(,)? $($lft:lifetime),*>
1841 { $($name:ident: $type:ty),* $(,)? }
1842 |$this:ident, $arg:ident: $arg_ty:ty| $body:expr $(,)?) => {{
1843 struct Callback<$tcx, $($lft),*> {
1844 $($name: $type,)*
1845 _phantom: std::marker::PhantomData<&$tcx ()>,
1846 }
1847
1848 impl<$tcx, $($lft),*> VisitProvenance for Callback<$tcx, $($lft),*> {
1849 fn visit_provenance(&self, _visit: &mut VisitWith<'_>) {
1850 $(
1851 self.$name.visit_provenance(_visit);
1852 )*
1853 }
1854 }
1855
1856 impl<$tcx, $($lft),*> MachineCallback<$tcx, $arg_ty> for Callback<$tcx, $($lft),*> {
1857 fn call(
1858 self: Box<Self>,
1859 $this: &mut MiriInterpCx<$tcx>,
1860 $arg: $arg_ty
1861 ) -> InterpResult<$tcx> {
1862 #[allow(unused_variables)]
1863 let Callback { $($name,)* _phantom } = *self;
1864 $body
1865 }
1866 }
1867
1868 Box::new(Callback {
1869 $($name,)*
1870 _phantom: std::marker::PhantomData
1871 })
1872 }};
1873}