1use std::sync::atomic::Ordering::Relaxed;
2
3use either::{Left, Right};
4use rustc_abi::{self as abi, BackendRepr};
5use rustc_errors::E0080;
6use rustc_hir::def::DefKind;
7use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
8use rustc_middle::mir::{self, ConstAlloc, ConstValue};
9use rustc_middle::query::TyCtxtAt;
10use rustc_middle::ty::layout::HasTypingEnv;
11use rustc_middle::ty::print::with_no_trimmed_paths;
12use rustc_middle::ty::{self, Ty, TyCtxt};
13use rustc_middle::{bug, throw_inval};
14use rustc_span::def_id::LocalDefId;
15use rustc_span::{DUMMY_SP, Span};
16use tracing::{debug, instrument, trace};
17
18use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
19use crate::const_eval::CheckAlignment;
20use crate::interpret::{
21 CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpErrorKind,
22 InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup, create_static_alloc,
23 eval_nullary_intrinsic, intern_const_alloc_recursive, interp_ok, throw_exhaust,
24};
25use crate::{CTRL_C_RECEIVED, errors};
26
27#[instrument(level = "trace", skip(ecx, body))]
29fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
30 ecx: &mut CompileTimeInterpCx<'tcx>,
31 cid: GlobalId<'tcx>,
32 body: &'tcx mir::Body<'tcx>,
33) -> InterpResult<'tcx, R> {
34 let tcx = *ecx.tcx;
35 assert!(
36 cid.promoted.is_some()
37 || matches!(
38 ecx.tcx.def_kind(cid.instance.def_id()),
39 DefKind::Const
40 | DefKind::Static { .. }
41 | DefKind::ConstParam
42 | DefKind::AnonConst
43 | DefKind::InlineConst
44 | DefKind::AssocConst
45 ),
46 "Unexpected DefKind: {:?}",
47 ecx.tcx.def_kind(cid.instance.def_id())
48 );
49 let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
50 assert!(layout.is_sized());
51
52 let intern_kind = if cid.promoted.is_some() {
53 InternKind::Promoted
54 } else {
55 match tcx.static_mutability(cid.instance.def_id()) {
56 Some(m) => InternKind::Static(m),
57 None => InternKind::Constant,
58 }
59 };
60
61 let ret = if let InternKind::Static(_) = intern_kind {
62 create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
63 } else {
64 ecx.allocate(layout, MemoryKind::Stack)?
65 };
66
67 trace!(
68 "eval_body_using_ecx: pushing stack frame for global: {}{}",
69 with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
70 cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
71 );
72
73 ecx.push_stack_frame_raw(
76 cid.instance,
77 body,
78 &ret.clone().into(),
79 StackPopCleanup::Root { cleanup: false },
80 )?;
81 ecx.storage_live_for_always_live_locals()?;
82
83 while ecx.step()? {
85 if CTRL_C_RECEIVED.load(Relaxed) {
86 throw_exhaust!(Interrupted);
87 }
88 }
89
90 let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
92
93 const_validate_mplace(ecx, &ret, cid)?;
95
96 match intern_result {
100 Ok(()) => {}
101 Err(InternResult::FoundDanglingPointer) => {
102 throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
103 ecx.tcx
104 .dcx()
105 .emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
106 )));
107 }
108 Err(InternResult::FoundBadMutablePointer) => {
109 throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
110 ecx.tcx
111 .dcx()
112 .emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
113 )));
114 }
115 }
116
117 interp_ok(R::make_result(ret, ecx))
118}
119
120pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
131 tcx: TyCtxt<'tcx>,
132 root_span: Span,
133 typing_env: ty::TypingEnv<'tcx>,
134 can_access_mut_global: CanAccessMutGlobal,
135) -> CompileTimeInterpCx<'tcx> {
136 debug!("mk_eval_cx: {:?}", typing_env);
137 InterpCx::new(
138 tcx,
139 root_span,
140 typing_env,
141 CompileTimeMachine::new(can_access_mut_global, CheckAlignment::No),
142 )
143}
144
145pub fn mk_eval_cx_for_const_val<'tcx>(
148 tcx: TyCtxtAt<'tcx>,
149 typing_env: ty::TypingEnv<'tcx>,
150 val: mir::ConstValue<'tcx>,
151 ty: Ty<'tcx>,
152) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
153 let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
154 let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
156 Some((ecx, op))
157}
158
159#[instrument(skip(ecx), level = "debug")]
166pub(super) fn op_to_const<'tcx>(
167 ecx: &CompileTimeInterpCx<'tcx>,
168 op: &OpTy<'tcx>,
169 for_diagnostics: bool,
170) -> ConstValue<'tcx> {
171 if op.layout.is_zst() {
173 return ConstValue::ZeroSized;
174 }
175
176 let force_as_immediate = match op.layout.backend_repr {
182 BackendRepr::Scalar(abi::Scalar::Initialized { .. }) => true,
183 _ => false,
191 };
192 let immediate = if force_as_immediate {
193 match ecx.read_immediate(op).report_err() {
194 Ok(imm) => Right(imm),
195 Err(err) => {
196 if for_diagnostics {
197 op.as_mplace_or_imm()
199 } else {
200 panic!("normalization works on validated constants: {err:?}")
201 }
202 }
203 }
204 } else {
205 op.as_mplace_or_imm()
206 };
207
208 debug!(?immediate);
209
210 match immediate {
211 Left(ref mplace) => {
212 let (prov, offset) = mplace.ptr().into_parts();
214 let alloc_id = prov.expect("cannot have `fake` place for non-ZST type").alloc_id();
215 ConstValue::Indirect { alloc_id, offset }
216 }
217 Right(imm) => match *imm {
219 Immediate::Scalar(x) => ConstValue::Scalar(x),
220 Immediate::ScalarPair(a, b) => {
221 debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
222 let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); debug_assert!(
227 matches!(
228 ecx.tcx.struct_tail_for_codegen(pointee_ty, ecx.typing_env()).kind(),
229 ty::Str | ty::Slice(..),
230 ),
231 "`ConstValue::Slice` is for slice-tailed types only, but got {}",
232 imm.layout.ty,
233 );
234 let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
235 let (prov, offset) = a.to_pointer(ecx).expect(msg).into_parts();
237 let alloc_id = prov.expect(msg).alloc_id();
238 let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
239 assert!(offset == abi::Size::ZERO, "{}", msg);
240 let meta = b.to_target_usize(ecx).expect(msg);
241 ConstValue::Slice { data, meta }
242 }
243 Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
244 },
245 }
246}
247
248#[instrument(skip(tcx), level = "debug", ret)]
249pub(crate) fn turn_into_const_value<'tcx>(
250 tcx: TyCtxt<'tcx>,
251 constant: ConstAlloc<'tcx>,
252 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
253) -> ConstValue<'tcx> {
254 let cid = key.value;
255 let def_id = cid.instance.def.def_id();
256 let is_static = tcx.is_static(def_id);
257 let ecx = mk_eval_cx_to_read_const_val(
259 tcx,
260 tcx.def_span(key.value.instance.def_id()),
261 key.typing_env,
262 CanAccessMutGlobal::from(is_static),
263 );
264
265 let mplace = ecx.raw_const_to_mplace(constant).expect(
266 "can only fail if layout computation failed, \
267 which should have given a good error before ever invoking this function",
268 );
269 assert!(
270 !is_static || cid.promoted.is_some(),
271 "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
272 );
273
274 op_to_const(&ecx, &mplace.into(), false)
276}
277
278#[instrument(skip(tcx), level = "debug")]
279pub fn eval_to_const_value_raw_provider<'tcx>(
280 tcx: TyCtxt<'tcx>,
281 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
282) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
283 if let ty::InstanceKind::Intrinsic(def_id) = key.value.instance.def {
286 let ty = key.value.instance.ty(tcx, key.typing_env);
287 let ty::FnDef(_, args) = ty.kind() else {
288 bug!("intrinsic with type {:?}", ty);
289 };
290 return eval_nullary_intrinsic(tcx, key.typing_env, def_id, args).report_err().map_err(
291 |error| {
292 let span = tcx.def_span(def_id);
293
294 super::report(
296 tcx,
297 error.into_kind(),
298 span,
299 || (span, vec![]),
300 |diag, span, _| {
301 diag.span_label(
302 span,
303 crate::fluent_generated::const_eval_nullary_intrinsic_fail,
304 );
305 },
306 )
307 },
308 );
309 }
310
311 tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
312}
313
314#[instrument(skip(tcx), level = "debug")]
315pub fn eval_static_initializer_provider<'tcx>(
316 tcx: TyCtxt<'tcx>,
317 def_id: LocalDefId,
318) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
319 assert!(tcx.is_static(def_id.to_def_id()));
320
321 let instance = ty::Instance::mono(tcx, def_id.to_def_id());
322 let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
323 eval_in_interpreter(tcx, cid, ty::TypingEnv::fully_monomorphized())
324}
325
326pub trait InterpretationResult<'tcx> {
327 fn make_result(
331 mplace: MPlaceTy<'tcx>,
332 ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
333 ) -> Self;
334}
335
336impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
337 fn make_result(
338 mplace: MPlaceTy<'tcx>,
339 _ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
340 ) -> Self {
341 ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
342 }
343}
344
345#[instrument(skip(tcx), level = "debug")]
346pub fn eval_to_allocation_raw_provider<'tcx>(
347 tcx: TyCtxt<'tcx>,
348 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
349) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
350 assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
353 debug_assert_eq!(key.typing_env.typing_mode, ty::TypingMode::PostAnalysis);
356 if cfg!(debug_assertions) {
357 let instance = with_no_trimmed_paths!(key.value.instance.to_string());
363 trace!("const eval: {:?} ({})", key, instance);
364 }
365
366 eval_in_interpreter(tcx, key.value, key.typing_env)
367}
368
369fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
370 tcx: TyCtxt<'tcx>,
371 cid: GlobalId<'tcx>,
372 typing_env: ty::TypingEnv<'tcx>,
373) -> Result<R, ErrorHandled> {
374 let def = cid.instance.def.def_id();
375 let is_static = tcx.is_static(def);
376
377 let mut ecx = InterpCx::new(
378 tcx,
379 tcx.def_span(def),
380 typing_env,
381 CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
386 );
387 let res = ecx.load_mir(cid.instance.def, cid.promoted);
388 res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
389 .report_err()
390 .map_err(|error| report_eval_error(&ecx, cid, error))
391}
392
393#[inline(always)]
394fn const_validate_mplace<'tcx>(
395 ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
396 mplace: &MPlaceTy<'tcx>,
397 cid: GlobalId<'tcx>,
398) -> Result<(), ErrorHandled> {
399 let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
400 let mut ref_tracking = RefTracking::new(mplace.clone());
401 let mut inner = false;
402 while let Some((mplace, path)) = ref_tracking.next() {
403 let mode = match ecx.tcx.static_mutability(cid.instance.def_id()) {
404 _ if cid.promoted.is_some() => CtfeValidationMode::Promoted,
405 Some(mutbl) => CtfeValidationMode::Static { mutbl }, None => {
407 CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
411 }
412 };
413 ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
414 .report_err()
415 .map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;
418 inner = true;
419 }
420
421 Ok(())
422}
423
424#[inline(never)]
425fn report_eval_error<'tcx>(
426 ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
427 cid: GlobalId<'tcx>,
428 error: InterpErrorInfo<'tcx>,
429) -> ErrorHandled {
430 let (error, backtrace) = error.into_parts();
431 backtrace.print_backtrace();
432
433 let instance = with_no_trimmed_paths!(cid.instance.to_string());
434
435 super::report(
436 *ecx.tcx,
437 error,
438 DUMMY_SP,
439 || super::get_span_and_frames(ecx.tcx, ecx.stack()),
440 |diag, span, frames| {
441 let num_frames = frames.len();
442 diag.code(E0080);
444 diag.span_label(span, crate::fluent_generated::const_eval_error);
445 for frame in frames {
446 diag.subdiagnostic(frame);
447 }
448 diag.arg("instance", instance);
450 diag.arg("num_frames", num_frames);
451 },
452 )
453}
454
455#[inline(never)]
456fn report_validation_error<'tcx>(
457 ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
458 cid: GlobalId<'tcx>,
459 error: InterpErrorInfo<'tcx>,
460 alloc_id: AllocId,
461) -> ErrorHandled {
462 if !matches!(error.kind(), InterpErrorKind::UndefinedBehavior(_)) {
463 return report_eval_error(ecx, cid, error);
465 }
466
467 let (error, backtrace) = error.into_parts();
468 backtrace.print_backtrace();
469
470 let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
471 let info = ecx.get_alloc_info(alloc_id);
472 let raw_bytes =
473 errors::RawBytesNote { size: info.size.bytes(), align: info.align.bytes(), bytes };
474
475 crate::const_eval::report(
476 *ecx.tcx,
477 error,
478 DUMMY_SP,
479 || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
480 move |diag, span, frames| {
481 diag.code(E0080);
483 diag.span_label(span, crate::fluent_generated::const_eval_validation_failure);
484 diag.note(crate::fluent_generated::const_eval_validation_failure_note);
485 for frame in frames {
486 diag.subdiagnostic(frame);
487 }
488 diag.subdiagnostic(raw_bytes);
489 },
490 )
491}