1use std::sync::atomic::Ordering::Relaxed;
2
3use either::{Left, Right};
4use rustc_abi::{self as abi, BackendRepr};
5use rustc_errors::E0080;
6use rustc_hir::def::DefKind;
7use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
8use rustc_middle::mir::{self, ConstAlloc, ConstValue};
9use rustc_middle::query::TyCtxtAt;
10use rustc_middle::ty::layout::HasTypingEnv;
11use rustc_middle::ty::print::with_no_trimmed_paths;
12use rustc_middle::ty::{self, Ty, TyCtxt};
13use rustc_middle::{bug, throw_inval};
14use rustc_span::def_id::LocalDefId;
15use rustc_span::{DUMMY_SP, Span};
16use tracing::{debug, instrument, trace};
17
18use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
19use crate::const_eval::CheckAlignment;
20use crate::interpret::{
21 CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpErrorKind,
22 InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, ReturnContinuation, create_static_alloc,
23 intern_const_alloc_recursive, interp_ok, throw_exhaust,
24};
25use crate::{CTRL_C_RECEIVED, errors};
26
27#[instrument(level = "trace", skip(ecx, body))]
29fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
30 ecx: &mut CompileTimeInterpCx<'tcx>,
31 cid: GlobalId<'tcx>,
32 body: &'tcx mir::Body<'tcx>,
33) -> InterpResult<'tcx, R> {
34 let tcx = *ecx.tcx;
35 assert!(
36 cid.promoted.is_some()
37 || matches!(
38 ecx.tcx.def_kind(cid.instance.def_id()),
39 DefKind::Const
40 | DefKind::Static { .. }
41 | DefKind::ConstParam
42 | DefKind::AnonConst
43 | DefKind::InlineConst
44 | DefKind::AssocConst
45 ),
46 "Unexpected DefKind: {:?}",
47 ecx.tcx.def_kind(cid.instance.def_id())
48 );
49 let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
50 assert!(layout.is_sized());
51
52 let intern_kind = if cid.promoted.is_some() {
53 InternKind::Promoted
54 } else {
55 match tcx.static_mutability(cid.instance.def_id()) {
56 Some(m) => InternKind::Static(m),
57 None => InternKind::Constant,
58 }
59 };
60
61 let ret = if let InternKind::Static(_) = intern_kind {
62 create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
63 } else {
64 ecx.allocate(layout, MemoryKind::Stack)?
65 };
66
67 trace!(
68 "eval_body_using_ecx: pushing stack frame for global: {}{}",
69 with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
70 cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
71 );
72
73 ecx.push_stack_frame_raw(
76 cid.instance,
77 body,
78 &ret.clone().into(),
79 ReturnContinuation::Stop { cleanup: false },
80 )?;
81 ecx.storage_live_for_always_live_locals()?;
82
83 while ecx.step()? {
85 if CTRL_C_RECEIVED.load(Relaxed) {
86 throw_exhaust!(Interrupted);
87 }
88 }
89
90 let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
92
93 const_validate_mplace(ecx, &ret, cid)?;
95
96 match intern_result {
100 Ok(()) => {}
101 Err(InternResult::FoundDanglingPointer) => {
102 throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
103 ecx.tcx
104 .dcx()
105 .emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
106 )));
107 }
108 Err(InternResult::FoundBadMutablePointer) => {
109 throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
110 ecx.tcx
111 .dcx()
112 .emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
113 )));
114 }
115 }
116
117 interp_ok(R::make_result(ret, ecx))
118}
119
120pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
131 tcx: TyCtxt<'tcx>,
132 root_span: Span,
133 typing_env: ty::TypingEnv<'tcx>,
134 can_access_mut_global: CanAccessMutGlobal,
135) -> CompileTimeInterpCx<'tcx> {
136 debug!("mk_eval_cx: {:?}", typing_env);
137 InterpCx::new(
138 tcx,
139 root_span,
140 typing_env,
141 CompileTimeMachine::new(can_access_mut_global, CheckAlignment::No),
142 )
143}
144
145pub fn mk_eval_cx_for_const_val<'tcx>(
148 tcx: TyCtxtAt<'tcx>,
149 typing_env: ty::TypingEnv<'tcx>,
150 val: mir::ConstValue<'tcx>,
151 ty: Ty<'tcx>,
152) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
153 let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
154 let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
156 Some((ecx, op))
157}
158
159#[instrument(skip(ecx), level = "debug")]
166pub(super) fn op_to_const<'tcx>(
167 ecx: &CompileTimeInterpCx<'tcx>,
168 op: &OpTy<'tcx>,
169 for_diagnostics: bool,
170) -> ConstValue<'tcx> {
171 if op.layout.is_zst() {
173 return ConstValue::ZeroSized;
174 }
175
176 let force_as_immediate = match op.layout.backend_repr {
182 BackendRepr::Scalar(abi::Scalar::Initialized { .. }) => true,
183 _ => false,
191 };
192 let immediate = if force_as_immediate {
193 match ecx.read_immediate(op).report_err() {
194 Ok(imm) => Right(imm),
195 Err(err) => {
196 if for_diagnostics {
197 op.as_mplace_or_imm()
199 } else {
200 panic!("normalization works on validated constants: {err:?}")
201 }
202 }
203 }
204 } else {
205 op.as_mplace_or_imm()
206 };
207
208 debug!(?immediate);
209
210 match immediate {
211 Left(ref mplace) => {
212 let (prov, offset) =
213 mplace.ptr().into_pointer_or_addr().unwrap().prov_and_relative_offset();
214 let alloc_id = prov.alloc_id();
215 ConstValue::Indirect { alloc_id, offset }
216 }
217 Right(imm) => match *imm {
219 Immediate::Scalar(x) => ConstValue::Scalar(x),
220 Immediate::ScalarPair(a, b) => {
221 debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
222 let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); debug_assert!(
227 matches!(
228 ecx.tcx.struct_tail_for_codegen(pointee_ty, ecx.typing_env()).kind(),
229 ty::Str | ty::Slice(..),
230 ),
231 "`ConstValue::Slice` is for slice-tailed types only, but got {}",
232 imm.layout.ty,
233 );
234 let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
235 let ptr = a.to_pointer(ecx).expect(msg);
236 let (prov, offset) =
237 ptr.into_pointer_or_addr().expect(msg).prov_and_relative_offset();
238 let alloc_id = prov.alloc_id();
239 let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
240 assert!(offset == abi::Size::ZERO, "{}", msg);
241 let meta = b.to_target_usize(ecx).expect(msg);
242 ConstValue::Slice { data, meta }
243 }
244 Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
245 },
246 }
247}
248
249#[instrument(skip(tcx), level = "debug", ret)]
250pub(crate) fn turn_into_const_value<'tcx>(
251 tcx: TyCtxt<'tcx>,
252 constant: ConstAlloc<'tcx>,
253 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
254) -> ConstValue<'tcx> {
255 let cid = key.value;
256 let def_id = cid.instance.def.def_id();
257 let is_static = tcx.is_static(def_id);
258 let ecx = mk_eval_cx_to_read_const_val(
260 tcx,
261 tcx.def_span(key.value.instance.def_id()),
262 key.typing_env,
263 CanAccessMutGlobal::from(is_static),
264 );
265
266 let mplace = ecx.raw_const_to_mplace(constant).expect(
267 "can only fail if layout computation failed, \
268 which should have given a good error before ever invoking this function",
269 );
270 assert!(
271 !is_static || cid.promoted.is_some(),
272 "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
273 );
274
275 op_to_const(&ecx, &mplace.into(), false)
277}
278
279#[instrument(skip(tcx), level = "debug")]
280pub fn eval_to_const_value_raw_provider<'tcx>(
281 tcx: TyCtxt<'tcx>,
282 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
283) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
284 tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
285}
286
287#[instrument(skip(tcx), level = "debug")]
288pub fn eval_static_initializer_provider<'tcx>(
289 tcx: TyCtxt<'tcx>,
290 def_id: LocalDefId,
291) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
292 assert!(tcx.is_static(def_id.to_def_id()));
293
294 let instance = ty::Instance::mono(tcx, def_id.to_def_id());
295 let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
296 eval_in_interpreter(tcx, cid, ty::TypingEnv::fully_monomorphized())
297}
298
299pub trait InterpretationResult<'tcx> {
300 fn make_result(
304 mplace: MPlaceTy<'tcx>,
305 ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
306 ) -> Self;
307}
308
309impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
310 fn make_result(
311 mplace: MPlaceTy<'tcx>,
312 _ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
313 ) -> Self {
314 ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
315 }
316}
317
318#[instrument(skip(tcx), level = "debug")]
319pub fn eval_to_allocation_raw_provider<'tcx>(
320 tcx: TyCtxt<'tcx>,
321 key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
322) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
323 assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
326 debug_assert_eq!(key.typing_env.typing_mode, ty::TypingMode::PostAnalysis);
329 if cfg!(debug_assertions) {
330 let instance = with_no_trimmed_paths!(key.value.instance.to_string());
336 trace!("const eval: {:?} ({})", key, instance);
337 }
338
339 eval_in_interpreter(tcx, key.value, key.typing_env)
340}
341
342fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
343 tcx: TyCtxt<'tcx>,
344 cid: GlobalId<'tcx>,
345 typing_env: ty::TypingEnv<'tcx>,
346) -> Result<R, ErrorHandled> {
347 let def = cid.instance.def.def_id();
348 let is_static = tcx.is_static(def);
349
350 let mut ecx = InterpCx::new(
351 tcx,
352 tcx.def_span(def),
353 typing_env,
354 CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
359 );
360 let res = ecx.load_mir(cid.instance.def, cid.promoted);
361 res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
362 .report_err()
363 .map_err(|error| report_eval_error(&ecx, cid, error))
364}
365
366#[inline(always)]
367fn const_validate_mplace<'tcx>(
368 ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
369 mplace: &MPlaceTy<'tcx>,
370 cid: GlobalId<'tcx>,
371) -> Result<(), ErrorHandled> {
372 let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
373 let mut ref_tracking = RefTracking::new(mplace.clone());
374 let mut inner = false;
375 while let Some((mplace, path)) = ref_tracking.next() {
376 let mode = match ecx.tcx.static_mutability(cid.instance.def_id()) {
377 _ if cid.promoted.is_some() => CtfeValidationMode::Promoted,
378 Some(mutbl) => CtfeValidationMode::Static { mutbl }, None => {
380 CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
384 }
385 };
386 ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
387 .report_err()
388 .map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;
391 inner = true;
392 }
393
394 Ok(())
395}
396
397#[inline(never)]
398fn report_eval_error<'tcx>(
399 ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
400 cid: GlobalId<'tcx>,
401 error: InterpErrorInfo<'tcx>,
402) -> ErrorHandled {
403 let (error, backtrace) = error.into_parts();
404 backtrace.print_backtrace();
405
406 let instance = with_no_trimmed_paths!(cid.instance.to_string());
407
408 super::report(
409 *ecx.tcx,
410 error,
411 DUMMY_SP,
412 || super::get_span_and_frames(ecx.tcx, ecx.stack()),
413 |diag, span, frames| {
414 let num_frames = frames.len();
415 diag.code(E0080);
417 diag.span_label(span, crate::fluent_generated::const_eval_error);
418 for frame in frames {
419 diag.subdiagnostic(frame);
420 }
421 diag.arg("instance", instance);
423 diag.arg("num_frames", num_frames);
424 },
425 )
426}
427
428#[inline(never)]
429fn report_validation_error<'tcx>(
430 ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
431 cid: GlobalId<'tcx>,
432 error: InterpErrorInfo<'tcx>,
433 alloc_id: AllocId,
434) -> ErrorHandled {
435 if !matches!(error.kind(), InterpErrorKind::UndefinedBehavior(_)) {
436 return report_eval_error(ecx, cid, error);
438 }
439
440 let (error, backtrace) = error.into_parts();
441 backtrace.print_backtrace();
442
443 let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
444 let info = ecx.get_alloc_info(alloc_id);
445 let raw_bytes =
446 errors::RawBytesNote { size: info.size.bytes(), align: info.align.bytes(), bytes };
447
448 crate::const_eval::report(
449 *ecx.tcx,
450 error,
451 DUMMY_SP,
452 || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
453 move |diag, span, frames| {
454 diag.code(E0080);
456 diag.span_label(span, crate::fluent_generated::const_eval_validation_failure);
457 diag.note(crate::fluent_generated::const_eval_validation_failure_note);
458 for frame in frames {
459 diag.subdiagnostic(frame);
460 }
461 diag.subdiagnostic(raw_bytes);
462 },
463 )
464}