rustc_const_eval/const_eval/
eval_queries.rs

1use std::sync::atomic::Ordering::Relaxed;
2
3use either::{Left, Right};
4use rustc_abi::{self as abi, BackendRepr};
5use rustc_errors::E0080;
6use rustc_hir::def::DefKind;
7use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
8use rustc_middle::mir::{self, ConstAlloc, ConstValue};
9use rustc_middle::query::TyCtxtAt;
10use rustc_middle::ty::layout::HasTypingEnv;
11use rustc_middle::ty::print::with_no_trimmed_paths;
12use rustc_middle::ty::{self, Ty, TyCtxt};
13use rustc_middle::{bug, throw_inval};
14use rustc_span::def_id::LocalDefId;
15use rustc_span::{DUMMY_SP, Span};
16use tracing::{debug, instrument, trace};
17
18use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
19use crate::const_eval::CheckAlignment;
20use crate::interpret::{
21    CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpErrorKind,
22    InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, ReturnContinuation, create_static_alloc,
23    intern_const_alloc_recursive, interp_ok, throw_exhaust,
24};
25use crate::{CTRL_C_RECEIVED, errors};
26
27// Returns a pointer to where the result lives
28#[instrument(level = "trace", skip(ecx, body))]
29fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
30    ecx: &mut CompileTimeInterpCx<'tcx>,
31    cid: GlobalId<'tcx>,
32    body: &'tcx mir::Body<'tcx>,
33) -> InterpResult<'tcx, R> {
34    let tcx = *ecx.tcx;
35    assert!(
36        cid.promoted.is_some()
37            || matches!(
38                ecx.tcx.def_kind(cid.instance.def_id()),
39                DefKind::Const
40                    | DefKind::Static { .. }
41                    | DefKind::ConstParam
42                    | DefKind::AnonConst
43                    | DefKind::InlineConst
44                    | DefKind::AssocConst
45            ),
46        "Unexpected DefKind: {:?}",
47        ecx.tcx.def_kind(cid.instance.def_id())
48    );
49    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
50    assert!(layout.is_sized());
51
52    let intern_kind = if cid.promoted.is_some() {
53        InternKind::Promoted
54    } else {
55        match tcx.static_mutability(cid.instance.def_id()) {
56            Some(m) => InternKind::Static(m),
57            None => InternKind::Constant,
58        }
59    };
60
61    let ret = if let InternKind::Static(_) = intern_kind {
62        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
63    } else {
64        ecx.allocate(layout, MemoryKind::Stack)?
65    };
66
67    trace!(
68        "eval_body_using_ecx: pushing stack frame for global: {}{}",
69        with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
70        cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
71    );
72
73    // This can't use `init_stack_frame` since `body` is not a function,
74    // so computing its ABI would fail. It's also not worth it since there are no arguments to pass.
75    ecx.push_stack_frame_raw(
76        cid.instance,
77        body,
78        &ret.clone().into(),
79        ReturnContinuation::Stop { cleanup: false },
80    )?;
81    ecx.storage_live_for_always_live_locals()?;
82
83    // The main interpreter loop.
84    while ecx.step()? {
85        if CTRL_C_RECEIVED.load(Relaxed) {
86            throw_exhaust!(Interrupted);
87        }
88    }
89
90    // Intern the result
91    let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
92
93    // Since evaluation had no errors, validate the resulting constant.
94    const_validate_mplace(ecx, &ret, cid)?;
95
96    // Only report this after validation, as validaiton produces much better diagnostics.
97    // FIXME: ensure validation always reports this and stop making interning care about it.
98
99    match intern_result {
100        Ok(()) => {}
101        Err(InternResult::FoundDanglingPointer) => {
102            throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
103                ecx.tcx
104                    .dcx()
105                    .emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
106            )));
107        }
108        Err(InternResult::FoundBadMutablePointer) => {
109            throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
110                ecx.tcx
111                    .dcx()
112                    .emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
113            )));
114        }
115    }
116
117    interp_ok(R::make_result(ret, ecx))
118}
119
120/// The `InterpCx` is only meant to be used to do field and index projections into constants for
121/// `simd_shuffle` and const patterns in match arms.
122///
123/// This should *not* be used to do any actual interpretation. In particular, alignment checks are
124/// turned off!
125///
126/// The function containing the `match` that is currently being analyzed may have generic bounds
127/// that inform us about the generic bounds of the constant. E.g., using an associated constant
128/// of a function's generic parameter will require knowledge about the bounds on the generic
129/// parameter. These bounds are passed to `mk_eval_cx` via the `ParamEnv` argument.
130pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
131    tcx: TyCtxt<'tcx>,
132    root_span: Span,
133    typing_env: ty::TypingEnv<'tcx>,
134    can_access_mut_global: CanAccessMutGlobal,
135) -> CompileTimeInterpCx<'tcx> {
136    debug!("mk_eval_cx: {:?}", typing_env);
137    InterpCx::new(
138        tcx,
139        root_span,
140        typing_env,
141        CompileTimeMachine::new(can_access_mut_global, CheckAlignment::No),
142    )
143}
144
145/// Create an interpreter context to inspect the given `ConstValue`.
146/// Returns both the context and an `OpTy` that represents the constant.
147pub fn mk_eval_cx_for_const_val<'tcx>(
148    tcx: TyCtxtAt<'tcx>,
149    typing_env: ty::TypingEnv<'tcx>,
150    val: mir::ConstValue<'tcx>,
151    ty: Ty<'tcx>,
152) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
153    let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
154    // FIXME: is it a problem to discard the error here?
155    let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
156    Some((ecx, op))
157}
158
159/// This function converts an interpreter value into a MIR constant.
160///
161/// The `for_diagnostics` flag turns the usual rules for returning `ConstValue::Scalar` into a
162/// best-effort attempt. This is not okay for use in const-eval sine it breaks invariants rustc
163/// relies on, but it is okay for diagnostics which will just give up gracefully when they
164/// encounter an `Indirect` they cannot handle.
165#[instrument(skip(ecx), level = "debug")]
166pub(super) fn op_to_const<'tcx>(
167    ecx: &CompileTimeInterpCx<'tcx>,
168    op: &OpTy<'tcx>,
169    for_diagnostics: bool,
170) -> ConstValue<'tcx> {
171    // Handle ZST consistently and early.
172    if op.layout.is_zst() {
173        return ConstValue::ZeroSized;
174    }
175
176    // All scalar types should be stored as `ConstValue::Scalar`. This is needed to make
177    // `ConstValue::try_to_scalar` efficient; we want that to work for *all* constants of scalar
178    // type (it's used throughout the compiler and having it work just on literals is not enough)
179    // and we want it to be fast (i.e., don't go to an `Allocation` and reconstruct the `Scalar`
180    // from its byte-serialized form).
181    let force_as_immediate = match op.layout.backend_repr {
182        BackendRepr::Scalar(abi::Scalar::Initialized { .. }) => true,
183        // We don't *force* `ConstValue::Slice` for `ScalarPair`. This has the advantage that if the
184        // input `op` is a place, then turning it into a `ConstValue` and back into a `OpTy` will
185        // not have to generate any duplicate allocations (we preserve the original `AllocId` in
186        // `ConstValue::Indirect`). It means accessing the contents of a slice can be slow (since
187        // they can be stored as `ConstValue::Indirect`), but that's not relevant since we barely
188        // ever have to do this. (`try_get_slice_bytes_for_diagnostics` exists to provide this
189        // functionality.)
190        _ => false,
191    };
192    let immediate = if force_as_immediate {
193        match ecx.read_immediate(op).report_err() {
194            Ok(imm) => Right(imm),
195            Err(err) => {
196                if for_diagnostics {
197                    // This discard the error, but for diagnostics that's okay.
198                    op.as_mplace_or_imm()
199                } else {
200                    panic!("normalization works on validated constants: {err:?}")
201                }
202            }
203        }
204    } else {
205        op.as_mplace_or_imm()
206    };
207
208    debug!(?immediate);
209
210    match immediate {
211        Left(ref mplace) => {
212            let (prov, offset) =
213                mplace.ptr().into_pointer_or_addr().unwrap().prov_and_relative_offset();
214            let alloc_id = prov.alloc_id();
215            ConstValue::Indirect { alloc_id, offset }
216        }
217        // see comment on `let force_as_immediate` above
218        Right(imm) => match *imm {
219            Immediate::Scalar(x) => ConstValue::Scalar(x),
220            Immediate::ScalarPair(a, b) => {
221                debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
222                // This codepath solely exists for `valtree_to_const_value` to not need to generate
223                // a `ConstValue::Indirect` for wide references, so it is tightly restricted to just
224                // that case.
225                let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); // `false` = no raw ptrs
226                debug_assert!(
227                    matches!(
228                        ecx.tcx.struct_tail_for_codegen(pointee_ty, ecx.typing_env()).kind(),
229                        ty::Str | ty::Slice(..),
230                    ),
231                    "`ConstValue::Slice` is for slice-tailed types only, but got {}",
232                    imm.layout.ty,
233                );
234                let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
235                let ptr = a.to_pointer(ecx).expect(msg);
236                let (prov, offset) =
237                    ptr.into_pointer_or_addr().expect(msg).prov_and_relative_offset();
238                let alloc_id = prov.alloc_id();
239                let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
240                assert!(offset == abi::Size::ZERO, "{}", msg);
241                let meta = b.to_target_usize(ecx).expect(msg);
242                ConstValue::Slice { data, meta }
243            }
244            Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
245        },
246    }
247}
248
249#[instrument(skip(tcx), level = "debug", ret)]
250pub(crate) fn turn_into_const_value<'tcx>(
251    tcx: TyCtxt<'tcx>,
252    constant: ConstAlloc<'tcx>,
253    key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
254) -> ConstValue<'tcx> {
255    let cid = key.value;
256    let def_id = cid.instance.def.def_id();
257    let is_static = tcx.is_static(def_id);
258    // This is just accessing an already computed constant, so no need to check alignment here.
259    let ecx = mk_eval_cx_to_read_const_val(
260        tcx,
261        tcx.def_span(key.value.instance.def_id()),
262        key.typing_env,
263        CanAccessMutGlobal::from(is_static),
264    );
265
266    let mplace = ecx.raw_const_to_mplace(constant).expect(
267        "can only fail if layout computation failed, \
268        which should have given a good error before ever invoking this function",
269    );
270    assert!(
271        !is_static || cid.promoted.is_some(),
272        "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
273    );
274
275    // Turn this into a proper constant.
276    op_to_const(&ecx, &mplace.into(), /* for diagnostics */ false)
277}
278
279#[instrument(skip(tcx), level = "debug")]
280pub fn eval_to_const_value_raw_provider<'tcx>(
281    tcx: TyCtxt<'tcx>,
282    key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
283) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
284    tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
285}
286
287#[instrument(skip(tcx), level = "debug")]
288pub fn eval_static_initializer_provider<'tcx>(
289    tcx: TyCtxt<'tcx>,
290    def_id: LocalDefId,
291) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
292    assert!(tcx.is_static(def_id.to_def_id()));
293
294    let instance = ty::Instance::mono(tcx, def_id.to_def_id());
295    let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
296    eval_in_interpreter(tcx, cid, ty::TypingEnv::fully_monomorphized())
297}
298
299pub trait InterpretationResult<'tcx> {
300    /// This function takes the place where the result of the evaluation is stored
301    /// and prepares it for returning it in the appropriate format needed by the specific
302    /// evaluation query.
303    fn make_result(
304        mplace: MPlaceTy<'tcx>,
305        ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
306    ) -> Self;
307}
308
309impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
310    fn make_result(
311        mplace: MPlaceTy<'tcx>,
312        _ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
313    ) -> Self {
314        ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
315    }
316}
317
318#[instrument(skip(tcx), level = "debug")]
319pub fn eval_to_allocation_raw_provider<'tcx>(
320    tcx: TyCtxt<'tcx>,
321    key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
322) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
323    // This shouldn't be used for statics, since statics are conceptually places,
324    // not values -- so what we do here could break pointer identity.
325    assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
326    // Const eval always happens in PostAnalysis mode . See the comment in
327    // `InterpCx::new` for more details.
328    debug_assert_eq!(key.typing_env.typing_mode, ty::TypingMode::PostAnalysis);
329    if cfg!(debug_assertions) {
330        // Make sure we format the instance even if we do not print it.
331        // This serves as a regression test against an ICE on printing.
332        // The next two lines concatenated contain some discussion:
333        // https://rust-lang.zulipchat.com/#narrow/stream/146212-t-compiler.2Fconst-eval/
334        // subject/anon_const_instance_printing/near/135980032
335        let instance = with_no_trimmed_paths!(key.value.instance.to_string());
336        trace!("const eval: {:?} ({})", key, instance);
337    }
338
339    eval_in_interpreter(tcx, key.value, key.typing_env)
340}
341
342fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
343    tcx: TyCtxt<'tcx>,
344    cid: GlobalId<'tcx>,
345    typing_env: ty::TypingEnv<'tcx>,
346) -> Result<R, ErrorHandled> {
347    let def = cid.instance.def.def_id();
348    let is_static = tcx.is_static(def);
349
350    let mut ecx = InterpCx::new(
351        tcx,
352        tcx.def_span(def),
353        typing_env,
354        // Statics (and promoteds inside statics) may access mutable global memory, because unlike consts
355        // they do not have to behave "as if" they were evaluated at runtime.
356        // For consts however we want to ensure they behave "as if" they were evaluated at runtime,
357        // so we have to reject reading mutable global memory.
358        CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
359    );
360    let res = ecx.load_mir(cid.instance.def, cid.promoted);
361    res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
362        .report_err()
363        .map_err(|error| report_eval_error(&ecx, cid, error))
364}
365
366#[inline(always)]
367fn const_validate_mplace<'tcx>(
368    ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
369    mplace: &MPlaceTy<'tcx>,
370    cid: GlobalId<'tcx>,
371) -> Result<(), ErrorHandled> {
372    let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
373    let mut ref_tracking = RefTracking::new(mplace.clone());
374    let mut inner = false;
375    while let Some((mplace, path)) = ref_tracking.next() {
376        let mode = match ecx.tcx.static_mutability(cid.instance.def_id()) {
377            _ if cid.promoted.is_some() => CtfeValidationMode::Promoted,
378            Some(mutbl) => CtfeValidationMode::Static { mutbl }, // a `static`
379            None => {
380                // This is a normal `const` (not promoted).
381                // The outermost allocation is always only copied, so having `UnsafeCell` in there
382                // is okay despite them being in immutable memory.
383                CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
384            }
385        };
386        ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
387            .report_err()
388            // Instead of just reporting the `InterpError` via the usual machinery, we give a more targeted
389            // error about the validation failure.
390            .map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;
391        inner = true;
392    }
393
394    Ok(())
395}
396
397#[inline(never)]
398fn report_eval_error<'tcx>(
399    ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
400    cid: GlobalId<'tcx>,
401    error: InterpErrorInfo<'tcx>,
402) -> ErrorHandled {
403    let (error, backtrace) = error.into_parts();
404    backtrace.print_backtrace();
405
406    let instance = with_no_trimmed_paths!(cid.instance.to_string());
407
408    super::report(
409        *ecx.tcx,
410        error,
411        DUMMY_SP,
412        || super::get_span_and_frames(ecx.tcx, ecx.stack()),
413        |diag, span, frames| {
414            let num_frames = frames.len();
415            // FIXME(oli-obk): figure out how to use structured diagnostics again.
416            diag.code(E0080);
417            diag.span_label(span, crate::fluent_generated::const_eval_error);
418            for frame in frames {
419                diag.subdiagnostic(frame);
420            }
421            // Add after the frame rendering above, as it adds its own `instance` args.
422            diag.arg("instance", instance);
423            diag.arg("num_frames", num_frames);
424        },
425    )
426}
427
428#[inline(never)]
429fn report_validation_error<'tcx>(
430    ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
431    cid: GlobalId<'tcx>,
432    error: InterpErrorInfo<'tcx>,
433    alloc_id: AllocId,
434) -> ErrorHandled {
435    if !matches!(error.kind(), InterpErrorKind::UndefinedBehavior(_)) {
436        // Some other error happened during validation, e.g. an unsupported operation.
437        return report_eval_error(ecx, cid, error);
438    }
439
440    let (error, backtrace) = error.into_parts();
441    backtrace.print_backtrace();
442
443    let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
444    let info = ecx.get_alloc_info(alloc_id);
445    let raw_bytes =
446        errors::RawBytesNote { size: info.size.bytes(), align: info.align.bytes(), bytes };
447
448    crate::const_eval::report(
449        *ecx.tcx,
450        error,
451        DUMMY_SP,
452        || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
453        move |diag, span, frames| {
454            // FIXME(oli-obk): figure out how to use structured diagnostics again.
455            diag.code(E0080);
456            diag.span_label(span, crate::fluent_generated::const_eval_validation_failure);
457            diag.note(crate::fluent_generated::const_eval_validation_failure_note);
458            for frame in frames {
459                diag.subdiagnostic(frame);
460            }
461            diag.subdiagnostic(raw_bytes);
462        },
463    )
464}