1use std::ops::Deref;
2
3use rustc_data_structures::sync::{AtomicU64, WorkerLocal};
4use rustc_hir::def_id::{DefId, LocalDefId};
5use rustc_hir::hir_id::OwnerId;
6use rustc_macros::HashStable;
7use rustc_query_system::HandleCycleError;
8use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
9pub(crate) use rustc_query_system::query::QueryJobId;
10use rustc_query_system::query::*;
11use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
12
13use crate::dep_graph;
14use crate::dep_graph::DepKind;
15use crate::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache};
16use crate::query::{
17 DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
18};
19use crate::ty::TyCtxt;
20
21pub struct DynamicQuery<'tcx, C: QueryCache> {
22 pub name: &'static str,
23 pub eval_always: bool,
24 pub dep_kind: DepKind,
25 pub handle_cycle_error: HandleCycleError,
26 pub query_state: usize,
28 pub query_cache: usize,
30 pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
31 pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
32 pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
33 pub can_load_from_disk: bool,
34 pub try_load_from_disk: fn(
35 tcx: TyCtxt<'tcx>,
36 key: &C::Key,
37 prev_index: SerializedDepNodeIndex,
38 index: DepNodeIndex,
39 ) -> Option<C::Value>,
40 pub loadable_from_disk:
41 fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool,
42 pub hash_result: HashResult<C::Value>,
43 pub value_from_cycle_error:
44 fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
45 pub format_value: fn(&C::Value) -> String,
46}
47
48pub struct QuerySystemFns {
49 pub engine: QueryEngine,
50 pub local_providers: Providers,
51 pub extern_providers: ExternProviders,
52 pub encode_query_results: for<'tcx> fn(
53 tcx: TyCtxt<'tcx>,
54 encoder: &mut CacheEncoder<'_, 'tcx>,
55 query_result_index: &mut EncodedDepNodeIndex,
56 ),
57 pub try_mark_green: for<'tcx> fn(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool,
58}
59
60pub struct QuerySystem<'tcx> {
61 pub states: QueryStates<'tcx>,
62 pub arenas: WorkerLocal<QueryArenas<'tcx>>,
63 pub caches: QueryCaches<'tcx>,
64 pub dynamic_queries: DynamicQueries<'tcx>,
65
66 pub on_disk_cache: Option<OnDiskCache>,
71
72 pub fns: QuerySystemFns,
73
74 pub jobs: AtomicU64,
75}
76
77#[derive(Copy, Clone)]
78pub struct TyCtxtAt<'tcx> {
79 pub tcx: TyCtxt<'tcx>,
80 pub span: Span,
81}
82
83impl<'tcx> Deref for TyCtxtAt<'tcx> {
84 type Target = TyCtxt<'tcx>;
85 #[inline(always)]
86 fn deref(&self) -> &Self::Target {
87 &self.tcx
88 }
89}
90
91#[derive(Copy, Clone)]
92#[must_use]
93pub struct TyCtxtEnsureOk<'tcx> {
94 pub tcx: TyCtxt<'tcx>,
95}
96
97#[derive(Copy, Clone)]
98#[must_use]
99pub struct TyCtxtEnsureDone<'tcx> {
100 pub tcx: TyCtxt<'tcx>,
101}
102
103impl<'tcx> TyCtxt<'tcx> {
104 #[inline(always)]
131 pub fn ensure_ok(self) -> TyCtxtEnsureOk<'tcx> {
132 TyCtxtEnsureOk { tcx: self }
133 }
134
135 #[inline(always)]
152 pub fn ensure_done(self) -> TyCtxtEnsureDone<'tcx> {
153 TyCtxtEnsureDone { tcx: self }
154 }
155
156 #[inline(always)]
159 pub fn at(self, span: Span) -> TyCtxtAt<'tcx> {
160 TyCtxtAt { tcx: self, span }
161 }
162
163 pub fn try_mark_green(self, dep_node: &dep_graph::DepNode) -> bool {
164 (self.query_system.fns.try_mark_green)(self, dep_node)
165 }
166}
167
168#[inline(always)]
169pub fn query_get_at<'tcx, Cache>(
170 tcx: TyCtxt<'tcx>,
171 execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
172 query_cache: &Cache,
173 span: Span,
174 key: Cache::Key,
175) -> Cache::Value
176where
177 Cache: QueryCache,
178{
179 let key = key.into_query_param();
180 match try_get_cached(tcx, query_cache, &key) {
181 Some(value) => value,
182 None => execute_query(tcx, span, key, QueryMode::Get).unwrap(),
183 }
184}
185
186#[inline]
187pub fn query_ensure<'tcx, Cache>(
188 tcx: TyCtxt<'tcx>,
189 execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
190 query_cache: &Cache,
191 key: Cache::Key,
192 check_cache: bool,
193) where
194 Cache: QueryCache,
195{
196 let key = key.into_query_param();
197 if try_get_cached(tcx, query_cache, &key).is_none() {
198 execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache });
199 }
200}
201
202#[inline]
203pub fn query_ensure_error_guaranteed<'tcx, Cache, T>(
204 tcx: TyCtxt<'tcx>,
205 execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
206 query_cache: &Cache,
207 key: Cache::Key,
208 check_cache: bool,
209) -> Result<(), ErrorGuaranteed>
210where
211 Cache: QueryCache<Value = super::erase::Erase<Result<T, ErrorGuaranteed>>>,
212 Result<T, ErrorGuaranteed>: EraseType,
213{
214 let key = key.into_query_param();
215 if let Some(res) = try_get_cached(tcx, query_cache, &key) {
216 super::erase::restore(res).map(drop)
217 } else {
218 execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache })
219 .map(super::erase::restore)
220 .map(|res| res.map(drop))
221 .unwrap_or(Ok(()))
228 }
229}
230
231macro_rules! query_ensure {
232 ([]$($args:tt)*) => {
233 query_ensure($($args)*)
234 };
235 ([(return_result_from_ensure_ok) $($rest:tt)*]$($args:tt)*) => {
236 query_ensure_error_guaranteed($($args)*).map(|_| ())
237 };
238 ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
239 query_ensure!([$($modifiers)*]$($args)*)
240 };
241}
242
243macro_rules! query_helper_param_ty {
244 (DefId) => { impl IntoQueryParam<DefId> };
245 (LocalDefId) => { impl IntoQueryParam<LocalDefId> };
246 ($K:ty) => { $K };
247}
248
249macro_rules! query_if_arena {
250 ([] $arena:tt $no_arena:tt) => {
251 $no_arena
252 };
253 ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
254 $arena
255 };
256 ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
257 query_if_arena!([$($modifiers)*]$($args)*)
258 };
259}
260
261macro_rules! local_key_if_separate_extern {
264 ([] $($K:tt)*) => {
265 $($K)*
266 };
267 ([(separate_provide_extern) $($rest:tt)*] $($K:tt)*) => {
268 <$($K)* as AsLocalKey>::LocalKey
269 };
270 ([$other:tt $($modifiers:tt)*] $($K:tt)*) => {
271 local_key_if_separate_extern!([$($modifiers)*] $($K)*)
272 };
273}
274
275macro_rules! separate_provide_extern_decl {
276 ([][$name:ident]) => {
277 ()
278 };
279 ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
280 for<'tcx> fn(
281 TyCtxt<'tcx>,
282 queries::$name::Key<'tcx>,
283 ) -> queries::$name::ProvidedValue<'tcx>
284 };
285 ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
286 separate_provide_extern_decl!([$($modifiers)*][$($args)*])
287 };
288}
289
290macro_rules! ensure_ok_result {
291 ( [] ) => {
292 ()
293 };
294 ( [(return_result_from_ensure_ok) $($rest:tt)*] ) => {
295 Result<(), ErrorGuaranteed>
296 };
297 ( [$other:tt $($modifiers:tt)*] ) => {
298 ensure_ok_result!( [$($modifiers)*] )
299 };
300}
301
302macro_rules! separate_provide_extern_default {
303 ([][$name:ident]) => {
304 ()
305 };
306 ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
307 |_, key| $crate::query::plumbing::default_extern_query(stringify!($name), &key)
308 };
309 ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
310 separate_provide_extern_default!([$($modifiers)*][$($args)*])
311 };
312}
313
314macro_rules! define_callbacks {
315 (
316 $(
317 $(#[$attr:meta])*
318 [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,
319 )*
320 ) => {
321
322 #[allow(unused_lifetimes)]
323 pub mod queries {
324 $(pub mod $name {
325 use super::super::*;
326
327 pub type Key<'tcx> = $($K)*;
328 pub type Value<'tcx> = $V;
329
330 pub type LocalKey<'tcx> = local_key_if_separate_extern!([$($modifiers)*] $($K)*);
331
332 pub type ProvidedValue<'tcx> = query_if_arena!(
336 [$($modifiers)*]
337 (<$V as $crate::query::arena_cached::ArenaCached<'tcx>>::Provided)
338 ($V)
339 );
340
341 #[inline(always)]
346 pub fn provided_to_erased<'tcx>(
347 _tcx: TyCtxt<'tcx>,
348 value: ProvidedValue<'tcx>,
349 ) -> Erase<Value<'tcx>> {
350 erase(query_if_arena!([$($modifiers)*]
351 {
352 use $crate::query::arena_cached::ArenaCached;
353
354 if mem::needs_drop::<<$V as ArenaCached<'tcx>>::Allocated>() {
355 <$V as ArenaCached>::alloc_in_arena(
356 |v| _tcx.query_system.arenas.$name.alloc(v),
357 value,
358 )
359 } else {
360 <$V as ArenaCached>::alloc_in_arena(
361 |v| _tcx.arena.dropless.alloc(v),
362 value,
363 )
364 }
365 }
366 (value)
367 ))
368 }
369
370 pub type Storage<'tcx> = <$($K)* as keys::Key>::Cache<Erase<$V>>;
371
372 #[cfg(target_pointer_width = "64")]
375 const _: () = {
376 if size_of::<Key<'static>>() > 88 {
377 panic!("{}", concat!(
378 "the query `",
379 stringify!($name),
380 "` has a key type `",
381 stringify!($($K)*),
382 "` that is too large"
383 ));
384 }
385 };
386
387 #[cfg(target_pointer_width = "64")]
390 #[cfg(not(feature = "rustc_randomized_layouts"))]
391 const _: () = {
392 if size_of::<Value<'static>>() > 64 {
393 panic!("{}", concat!(
394 "the query `",
395 stringify!($name),
396 "` has a value type `",
397 stringify!($V),
398 "` that is too large"
399 ));
400 }
401 };
402 })*
403 }
404
405 pub struct QueryArenas<'tcx> {
406 $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
407 (TypedArena<<$V as $crate::query::arena_cached::ArenaCached<'tcx>>::Allocated>)
408 ()
409 ),)*
410 }
411
412 impl Default for QueryArenas<'_> {
413 fn default() -> Self {
414 Self {
415 $($name: query_if_arena!([$($modifiers)*]
416 (Default::default())
417 ()
418 ),)*
419 }
420 }
421 }
422
423 #[derive(Default)]
424 pub struct QueryCaches<'tcx> {
425 $($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
426 }
427
428 impl<'tcx> TyCtxtEnsureOk<'tcx> {
429 $($(#[$attr])*
430 #[inline(always)]
431 pub fn $name(
432 self,
433 key: query_helper_param_ty!($($K)*),
434 ) -> ensure_ok_result!([$($modifiers)*]) {
435 query_ensure!(
436 [$($modifiers)*]
437 self.tcx,
438 self.tcx.query_system.fns.engine.$name,
439 &self.tcx.query_system.caches.$name,
440 key.into_query_param(),
441 false,
442 )
443 })*
444 }
445
446 impl<'tcx> TyCtxtEnsureDone<'tcx> {
447 $($(#[$attr])*
448 #[inline(always)]
449 pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
450 query_ensure(
451 self.tcx,
452 self.tcx.query_system.fns.engine.$name,
453 &self.tcx.query_system.caches.$name,
454 key.into_query_param(),
455 true,
456 );
457 })*
458 }
459
460 impl<'tcx> TyCtxt<'tcx> {
461 $($(#[$attr])*
462 #[inline(always)]
463 #[must_use]
464 pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
465 {
466 self.at(DUMMY_SP).$name(key)
467 })*
468 }
469
470 impl<'tcx> TyCtxtAt<'tcx> {
471 $($(#[$attr])*
472 #[inline(always)]
473 pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
474 {
475 restore::<$V>(query_get_at(
476 self.tcx,
477 self.tcx.query_system.fns.engine.$name,
478 &self.tcx.query_system.caches.$name,
479 self.span,
480 key.into_query_param(),
481 ))
482 })*
483 }
484
485 pub struct DynamicQueries<'tcx> {
486 $(
487 pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
488 )*
489 }
490
491 #[derive(Default)]
492 pub struct QueryStates<'tcx> {
493 $(
494 pub $name: QueryState<$($K)*, QueryStackDeferred<'tcx>>,
495 )*
496 }
497
498 pub struct Providers {
499 $(pub $name: for<'tcx> fn(
500 TyCtxt<'tcx>,
501 queries::$name::LocalKey<'tcx>,
502 ) -> queries::$name::ProvidedValue<'tcx>,)*
503 }
504
505 pub struct ExternProviders {
506 $(pub $name: separate_provide_extern_decl!([$($modifiers)*][$name]),)*
507 }
508
509 impl Default for Providers {
510 fn default() -> Self {
511 Providers {
512 $($name: |_, key| $crate::query::plumbing::default_query(stringify!($name), &key)),*
513 }
514 }
515 }
516
517 impl Default for ExternProviders {
518 fn default() -> Self {
519 ExternProviders {
520 $($name: separate_provide_extern_default!([$($modifiers)*][$name]),)*
521 }
522 }
523 }
524
525 impl Copy for Providers {}
526 impl Clone for Providers {
527 fn clone(&self) -> Self { *self }
528 }
529
530 impl Copy for ExternProviders {}
531 impl Clone for ExternProviders {
532 fn clone(&self) -> Self { *self }
533 }
534
535 pub struct QueryEngine {
536 $(pub $name: for<'tcx> fn(
537 TyCtxt<'tcx>,
538 Span,
539 queries::$name::Key<'tcx>,
540 QueryMode,
541 ) -> Option<Erase<$V>>,)*
542 }
543 };
544}
545
546macro_rules! hash_result {
547 ([]) => {{
548 Some(dep_graph::hash_result)
549 }};
550 ([(no_hash) $($rest:tt)*]) => {{
551 None
552 }};
553 ([$other:tt $($modifiers:tt)*]) => {
554 hash_result!([$($modifiers)*])
555 };
556}
557
558macro_rules! define_feedable {
559 ($($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
560 $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
561 $(#[$attr])*
562 #[inline(always)]
563 pub fn $name(self, value: queries::$name::ProvidedValue<'tcx>) {
564 let key = self.key().into_query_param();
565
566 let tcx = self.tcx;
567 let erased = queries::$name::provided_to_erased(tcx, value);
568 let value = restore::<$V>(erased);
569 let cache = &tcx.query_system.caches.$name;
570
571 let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
572 match try_get_cached(tcx, cache, &key) {
573 Some(old) => {
574 let old = restore::<$V>(old);
575 if let Some(hasher) = hasher {
576 let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx.with_stable_hashing_context(|mut hcx|
577 (hasher(&mut hcx, &value), hasher(&mut hcx, &old))
578 );
579 if old_hash != value_hash {
580 tcx.dcx().delayed_bug(format!(
584 "Trying to feed an already recorded value for query {} key={key:?}:\n\
585 old value: {old:?}\nnew value: {value:?}",
586 stringify!($name),
587 ));
588 }
589 } else {
590 bug!(
594 "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
595 stringify!($name),
596 )
597 }
598 }
599 None => {
600 let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::dep_kinds::$name, &key);
601 let dep_node_index = tcx.dep_graph.with_feed_task(
602 dep_node,
603 tcx,
604 &value,
605 hash_result!([$($modifiers)*]),
606 );
607 cache.complete(key, erased, dep_node_index);
608 }
609 }
610 }
611 })*
612 }
613}
614
615mod sealed {
628 use rustc_hir::def_id::{LocalModDefId, ModDefId};
629
630 use super::{DefId, LocalDefId, OwnerId};
631
632 pub trait IntoQueryParam<P> {
637 fn into_query_param(self) -> P;
638 }
639
640 impl<P> IntoQueryParam<P> for P {
641 #[inline(always)]
642 fn into_query_param(self) -> P {
643 self
644 }
645 }
646
647 impl<'a, P: Copy> IntoQueryParam<P> for &'a P {
648 #[inline(always)]
649 fn into_query_param(self) -> P {
650 *self
651 }
652 }
653
654 impl IntoQueryParam<LocalDefId> for OwnerId {
655 #[inline(always)]
656 fn into_query_param(self) -> LocalDefId {
657 self.def_id
658 }
659 }
660
661 impl IntoQueryParam<DefId> for LocalDefId {
662 #[inline(always)]
663 fn into_query_param(self) -> DefId {
664 self.to_def_id()
665 }
666 }
667
668 impl IntoQueryParam<DefId> for OwnerId {
669 #[inline(always)]
670 fn into_query_param(self) -> DefId {
671 self.to_def_id()
672 }
673 }
674
675 impl IntoQueryParam<DefId> for ModDefId {
676 #[inline(always)]
677 fn into_query_param(self) -> DefId {
678 self.to_def_id()
679 }
680 }
681
682 impl IntoQueryParam<DefId> for LocalModDefId {
683 #[inline(always)]
684 fn into_query_param(self) -> DefId {
685 self.to_def_id()
686 }
687 }
688
689 impl IntoQueryParam<LocalDefId> for LocalModDefId {
690 #[inline(always)]
691 fn into_query_param(self) -> LocalDefId {
692 self.into()
693 }
694 }
695}
696
697pub use sealed::IntoQueryParam;
698
699use super::erase::EraseType;
700
701#[derive(Copy, Clone, Debug, HashStable)]
702pub struct CyclePlaceholder(pub ErrorGuaranteed);
703
704#[cold]
705pub(crate) fn default_query(name: &str, key: &dyn std::fmt::Debug) -> ! {
706 bug!(
707 "`tcx.{name}({key:?})` is not supported for this key;\n\
708 hint: Queries can be either made to the local crate, or the external crate. \
709 This error means you tried to use it for one that's not supported.\n\
710 If that's not the case, {name} was likely never assigned to a provider function.\n",
711 )
712}
713
714#[cold]
715pub(crate) fn default_extern_query(name: &str, key: &dyn std::fmt::Debug) -> ! {
716 bug!(
717 "`tcx.{name}({key:?})` unsupported by its crate; \
718 perhaps the `{name}` query was never assigned a provider function",
719 )
720}