rustc_query_impl/
plumbing.rs

1//! The implementation of the query system itself. This defines the macros that
2//! generate the actual methods on tcx which find and execute the provider,
3//! manage the caches, and so forth.
4
5use std::num::NonZero;
6
7use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
8use rustc_data_structures::sync::{DynSend, DynSync};
9use rustc_data_structures::unord::UnordMap;
10use rustc_hashes::Hash64;
11use rustc_index::Idx;
12use rustc_middle::bug;
13use rustc_middle::dep_graph::{
14    self, DepContext, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex,
15    dep_kinds,
16};
17use rustc_middle::query::Key;
18use rustc_middle::query::on_disk_cache::{
19    AbsoluteBytePos, CacheDecoder, CacheEncoder, EncodedDepNodeIndex,
20};
21use rustc_middle::ty::codec::TyEncoder;
22use rustc_middle::ty::print::with_reduced_queries;
23use rustc_middle::ty::tls::{self, ImplicitCtxt};
24use rustc_middle::ty::{self, TyCtxt};
25use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext};
26use rustc_query_system::ich::StableHashingContext;
27use rustc_query_system::query::{
28    QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffect,
29    QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra, force_query,
30};
31use rustc_query_system::{QueryOverflow, QueryOverflowNote};
32use rustc_serialize::{Decodable, Encodable};
33use rustc_session::Limit;
34use rustc_span::def_id::LOCAL_CRATE;
35
36use crate::QueryConfigRestored;
37
38#[derive(Copy, Clone)]
39pub struct QueryCtxt<'tcx> {
40    pub tcx: TyCtxt<'tcx>,
41}
42
43impl<'tcx> QueryCtxt<'tcx> {
44    #[inline]
45    pub fn new(tcx: TyCtxt<'tcx>) -> Self {
46        QueryCtxt { tcx }
47    }
48}
49
50impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
51    type Target = TyCtxt<'tcx>;
52
53    #[inline]
54    fn deref(&self) -> &Self::Target {
55        &self.tcx
56    }
57}
58
59impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
60    type Deps = rustc_middle::dep_graph::DepsType;
61    type DepContext = TyCtxt<'tcx>;
62
63    #[inline]
64    fn dep_context(&self) -> &Self::DepContext {
65        &self.tcx
66    }
67}
68
69impl<'tcx> QueryContext for QueryCtxt<'tcx> {
70    type QueryInfo = QueryStackDeferred<'tcx>;
71
72    #[inline]
73    fn next_job_id(self) -> QueryJobId {
74        QueryJobId(
75            NonZero::new(self.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed))
76                .unwrap(),
77        )
78    }
79
80    #[inline]
81    fn current_query_job(self) -> Option<QueryJobId> {
82        tls::with_related_context(self.tcx, |icx| icx.query)
83    }
84
85    /// Returns a query map representing active query jobs.
86    /// It returns an incomplete map as an error if it fails
87    /// to take locks.
88    fn collect_active_jobs(
89        self,
90    ) -> Result<QueryMap<QueryStackDeferred<'tcx>>, QueryMap<QueryStackDeferred<'tcx>>> {
91        let mut jobs = QueryMap::default();
92        let mut complete = true;
93
94        for collect in super::TRY_COLLECT_ACTIVE_JOBS.iter() {
95            if collect(self.tcx, &mut jobs).is_none() {
96                complete = false;
97            }
98        }
99
100        if complete { Ok(jobs) } else { Err(jobs) }
101    }
102
103    fn lift_query_info(
104        self,
105        info: &QueryStackDeferred<'tcx>,
106    ) -> rustc_query_system::query::QueryStackFrameExtra {
107        info.extract()
108    }
109
110    // Interactions with on_disk_cache
111    fn load_side_effect(
112        self,
113        prev_dep_node_index: SerializedDepNodeIndex,
114    ) -> Option<QuerySideEffect> {
115        self.query_system
116            .on_disk_cache
117            .as_ref()
118            .and_then(|c| c.load_side_effect(self.tcx, prev_dep_node_index))
119    }
120
121    #[inline(never)]
122    #[cold]
123    fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect) {
124        if let Some(c) = self.query_system.on_disk_cache.as_ref() {
125            c.store_side_effect(dep_node_index, side_effect)
126        }
127    }
128
129    /// Executes a job by changing the `ImplicitCtxt` to point to the
130    /// new query job while it executes.
131    #[inline(always)]
132    fn start_query<R>(
133        self,
134        token: QueryJobId,
135        depth_limit: bool,
136        compute: impl FnOnce() -> R,
137    ) -> R {
138        // The `TyCtxt` stored in TLS has the same global interner lifetime
139        // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
140        // when accessing the `ImplicitCtxt`.
141        tls::with_related_context(self.tcx, move |current_icx| {
142            if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) {
143                self.depth_limit_error(token);
144            }
145
146            // Update the `ImplicitCtxt` to point to our new query job.
147            let new_icx = ImplicitCtxt {
148                tcx: self.tcx,
149                query: Some(token),
150                query_depth: current_icx.query_depth + depth_limit as usize,
151                task_deps: current_icx.task_deps,
152            };
153
154            // Use the `ImplicitCtxt` while we execute the query.
155            tls::enter_context(&new_icx, compute)
156        })
157    }
158
159    fn depth_limit_error(self, job: QueryJobId) {
160        // FIXME: `collect_active_jobs` expects no locks to be held, which doesn't hold for this call.
161        let query_map = match self.collect_active_jobs() {
162            Ok(query_map) => query_map,
163            Err(query_map) => query_map,
164        };
165        let (info, depth) = job.find_dep_kind_root(query_map);
166
167        let suggested_limit = match self.recursion_limit() {
168            Limit(0) => Limit(2),
169            limit => limit * 2,
170        };
171
172        self.sess.dcx().emit_fatal(QueryOverflow {
173            span: info.job.span,
174            note: QueryOverflowNote {
175                desc: self.lift_query_info(&info.query.info).description,
176                depth,
177            },
178            suggested_limit,
179            crate_name: self.crate_name(LOCAL_CRATE),
180        });
181    }
182}
183
184pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
185    tcx.dep_graph.try_mark_green(QueryCtxt::new(tcx), dep_node).is_some()
186}
187
188pub(super) fn encode_all_query_results<'tcx>(
189    tcx: TyCtxt<'tcx>,
190    encoder: &mut CacheEncoder<'_, 'tcx>,
191    query_result_index: &mut EncodedDepNodeIndex,
192) {
193    for encode in super::ENCODE_QUERY_RESULTS.iter().copied().flatten() {
194        encode(tcx, encoder, query_result_index);
195    }
196}
197
198pub fn query_key_hash_verify_all<'tcx>(tcx: TyCtxt<'tcx>) {
199    if tcx.sess().opts.unstable_opts.incremental_verify_ich || cfg!(debug_assertions) {
200        tcx.sess.time("query_key_hash_verify_all", || {
201            for verify in super::QUERY_KEY_HASH_VERIFY.iter() {
202                verify(tcx);
203            }
204        })
205    }
206}
207
208macro_rules! handle_cycle_error {
209    ([]) => {{
210        rustc_query_system::HandleCycleError::Error
211    }};
212    ([(fatal_cycle) $($rest:tt)*]) => {{
213        rustc_query_system::HandleCycleError::Fatal
214    }};
215    ([(cycle_stash) $($rest:tt)*]) => {{
216        rustc_query_system::HandleCycleError::Stash
217    }};
218    ([(cycle_delay_bug) $($rest:tt)*]) => {{
219        rustc_query_system::HandleCycleError::DelayBug
220    }};
221    ([$other:tt $($modifiers:tt)*]) => {
222        handle_cycle_error!([$($modifiers)*])
223    };
224}
225
226macro_rules! is_anon {
227    ([]) => {{
228        false
229    }};
230    ([(anon) $($rest:tt)*]) => {{
231        true
232    }};
233    ([$other:tt $($modifiers:tt)*]) => {
234        is_anon!([$($modifiers)*])
235    };
236}
237
238macro_rules! is_eval_always {
239    ([]) => {{
240        false
241    }};
242    ([(eval_always) $($rest:tt)*]) => {{
243        true
244    }};
245    ([$other:tt $($modifiers:tt)*]) => {
246        is_eval_always!([$($modifiers)*])
247    };
248}
249
250macro_rules! depth_limit {
251    ([]) => {{
252        false
253    }};
254    ([(depth_limit) $($rest:tt)*]) => {{
255        true
256    }};
257    ([$other:tt $($modifiers:tt)*]) => {
258        depth_limit!([$($modifiers)*])
259    };
260}
261
262macro_rules! feedable {
263    ([]) => {{
264        false
265    }};
266    ([(feedable) $($rest:tt)*]) => {{
267        true
268    }};
269    ([$other:tt $($modifiers:tt)*]) => {
270        feedable!([$($modifiers)*])
271    };
272}
273
274macro_rules! hash_result {
275    ([][$V:ty]) => {{
276        Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
277    }};
278    ([(no_hash) $($rest:tt)*][$V:ty]) => {{
279        None
280    }};
281    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
282        hash_result!([$($modifiers)*][$($args)*])
283    };
284}
285
286macro_rules! call_provider {
287    ([][$tcx:expr, $name:ident, $key:expr]) => {{
288        ($tcx.query_system.fns.local_providers.$name)($tcx, $key)
289    }};
290    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
291        if let Some(key) = $key.as_local_key() {
292            ($tcx.query_system.fns.local_providers.$name)($tcx, key)
293        } else {
294            ($tcx.query_system.fns.extern_providers.$name)($tcx, $key)
295        }
296    }};
297    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
298        call_provider!([$($modifiers)*][$($args)*])
299    };
300}
301
302macro_rules! should_ever_cache_on_disk {
303    ([]$yes:tt $no:tt) => {{
304        $no
305    }};
306    ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
307        $yes
308    }};
309    ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
310        should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
311    };
312}
313
314fn create_query_frame_extra<'tcx, K: Key + Copy + 'tcx>(
315    (tcx, key, kind, name, do_describe): (
316        TyCtxt<'tcx>,
317        K,
318        DepKind,
319        &'static str,
320        fn(TyCtxt<'tcx>, K) -> String,
321    ),
322) -> QueryStackFrameExtra {
323    let def_id = key.key_as_def_id();
324
325    // If reduced queries are requested, we may be printing a query stack due
326    // to a panic. Avoid using `default_span` and `def_kind` in that case.
327    let reduce_queries = with_reduced_queries();
328
329    // Avoid calling queries while formatting the description
330    let description = ty::print::with_no_queries!(do_describe(tcx, key));
331    let description = if tcx.sess.verbose_internals() {
332        format!("{description} [{name:?}]")
333    } else {
334        description
335    };
336    let span = if kind == dep_graph::dep_kinds::def_span || reduce_queries {
337        // The `def_span` query is used to calculate `default_span`,
338        // so exit to avoid infinite recursion.
339        None
340    } else {
341        Some(key.default_span(tcx))
342    };
343
344    let def_kind = if kind == dep_graph::dep_kinds::def_kind || reduce_queries {
345        // Try to avoid infinite recursion.
346        None
347    } else {
348        def_id.and_then(|def_id| def_id.as_local()).map(|def_id| tcx.def_kind(def_id))
349    };
350    QueryStackFrameExtra::new(description, span, def_kind)
351}
352
353pub(crate) fn create_query_frame<
354    'tcx,
355    K: Copy + DynSend + DynSync + Key + for<'a> HashStable<StableHashingContext<'a>> + 'tcx,
356>(
357    tcx: TyCtxt<'tcx>,
358    do_describe: fn(TyCtxt<'tcx>, K) -> String,
359    key: K,
360    kind: DepKind,
361    name: &'static str,
362) -> QueryStackFrame<QueryStackDeferred<'tcx>> {
363    let def_id = key.key_as_def_id();
364
365    let hash = || {
366        tcx.with_stable_hashing_context(|mut hcx| {
367            let mut hasher = StableHasher::new();
368            kind.as_usize().hash_stable(&mut hcx, &mut hasher);
369            key.hash_stable(&mut hcx, &mut hasher);
370            hasher.finish::<Hash64>()
371        })
372    };
373    let def_id_for_ty_in_cycle = key.def_id_for_ty_in_cycle();
374
375    let info =
376        QueryStackDeferred::new((tcx, key, kind, name, do_describe), create_query_frame_extra);
377
378    QueryStackFrame::new(info, kind, hash, def_id, def_id_for_ty_in_cycle)
379}
380
381pub(crate) fn encode_query_results<'a, 'tcx, Q>(
382    query: Q::Config,
383    qcx: QueryCtxt<'tcx>,
384    encoder: &mut CacheEncoder<'a, 'tcx>,
385    query_result_index: &mut EncodedDepNodeIndex,
386) where
387    Q: super::QueryConfigRestored<'tcx>,
388    Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
389{
390    let _timer = qcx.profiler().generic_activity_with_arg("encode_query_results_for", query.name());
391
392    assert!(query.query_state(qcx).all_inactive());
393    let cache = query.query_cache(qcx);
394    cache.iter(&mut |key, value, dep_node| {
395        if query.cache_on_disk(qcx.tcx, key) {
396            let dep_node = SerializedDepNodeIndex::new(dep_node.index());
397
398            // Record position of the cache entry.
399            query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
400
401            // Encode the type check tables with the `SerializedDepNodeIndex`
402            // as tag.
403            encoder.encode_tagged(dep_node, &Q::restore(*value));
404        }
405    });
406}
407
408pub(crate) fn query_key_hash_verify<'tcx>(
409    query: impl QueryConfig<QueryCtxt<'tcx>>,
410    qcx: QueryCtxt<'tcx>,
411) {
412    let _timer =
413        qcx.profiler().generic_activity_with_arg("query_key_hash_verify_for", query.name());
414
415    let mut map = UnordMap::default();
416
417    let cache = query.query_cache(qcx);
418    cache.iter(&mut |key, _, _| {
419        let node = DepNode::construct(qcx.tcx, query.dep_kind(), key);
420        if let Some(other_key) = map.insert(node, *key) {
421            bug!(
422                "query key:\n\
423                `{:?}`\n\
424                and key:\n\
425                `{:?}`\n\
426                mapped to the same dep node:\n\
427                {:?}",
428                key,
429                other_key,
430                node
431            );
432        }
433    });
434}
435
436fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
437where
438    Q: QueryConfig<QueryCtxt<'tcx>>,
439{
440    debug_assert!(tcx.dep_graph.is_green(&dep_node));
441
442    let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
443        panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
444    });
445    if query.cache_on_disk(tcx, &key) {
446        let _ = query.execute_query(tcx, key);
447    }
448}
449
450pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool {
451    if let Some(cache) = tcx.query_system.on_disk_cache.as_ref() {
452        cache.loadable_from_disk(id)
453    } else {
454        false
455    }
456}
457
458pub(crate) fn try_load_from_disk<'tcx, V>(
459    tcx: TyCtxt<'tcx>,
460    prev_index: SerializedDepNodeIndex,
461    index: DepNodeIndex,
462) -> Option<V>
463where
464    V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
465{
466    let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?;
467
468    let prof_timer = tcx.prof.incr_cache_loading();
469
470    // The call to `with_query_deserialization` enforces that no new `DepNodes`
471    // are created during deserialization. See the docs of that method for more
472    // details.
473    let value = tcx
474        .dep_graph
475        .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index));
476
477    prof_timer.finish_with_query_invocation_id(index.into());
478
479    value
480}
481
482fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
483where
484    Q: QueryConfig<QueryCtxt<'tcx>>,
485{
486    // We must avoid ever having to call `force_from_dep_node()` for a
487    // `DepNode::codegen_unit`:
488    // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
489    // would always end up having to evaluate the first caller of the
490    // `codegen_unit` query that *is* reconstructible. This might very well be
491    // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
492    // to re-trigger calling the `codegen_unit` query with the right key. At
493    // that point we would already have re-done all the work we are trying to
494    // avoid doing in the first place.
495    // The solution is simple: Just explicitly call the `codegen_unit` query for
496    // each CGU, right after partitioning. This way `try_mark_green` will always
497    // hit the cache instead of having to go through `force_from_dep_node`.
498    // This assertion makes sure, we actually keep applying the solution above.
499    debug_assert!(
500        dep_node.kind != dep_kinds::codegen_unit,
501        "calling force_from_dep_node() on dep_kinds::codegen_unit"
502    );
503
504    if let Some(key) = Q::Key::recover(tcx, &dep_node) {
505        force_query(query, QueryCtxt::new(tcx), key, dep_node);
506        true
507    } else {
508        false
509    }
510}
511
512pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
513where
514    Q: QueryConfigRestored<'tcx>,
515{
516    let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style();
517
518    if is_anon || !fingerprint_style.reconstructible() {
519        return DepKindStruct {
520            is_anon,
521            is_eval_always,
522            fingerprint_style,
523            force_from_dep_node: None,
524            try_load_from_on_disk_cache: None,
525            name: Q::NAME,
526        };
527    }
528
529    DepKindStruct {
530        is_anon,
531        is_eval_always,
532        fingerprint_style,
533        force_from_dep_node: Some(|tcx, dep_node, _| {
534            force_from_dep_node(Q::config(tcx), tcx, dep_node)
535        }),
536        try_load_from_on_disk_cache: Some(|tcx, dep_node| {
537            try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node)
538        }),
539        name: Q::NAME,
540    }
541}
542
543macro_rules! item_if_cached {
544    ([] $tokens:tt) => {};
545    ([(cache) $($rest:tt)*] { $($tokens:tt)* }) => {
546        $($tokens)*
547    };
548    ([$other:tt $($modifiers:tt)*] $tokens:tt) => {
549        item_if_cached! { [$($modifiers)*] $tokens }
550    };
551}
552
553macro_rules! expand_if_cached {
554    ([], $tokens:expr) => {{
555        None
556    }};
557    ([(cache) $($rest:tt)*], $tokens:expr) => {{
558        Some($tokens)
559    }};
560    ([$other:tt $($modifiers:tt)*], $tokens:expr) => {
561        expand_if_cached!([$($modifiers)*], $tokens)
562    };
563}
564
565/// Don't show the backtrace for query system by default
566/// use `RUST_BACKTRACE=full` to show all the backtraces
567#[inline(never)]
568pub(crate) fn __rust_begin_short_backtrace<F, T>(f: F) -> T
569where
570    F: FnOnce() -> T,
571{
572    let result = f();
573    std::hint::black_box(());
574    result
575}
576
577// NOTE: `$V` isn't used here, but we still need to match on it so it can be passed to other macros
578// invoked by `rustc_query_append`.
579macro_rules! define_queries {
580    (
581     $($(#[$attr:meta])*
582        [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
583
584        pub(crate) mod query_impl { $(pub(crate) mod $name {
585            use super::super::*;
586            use std::marker::PhantomData;
587
588            pub(crate) mod get_query_incr {
589                use super::*;
590
591                // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames
592                // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming
593                #[inline(never)]
594                pub(crate) fn __rust_end_short_backtrace<'tcx>(
595                    tcx: TyCtxt<'tcx>,
596                    span: Span,
597                    key: queries::$name::Key<'tcx>,
598                    mode: QueryMode,
599                ) -> Option<Erase<queries::$name::Value<'tcx>>> {
600                    #[cfg(debug_assertions)]
601                    let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
602                    get_query_incr(
603                        QueryType::config(tcx),
604                        QueryCtxt::new(tcx),
605                        span,
606                        key,
607                        mode
608                    )
609                }
610            }
611
612            pub(crate) mod get_query_non_incr {
613                use super::*;
614
615                #[inline(never)]
616                pub(crate) fn __rust_end_short_backtrace<'tcx>(
617                    tcx: TyCtxt<'tcx>,
618                    span: Span,
619                    key: queries::$name::Key<'tcx>,
620                    __mode: QueryMode,
621                ) -> Option<Erase<queries::$name::Value<'tcx>>> {
622                    Some(get_query_non_incr(
623                        QueryType::config(tcx),
624                        QueryCtxt::new(tcx),
625                        span,
626                        key,
627                    ))
628                }
629            }
630
631            pub(crate) fn dynamic_query<'tcx>()
632                -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>>
633            {
634                DynamicQuery {
635                    name: stringify!($name),
636                    eval_always: is_eval_always!([$($modifiers)*]),
637                    dep_kind: dep_graph::dep_kinds::$name,
638                    handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
639                    query_state: std::mem::offset_of!(QueryStates<'tcx>, $name),
640                    query_cache: std::mem::offset_of!(QueryCaches<'tcx>, $name),
641                    cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
642                    execute_query: |tcx, key| erase(tcx.$name(key)),
643                    compute: |tcx, key| {
644                        #[cfg(debug_assertions)]
645                        let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
646                        __rust_begin_short_backtrace(||
647                            queries::$name::provided_to_erased(
648                                tcx,
649                                {
650                                    let ret = call_provider!([$($modifiers)*][tcx, $name, key]);
651                                    rustc_middle::ty::print::with_reduced_queries!({
652                                        tracing::trace!(?ret);
653                                    });
654                                    ret
655                                }
656                            )
657                        )
658                    },
659                    can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false),
660                    try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] {
661                        |tcx, key, prev_index, index| {
662                            if ::rustc_middle::query::cached::$name(tcx, key) {
663                                let value = $crate::plumbing::try_load_from_disk::<
664                                    queries::$name::ProvidedValue<'tcx>
665                                >(
666                                    tcx,
667                                    prev_index,
668                                    index,
669                                );
670                                value.map(|value| queries::$name::provided_to_erased(tcx, value))
671                            } else {
672                                None
673                            }
674                        }
675                    } {
676                        |_tcx, _key, _prev_index, _index| None
677                    }),
678                    value_from_cycle_error: |tcx, cycle, guar| {
679                        let result: queries::$name::Value<'tcx> = Value::from_cycle_error(tcx, cycle, guar);
680                        erase(result)
681                    },
682                    loadable_from_disk: |_tcx, _key, _index| {
683                        should_ever_cache_on_disk!([$($modifiers)*] {
684                            ::rustc_middle::query::cached::$name(_tcx, _key) &&
685                                $crate::plumbing::loadable_from_disk(_tcx, _index)
686                        } {
687                            false
688                        })
689                    },
690                    hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]),
691                    format_value: |value| format!("{:?}", restore::<queries::$name::Value<'tcx>>(*value)),
692                }
693            }
694
695            #[derive(Copy, Clone, Default)]
696            pub(crate) struct QueryType<'tcx> {
697                data: PhantomData<&'tcx ()>
698            }
699
700            impl<'tcx> QueryConfigRestored<'tcx> for QueryType<'tcx> {
701                type RestoredValue = queries::$name::Value<'tcx>;
702                type Config = DynamicConfig<
703                    'tcx,
704                    queries::$name::Storage<'tcx>,
705                    { is_anon!([$($modifiers)*]) },
706                    { depth_limit!([$($modifiers)*]) },
707                    { feedable!([$($modifiers)*]) },
708                >;
709
710                const NAME: &'static &'static str = &stringify!($name);
711
712                #[inline(always)]
713                fn config(tcx: TyCtxt<'tcx>) -> Self::Config {
714                    DynamicConfig {
715                        dynamic: &tcx.query_system.dynamic_queries.$name,
716                    }
717                }
718
719                #[inline(always)]
720                fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
721                    restore::<queries::$name::Value<'tcx>>(value)
722                }
723            }
724
725            pub(crate) fn try_collect_active_jobs<'tcx>(
726                tcx: TyCtxt<'tcx>,
727                qmap: &mut QueryMap<QueryStackDeferred<'tcx>>,
728            ) -> Option<()> {
729                let make_query = |tcx, key| {
730                    let kind = rustc_middle::dep_graph::dep_kinds::$name;
731                    let name = stringify!($name);
732                    $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name)
733                };
734                let res = tcx.query_system.states.$name.try_collect_active_jobs(
735                    tcx,
736                    make_query,
737                    qmap,
738                );
739                // this can be called during unwinding, and the function has a `try_`-prefix, so
740                // don't `unwrap()` here, just manually check for `None` and do best-effort error
741                // reporting.
742                if res.is_none() {
743                    tracing::warn!(
744                        "Failed to collect active jobs for query with name `{}`!",
745                        stringify!($name)
746                    );
747                }
748                res
749            }
750
751            pub(crate) fn alloc_self_profile_query_strings<'tcx>(
752                tcx: TyCtxt<'tcx>,
753                string_cache: &mut QueryKeyStringCache
754            ) {
755                $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
756                    tcx,
757                    stringify!($name),
758                    &tcx.query_system.caches.$name,
759                    string_cache,
760                )
761            }
762
763            item_if_cached! { [$($modifiers)*] {
764                pub(crate) fn encode_query_results<'tcx>(
765                    tcx: TyCtxt<'tcx>,
766                    encoder: &mut CacheEncoder<'_, 'tcx>,
767                    query_result_index: &mut EncodedDepNodeIndex
768                ) {
769                    $crate::plumbing::encode_query_results::<query_impl::$name::QueryType<'tcx>>(
770                        query_impl::$name::QueryType::config(tcx),
771                        QueryCtxt::new(tcx),
772                        encoder,
773                        query_result_index,
774                    )
775                }
776            }}
777
778            pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) {
779                $crate::plumbing::query_key_hash_verify(
780                    query_impl::$name::QueryType::config(tcx),
781                    QueryCtxt::new(tcx),
782                )
783            }
784        })*}
785
786        pub(crate) fn engine(incremental: bool) -> QueryEngine {
787            if incremental {
788                QueryEngine {
789                    $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)*
790                }
791            } else {
792                QueryEngine {
793                    $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)*
794                }
795            }
796        }
797
798        pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> {
799            DynamicQueries {
800                $(
801                    $name: query_impl::$name::dynamic_query(),
802                )*
803            }
804        }
805
806        // These arrays are used for iteration and can't be indexed by `DepKind`.
807
808        const TRY_COLLECT_ACTIVE_JOBS: &[
809            for<'tcx> fn(TyCtxt<'tcx>, &mut QueryMap<QueryStackDeferred<'tcx>>) -> Option<()>
810        ] =
811            &[$(query_impl::$name::try_collect_active_jobs),*];
812
813        const ALLOC_SELF_PROFILE_QUERY_STRINGS: &[
814            for<'tcx> fn(TyCtxt<'tcx>, &mut QueryKeyStringCache)
815        ] = &[$(query_impl::$name::alloc_self_profile_query_strings),*];
816
817        const ENCODE_QUERY_RESULTS: &[
818            Option<for<'tcx> fn(
819                TyCtxt<'tcx>,
820                &mut CacheEncoder<'_, 'tcx>,
821                &mut EncodedDepNodeIndex)
822            >
823        ] = &[$(expand_if_cached!([$($modifiers)*], query_impl::$name::encode_query_results)),*];
824
825        const QUERY_KEY_HASH_VERIFY: &[
826            for<'tcx> fn(TyCtxt<'tcx>)
827        ] = &[$(query_impl::$name::query_key_hash_verify),*];
828
829        #[allow(nonstandard_style)]
830        mod query_callbacks {
831            use super::*;
832            use rustc_middle::bug;
833            use rustc_query_system::dep_graph::FingerprintStyle;
834
835            // We use this for most things when incr. comp. is turned off.
836            pub(crate) fn Null<'tcx>() -> DepKindStruct<'tcx> {
837                DepKindStruct {
838                    is_anon: false,
839                    is_eval_always: false,
840                    fingerprint_style: FingerprintStyle::Unit,
841                    force_from_dep_node: Some(|_, dep_node, _| bug!("force_from_dep_node: encountered {:?}", dep_node)),
842                    try_load_from_on_disk_cache: None,
843                    name: &"Null",
844                }
845            }
846
847            // We use this for the forever-red node.
848            pub(crate) fn Red<'tcx>() -> DepKindStruct<'tcx> {
849                DepKindStruct {
850                    is_anon: false,
851                    is_eval_always: false,
852                    fingerprint_style: FingerprintStyle::Unit,
853                    force_from_dep_node: Some(|_, dep_node, _| bug!("force_from_dep_node: encountered {:?}", dep_node)),
854                    try_load_from_on_disk_cache: None,
855                    name: &"Red",
856                }
857            }
858
859            pub(crate) fn SideEffect<'tcx>() -> DepKindStruct<'tcx> {
860                DepKindStruct {
861                    is_anon: false,
862                    is_eval_always: false,
863                    fingerprint_style: FingerprintStyle::Unit,
864                    force_from_dep_node: Some(|tcx, _, prev_index| {
865                        tcx.dep_graph.force_diagnostic_node(QueryCtxt::new(tcx), prev_index);
866                        true
867                    }),
868                    try_load_from_on_disk_cache: None,
869                    name: &"SideEffect",
870                }
871            }
872
873            pub(crate) fn TraitSelect<'tcx>() -> DepKindStruct<'tcx> {
874                DepKindStruct {
875                    is_anon: true,
876                    is_eval_always: false,
877                    fingerprint_style: FingerprintStyle::Unit,
878                    force_from_dep_node: None,
879                    try_load_from_on_disk_cache: None,
880                    name: &"TraitSelect",
881                }
882            }
883
884            pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindStruct<'tcx> {
885                DepKindStruct {
886                    is_anon: false,
887                    is_eval_always: false,
888                    fingerprint_style: FingerprintStyle::Opaque,
889                    force_from_dep_node: None,
890                    try_load_from_on_disk_cache: None,
891                    name: &"CompileCodegenUnit",
892                }
893            }
894
895            pub(crate) fn CompileMonoItem<'tcx>() -> DepKindStruct<'tcx> {
896                DepKindStruct {
897                    is_anon: false,
898                    is_eval_always: false,
899                    fingerprint_style: FingerprintStyle::Opaque,
900                    force_from_dep_node: None,
901                    try_load_from_on_disk_cache: None,
902                    name: &"CompileMonoItem",
903                }
904            }
905
906            $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> {
907                $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>(
908                    is_anon!([$($modifiers)*]),
909                    is_eval_always!([$($modifiers)*]),
910                )
911            })*
912        }
913
914        pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'tcx>] {
915            arena.alloc_from_iter(rustc_middle::make_dep_kind_array!(query_callbacks))
916        }
917
918        pub fn dep_kind_names() -> Vec<&'static str> {
919            rustc_middle::make_dep_kind_name_array!(query_callbacks)
920        }
921    }
922}