rustc_query_impl/
plumbing.rs

1//! The implementation of the query system itself. This defines the macros that
2//! generate the actual methods on tcx which find and execute the provider,
3//! manage the caches, and so forth.
4
5use std::num::NonZero;
6
7use rustc_data_structures::jobserver::Proxy;
8use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
9use rustc_data_structures::sync::{DynSend, DynSync};
10use rustc_data_structures::unord::UnordMap;
11use rustc_hashes::Hash64;
12use rustc_index::Idx;
13use rustc_middle::bug;
14use rustc_middle::dep_graph::{
15    self, DepContext, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex,
16    dep_kinds,
17};
18use rustc_middle::query::Key;
19use rustc_middle::query::on_disk_cache::{
20    AbsoluteBytePos, CacheDecoder, CacheEncoder, EncodedDepNodeIndex,
21};
22use rustc_middle::ty::codec::TyEncoder;
23use rustc_middle::ty::print::with_reduced_queries;
24use rustc_middle::ty::tls::{self, ImplicitCtxt};
25use rustc_middle::ty::{self, TyCtxt};
26use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext};
27use rustc_query_system::ich::StableHashingContext;
28use rustc_query_system::query::{
29    QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffect,
30    QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra, force_query,
31};
32use rustc_query_system::{QueryOverflow, QueryOverflowNote};
33use rustc_serialize::{Decodable, Encodable};
34use rustc_session::Limit;
35use rustc_span::def_id::LOCAL_CRATE;
36
37use crate::QueryConfigRestored;
38
39#[derive(Copy, Clone)]
40pub struct QueryCtxt<'tcx> {
41    pub tcx: TyCtxt<'tcx>,
42}
43
44impl<'tcx> QueryCtxt<'tcx> {
45    #[inline]
46    pub fn new(tcx: TyCtxt<'tcx>) -> Self {
47        QueryCtxt { tcx }
48    }
49}
50
51impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
52    type Target = TyCtxt<'tcx>;
53
54    #[inline]
55    fn deref(&self) -> &Self::Target {
56        &self.tcx
57    }
58}
59
60impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
61    type Deps = rustc_middle::dep_graph::DepsType;
62    type DepContext = TyCtxt<'tcx>;
63
64    #[inline]
65    fn dep_context(&self) -> &Self::DepContext {
66        &self.tcx
67    }
68}
69
70impl<'tcx> QueryContext for QueryCtxt<'tcx> {
71    type QueryInfo = QueryStackDeferred<'tcx>;
72
73    #[inline]
74    fn jobserver_proxy(&self) -> &Proxy {
75        &*self.jobserver_proxy
76    }
77
78    #[inline]
79    fn next_job_id(self) -> QueryJobId {
80        QueryJobId(
81            NonZero::new(self.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed))
82                .unwrap(),
83        )
84    }
85
86    #[inline]
87    fn current_query_job(self) -> Option<QueryJobId> {
88        tls::with_related_context(self.tcx, |icx| icx.query)
89    }
90
91    /// Returns a query map representing active query jobs.
92    /// It returns an incomplete map as an error if it fails
93    /// to take locks.
94    fn collect_active_jobs(
95        self,
96    ) -> Result<QueryMap<QueryStackDeferred<'tcx>>, QueryMap<QueryStackDeferred<'tcx>>> {
97        let mut jobs = QueryMap::default();
98        let mut complete = true;
99
100        for collect in super::TRY_COLLECT_ACTIVE_JOBS.iter() {
101            if collect(self.tcx, &mut jobs).is_none() {
102                complete = false;
103            }
104        }
105
106        if complete { Ok(jobs) } else { Err(jobs) }
107    }
108
109    fn lift_query_info(
110        self,
111        info: &QueryStackDeferred<'tcx>,
112    ) -> rustc_query_system::query::QueryStackFrameExtra {
113        info.extract()
114    }
115
116    // Interactions with on_disk_cache
117    fn load_side_effect(
118        self,
119        prev_dep_node_index: SerializedDepNodeIndex,
120    ) -> Option<QuerySideEffect> {
121        self.query_system
122            .on_disk_cache
123            .as_ref()
124            .and_then(|c| c.load_side_effect(self.tcx, prev_dep_node_index))
125    }
126
127    #[inline(never)]
128    #[cold]
129    fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect) {
130        if let Some(c) = self.query_system.on_disk_cache.as_ref() {
131            c.store_side_effect(dep_node_index, side_effect)
132        }
133    }
134
135    /// Executes a job by changing the `ImplicitCtxt` to point to the
136    /// new query job while it executes.
137    #[inline(always)]
138    fn start_query<R>(
139        self,
140        token: QueryJobId,
141        depth_limit: bool,
142        compute: impl FnOnce() -> R,
143    ) -> R {
144        // The `TyCtxt` stored in TLS has the same global interner lifetime
145        // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
146        // when accessing the `ImplicitCtxt`.
147        tls::with_related_context(self.tcx, move |current_icx| {
148            if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) {
149                self.depth_limit_error(token);
150            }
151
152            // Update the `ImplicitCtxt` to point to our new query job.
153            let new_icx = ImplicitCtxt {
154                tcx: self.tcx,
155                query: Some(token),
156                query_depth: current_icx.query_depth + depth_limit as usize,
157                task_deps: current_icx.task_deps,
158            };
159
160            // Use the `ImplicitCtxt` while we execute the query.
161            tls::enter_context(&new_icx, compute)
162        })
163    }
164
165    fn depth_limit_error(self, job: QueryJobId) {
166        // FIXME: `collect_active_jobs` expects no locks to be held, which doesn't hold for this call.
167        let query_map = match self.collect_active_jobs() {
168            Ok(query_map) => query_map,
169            Err(query_map) => query_map,
170        };
171        let (info, depth) = job.find_dep_kind_root(query_map);
172
173        let suggested_limit = match self.recursion_limit() {
174            Limit(0) => Limit(2),
175            limit => limit * 2,
176        };
177
178        self.sess.dcx().emit_fatal(QueryOverflow {
179            span: info.job.span,
180            note: QueryOverflowNote {
181                desc: self.lift_query_info(&info.query.info).description,
182                depth,
183            },
184            suggested_limit,
185            crate_name: self.crate_name(LOCAL_CRATE),
186        });
187    }
188}
189
190pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
191    tcx.dep_graph.try_mark_green(QueryCtxt::new(tcx), dep_node).is_some()
192}
193
194pub(super) fn encode_all_query_results<'tcx>(
195    tcx: TyCtxt<'tcx>,
196    encoder: &mut CacheEncoder<'_, 'tcx>,
197    query_result_index: &mut EncodedDepNodeIndex,
198) {
199    for encode in super::ENCODE_QUERY_RESULTS.iter().copied().flatten() {
200        encode(tcx, encoder, query_result_index);
201    }
202}
203
204pub fn query_key_hash_verify_all<'tcx>(tcx: TyCtxt<'tcx>) {
205    if tcx.sess().opts.unstable_opts.incremental_verify_ich || cfg!(debug_assertions) {
206        tcx.sess.time("query_key_hash_verify_all", || {
207            for verify in super::QUERY_KEY_HASH_VERIFY.iter() {
208                verify(tcx);
209            }
210        })
211    }
212}
213
214macro_rules! handle_cycle_error {
215    ([]) => {{
216        rustc_query_system::HandleCycleError::Error
217    }};
218    ([(fatal_cycle) $($rest:tt)*]) => {{
219        rustc_query_system::HandleCycleError::Fatal
220    }};
221    ([(cycle_stash) $($rest:tt)*]) => {{
222        rustc_query_system::HandleCycleError::Stash
223    }};
224    ([(cycle_delay_bug) $($rest:tt)*]) => {{
225        rustc_query_system::HandleCycleError::DelayBug
226    }};
227    ([$other:tt $($modifiers:tt)*]) => {
228        handle_cycle_error!([$($modifiers)*])
229    };
230}
231
232macro_rules! is_anon {
233    ([]) => {{
234        false
235    }};
236    ([(anon) $($rest:tt)*]) => {{
237        true
238    }};
239    ([$other:tt $($modifiers:tt)*]) => {
240        is_anon!([$($modifiers)*])
241    };
242}
243
244macro_rules! is_eval_always {
245    ([]) => {{
246        false
247    }};
248    ([(eval_always) $($rest:tt)*]) => {{
249        true
250    }};
251    ([$other:tt $($modifiers:tt)*]) => {
252        is_eval_always!([$($modifiers)*])
253    };
254}
255
256macro_rules! depth_limit {
257    ([]) => {{
258        false
259    }};
260    ([(depth_limit) $($rest:tt)*]) => {{
261        true
262    }};
263    ([$other:tt $($modifiers:tt)*]) => {
264        depth_limit!([$($modifiers)*])
265    };
266}
267
268macro_rules! feedable {
269    ([]) => {{
270        false
271    }};
272    ([(feedable) $($rest:tt)*]) => {{
273        true
274    }};
275    ([$other:tt $($modifiers:tt)*]) => {
276        feedable!([$($modifiers)*])
277    };
278}
279
280macro_rules! hash_result {
281    ([][$V:ty]) => {{
282        Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
283    }};
284    ([(no_hash) $($rest:tt)*][$V:ty]) => {{
285        None
286    }};
287    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
288        hash_result!([$($modifiers)*][$($args)*])
289    };
290}
291
292macro_rules! call_provider {
293    ([][$tcx:expr, $name:ident, $key:expr]) => {{
294        ($tcx.query_system.fns.local_providers.$name)($tcx, $key)
295    }};
296    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
297        if let Some(key) = $key.as_local_key() {
298            ($tcx.query_system.fns.local_providers.$name)($tcx, key)
299        } else {
300            ($tcx.query_system.fns.extern_providers.$name)($tcx, $key)
301        }
302    }};
303    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
304        call_provider!([$($modifiers)*][$($args)*])
305    };
306}
307
308macro_rules! should_ever_cache_on_disk {
309    ([]$yes:tt $no:tt) => {{
310        $no
311    }};
312    ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
313        $yes
314    }};
315    ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
316        should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
317    };
318}
319
320fn create_query_frame_extra<'tcx, K: Key + Copy + 'tcx>(
321    (tcx, key, kind, name, do_describe): (
322        TyCtxt<'tcx>,
323        K,
324        DepKind,
325        &'static str,
326        fn(TyCtxt<'tcx>, K) -> String,
327    ),
328) -> QueryStackFrameExtra {
329    let def_id = key.key_as_def_id();
330
331    // If reduced queries are requested, we may be printing a query stack due
332    // to a panic. Avoid using `default_span` and `def_kind` in that case.
333    let reduce_queries = with_reduced_queries();
334
335    // Avoid calling queries while formatting the description
336    let description = ty::print::with_no_queries!(do_describe(tcx, key));
337    let description = if tcx.sess.verbose_internals() {
338        format!("{description} [{name:?}]")
339    } else {
340        description
341    };
342    let span = if kind == dep_graph::dep_kinds::def_span || reduce_queries {
343        // The `def_span` query is used to calculate `default_span`,
344        // so exit to avoid infinite recursion.
345        None
346    } else {
347        Some(key.default_span(tcx))
348    };
349
350    let def_kind = if kind == dep_graph::dep_kinds::def_kind || reduce_queries {
351        // Try to avoid infinite recursion.
352        None
353    } else {
354        def_id.and_then(|def_id| def_id.as_local()).map(|def_id| tcx.def_kind(def_id))
355    };
356    QueryStackFrameExtra::new(description, span, def_kind)
357}
358
359pub(crate) fn create_query_frame<
360    'tcx,
361    K: Copy + DynSend + DynSync + Key + for<'a> HashStable<StableHashingContext<'a>> + 'tcx,
362>(
363    tcx: TyCtxt<'tcx>,
364    do_describe: fn(TyCtxt<'tcx>, K) -> String,
365    key: K,
366    kind: DepKind,
367    name: &'static str,
368) -> QueryStackFrame<QueryStackDeferred<'tcx>> {
369    let def_id = key.key_as_def_id();
370
371    let hash = || {
372        tcx.with_stable_hashing_context(|mut hcx| {
373            let mut hasher = StableHasher::new();
374            kind.as_usize().hash_stable(&mut hcx, &mut hasher);
375            key.hash_stable(&mut hcx, &mut hasher);
376            hasher.finish::<Hash64>()
377        })
378    };
379    let def_id_for_ty_in_cycle = key.def_id_for_ty_in_cycle();
380
381    let info =
382        QueryStackDeferred::new((tcx, key, kind, name, do_describe), create_query_frame_extra);
383
384    QueryStackFrame::new(info, kind, hash, def_id, def_id_for_ty_in_cycle)
385}
386
387pub(crate) fn encode_query_results<'a, 'tcx, Q>(
388    query: Q::Config,
389    qcx: QueryCtxt<'tcx>,
390    encoder: &mut CacheEncoder<'a, 'tcx>,
391    query_result_index: &mut EncodedDepNodeIndex,
392) where
393    Q: super::QueryConfigRestored<'tcx>,
394    Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
395{
396    let _timer = qcx.profiler().generic_activity_with_arg("encode_query_results_for", query.name());
397
398    assert!(query.query_state(qcx).all_inactive());
399    let cache = query.query_cache(qcx);
400    cache.iter(&mut |key, value, dep_node| {
401        if query.cache_on_disk(qcx.tcx, key) {
402            let dep_node = SerializedDepNodeIndex::new(dep_node.index());
403
404            // Record position of the cache entry.
405            query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
406
407            // Encode the type check tables with the `SerializedDepNodeIndex`
408            // as tag.
409            encoder.encode_tagged(dep_node, &Q::restore(*value));
410        }
411    });
412}
413
414pub(crate) fn query_key_hash_verify<'tcx>(
415    query: impl QueryConfig<QueryCtxt<'tcx>>,
416    qcx: QueryCtxt<'tcx>,
417) {
418    let _timer =
419        qcx.profiler().generic_activity_with_arg("query_key_hash_verify_for", query.name());
420
421    let mut map = UnordMap::default();
422
423    let cache = query.query_cache(qcx);
424    cache.iter(&mut |key, _, _| {
425        let node = DepNode::construct(qcx.tcx, query.dep_kind(), key);
426        if let Some(other_key) = map.insert(node, *key) {
427            bug!(
428                "query key:\n\
429                `{:?}`\n\
430                and key:\n\
431                `{:?}`\n\
432                mapped to the same dep node:\n\
433                {:?}",
434                key,
435                other_key,
436                node
437            );
438        }
439    });
440}
441
442fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
443where
444    Q: QueryConfig<QueryCtxt<'tcx>>,
445{
446    debug_assert!(tcx.dep_graph.is_green(&dep_node));
447
448    let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
449        panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
450    });
451    if query.cache_on_disk(tcx, &key) {
452        let _ = query.execute_query(tcx, key);
453    }
454}
455
456pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool {
457    if let Some(cache) = tcx.query_system.on_disk_cache.as_ref() {
458        cache.loadable_from_disk(id)
459    } else {
460        false
461    }
462}
463
464pub(crate) fn try_load_from_disk<'tcx, V>(
465    tcx: TyCtxt<'tcx>,
466    prev_index: SerializedDepNodeIndex,
467    index: DepNodeIndex,
468) -> Option<V>
469where
470    V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
471{
472    let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?;
473
474    let prof_timer = tcx.prof.incr_cache_loading();
475
476    // The call to `with_query_deserialization` enforces that no new `DepNodes`
477    // are created during deserialization. See the docs of that method for more
478    // details.
479    let value = tcx
480        .dep_graph
481        .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index));
482
483    prof_timer.finish_with_query_invocation_id(index.into());
484
485    value
486}
487
488fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
489where
490    Q: QueryConfig<QueryCtxt<'tcx>>,
491{
492    // We must avoid ever having to call `force_from_dep_node()` for a
493    // `DepNode::codegen_unit`:
494    // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
495    // would always end up having to evaluate the first caller of the
496    // `codegen_unit` query that *is* reconstructible. This might very well be
497    // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
498    // to re-trigger calling the `codegen_unit` query with the right key. At
499    // that point we would already have re-done all the work we are trying to
500    // avoid doing in the first place.
501    // The solution is simple: Just explicitly call the `codegen_unit` query for
502    // each CGU, right after partitioning. This way `try_mark_green` will always
503    // hit the cache instead of having to go through `force_from_dep_node`.
504    // This assertion makes sure, we actually keep applying the solution above.
505    debug_assert!(
506        dep_node.kind != dep_kinds::codegen_unit,
507        "calling force_from_dep_node() on dep_kinds::codegen_unit"
508    );
509
510    if let Some(key) = Q::Key::recover(tcx, &dep_node) {
511        force_query(query, QueryCtxt::new(tcx), key, dep_node);
512        true
513    } else {
514        false
515    }
516}
517
518pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
519where
520    Q: QueryConfigRestored<'tcx>,
521{
522    let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style();
523
524    if is_anon || !fingerprint_style.reconstructible() {
525        return DepKindStruct {
526            is_anon,
527            is_eval_always,
528            fingerprint_style,
529            force_from_dep_node: None,
530            try_load_from_on_disk_cache: None,
531            name: Q::NAME,
532        };
533    }
534
535    DepKindStruct {
536        is_anon,
537        is_eval_always,
538        fingerprint_style,
539        force_from_dep_node: Some(|tcx, dep_node, _| {
540            force_from_dep_node(Q::config(tcx), tcx, dep_node)
541        }),
542        try_load_from_on_disk_cache: Some(|tcx, dep_node| {
543            try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node)
544        }),
545        name: Q::NAME,
546    }
547}
548
549macro_rules! item_if_cached {
550    ([] $tokens:tt) => {};
551    ([(cache) $($rest:tt)*] { $($tokens:tt)* }) => {
552        $($tokens)*
553    };
554    ([$other:tt $($modifiers:tt)*] $tokens:tt) => {
555        item_if_cached! { [$($modifiers)*] $tokens }
556    };
557}
558
559macro_rules! expand_if_cached {
560    ([], $tokens:expr) => {{
561        None
562    }};
563    ([(cache) $($rest:tt)*], $tokens:expr) => {{
564        Some($tokens)
565    }};
566    ([$other:tt $($modifiers:tt)*], $tokens:expr) => {
567        expand_if_cached!([$($modifiers)*], $tokens)
568    };
569}
570
571/// Don't show the backtrace for query system by default
572/// use `RUST_BACKTRACE=full` to show all the backtraces
573#[inline(never)]
574pub(crate) fn __rust_begin_short_backtrace<F, T>(f: F) -> T
575where
576    F: FnOnce() -> T,
577{
578    let result = f();
579    std::hint::black_box(());
580    result
581}
582
583// NOTE: `$V` isn't used here, but we still need to match on it so it can be passed to other macros
584// invoked by `rustc_with_all_queries`.
585macro_rules! define_queries {
586    (
587        $(
588            $(#[$attr:meta])*
589            [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,
590        )*
591    ) => {
592
593        pub(crate) mod query_impl { $(pub(crate) mod $name {
594            use super::super::*;
595            use std::marker::PhantomData;
596
597            pub(crate) mod get_query_incr {
598                use super::*;
599
600                // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames
601                // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming
602                #[inline(never)]
603                pub(crate) fn __rust_end_short_backtrace<'tcx>(
604                    tcx: TyCtxt<'tcx>,
605                    span: Span,
606                    key: queries::$name::Key<'tcx>,
607                    mode: QueryMode,
608                ) -> Option<Erase<queries::$name::Value<'tcx>>> {
609                    #[cfg(debug_assertions)]
610                    let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
611                    get_query_incr(
612                        QueryType::config(tcx),
613                        QueryCtxt::new(tcx),
614                        span,
615                        key,
616                        mode
617                    )
618                }
619            }
620
621            pub(crate) mod get_query_non_incr {
622                use super::*;
623
624                #[inline(never)]
625                pub(crate) fn __rust_end_short_backtrace<'tcx>(
626                    tcx: TyCtxt<'tcx>,
627                    span: Span,
628                    key: queries::$name::Key<'tcx>,
629                    __mode: QueryMode,
630                ) -> Option<Erase<queries::$name::Value<'tcx>>> {
631                    Some(get_query_non_incr(
632                        QueryType::config(tcx),
633                        QueryCtxt::new(tcx),
634                        span,
635                        key,
636                    ))
637                }
638            }
639
640            pub(crate) fn dynamic_query<'tcx>()
641                -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>>
642            {
643                DynamicQuery {
644                    name: stringify!($name),
645                    eval_always: is_eval_always!([$($modifiers)*]),
646                    dep_kind: dep_graph::dep_kinds::$name,
647                    handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
648                    query_state: std::mem::offset_of!(QueryStates<'tcx>, $name),
649                    query_cache: std::mem::offset_of!(QueryCaches<'tcx>, $name),
650                    cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
651                    execute_query: |tcx, key| erase(tcx.$name(key)),
652                    compute: |tcx, key| {
653                        #[cfg(debug_assertions)]
654                        let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
655                        __rust_begin_short_backtrace(||
656                            queries::$name::provided_to_erased(
657                                tcx,
658                                {
659                                    let ret = call_provider!([$($modifiers)*][tcx, $name, key]);
660                                    rustc_middle::ty::print::with_reduced_queries!({
661                                        tracing::trace!(?ret);
662                                    });
663                                    ret
664                                }
665                            )
666                        )
667                    },
668                    can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false),
669                    try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] {
670                        |tcx, key, prev_index, index| {
671                            if ::rustc_middle::query::cached::$name(tcx, key) {
672                                let value = $crate::plumbing::try_load_from_disk::<
673                                    queries::$name::ProvidedValue<'tcx>
674                                >(
675                                    tcx,
676                                    prev_index,
677                                    index,
678                                );
679                                value.map(|value| queries::$name::provided_to_erased(tcx, value))
680                            } else {
681                                None
682                            }
683                        }
684                    } {
685                        |_tcx, _key, _prev_index, _index| None
686                    }),
687                    value_from_cycle_error: |tcx, cycle, guar| {
688                        let result: queries::$name::Value<'tcx> = Value::from_cycle_error(tcx, cycle, guar);
689                        erase(result)
690                    },
691                    loadable_from_disk: |_tcx, _key, _index| {
692                        should_ever_cache_on_disk!([$($modifiers)*] {
693                            ::rustc_middle::query::cached::$name(_tcx, _key) &&
694                                $crate::plumbing::loadable_from_disk(_tcx, _index)
695                        } {
696                            false
697                        })
698                    },
699                    hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]),
700                    format_value: |value| format!("{:?}", restore::<queries::$name::Value<'tcx>>(*value)),
701                }
702            }
703
704            #[derive(Copy, Clone, Default)]
705            pub(crate) struct QueryType<'tcx> {
706                data: PhantomData<&'tcx ()>
707            }
708
709            impl<'tcx> QueryConfigRestored<'tcx> for QueryType<'tcx> {
710                type RestoredValue = queries::$name::Value<'tcx>;
711                type Config = DynamicConfig<
712                    'tcx,
713                    queries::$name::Storage<'tcx>,
714                    { is_anon!([$($modifiers)*]) },
715                    { depth_limit!([$($modifiers)*]) },
716                    { feedable!([$($modifiers)*]) },
717                >;
718
719                const NAME: &'static &'static str = &stringify!($name);
720
721                #[inline(always)]
722                fn config(tcx: TyCtxt<'tcx>) -> Self::Config {
723                    DynamicConfig {
724                        dynamic: &tcx.query_system.dynamic_queries.$name,
725                    }
726                }
727
728                #[inline(always)]
729                fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
730                    restore::<queries::$name::Value<'tcx>>(value)
731                }
732            }
733
734            pub(crate) fn try_collect_active_jobs<'tcx>(
735                tcx: TyCtxt<'tcx>,
736                qmap: &mut QueryMap<QueryStackDeferred<'tcx>>,
737            ) -> Option<()> {
738                let make_query = |tcx, key| {
739                    let kind = rustc_middle::dep_graph::dep_kinds::$name;
740                    let name = stringify!($name);
741                    $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name)
742                };
743                let res = tcx.query_system.states.$name.try_collect_active_jobs(
744                    tcx,
745                    make_query,
746                    qmap,
747                );
748                // this can be called during unwinding, and the function has a `try_`-prefix, so
749                // don't `unwrap()` here, just manually check for `None` and do best-effort error
750                // reporting.
751                if res.is_none() {
752                    tracing::warn!(
753                        "Failed to collect active jobs for query with name `{}`!",
754                        stringify!($name)
755                    );
756                }
757                res
758            }
759
760            pub(crate) fn alloc_self_profile_query_strings<'tcx>(
761                tcx: TyCtxt<'tcx>,
762                string_cache: &mut QueryKeyStringCache
763            ) {
764                $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
765                    tcx,
766                    stringify!($name),
767                    &tcx.query_system.caches.$name,
768                    string_cache,
769                )
770            }
771
772            item_if_cached! { [$($modifiers)*] {
773                pub(crate) fn encode_query_results<'tcx>(
774                    tcx: TyCtxt<'tcx>,
775                    encoder: &mut CacheEncoder<'_, 'tcx>,
776                    query_result_index: &mut EncodedDepNodeIndex
777                ) {
778                    $crate::plumbing::encode_query_results::<query_impl::$name::QueryType<'tcx>>(
779                        query_impl::$name::QueryType::config(tcx),
780                        QueryCtxt::new(tcx),
781                        encoder,
782                        query_result_index,
783                    )
784                }
785            }}
786
787            pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) {
788                $crate::plumbing::query_key_hash_verify(
789                    query_impl::$name::QueryType::config(tcx),
790                    QueryCtxt::new(tcx),
791                )
792            }
793        })*}
794
795        pub(crate) fn engine(incremental: bool) -> QueryEngine {
796            if incremental {
797                QueryEngine {
798                    $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)*
799                }
800            } else {
801                QueryEngine {
802                    $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)*
803                }
804            }
805        }
806
807        pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> {
808            DynamicQueries {
809                $(
810                    $name: query_impl::$name::dynamic_query(),
811                )*
812            }
813        }
814
815        // These arrays are used for iteration and can't be indexed by `DepKind`.
816
817        const TRY_COLLECT_ACTIVE_JOBS: &[
818            for<'tcx> fn(TyCtxt<'tcx>, &mut QueryMap<QueryStackDeferred<'tcx>>) -> Option<()>
819        ] =
820            &[$(query_impl::$name::try_collect_active_jobs),*];
821
822        const ALLOC_SELF_PROFILE_QUERY_STRINGS: &[
823            for<'tcx> fn(TyCtxt<'tcx>, &mut QueryKeyStringCache)
824        ] = &[$(query_impl::$name::alloc_self_profile_query_strings),*];
825
826        const ENCODE_QUERY_RESULTS: &[
827            Option<for<'tcx> fn(
828                TyCtxt<'tcx>,
829                &mut CacheEncoder<'_, 'tcx>,
830                &mut EncodedDepNodeIndex)
831            >
832        ] = &[$(expand_if_cached!([$($modifiers)*], query_impl::$name::encode_query_results)),*];
833
834        const QUERY_KEY_HASH_VERIFY: &[
835            for<'tcx> fn(TyCtxt<'tcx>)
836        ] = &[$(query_impl::$name::query_key_hash_verify),*];
837
838        #[allow(nonstandard_style)]
839        mod query_callbacks {
840            use super::*;
841            use rustc_middle::bug;
842            use rustc_query_system::dep_graph::FingerprintStyle;
843
844            // We use this for most things when incr. comp. is turned off.
845            pub(crate) fn Null<'tcx>() -> DepKindStruct<'tcx> {
846                DepKindStruct {
847                    is_anon: false,
848                    is_eval_always: false,
849                    fingerprint_style: FingerprintStyle::Unit,
850                    force_from_dep_node: Some(|_, dep_node, _| bug!("force_from_dep_node: encountered {:?}", dep_node)),
851                    try_load_from_on_disk_cache: None,
852                    name: &"Null",
853                }
854            }
855
856            // We use this for the forever-red node.
857            pub(crate) fn Red<'tcx>() -> DepKindStruct<'tcx> {
858                DepKindStruct {
859                    is_anon: false,
860                    is_eval_always: false,
861                    fingerprint_style: FingerprintStyle::Unit,
862                    force_from_dep_node: Some(|_, dep_node, _| bug!("force_from_dep_node: encountered {:?}", dep_node)),
863                    try_load_from_on_disk_cache: None,
864                    name: &"Red",
865                }
866            }
867
868            pub(crate) fn SideEffect<'tcx>() -> DepKindStruct<'tcx> {
869                DepKindStruct {
870                    is_anon: false,
871                    is_eval_always: false,
872                    fingerprint_style: FingerprintStyle::Unit,
873                    force_from_dep_node: Some(|tcx, _, prev_index| {
874                        tcx.dep_graph.force_diagnostic_node(QueryCtxt::new(tcx), prev_index);
875                        true
876                    }),
877                    try_load_from_on_disk_cache: None,
878                    name: &"SideEffect",
879                }
880            }
881
882            pub(crate) fn AnonZeroDeps<'tcx>() -> DepKindStruct<'tcx> {
883                DepKindStruct {
884                    is_anon: true,
885                    is_eval_always: false,
886                    fingerprint_style: FingerprintStyle::Opaque,
887                    force_from_dep_node: Some(|_, _, _| bug!("cannot force an anon node")),
888                    try_load_from_on_disk_cache: None,
889                    name: &"AnonZeroDeps",
890                }
891            }
892
893            pub(crate) fn TraitSelect<'tcx>() -> DepKindStruct<'tcx> {
894                DepKindStruct {
895                    is_anon: true,
896                    is_eval_always: false,
897                    fingerprint_style: FingerprintStyle::Unit,
898                    force_from_dep_node: None,
899                    try_load_from_on_disk_cache: None,
900                    name: &"TraitSelect",
901                }
902            }
903
904            pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindStruct<'tcx> {
905                DepKindStruct {
906                    is_anon: false,
907                    is_eval_always: false,
908                    fingerprint_style: FingerprintStyle::Opaque,
909                    force_from_dep_node: None,
910                    try_load_from_on_disk_cache: None,
911                    name: &"CompileCodegenUnit",
912                }
913            }
914
915            pub(crate) fn CompileMonoItem<'tcx>() -> DepKindStruct<'tcx> {
916                DepKindStruct {
917                    is_anon: false,
918                    is_eval_always: false,
919                    fingerprint_style: FingerprintStyle::Opaque,
920                    force_from_dep_node: None,
921                    try_load_from_on_disk_cache: None,
922                    name: &"CompileMonoItem",
923                }
924            }
925
926            $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> {
927                $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>(
928                    is_anon!([$($modifiers)*]),
929                    is_eval_always!([$($modifiers)*]),
930                )
931            })*
932        }
933
934        pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'tcx>] {
935            arena.alloc_from_iter(rustc_middle::make_dep_kind_array!(query_callbacks))
936        }
937
938        pub fn dep_kind_names() -> Vec<&'static str> {
939            rustc_middle::make_dep_kind_name_array!(query_callbacks)
940        }
941    }
942}