rustc_macros/
symbols.rs

1//! Proc macro which builds the Symbol table
2//!
3//! # Debugging
4//!
5//! Since this proc-macro does some non-trivial work, debugging it is important.
6//! This proc-macro can be invoked as an ordinary unit test, like so:
7//!
8//! ```bash
9//! cd compiler/rustc_macros
10//! cargo test symbols::test_symbols -- --nocapture
11//! ```
12//!
13//! This unit test finds the `symbols!` invocation in `compiler/rustc_span/src/symbol.rs`
14//! and runs it. It verifies that the output token stream can be parsed as valid module
15//! items and that no errors were produced.
16//!
17//! You can also view the generated code by using `cargo expand`:
18//!
19//! ```bash
20//! cargo install cargo-expand          # this is necessary only once
21//! cd compiler/rustc_span
22//! # The specific version number in CFG_RELEASE doesn't matter.
23//! # The output is large.
24//! CFG_RELEASE="0.0.0" cargo +nightly expand > /tmp/rustc_span.rs
25//! ```
26
27use std::collections::HashMap;
28
29use proc_macro2::{Span, TokenStream};
30use quote::quote;
31use syn::parse::{Parse, ParseStream, Result};
32use syn::punctuated::Punctuated;
33use syn::{Expr, Ident, Lit, LitStr, Macro, Token, braced};
34
35#[cfg(test)]
36mod tests;
37
38mod kw {
39    syn::custom_keyword!(Keywords);
40    syn::custom_keyword!(Symbols);
41}
42
43struct Keyword {
44    name: Ident,
45    value: LitStr,
46}
47
48impl Parse for Keyword {
49    fn parse(input: ParseStream<'_>) -> Result<Self> {
50        let name = input.parse()?;
51        input.parse::<Token![:]>()?;
52        let value = input.parse()?;
53
54        Ok(Keyword { name, value })
55    }
56}
57
58struct Symbol {
59    name: Ident,
60    value: Value,
61}
62
63enum Value {
64    SameAsName,
65    String(LitStr),
66    Env(LitStr, Macro),
67    Unsupported(Expr),
68}
69
70impl Parse for Symbol {
71    fn parse(input: ParseStream<'_>) -> Result<Self> {
72        let name = input.parse()?;
73        let colon_token: Option<Token![:]> = input.parse()?;
74        let value = if colon_token.is_some() { input.parse()? } else { Value::SameAsName };
75
76        Ok(Symbol { name, value })
77    }
78}
79
80impl Parse for Value {
81    fn parse(input: ParseStream<'_>) -> Result<Self> {
82        let expr: Expr = input.parse()?;
83        match &expr {
84            Expr::Lit(expr) => {
85                if let Lit::Str(lit) = &expr.lit {
86                    return Ok(Value::String(lit.clone()));
87                }
88            }
89            Expr::Macro(expr) => {
90                if expr.mac.path.is_ident("env")
91                    && let Ok(lit) = expr.mac.parse_body()
92                {
93                    return Ok(Value::Env(lit, expr.mac.clone()));
94                }
95            }
96            _ => {}
97        }
98        Ok(Value::Unsupported(expr))
99    }
100}
101
102struct Input {
103    keywords: Punctuated<Keyword, Token![,]>,
104    symbols: Punctuated<Symbol, Token![,]>,
105}
106
107impl Parse for Input {
108    fn parse(input: ParseStream<'_>) -> Result<Self> {
109        input.parse::<kw::Keywords>()?;
110        let content;
111        braced!(content in input);
112        let keywords = Punctuated::parse_terminated(&content)?;
113
114        input.parse::<kw::Symbols>()?;
115        let content;
116        braced!(content in input);
117        let symbols = Punctuated::parse_terminated(&content)?;
118
119        Ok(Input { keywords, symbols })
120    }
121}
122
123#[derive(Default)]
124struct Errors {
125    list: Vec<syn::Error>,
126}
127
128impl Errors {
129    fn error(&mut self, span: Span, message: String) {
130        self.list.push(syn::Error::new(span, message));
131    }
132}
133
134pub(super) fn symbols(input: TokenStream) -> TokenStream {
135    let (mut output, errors) = symbols_with_errors(input);
136
137    // If we generated any errors, then report them as compiler_error!() macro calls.
138    // This lets the errors point back to the most relevant span. It also allows us
139    // to report as many errors as we can during a single run.
140    output.extend(errors.into_iter().map(|e| e.to_compile_error()));
141
142    output
143}
144
145struct Predefined {
146    idx: u32,
147    span_of_name: Span,
148}
149
150struct Entries {
151    map: HashMap<String, Predefined>,
152}
153
154impl Entries {
155    fn with_capacity(capacity: usize) -> Self {
156        Entries { map: HashMap::with_capacity(capacity) }
157    }
158
159    fn insert(&mut self, span: Span, s: &str, errors: &mut Errors) -> u32 {
160        if let Some(prev) = self.map.get(s) {
161            errors.error(span, format!("Symbol `{s}` is duplicated"));
162            errors.error(prev.span_of_name, "location of previous definition".to_string());
163            prev.idx
164        } else {
165            let idx = self.len();
166            self.map.insert(s.to_string(), Predefined { idx, span_of_name: span });
167            idx
168        }
169    }
170
171    fn len(&self) -> u32 {
172        u32::try_from(self.map.len()).expect("way too many symbols")
173    }
174}
175
176fn symbols_with_errors(input: TokenStream) -> (TokenStream, Vec<syn::Error>) {
177    let mut errors = Errors::default();
178
179    let input: Input = match syn::parse2(input) {
180        Ok(input) => input,
181        Err(e) => {
182            // This allows us to display errors at the proper span, while minimizing
183            // unrelated errors caused by bailing out (and not generating code).
184            errors.list.push(e);
185            Input { keywords: Default::default(), symbols: Default::default() }
186        }
187    };
188
189    let mut keyword_stream = quote! {};
190    let mut symbols_stream = quote! {};
191    let mut prefill_stream = quote! {};
192    let mut entries = Entries::with_capacity(input.keywords.len() + input.symbols.len() + 10);
193
194    // Generate the listed keywords.
195    for keyword in input.keywords.iter() {
196        let name = &keyword.name;
197        let value = &keyword.value;
198        let value_string = value.value();
199        let idx = entries.insert(keyword.name.span(), &value_string, &mut errors);
200        prefill_stream.extend(quote! {
201            #value,
202        });
203        keyword_stream.extend(quote! {
204            pub const #name: Symbol = Symbol::new(#idx);
205        });
206    }
207
208    // Generate the listed symbols.
209    for symbol in input.symbols.iter() {
210        let name = &symbol.name;
211
212        let value = match &symbol.value {
213            Value::SameAsName => name.to_string(),
214            Value::String(lit) => lit.value(),
215            Value::Env(..) => continue, // in another loop below
216            Value::Unsupported(expr) => {
217                errors.list.push(syn::Error::new_spanned(
218                    expr,
219                    concat!(
220                        "unsupported expression for symbol value; implement support for this in ",
221                        file!(),
222                    ),
223                ));
224                continue;
225            }
226        };
227        let idx = entries.insert(symbol.name.span(), &value, &mut errors);
228
229        prefill_stream.extend(quote! {
230            #value,
231        });
232        symbols_stream.extend(quote! {
233            pub const #name: Symbol = Symbol::new(#idx);
234        });
235    }
236
237    // Generate symbols for the strings "0", "1", ..., "9".
238    for n in 0..10 {
239        let n = n.to_string();
240        entries.insert(Span::call_site(), &n, &mut errors);
241        prefill_stream.extend(quote! {
242            #n,
243        });
244    }
245
246    // Symbols whose value comes from an environment variable. It's allowed for
247    // these to have the same value as another symbol.
248    for symbol in &input.symbols {
249        let (env_var, expr) = match &symbol.value {
250            Value::Env(lit, expr) => (lit, expr),
251            Value::SameAsName | Value::String(_) | Value::Unsupported(_) => continue,
252        };
253
254        if !proc_macro::is_available() {
255            errors.error(
256                Span::call_site(),
257                "proc_macro::tracked_env is not available in unit test".to_owned(),
258            );
259            break;
260        }
261
262        let value = match proc_macro::tracked_env::var(env_var.value()) {
263            Ok(value) => value,
264            Err(err) => {
265                errors.list.push(syn::Error::new_spanned(expr, err));
266                continue;
267            }
268        };
269
270        let idx = if let Some(prev) = entries.map.get(&value) {
271            prev.idx
272        } else {
273            prefill_stream.extend(quote! {
274                #value,
275            });
276            entries.insert(symbol.name.span(), &value, &mut errors)
277        };
278
279        let name = &symbol.name;
280        symbols_stream.extend(quote! {
281            pub const #name: Symbol = Symbol::new(#idx);
282        });
283    }
284
285    let symbol_digits_base = entries.map["0"].idx;
286    let predefined_symbols_count = entries.len();
287    let output = quote! {
288        const SYMBOL_DIGITS_BASE: u32 = #symbol_digits_base;
289
290        /// The number of predefined symbols; this is the first index for
291        /// extra pre-interned symbols in an Interner created via
292        /// [`Interner::with_extra_symbols`].
293        pub const PREDEFINED_SYMBOLS_COUNT: u32 = #predefined_symbols_count;
294
295        #[doc(hidden)]
296        #[allow(non_upper_case_globals)]
297        mod kw_generated {
298            use super::Symbol;
299            #keyword_stream
300        }
301
302        #[allow(non_upper_case_globals)]
303        #[doc(hidden)]
304        pub mod sym_generated {
305            use super::Symbol;
306            #symbols_stream
307        }
308
309        impl Interner {
310            /// Creates an `Interner` with the predefined symbols from the `symbols!` macro and
311            /// any extra symbols provided by external drivers such as Clippy
312            pub(crate) fn with_extra_symbols(extra_symbols: &[&'static str]) -> Self {
313                Interner::prefill(
314                    &[#prefill_stream],
315                    extra_symbols,
316                )
317            }
318        }
319    };
320
321    (output, errors.list)
322}