rustc_proc_macro/
quote.rs

1//! # Quasiquoter
2//! This file contains the implementation internals of the quasiquoter provided by `quote!`.
3
4//! This quasiquoter uses macros 2.0 hygiene to reliably access
5//! items from `proc_macro`, to build a `proc_macro::TokenStream`.
6
7use crate::{
8    BitOr, Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
9};
10
11#[doc(hidden)]
12pub struct HasIterator; // True
13#[doc(hidden)]
14pub struct ThereIsNoIteratorInRepetition; // False
15
16impl BitOr<ThereIsNoIteratorInRepetition> for ThereIsNoIteratorInRepetition {
17    type Output = ThereIsNoIteratorInRepetition;
18    fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> ThereIsNoIteratorInRepetition {
19        ThereIsNoIteratorInRepetition
20    }
21}
22
23impl BitOr<ThereIsNoIteratorInRepetition> for HasIterator {
24    type Output = HasIterator;
25    fn bitor(self, _rhs: ThereIsNoIteratorInRepetition) -> HasIterator {
26        HasIterator
27    }
28}
29
30impl BitOr<HasIterator> for ThereIsNoIteratorInRepetition {
31    type Output = HasIterator;
32    fn bitor(self, _rhs: HasIterator) -> HasIterator {
33        HasIterator
34    }
35}
36
37impl BitOr<HasIterator> for HasIterator {
38    type Output = HasIterator;
39    fn bitor(self, _rhs: HasIterator) -> HasIterator {
40        HasIterator
41    }
42}
43
44/// Extension traits used by the implementation of `quote!`. These are defined
45/// in separate traits, rather than as a single trait due to ambiguity issues.
46///
47/// These traits expose a `quote_into_iter` method which should allow calling
48/// whichever impl happens to be applicable. Calling that method repeatedly on
49/// the returned value should be idempotent.
50#[doc(hidden)]
51pub mod ext {
52    use core::slice;
53    use std::collections::btree_set::{self, BTreeSet};
54
55    use super::{
56        HasIterator as HasIter, RepInterp, ThereIsNoIteratorInRepetition as DoesNotHaveIter,
57    };
58    use crate::ToTokens;
59
60    /// Extension trait providing the `quote_into_iter` method on iterators.
61    #[doc(hidden)]
62    pub trait RepIteratorExt: Iterator + Sized {
63        fn quote_into_iter(self) -> (Self, HasIter) {
64            (self, HasIter)
65        }
66    }
67
68    impl<T: Iterator> RepIteratorExt for T {}
69
70    /// Extension trait providing the `quote_into_iter` method for
71    /// non-iterable types. These types interpolate the same value in each
72    /// iteration of the repetition.
73    #[doc(hidden)]
74    pub trait RepToTokensExt {
75        /// Pretend to be an iterator for the purposes of `quote_into_iter`.
76        /// This allows repeated calls to `quote_into_iter` to continue
77        /// correctly returning DoesNotHaveIter.
78        fn next(&self) -> Option<&Self> {
79            Some(self)
80        }
81
82        fn quote_into_iter(&self) -> (&Self, DoesNotHaveIter) {
83            (self, DoesNotHaveIter)
84        }
85    }
86
87    impl<T: ToTokens + ?Sized> RepToTokensExt for T {}
88
89    /// Extension trait providing the `quote_into_iter` method for types that
90    /// can be referenced as an iterator.
91    #[doc(hidden)]
92    pub trait RepAsIteratorExt<'q> {
93        type Iter: Iterator;
94
95        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter);
96    }
97
98    impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &T {
99        type Iter = T::Iter;
100
101        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
102            <T as RepAsIteratorExt>::quote_into_iter(*self)
103        }
104    }
105
106    impl<'q, T: RepAsIteratorExt<'q> + ?Sized> RepAsIteratorExt<'q> for &mut T {
107        type Iter = T::Iter;
108
109        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
110            <T as RepAsIteratorExt>::quote_into_iter(*self)
111        }
112    }
113
114    impl<'q, T: 'q> RepAsIteratorExt<'q> for [T] {
115        type Iter = slice::Iter<'q, T>;
116
117        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
118            (self.iter(), HasIter)
119        }
120    }
121
122    impl<'q, T: 'q, const N: usize> RepAsIteratorExt<'q> for [T; N] {
123        type Iter = slice::Iter<'q, T>;
124
125        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
126            (self.iter(), HasIter)
127        }
128    }
129
130    impl<'q, T: 'q> RepAsIteratorExt<'q> for Vec<T> {
131        type Iter = slice::Iter<'q, T>;
132
133        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
134            (self.iter(), HasIter)
135        }
136    }
137
138    impl<'q, T: 'q> RepAsIteratorExt<'q> for BTreeSet<T> {
139        type Iter = btree_set::Iter<'q, T>;
140
141        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
142            (self.iter(), HasIter)
143        }
144    }
145
146    impl<'q, T: RepAsIteratorExt<'q>> RepAsIteratorExt<'q> for RepInterp<T> {
147        type Iter = T::Iter;
148
149        fn quote_into_iter(&'q self) -> (Self::Iter, HasIter) {
150            self.0.quote_into_iter()
151        }
152    }
153}
154
155// Helper type used within interpolations to allow for repeated binding names.
156// Implements the relevant traits, and exports a dummy `next()` method.
157#[derive(Copy, Clone)]
158#[doc(hidden)]
159pub struct RepInterp<T>(pub T);
160
161impl<T> RepInterp<T> {
162    // This method is intended to look like `Iterator::next`, and is called when
163    // a name is bound multiple times, as the previous binding will shadow the
164    // original `Iterator` object. This allows us to avoid advancing the
165    // iterator multiple times per iteration.
166    pub fn next(self) -> Option<T> {
167        Some(self.0)
168    }
169}
170
171impl<T: Iterator> Iterator for RepInterp<T> {
172    type Item = T::Item;
173
174    fn next(&mut self) -> Option<Self::Item> {
175        self.0.next()
176    }
177}
178
179impl<T: ToTokens> ToTokens for RepInterp<T> {
180    fn to_tokens(&self, tokens: &mut TokenStream) {
181        self.0.to_tokens(tokens);
182    }
183}
184
185macro_rules! minimal_quote_tt {
186    (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) };
187    ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, minimal_quote!($($t)*)) };
188    ({$($t:tt)*}) => { Group::new(Delimiter::Brace, minimal_quote!($($t)*)) };
189    (,) => { Punct::new(',', Spacing::Alone) };
190    (.) => { Punct::new('.', Spacing::Alone) };
191    (;) => { Punct::new(';', Spacing::Alone) };
192    (!) => { Punct::new('!', Spacing::Alone) };
193    (<) => { Punct::new('<', Spacing::Alone) };
194    (>) => { Punct::new('>', Spacing::Alone) };
195    (&) => { Punct::new('&', Spacing::Alone) };
196    (=) => { Punct::new('=', Spacing::Alone) };
197    (#) => { Punct::new('#', Spacing::Alone) };
198    (|) => { Punct::new('|', Spacing::Alone) };
199    (:) => { Punct::new(':', Spacing::Alone) };
200    (*) => { Punct::new('*', Spacing::Alone) };
201    (_) => { Ident::new("_", Span::def_site()) };
202    ($i:ident) => { Ident::new(stringify!($i), Span::def_site()) };
203    ($lit:literal) => { stringify!($lit).parse::<Literal>().unwrap() };
204}
205
206macro_rules! minimal_quote_ts {
207    ((@ $($t:tt)*)) => { $($t)* };
208    (::) => {
209        {
210            let mut c = (
211                TokenTree::from(Punct::new(':', Spacing::Joint)),
212                TokenTree::from(Punct::new(':', Spacing::Alone))
213            );
214            c.0.set_span(Span::def_site());
215            c.1.set_span(Span::def_site());
216            [c.0, c.1].into_iter().collect::<TokenStream>()
217        }
218    };
219    (=>) => {
220        {
221            let mut c = (
222                TokenTree::from(Punct::new('=', Spacing::Joint)),
223                TokenTree::from(Punct::new('>', Spacing::Alone))
224            );
225            c.0.set_span(Span::def_site());
226            c.1.set_span(Span::def_site());
227            [c.0, c.1].into_iter().collect::<TokenStream>()
228        }
229    };
230    (+=) => {
231        {
232            let mut c = (
233                TokenTree::from(Punct::new('+', Spacing::Joint)),
234                TokenTree::from(Punct::new('=', Spacing::Alone))
235            );
236            c.0.set_span(Span::def_site());
237            c.1.set_span(Span::def_site());
238            [c.0, c.1].into_iter().collect::<TokenStream>()
239        }
240    };
241    (!=) => {
242        {
243            let mut c = (
244                TokenTree::from(Punct::new('!', Spacing::Joint)),
245                TokenTree::from(Punct::new('=', Spacing::Alone))
246            );
247            c.0.set_span(Span::def_site());
248            c.1.set_span(Span::def_site());
249            [c.0, c.1].into_iter().collect::<TokenStream>()
250        }
251    };
252    ($t:tt) => { TokenTree::from(minimal_quote_tt!($t)) };
253}
254
255/// Simpler version of the real `quote!` macro, implemented solely
256/// through `macro_rules`, for bootstrapping the real implementation
257/// (see the `quote` function), which does not have access to the
258/// real `quote!` macro due to the `proc_macro` crate not being
259/// able to depend on itself.
260///
261/// Note: supported tokens are a subset of the real `quote!`, but
262/// unquoting is different: instead of `$x`, this uses `(@ expr)`.
263macro_rules! minimal_quote {
264    ($($t:tt)*) => {
265        {
266            #[allow(unused_mut)] // In case the expansion is empty
267            let mut ts = TokenStream::new();
268            $(ToTokens::to_tokens(&minimal_quote_ts!($t), &mut ts);)*
269            ts
270        }
271    };
272}
273
274/// Quote a `TokenStream` into a `TokenStream`.
275/// This is the actual implementation of the `quote!()` proc macro.
276///
277/// It is loaded by the compiler in `register_builtin_macros`.
278#[unstable(feature = "proc_macro_quote", issue = "54722")]
279pub fn quote(stream: TokenStream) -> TokenStream {
280    if stream.is_empty() {
281        return minimal_quote!(crate::TokenStream::new());
282    }
283    let proc_macro_crate = minimal_quote!(crate);
284    let mut after_dollar = false;
285
286    let mut tokens = crate::TokenStream::new();
287    let mut iter = stream.into_iter().peekable();
288    while let Some(tree) = iter.next() {
289        if after_dollar {
290            after_dollar = false;
291            match tree {
292                TokenTree::Group(tt) => {
293                    // Handles repetition by expanding `$( CONTENTS ) SEP_OPT *` to `{ REP_EXPANDED }`.
294                    let contents = tt.stream();
295
296                    // The `*` token is also consumed here.
297                    let sep_opt: Option<Punct> = match (iter.next(), iter.peek()) {
298                        (Some(TokenTree::Punct(sep)), Some(TokenTree::Punct(star)))
299                            if sep.spacing() == Spacing::Joint && star.as_char() == '*' =>
300                        {
301                            iter.next();
302                            Some(sep)
303                        }
304                        (Some(TokenTree::Punct(star)), _) if star.as_char() == '*' => None,
305                        _ => panic!("`$(...)` must be followed by `*` in `quote!`"),
306                    };
307
308                    let mut rep_expanded = TokenStream::new();
309
310                    // Append setup code for a `while`, where recursively quoted `CONTENTS`
311                    // and `SEP_OPT` are repeatedly processed, to `REP_EXPANDED`.
312                    let meta_vars = collect_meta_vars(contents.clone());
313                    minimal_quote!(
314                        use crate::ext::*;
315                        (@ if sep_opt.is_some() {
316                            minimal_quote!(let mut _i = 0usize;)
317                        } else {
318                            minimal_quote!(();)
319                        })
320                        let has_iter = crate::ThereIsNoIteratorInRepetition;
321                    )
322                    .to_tokens(&mut rep_expanded);
323                    for meta_var in &meta_vars {
324                        minimal_quote!(
325                            #[allow(unused_mut)]
326                            let (mut (@ meta_var), i) = (@ meta_var).quote_into_iter();
327                            let has_iter = has_iter | i;
328                        )
329                        .to_tokens(&mut rep_expanded);
330                    }
331                    minimal_quote!(let _: crate::HasIterator = has_iter;)
332                        .to_tokens(&mut rep_expanded);
333
334                    // Append the `while` to `REP_EXPANDED`.
335                    let mut while_body = TokenStream::new();
336                    for meta_var in &meta_vars {
337                        minimal_quote!(
338                            let (@ meta_var) = match (@ meta_var).next() {
339                                Some(_x) => crate::RepInterp(_x),
340                                None => break,
341                            };
342                        )
343                        .to_tokens(&mut while_body);
344                    }
345                    minimal_quote!(
346                        (@ if let Some(sep) = sep_opt {
347                            minimal_quote!(
348                                if _i > 0 {
349                                    (@ minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
350                                        (@ TokenTree::from(Literal::character(sep.as_char()))),
351                                        (@ minimal_quote!(crate::Spacing::Alone)),
352                                    )), &mut ts);))
353                                }
354                                _i += 1;
355                            )
356                        } else {
357                            minimal_quote!(();)
358                        })
359                        (@ quote(contents.clone())).to_tokens(&mut ts);
360                    )
361                        .to_tokens(&mut while_body);
362                    rep_expanded.extend(vec![
363                        TokenTree::Ident(Ident::new("while", Span::call_site())),
364                        TokenTree::Ident(Ident::new("true", Span::call_site())),
365                        TokenTree::Group(Group::new(Delimiter::Brace, while_body)),
366                    ]);
367
368                    minimal_quote!((@ TokenTree::Group(Group::new(Delimiter::Brace, rep_expanded)))).to_tokens(&mut tokens);
369                    continue;
370                }
371                TokenTree::Ident(_) => {
372                    minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);)
373                        .to_tokens(&mut tokens);
374                    continue;
375                }
376                TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
377                _ => panic!(
378                    "`$` must be followed by an ident or `$` or a repetition group in `quote!`"
379                ),
380            }
381        } else if let TokenTree::Punct(ref tt) = tree {
382            if tt.as_char() == '$' {
383                after_dollar = true;
384                continue;
385            }
386        }
387
388        match tree {
389            TokenTree::Punct(tt) => {
390                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
391                    (@ TokenTree::from(Literal::character(tt.as_char()))),
392                    (@ match tt.spacing() {
393                        Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
394                        Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
395                    }),
396                )), &mut ts);)
397            }
398            TokenTree::Group(tt) => {
399                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
400                    (@ match tt.delimiter() {
401                        Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
402                        Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
403                        Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
404                        Delimiter::None => minimal_quote!(crate::Delimiter::None),
405                    }),
406                    (@ quote(tt.stream())),
407                )), &mut ts);)
408            }
409            TokenTree::Ident(tt) => {
410                let literal = tt.to_string();
411                let (literal, ctor) = if let Some(stripped) = literal.strip_prefix("r#") {
412                    (stripped, minimal_quote!(crate::Ident::new_raw))
413                } else {
414                    (literal.as_str(), minimal_quote!(crate::Ident::new))
415                };
416                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident((@ ctor)(
417                    (@ TokenTree::from(Literal::string(literal))),
418                    (@ quote_span(proc_macro_crate.clone(), tt.span())),
419                )), &mut ts);)
420            }
421            TokenTree::Literal(tt) => {
422                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
423                    let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
424                        .parse::<crate::TokenStream>()
425                        .unwrap()
426                        .into_iter();
427                    if let (Some(crate::TokenTree::Literal(mut lit)), None) =
428                        (iter.next(), iter.next())
429                    {
430                        lit.set_span((@ quote_span(proc_macro_crate.clone(), tt.span())));
431                        lit
432                    } else {
433                        unreachable!()
434                    }
435                }), &mut ts);)
436            }
437        }
438        .to_tokens(&mut tokens);
439    }
440    if after_dollar {
441        panic!("unexpected trailing `$` in `quote!`");
442    }
443
444    minimal_quote! {
445        {
446            let mut ts = crate::TokenStream::new();
447            (@ tokens)
448            ts
449        }
450    }
451}
452
453/// Helper function to support macro repetitions like `$( CONTENTS ) SEP_OPT *` in `quote!`.
454/// Recursively collects all `Ident`s (meta-variables) that follow a `$`
455/// from the given `CONTENTS` stream, preserving their order of appearance.
456fn collect_meta_vars(content_stream: TokenStream) -> Vec<Ident> {
457    fn helper(stream: TokenStream, out: &mut Vec<Ident>) {
458        let mut iter = stream.into_iter().peekable();
459        while let Some(tree) = iter.next() {
460            match &tree {
461                TokenTree::Punct(tt) if tt.as_char() == '$' => {
462                    if let Some(TokenTree::Ident(id)) = iter.peek() {
463                        out.push(id.clone());
464                        iter.next();
465                    }
466                }
467                TokenTree::Group(tt) => {
468                    helper(tt.stream(), out);
469                }
470                _ => {}
471            }
472        }
473    }
474
475    let mut vars = Vec::new();
476    helper(content_stream, &mut vars);
477    vars
478}
479
480/// Quote a `Span` into a `TokenStream`.
481/// This is needed to implement a custom quoter.
482#[unstable(feature = "proc_macro_quote", issue = "54722")]
483pub fn quote_span(proc_macro_crate: TokenStream, span: Span) -> TokenStream {
484    let id = span.save_span();
485    minimal_quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
486}
rustc_proc_macro/quote.rs

rustc_proc_macro/
quote.rs
