rustc_ast/util/
literal.rs

1//! Code related to parsing literals.
2
3use std::{ascii, fmt, str};
4
5use rustc_literal_escaper::{
6    MixedUnit, unescape_byte, unescape_byte_str, unescape_c_str, unescape_char, unescape_str,
7};
8use rustc_span::{ByteSymbol, Span, Symbol, kw, sym};
9use tracing::debug;
10
11use crate::ast::{self, LitKind, MetaItemLit, StrStyle};
12use crate::token::{self, Token};
13
14// Escapes a string, represented as a symbol. Reuses the original symbol,
15// avoiding interning, if no changes are required.
16pub fn escape_string_symbol(symbol: Symbol) -> Symbol {
17    let s = symbol.as_str();
18    let escaped = s.escape_default().to_string();
19    if s == escaped { symbol } else { Symbol::intern(&escaped) }
20}
21
22// Escapes a char.
23pub fn escape_char_symbol(ch: char) -> Symbol {
24    let s: String = ch.escape_default().map(Into::<char>::into).collect();
25    Symbol::intern(&s)
26}
27
28// Escapes a byte string.
29pub fn escape_byte_str_symbol(bytes: &[u8]) -> Symbol {
30    let s = bytes.escape_ascii().to_string();
31    Symbol::intern(&s)
32}
33
34#[derive(Debug)]
35pub enum LitError {
36    InvalidSuffix(Symbol),
37    InvalidIntSuffix(Symbol),
38    InvalidFloatSuffix(Symbol),
39    NonDecimalFloat(u32), // u32 is the base
40    IntTooLarge(u32),     // u32 is the base
41}
42
43impl LitKind {
44    /// Converts literal token into a semantic literal.
45    pub fn from_token_lit(lit: token::Lit) -> Result<LitKind, LitError> {
46        let token::Lit { kind, symbol, suffix } = lit;
47        if let Some(suffix) = suffix
48            && !kind.may_have_suffix()
49        {
50            return Err(LitError::InvalidSuffix(suffix));
51        }
52
53        // For byte/char/string literals, chars and escapes have already been
54        // checked in the lexer (in `cook_lexer_literal`). So we can assume all
55        // chars and escapes are valid here.
56        Ok(match kind {
57            token::Bool => {
58                assert!(symbol.is_bool_lit());
59                LitKind::Bool(symbol == kw::True)
60            }
61            token::Byte => {
62                return unescape_byte(symbol.as_str())
63                    .map(LitKind::Byte)
64                    .map_err(|_| panic!("failed to unescape byte literal"));
65            }
66            token::Char => {
67                return unescape_char(symbol.as_str())
68                    .map(LitKind::Char)
69                    .map_err(|_| panic!("failed to unescape char literal"));
70            }
71
72            // There are some valid suffixes for integer and float literals,
73            // so all the handling is done internally.
74            token::Integer => return integer_lit(symbol, suffix),
75            token::Float => return float_lit(symbol, suffix),
76
77            token::Str => {
78                // If there are no characters requiring special treatment we can
79                // reuse the symbol from the token. Otherwise, we must generate a
80                // new symbol because the string in the LitKind is different to the
81                // string in the token.
82                let s = symbol.as_str();
83                // Vanilla strings are so common we optimize for the common case where no chars
84                // requiring special behaviour are present.
85                let symbol = if s.contains('\\') {
86                    let mut buf = String::with_capacity(s.len());
87                    // Force-inlining here is aggressive but the closure is
88                    // called on every char in the string, so it can be hot in
89                    // programs with many long strings containing escapes.
90                    unescape_str(
91                        s,
92                        #[inline(always)]
93                        |_, res| match res {
94                            Ok(c) => buf.push(c),
95                            Err(err) => {
96                                assert!(!err.is_fatal(), "failed to unescape string literal")
97                            }
98                        },
99                    );
100                    Symbol::intern(&buf)
101                } else {
102                    symbol
103                };
104                LitKind::Str(symbol, ast::StrStyle::Cooked)
105            }
106            token::StrRaw(n) => {
107                // Raw strings have no escapes so no work is needed here.
108                LitKind::Str(symbol, ast::StrStyle::Raw(n))
109            }
110            token::ByteStr => {
111                let s = symbol.as_str();
112                let mut buf = Vec::with_capacity(s.len());
113                unescape_byte_str(s, |_, res| match res {
114                    Ok(b) => buf.push(b),
115                    Err(err) => {
116                        assert!(!err.is_fatal(), "failed to unescape string literal")
117                    }
118                });
119                LitKind::ByteStr(ByteSymbol::intern(&buf), StrStyle::Cooked)
120            }
121            token::ByteStrRaw(n) => {
122                // Raw byte strings have no escapes so no work is needed here.
123                let buf = symbol.as_str().to_owned().into_bytes();
124                LitKind::ByteStr(ByteSymbol::intern(&buf), StrStyle::Raw(n))
125            }
126            token::CStr => {
127                let s = symbol.as_str();
128                let mut buf = Vec::with_capacity(s.len());
129                unescape_c_str(s, |_span, c| match c {
130                    Ok(MixedUnit::Char(c)) => {
131                        buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
132                    }
133                    Ok(MixedUnit::HighByte(b)) => buf.push(b),
134                    Err(err) => {
135                        assert!(!err.is_fatal(), "failed to unescape C string literal")
136                    }
137                });
138                buf.push(0);
139                LitKind::CStr(ByteSymbol::intern(&buf), StrStyle::Cooked)
140            }
141            token::CStrRaw(n) => {
142                // Raw strings have no escapes so we can convert the symbol
143                // directly to a `Arc<u8>` after appending the terminating NUL
144                // char.
145                let mut buf = symbol.as_str().to_owned().into_bytes();
146                buf.push(0);
147                LitKind::CStr(ByteSymbol::intern(&buf), StrStyle::Raw(n))
148            }
149            token::Err(guar) => LitKind::Err(guar),
150        })
151    }
152}
153
154impl fmt::Display for LitKind {
155    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
156        match *self {
157            LitKind::Byte(b) => {
158                let b: String = ascii::escape_default(b).map(Into::<char>::into).collect();
159                write!(f, "b'{b}'")?;
160            }
161            LitKind::Char(ch) => write!(f, "'{}'", escape_char_symbol(ch))?,
162            LitKind::Str(sym, StrStyle::Cooked) => write!(f, "\"{}\"", escape_string_symbol(sym))?,
163            LitKind::Str(sym, StrStyle::Raw(n)) => write!(
164                f,
165                "r{delim}\"{string}\"{delim}",
166                delim = "#".repeat(n as usize),
167                string = sym
168            )?,
169            LitKind::ByteStr(ref byte_sym, StrStyle::Cooked) => {
170                write!(f, "b\"{}\"", escape_byte_str_symbol(byte_sym.as_byte_str()))?
171            }
172            LitKind::ByteStr(ref byte_sym, StrStyle::Raw(n)) => {
173                // Unwrap because raw byte string literals can only contain ASCII.
174                let symbol = str::from_utf8(byte_sym.as_byte_str()).unwrap();
175                write!(
176                    f,
177                    "br{delim}\"{string}\"{delim}",
178                    delim = "#".repeat(n as usize),
179                    string = symbol
180                )?;
181            }
182            LitKind::CStr(ref bytes, StrStyle::Cooked) => {
183                write!(f, "c\"{}\"", escape_byte_str_symbol(bytes.as_byte_str()))?
184            }
185            LitKind::CStr(ref bytes, StrStyle::Raw(n)) => {
186                // This can only be valid UTF-8.
187                let symbol = str::from_utf8(bytes.as_byte_str()).unwrap();
188                write!(f, "cr{delim}\"{symbol}\"{delim}", delim = "#".repeat(n as usize),)?;
189            }
190            LitKind::Int(n, ty) => {
191                write!(f, "{n}")?;
192                match ty {
193                    ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name())?,
194                    ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name())?,
195                    ast::LitIntType::Unsuffixed => {}
196                }
197            }
198            LitKind::Float(symbol, ty) => {
199                write!(f, "{symbol}")?;
200                match ty {
201                    ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name())?,
202                    ast::LitFloatType::Unsuffixed => {}
203                }
204            }
205            LitKind::Bool(b) => write!(f, "{}", if b { "true" } else { "false" })?,
206            LitKind::Err(_) => {
207                // This only shows up in places like `-Zunpretty=hir` output, so we
208                // don't bother to produce something useful.
209                write!(f, "<bad-literal>")?;
210            }
211        }
212
213        Ok(())
214    }
215}
216
217impl MetaItemLit {
218    /// Converts a token literal into a meta item literal.
219    pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<MetaItemLit, LitError> {
220        Ok(MetaItemLit {
221            symbol: token_lit.symbol,
222            suffix: token_lit.suffix,
223            kind: LitKind::from_token_lit(token_lit)?,
224            span,
225        })
226    }
227
228    /// Cheaply converts a meta item literal into a token literal.
229    pub fn as_token_lit(&self) -> token::Lit {
230        let kind = match self.kind {
231            LitKind::Bool(_) => token::Bool,
232            LitKind::Str(_, ast::StrStyle::Cooked) => token::Str,
233            LitKind::Str(_, ast::StrStyle::Raw(n)) => token::StrRaw(n),
234            LitKind::ByteStr(_, ast::StrStyle::Cooked) => token::ByteStr,
235            LitKind::ByteStr(_, ast::StrStyle::Raw(n)) => token::ByteStrRaw(n),
236            LitKind::CStr(_, ast::StrStyle::Cooked) => token::CStr,
237            LitKind::CStr(_, ast::StrStyle::Raw(n)) => token::CStrRaw(n),
238            LitKind::Byte(_) => token::Byte,
239            LitKind::Char(_) => token::Char,
240            LitKind::Int(..) => token::Integer,
241            LitKind::Float(..) => token::Float,
242            LitKind::Err(guar) => token::Err(guar),
243        };
244
245        token::Lit::new(kind, self.symbol, self.suffix)
246    }
247
248    /// Converts an arbitrary token into meta item literal.
249    pub fn from_token(token: &Token) -> Option<MetaItemLit> {
250        token::Lit::from_token(token)
251            .and_then(|token_lit| MetaItemLit::from_token_lit(token_lit, token.span).ok())
252    }
253}
254
255fn strip_underscores(symbol: Symbol) -> Symbol {
256    // Do not allocate a new string unless necessary.
257    let s = symbol.as_str();
258    if s.contains('_') {
259        let mut s = s.to_string();
260        s.retain(|c| c != '_');
261        return Symbol::intern(&s);
262    }
263    symbol
264}
265
266fn filtered_float_lit(
267    symbol: Symbol,
268    suffix: Option<Symbol>,
269    base: u32,
270) -> Result<LitKind, LitError> {
271    debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base);
272    if base != 10 {
273        return Err(LitError::NonDecimalFloat(base));
274    }
275    Ok(match suffix {
276        Some(suffix) => LitKind::Float(
277            symbol,
278            ast::LitFloatType::Suffixed(match suffix {
279                sym::f16 => ast::FloatTy::F16,
280                sym::f32 => ast::FloatTy::F32,
281                sym::f64 => ast::FloatTy::F64,
282                sym::f128 => ast::FloatTy::F128,
283                _ => return Err(LitError::InvalidFloatSuffix(suffix)),
284            }),
285        ),
286        None => LitKind::Float(symbol, ast::LitFloatType::Unsuffixed),
287    })
288}
289
290fn float_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
291    debug!("float_lit: {:?}, {:?}", symbol, suffix);
292    filtered_float_lit(strip_underscores(symbol), suffix, 10)
293}
294
295fn integer_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
296    debug!("integer_lit: {:?}, {:?}", symbol, suffix);
297    let symbol = strip_underscores(symbol);
298    let s = symbol.as_str();
299
300    let base = match s.as_bytes() {
301        [b'0', b'x', ..] => 16,
302        [b'0', b'o', ..] => 8,
303        [b'0', b'b', ..] => 2,
304        _ => 10,
305    };
306
307    let ty = match suffix {
308        Some(suf) => match suf {
309            sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize),
310            sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8),
311            sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16),
312            sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32),
313            sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64),
314            sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128),
315            sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize),
316            sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8),
317            sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16),
318            sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32),
319            sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64),
320            sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128),
321            // `1f64` and `2f32` etc. are valid float literals, and
322            // `fxxx` looks more like an invalid float literal than invalid integer literal.
323            _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base),
324            _ => return Err(LitError::InvalidIntSuffix(suf)),
325        },
326        _ => ast::LitIntType::Unsuffixed,
327    };
328
329    let s = &s[if base != 10 { 2 } else { 0 }..];
330    u128::from_str_radix(s, base)
331        .map(|i| LitKind::Int(i.into(), ty))
332        .map_err(|_| LitError::IntTooLarge(base))
333}
rustc_ast/util/literal.rs

rustc_ast/util/
literal.rs
