tstr_proc_macros/
non_syn_parsing.rs

Help
1use std::iter::once;
2
3#[allow(unused_imports)]
4use proc_macro::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
5
6use proc_macro::token_stream::IntoIter as TSIterator;
7
8use super::{Inputs, TStr};
9
10pub(crate) fn parse_inputs(ts: TokenStream) -> Result<Inputs, Error> {
11    let iter = &mut ts.into_iter();
12
13    let crate_path = match iter.next() {
14        Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Parenthesis => {
15            group.stream()
16        }
17        Some(x) => {
18            return Err(Error::new(
19                x.span(),
20                &format!("Expected parentheses: found {}", x),
21            ))
22        }
23        None => {
24            return Err(Error::new(
25                Span::call_site(),
26                "Expected parentheses, found nothing",
27            ))
28        }
29    };
30
31    let mut strings = Vec::<TStr>::with_capacity(1);
32
33    while let Some(x) = parse_tstr(iter)? {
34        strings.push(x);
35    }
36
37    Ok(Inputs {
38        crate_path,
39        strings,
40    })
41}
42
43fn parse_tstr(iter: &mut TSIterator) -> Result<Option<TStr>, Error> {
44    const IN_MSG: &str = "Expected one of: string literal, integer literal, identifier";
45    match iter.next() {
46        Some(TokenTree::Ident(ident)) => {
47            let mut string = ident.to_string();
48            if string == "concat" {
49                let (span, ts) = parse_post_macro_name(iter)?;
50
51                let mut string = String::new();
52                let iter = &mut ts.into_iter();
53
54                while let Some(tstr) = parse_tstr(iter)? {
55                    string.push_str(&tstr.string);
56
57                    if let sep @ Some(_) = iter.next() {
58                        assert_punct(sep, ',')?;
59                    }
60                }
61
62                Ok(Some(TStr { string, span }))
63            } else if string == "stringify" {
64                let (span, ts) = parse_post_macro_name(iter)?;
65
66                let string = ts.to_string();
67
68                Ok(Some(TStr { string, span }))
69            } else {
70                let trimmed = string.trim_start_matches("r#");
71                if trimmed.len() != string.len() {
72                    string = trimmed.to_string();
73                }
74
75                Ok(Some(TStr {
76                    string,
77                    span: ident.span(),
78                }))
79            }
80        }
81        Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::None => {
82            parse_tstr(&mut group.stream().into_iter())
83        }
84        Some(TokenTree::Literal(lit)) => parse_literal(lit).map(Some),
85        Some(x) => Err(Error::new(x.span(), &format!("{}\nFound: {}", IN_MSG, x))),
86        None => Ok(None),
87    }
88}
89
90fn parse_post_macro_name(iter: &mut TSIterator) -> Result<(Span, TokenStream), Error> {
91    let bang_span = assert_punct(iter.next(), '!')?;
92    match iter.next() {
93        Some(TokenTree::Group(g)) if g.delimiter() == Delimiter::Parenthesis => {
94            Ok((g.span(), g.stream()))
95        }
96        _ => Err(Error::new(bang_span, "Expected `( ..... )` after `!`")),
97    }
98}
99
100fn parse_literal(lit: Literal) -> Result<TStr, Error> {
101    let span = lit.span();
102    let string = lit.to_string();
103
104    let string = if string.starts_with('"') {
105        parse_string(&string, span)?
106    } else if string.starts_with('r') {
107        parse_raw_string(&string, span)?
108    } else {
109        parse_integer(&string, span)?
110    };
111
112    Ok(TStr { string, span })
113}
114
115fn parse_string(input: &str, span: Span) -> Result<String, Error> {
116    if !input.ends_with('"') {
117        return Err(Error::new(
118            span,
119            "Somehow there's no terminating quote character?",
120        ));
121    }
122
123    let make_err = |rem: &str, error: &str| -> Error {
124        let pos = rem.as_ptr() as usize - input.as_ptr() as usize;
125
126        let upto = input[..pos]
127            .chars()
128            .rev()
129            .take(10)
130            .collect::<String>()
131            .chars()
132            .rev()
133            .collect::<String>();
134
135        Error::new(span, &format!("Error: {}    After: {}", error, upto,))
136    };
137
138    let mut rem = &input[1..input.len() - 1];
139    let mut out = String::new();
140
141    loop {
142        let end_copied = rem.find('\\').unwrap_or(rem.len());
143        out.push_str(&rem[..end_copied]);
144
145        rem = &rem[end_copied..];
146
147        if rem.is_empty() {
148            break;
149        }
150
151        // The byte after the '\\' character
152        let b = get_byte(rem, 1);
153
154        // Now we're at the character right after the matched one.
155        rem = &rem[2..];
156
157        out.push(match b {
158            b'x' => {
159                if let Some(hex) = rem.get(..2) {
160                    let num = u8::from_str_radix(hex, 16)
161                        .ok()
162                        .filter(|&x| x < 128)
163                        .ok_or_else(|| {
164                            make_err(
165                                rem,
166                                &format!("expected values from \\x00 to \\x7F, found: {}", hex),
167                            )
168                        })?;
169                    out.push(num as char);
170                } else {
171                    return Err(make_err(rem, "invalid ascii escape"));
172                }
173                rem = &rem[2..];
174                continue;
175            }
176            b'u' => {
177                if let Some(end_brace) = rem.bytes().position(|b| b == b'}') {
178                    let c: char = u32::from_str_radix(&rem[1..end_brace], 16)
179                        .ok()
180                        .and_then(std::char::from_u32)
181                        .ok_or_else(|| {
182                            make_err(
183                                rem,
184                                &format!("Invalid unicode escape: {}", &rem[..end_brace]),
185                            )
186                        })?;
187                    out.push(c);
188
189                    rem = &rem[end_brace + 1..];
190                } else {
191                    return Err(make_err(rem, "Expected closing brace for unicode escape"));
192                }
193                continue;
194            }
195            b'n' => '\n',
196            b'r' => '\r',
197            b't' => '\t',
198            b'\\' => '\\',
199            b'0' => '\0',
200            b'\'' => '\'',
201            b'"' => '"',
202            b'\r' | b'\n' => {
203                rem = rem.trim_start();
204                continue;
205            }
206            _ => return Err(make_err(rem, "invalid escape")),
207        });
208    }
209
210    Ok(out)
211}
212
213fn get_byte(s: &str, at: usize) -> u8 {
214    match s.as_bytes().get(at) {
215        Some(&x) => x,
216        None => 0,
217    }
218}
219
220fn parse_raw_string(input: &str, span: Span) -> Result<String, Error> {
221    let input = &input[1..];
222
223    let hash_count = input
224        .bytes()
225        .position(|b| b != b'#')
226        .filter(|&p| input.as_bytes()[p] == b'"')
227        .ok_or_else(|| {
228            Error::new(
229                span,
230                "Couldn't find initial '\"' character in raw string literal.",
231            )
232        })?;
233
234    let end_quote = input
235        .bytes()
236        .rev()
237        .position(|b| b != b'#')
238        .map(|p| input.len() - 1 - p)
239        .filter(|&p| input.as_bytes()[p] == b'"')
240        .ok_or_else(|| {
241            Error::new(
242                span,
243                "Couldn't find final '\"' character in raw string literal.",
244            )
245        })?;
246
247    Ok(input[hash_count + 1..end_quote].to_string())
248}
249
250fn parse_integer(input: &str, span: Span) -> Result<String, Error> {
251    fn make_err(input: &str, span: Span) -> Error {
252        Error::new(
253            span,
254            &format!("could not parse as integer literal: {}", input),
255        )
256    }
257
258    let input = input.replace('_', "");
259    let input = input.as_str();
260
261    let input_bytes = input.as_bytes();
262    if input_bytes.get(0) == Some(&b'0') {
263        let radix = match input_bytes.get(1) {
264            Some(b'x') => 16,
265            Some(b'o') => 8,
266            Some(b'b') => 2,
267            Some(_) => {
268                return Err(Error::new(
269                    span,
270                    &format!("Unknown integer prefix: {}", &input[..2]),
271                ))
272            }
273            None => return Ok(String::from("0")),
274        };
275        u128::from_str_radix(&input[2..], radix).map_err(|_| make_err(input, span))
276    } else {
277        input.parse::<u128>().map_err(|_| make_err(input, span))
278    }
279    .map(|i| i.to_string())
280}
281
282fn assert_punct(tt: Option<TokenTree>, c: char) -> Result<Span, Error> {
283    match tt {
284        Some(TokenTree::Punct(p)) if p.as_char() == c => Ok(p.span()),
285        Some(x) => Err(Error::new(
286            x.span(),
287            &format!("Expected `{}`, found `{}`", c, x),
288        )),
289        None => Err(Error::new(Span::call_site(), "Expected some token")),
290    }
291}
292
293pub(crate) struct Error {
294    span: Span,
295    message: String,
296}
297
298impl Error {
299    fn new(span: Span, message: &str) -> Self {
300        Self {
301            span,
302            message: message.to_string(),
303        }
304    }
305
306    pub(crate) fn to_compile_error(&self) -> TokenStream {
307        let Error { ref message, span } = *self;
308
309        let mut out = TokenStream::new();
310
311        out.extend(crate::utils::ident_token("compile_error", span));
312
313        out.extend(crate::utils::punct_token('!', span));
314
315        let msg_paren = crate::utils::paren(span, |ts| {
316            let mut msg = Literal::string(message);
317            msg.set_span(self.span);
318            let msg = TokenTree::from(msg);
319            ts.extend(once(msg))
320        });
321        out.extend(once(msg_paren));
322
323        out
324    }
325}
326
327trait TokenTreeExt: Sized {
328    fn into_token_tree(self) -> TokenTree;
329
330    fn set_span_recursive(self, span: Span) -> TokenTree {
331        let mut tt = self.into_token_tree();
332
333        tt.set_span(span);
334        if let TokenTree::Group(group) = tt {
335            let delim = group.delimiter();
336            let stream = group.stream().set_span_recursive(span);
337            tt = TokenTree::Group(Group::new(delim, stream));
338        }
339        tt.set_span(span);
340        tt
341    }
342}
343
344impl TokenTreeExt for TokenTree {
345    fn into_token_tree(self) -> TokenTree {
346        self
347    }
348}
349
350pub trait TokenStreamExt: Sized {
351    fn into_token_stream(self) -> TokenStream;
352
353    fn set_span_recursive(self, span: Span) -> TokenStream {
354        self.into_token_stream()
355            .into_iter()
356            .map(|tt| tt.set_span_recursive(span))
357            .collect()
358    }
359}
360
361impl TokenStreamExt for TokenStream {
362    fn into_token_stream(self) -> TokenStream {
363        self
364    }
365}
tstr_proc_macros/non_syn_parsing.rs

tstr_proc_macros/
non_syn_parsing.rs
