1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 #[cfg(feature = "parsing")] 4 use crate::lookahead; 5 #[cfg(feature = "parsing")] 6 use crate::parse::{Parse, Parser}; 7 use crate::{Error, Result}; 8 use proc_macro2::{Ident, Literal, Span}; 9 #[cfg(feature = "parsing")] 10 use proc_macro2::{TokenStream, TokenTree}; 11 use std::ffi::{CStr, CString}; 12 use std::fmt::{self, Display}; 13 #[cfg(feature = "extra-traits")] 14 use std::hash::{Hash, Hasher}; 15 use std::str::{self, FromStr}; 16 17 ast_enum_of_structs! { 18 /// A Rust literal such as a string or integer or boolean. 19 /// 20 /// # Syntax tree enum 21 /// 22 /// This type is a [syntax tree enum]. 23 /// 24 /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums 25 #[non_exhaustive] 26 pub enum Lit { 27 /// A UTF-8 string literal: `"foo"`. 28 Str(LitStr), 29 30 /// A byte string literal: `b"foo"`. 31 ByteStr(LitByteStr), 32 33 /// A nul-terminated C-string literal: `c"foo"`. 34 CStr(LitCStr), 35 36 /// A byte literal: `b'f'`. 37 Byte(LitByte), 38 39 /// A character literal: `'a'`. 40 Char(LitChar), 41 42 /// An integer literal: `1` or `1u16`. 43 Int(LitInt), 44 45 /// A floating point literal: `1f64` or `1.0e10f64`. 46 /// 47 /// Must be finite. May not be infinite or NaN. 48 Float(LitFloat), 49 50 /// A boolean literal: `true` or `false`. 51 Bool(LitBool), 52 53 /// A raw token literal not interpreted by Syn. 54 Verbatim(Literal), 55 } 56 } 57 58 ast_struct! { 59 /// A UTF-8 string literal: `"foo"`. 60 pub struct LitStr { 61 repr: Box<LitRepr>, 62 } 63 } 64 65 ast_struct! { 66 /// A byte string literal: `b"foo"`. 67 pub struct LitByteStr { 68 repr: Box<LitRepr>, 69 } 70 } 71 72 ast_struct! { 73 /// A nul-terminated C-string literal: `c"foo"`. 74 pub struct LitCStr { 75 repr: Box<LitRepr>, 76 } 77 } 78 79 ast_struct! { 80 /// A byte literal: `b'f'`. 81 pub struct LitByte { 82 repr: Box<LitRepr>, 83 } 84 } 85 86 ast_struct! { 87 /// A character literal: `'a'`. 88 pub struct LitChar { 89 repr: Box<LitRepr>, 90 } 91 } 92 93 struct LitRepr { 94 token: Literal, 95 suffix: Box<str>, 96 } 97 98 ast_struct! { 99 /// An integer literal: `1` or `1u16`. 100 pub struct LitInt { 101 repr: Box<LitIntRepr>, 102 } 103 } 104 105 struct LitIntRepr { 106 token: Literal, 107 digits: Box<str>, 108 suffix: Box<str>, 109 } 110 111 ast_struct! { 112 /// A floating point literal: `1f64` or `1.0e10f64`. 113 /// 114 /// Must be finite. May not be infinite or NaN. 115 pub struct LitFloat { 116 repr: Box<LitFloatRepr>, 117 } 118 } 119 120 struct LitFloatRepr { 121 token: Literal, 122 digits: Box<str>, 123 suffix: Box<str>, 124 } 125 126 ast_struct! { 127 /// A boolean literal: `true` or `false`. 128 pub struct LitBool { 129 pub value: bool, 130 pub span: Span, 131 } 132 } 133 134 impl LitStr { 135 pub fn new(value: &str, span: Span) -> Self { 136 let mut token = Literal::string(value); 137 token.set_span(span); 138 LitStr { 139 repr: Box::new(LitRepr { 140 token, 141 suffix: Box::<str>::default(), 142 }), 143 } 144 } 145 146 pub fn value(&self) -> String { 147 let repr = self.repr.token.to_string(); 148 let (value, _suffix) = value::parse_lit_str(&repr); 149 String::from(value) 150 } 151 152 /// Parse a syntax tree node from the content of this string literal. 153 /// 154 /// All spans in the syntax tree will point to the span of this `LitStr`. 155 /// 156 /// # Example 157 /// 158 /// ``` 159 /// use syn::{Attribute, Error, Expr, Lit, Meta, Path, Result}; 160 /// 161 /// // Parses the path from an attribute that looks like: 162 /// // 163 /// // #[path = "a::b::c"] 164 /// // 165 /// // or returns `None` if the input is some other attribute. 166 /// fn get_path(attr: &Attribute) -> Result<Option<Path>> { 167 /// if !attr.path().is_ident("path") { 168 /// return Ok(None); 169 /// } 170 /// 171 /// if let Meta::NameValue(meta) = &attr.meta { 172 /// if let Expr::Lit(expr) = &meta.value { 173 /// if let Lit::Str(lit_str) = &expr.lit { 174 /// return lit_str.parse().map(Some); 175 /// } 176 /// } 177 /// } 178 /// 179 /// let message = "expected #[path = \"...\"]"; 180 /// Err(Error::new_spanned(attr, message)) 181 /// } 182 /// ``` 183 #[cfg(feature = "parsing")] 184 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 185 pub fn parse<T: Parse>(&self) -> Result<T> { 186 self.parse_with(T::parse) 187 } 188 189 /// Invoke parser on the content of this string literal. 190 /// 191 /// All spans in the syntax tree will point to the span of this `LitStr`. 192 /// 193 /// # Example 194 /// 195 /// ``` 196 /// # use proc_macro2::Span; 197 /// # use syn::{LitStr, Result}; 198 /// # 199 /// # fn main() -> Result<()> { 200 /// # let lit_str = LitStr::new("a::b::c", Span::call_site()); 201 /// # 202 /// # const IGNORE: &str = stringify! { 203 /// let lit_str: LitStr = /* ... */; 204 /// # }; 205 /// 206 /// // Parse a string literal like "a::b::c" into a Path, not allowing 207 /// // generic arguments on any of the path segments. 208 /// let basic_path = lit_str.parse_with(syn::Path::parse_mod_style)?; 209 /// # 210 /// # Ok(()) 211 /// # } 212 /// ``` 213 #[cfg(feature = "parsing")] 214 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 215 pub fn parse_with<F: Parser>(&self, parser: F) -> Result<F::Output> { 216 use proc_macro2::Group; 217 218 // Token stream with every span replaced by the given one. 219 fn respan_token_stream(stream: TokenStream, span: Span) -> TokenStream { 220 stream 221 .into_iter() 222 .map(|token| respan_token_tree(token, span)) 223 .collect() 224 } 225 226 // Token tree with every span replaced by the given one. 227 fn respan_token_tree(mut token: TokenTree, span: Span) -> TokenTree { 228 match &mut token { 229 TokenTree::Group(g) => { 230 let stream = respan_token_stream(g.stream(), span); 231 *g = Group::new(g.delimiter(), stream); 232 g.set_span(span); 233 } 234 other => other.set_span(span), 235 } 236 token 237 } 238 239 // Parse string literal into a token stream with every span equal to the 240 // original literal's span. 241 let span = self.span(); 242 let mut tokens = TokenStream::from_str(&self.value())?; 243 tokens = respan_token_stream(tokens, span); 244 245 let result = crate::parse::parse_scoped(parser, span, tokens)?; 246 247 let suffix = self.suffix(); 248 if !suffix.is_empty() { 249 return Err(Error::new( 250 self.span(), 251 format!("unexpected suffix `{}` on string literal", suffix), 252 )); 253 } 254 255 Ok(result) 256 } 257 258 pub fn span(&self) -> Span { 259 self.repr.token.span() 260 } 261 262 pub fn set_span(&mut self, span: Span) { 263 self.repr.token.set_span(span); 264 } 265 266 pub fn suffix(&self) -> &str { 267 &self.repr.suffix 268 } 269 270 pub fn token(&self) -> Literal { 271 self.repr.token.clone() 272 } 273 } 274 275 impl LitByteStr { 276 pub fn new(value: &[u8], span: Span) -> Self { 277 let mut token = Literal::byte_string(value); 278 token.set_span(span); 279 LitByteStr { 280 repr: Box::new(LitRepr { 281 token, 282 suffix: Box::<str>::default(), 283 }), 284 } 285 } 286 287 pub fn value(&self) -> Vec<u8> { 288 let repr = self.repr.token.to_string(); 289 let (value, _suffix) = value::parse_lit_byte_str(&repr); 290 value 291 } 292 293 pub fn span(&self) -> Span { 294 self.repr.token.span() 295 } 296 297 pub fn set_span(&mut self, span: Span) { 298 self.repr.token.set_span(span); 299 } 300 301 pub fn suffix(&self) -> &str { 302 &self.repr.suffix 303 } 304 305 pub fn token(&self) -> Literal { 306 self.repr.token.clone() 307 } 308 } 309 310 impl LitCStr { 311 pub fn new(value: &CStr, span: Span) -> Self { 312 let mut token = Literal::c_string(value); 313 token.set_span(span); 314 LitCStr { 315 repr: Box::new(LitRepr { 316 token, 317 suffix: Box::<str>::default(), 318 }), 319 } 320 } 321 322 pub fn value(&self) -> CString { 323 let repr = self.repr.token.to_string(); 324 let (value, _suffix) = value::parse_lit_c_str(&repr); 325 value 326 } 327 328 pub fn span(&self) -> Span { 329 self.repr.token.span() 330 } 331 332 pub fn set_span(&mut self, span: Span) { 333 self.repr.token.set_span(span); 334 } 335 336 pub fn suffix(&self) -> &str { 337 &self.repr.suffix 338 } 339 340 pub fn token(&self) -> Literal { 341 self.repr.token.clone() 342 } 343 } 344 345 impl LitByte { 346 pub fn new(value: u8, span: Span) -> Self { 347 let mut token = Literal::u8_suffixed(value); 348 token.set_span(span); 349 LitByte { 350 repr: Box::new(LitRepr { 351 token, 352 suffix: Box::<str>::default(), 353 }), 354 } 355 } 356 357 pub fn value(&self) -> u8 { 358 let repr = self.repr.token.to_string(); 359 let (value, _suffix) = value::parse_lit_byte(&repr); 360 value 361 } 362 363 pub fn span(&self) -> Span { 364 self.repr.token.span() 365 } 366 367 pub fn set_span(&mut self, span: Span) { 368 self.repr.token.set_span(span); 369 } 370 371 pub fn suffix(&self) -> &str { 372 &self.repr.suffix 373 } 374 375 pub fn token(&self) -> Literal { 376 self.repr.token.clone() 377 } 378 } 379 380 impl LitChar { 381 pub fn new(value: char, span: Span) -> Self { 382 let mut token = Literal::character(value); 383 token.set_span(span); 384 LitChar { 385 repr: Box::new(LitRepr { 386 token, 387 suffix: Box::<str>::default(), 388 }), 389 } 390 } 391 392 pub fn value(&self) -> char { 393 let repr = self.repr.token.to_string(); 394 let (value, _suffix) = value::parse_lit_char(&repr); 395 value 396 } 397 398 pub fn span(&self) -> Span { 399 self.repr.token.span() 400 } 401 402 pub fn set_span(&mut self, span: Span) { 403 self.repr.token.set_span(span); 404 } 405 406 pub fn suffix(&self) -> &str { 407 &self.repr.suffix 408 } 409 410 pub fn token(&self) -> Literal { 411 self.repr.token.clone() 412 } 413 } 414 415 impl LitInt { 416 pub fn new(repr: &str, span: Span) -> Self { 417 let (digits, suffix) = match value::parse_lit_int(repr) { 418 Some(parse) => parse, 419 None => panic!("not an integer literal: `{}`", repr), 420 }; 421 422 let mut token: Literal = repr.parse().unwrap(); 423 token.set_span(span); 424 LitInt { 425 repr: Box::new(LitIntRepr { 426 token, 427 digits, 428 suffix, 429 }), 430 } 431 } 432 433 pub fn base10_digits(&self) -> &str { 434 &self.repr.digits 435 } 436 437 /// Parses the literal into a selected number type. 438 /// 439 /// This is equivalent to `lit.base10_digits().parse()` except that the 440 /// resulting errors will be correctly spanned to point to the literal token 441 /// in the macro input. 442 /// 443 /// ``` 444 /// use syn::LitInt; 445 /// use syn::parse::{Parse, ParseStream, Result}; 446 /// 447 /// struct Port { 448 /// value: u16, 449 /// } 450 /// 451 /// impl Parse for Port { 452 /// fn parse(input: ParseStream) -> Result<Self> { 453 /// let lit: LitInt = input.parse()?; 454 /// let value = lit.base10_parse::<u16>()?; 455 /// Ok(Port { value }) 456 /// } 457 /// } 458 /// ``` 459 pub fn base10_parse<N>(&self) -> Result<N> 460 where 461 N: FromStr, 462 N::Err: Display, 463 { 464 self.base10_digits() 465 .parse() 466 .map_err(|err| Error::new(self.span(), err)) 467 } 468 469 pub fn suffix(&self) -> &str { 470 &self.repr.suffix 471 } 472 473 pub fn span(&self) -> Span { 474 self.repr.token.span() 475 } 476 477 pub fn set_span(&mut self, span: Span) { 478 self.repr.token.set_span(span); 479 } 480 481 pub fn token(&self) -> Literal { 482 self.repr.token.clone() 483 } 484 } 485 486 impl From<Literal> for LitInt { 487 fn from(token: Literal) -> Self { 488 let repr = token.to_string(); 489 if let Some((digits, suffix)) = value::parse_lit_int(&repr) { 490 LitInt { 491 repr: Box::new(LitIntRepr { 492 token, 493 digits, 494 suffix, 495 }), 496 } 497 } else { 498 panic!("not an integer literal: `{}`", repr); 499 } 500 } 501 } 502 503 impl Display for LitInt { 504 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 505 self.repr.token.fmt(formatter) 506 } 507 } 508 509 impl LitFloat { 510 pub fn new(repr: &str, span: Span) -> Self { 511 let (digits, suffix) = match value::parse_lit_float(repr) { 512 Some(parse) => parse, 513 None => panic!("not a float literal: `{}`", repr), 514 }; 515 516 let mut token: Literal = repr.parse().unwrap(); 517 token.set_span(span); 518 LitFloat { 519 repr: Box::new(LitFloatRepr { 520 token, 521 digits, 522 suffix, 523 }), 524 } 525 } 526 527 pub fn base10_digits(&self) -> &str { 528 &self.repr.digits 529 } 530 531 pub fn base10_parse<N>(&self) -> Result<N> 532 where 533 N: FromStr, 534 N::Err: Display, 535 { 536 self.base10_digits() 537 .parse() 538 .map_err(|err| Error::new(self.span(), err)) 539 } 540 541 pub fn suffix(&self) -> &str { 542 &self.repr.suffix 543 } 544 545 pub fn span(&self) -> Span { 546 self.repr.token.span() 547 } 548 549 pub fn set_span(&mut self, span: Span) { 550 self.repr.token.set_span(span); 551 } 552 553 pub fn token(&self) -> Literal { 554 self.repr.token.clone() 555 } 556 } 557 558 impl From<Literal> for LitFloat { 559 fn from(token: Literal) -> Self { 560 let repr = token.to_string(); 561 if let Some((digits, suffix)) = value::parse_lit_float(&repr) { 562 LitFloat { 563 repr: Box::new(LitFloatRepr { 564 token, 565 digits, 566 suffix, 567 }), 568 } 569 } else { 570 panic!("not a float literal: `{}`", repr); 571 } 572 } 573 } 574 575 impl Display for LitFloat { 576 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 577 self.repr.token.fmt(formatter) 578 } 579 } 580 581 impl LitBool { 582 pub fn new(value: bool, span: Span) -> Self { 583 LitBool { value, span } 584 } 585 586 pub fn value(&self) -> bool { 587 self.value 588 } 589 590 pub fn span(&self) -> Span { 591 self.span 592 } 593 594 pub fn set_span(&mut self, span: Span) { 595 self.span = span; 596 } 597 598 pub fn token(&self) -> Ident { 599 let s = if self.value { "true" } else { "false" }; 600 Ident::new(s, self.span) 601 } 602 } 603 604 #[cfg(feature = "extra-traits")] 605 mod debug_impls { 606 use crate::lit::{LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitInt, LitStr}; 607 use std::fmt::{self, Debug}; 608 609 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 610 impl Debug for LitStr { 611 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 612 self.debug(formatter, "LitStr") 613 } 614 } 615 616 impl LitStr { 617 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 618 formatter 619 .debug_struct(name) 620 .field("token", &format_args!("{}", self.repr.token)) 621 .finish() 622 } 623 } 624 625 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 626 impl Debug for LitByteStr { 627 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 628 self.debug(formatter, "LitByteStr") 629 } 630 } 631 632 impl LitByteStr { 633 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 634 formatter 635 .debug_struct(name) 636 .field("token", &format_args!("{}", self.repr.token)) 637 .finish() 638 } 639 } 640 641 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 642 impl Debug for LitCStr { 643 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 644 self.debug(formatter, "LitCStr") 645 } 646 } 647 648 impl LitCStr { 649 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 650 formatter 651 .debug_struct(name) 652 .field("token", &format_args!("{}", self.repr.token)) 653 .finish() 654 } 655 } 656 657 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 658 impl Debug for LitByte { 659 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 660 self.debug(formatter, "LitByte") 661 } 662 } 663 664 impl LitByte { 665 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 666 formatter 667 .debug_struct(name) 668 .field("token", &format_args!("{}", self.repr.token)) 669 .finish() 670 } 671 } 672 673 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 674 impl Debug for LitChar { 675 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 676 self.debug(formatter, "LitChar") 677 } 678 } 679 680 impl LitChar { 681 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 682 formatter 683 .debug_struct(name) 684 .field("token", &format_args!("{}", self.repr.token)) 685 .finish() 686 } 687 } 688 689 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 690 impl Debug for LitInt { 691 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 692 self.debug(formatter, "LitInt") 693 } 694 } 695 696 impl LitInt { 697 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 698 formatter 699 .debug_struct(name) 700 .field("token", &format_args!("{}", self.repr.token)) 701 .finish() 702 } 703 } 704 705 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 706 impl Debug for LitFloat { 707 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 708 self.debug(formatter, "LitFloat") 709 } 710 } 711 712 impl LitFloat { 713 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 714 formatter 715 .debug_struct(name) 716 .field("token", &format_args!("{}", self.repr.token)) 717 .finish() 718 } 719 } 720 721 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 722 impl Debug for LitBool { 723 fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 724 self.debug(formatter, "LitBool") 725 } 726 } 727 728 impl LitBool { 729 pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { 730 formatter 731 .debug_struct(name) 732 .field("value", &self.value) 733 .finish() 734 } 735 } 736 } 737 738 #[cfg(feature = "clone-impls")] 739 #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] 740 impl Clone for LitRepr { 741 fn clone(&self) -> Self { 742 LitRepr { 743 token: self.token.clone(), 744 suffix: self.suffix.clone(), 745 } 746 } 747 } 748 749 #[cfg(feature = "clone-impls")] 750 #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] 751 impl Clone for LitIntRepr { 752 fn clone(&self) -> Self { 753 LitIntRepr { 754 token: self.token.clone(), 755 digits: self.digits.clone(), 756 suffix: self.suffix.clone(), 757 } 758 } 759 } 760 761 #[cfg(feature = "clone-impls")] 762 #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] 763 impl Clone for LitFloatRepr { 764 fn clone(&self) -> Self { 765 LitFloatRepr { 766 token: self.token.clone(), 767 digits: self.digits.clone(), 768 suffix: self.suffix.clone(), 769 } 770 } 771 } 772 773 macro_rules! lit_extra_traits { 774 ($ty:ident) => { 775 #[cfg(feature = "clone-impls")] 776 #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] 777 impl Clone for $ty { 778 fn clone(&self) -> Self { 779 $ty { 780 repr: self.repr.clone(), 781 } 782 } 783 } 784 785 #[cfg(feature = "extra-traits")] 786 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 787 impl PartialEq for $ty { 788 fn eq(&self, other: &Self) -> bool { 789 self.repr.token.to_string() == other.repr.token.to_string() 790 } 791 } 792 793 #[cfg(feature = "extra-traits")] 794 #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] 795 impl Hash for $ty { 796 fn hash<H>(&self, state: &mut H) 797 where 798 H: Hasher, 799 { 800 self.repr.token.to_string().hash(state); 801 } 802 } 803 804 #[cfg(feature = "parsing")] 805 pub_if_not_doc! { 806 #[doc(hidden)] 807 #[allow(non_snake_case)] 808 pub fn $ty(marker: lookahead::TokenMarker) -> $ty { 809 match marker {} 810 } 811 } 812 }; 813 } 814 815 lit_extra_traits!(LitStr); 816 lit_extra_traits!(LitByteStr); 817 lit_extra_traits!(LitCStr); 818 lit_extra_traits!(LitByte); 819 lit_extra_traits!(LitChar); 820 lit_extra_traits!(LitInt); 821 lit_extra_traits!(LitFloat); 822 823 #[cfg(feature = "parsing")] 824 pub_if_not_doc! { 825 #[doc(hidden)] 826 #[allow(non_snake_case)] 827 pub fn LitBool(marker: lookahead::TokenMarker) -> LitBool { 828 match marker {} 829 } 830 } 831 832 /// The style of a string literal, either plain quoted or a raw string like 833 /// `r##"data"##`. 834 #[doc(hidden)] // https://github.com/dtolnay/syn/issues/1566 835 pub enum StrStyle { 836 /// An ordinary string like `"data"`. 837 Cooked, 838 /// A raw string like `r##"data"##`. 839 /// 840 /// The unsigned integer is the number of `#` symbols used. 841 Raw(usize), 842 } 843 844 #[cfg(feature = "parsing")] 845 pub_if_not_doc! { 846 #[doc(hidden)] 847 #[allow(non_snake_case)] 848 pub fn Lit(marker: lookahead::TokenMarker) -> Lit { 849 match marker {} 850 } 851 } 852 853 #[cfg(feature = "parsing")] 854 pub(crate) mod parsing { 855 use crate::buffer::Cursor; 856 use crate::error::Result; 857 use crate::lit::{ 858 value, Lit, LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitFloatRepr, LitInt, 859 LitIntRepr, LitStr, 860 }; 861 use crate::parse::{Parse, ParseStream, Unexpected}; 862 use crate::token::{self, Token}; 863 use proc_macro2::{Literal, Punct, Span}; 864 use std::cell::Cell; 865 use std::rc::Rc; 866 867 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 868 impl Parse for Lit { 869 fn parse(input: ParseStream) -> Result<Self> { 870 input.step(|cursor| { 871 if let Some((lit, rest)) = cursor.literal() { 872 return Ok((Lit::new(lit), rest)); 873 } 874 875 if let Some((ident, rest)) = cursor.ident() { 876 let value = ident == "true"; 877 if value || ident == "false" { 878 let lit_bool = LitBool { 879 value, 880 span: ident.span(), 881 }; 882 return Ok((Lit::Bool(lit_bool), rest)); 883 } 884 } 885 886 if let Some((punct, rest)) = cursor.punct() { 887 if punct.as_char() == '-' { 888 if let Some((lit, rest)) = parse_negative_lit(punct, rest) { 889 return Ok((lit, rest)); 890 } 891 } 892 } 893 894 Err(cursor.error("expected literal")) 895 }) 896 } 897 } 898 899 fn parse_negative_lit(neg: Punct, cursor: Cursor) -> Option<(Lit, Cursor)> { 900 let (lit, rest) = cursor.literal()?; 901 902 let mut span = neg.span(); 903 span = span.join(lit.span()).unwrap_or(span); 904 905 let mut repr = lit.to_string(); 906 repr.insert(0, '-'); 907 908 if let Some((digits, suffix)) = value::parse_lit_int(&repr) { 909 let mut token: Literal = repr.parse().unwrap(); 910 token.set_span(span); 911 return Some(( 912 Lit::Int(LitInt { 913 repr: Box::new(LitIntRepr { 914 token, 915 digits, 916 suffix, 917 }), 918 }), 919 rest, 920 )); 921 } 922 923 let (digits, suffix) = value::parse_lit_float(&repr)?; 924 let mut token: Literal = repr.parse().unwrap(); 925 token.set_span(span); 926 Some(( 927 Lit::Float(LitFloat { 928 repr: Box::new(LitFloatRepr { 929 token, 930 digits, 931 suffix, 932 }), 933 }), 934 rest, 935 )) 936 } 937 938 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 939 impl Parse for LitStr { 940 fn parse(input: ParseStream) -> Result<Self> { 941 let head = input.fork(); 942 match input.parse() { 943 Ok(Lit::Str(lit)) => Ok(lit), 944 _ => Err(head.error("expected string literal")), 945 } 946 } 947 } 948 949 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 950 impl Parse for LitByteStr { 951 fn parse(input: ParseStream) -> Result<Self> { 952 let head = input.fork(); 953 match input.parse() { 954 Ok(Lit::ByteStr(lit)) => Ok(lit), 955 _ => Err(head.error("expected byte string literal")), 956 } 957 } 958 } 959 960 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 961 impl Parse for LitCStr { 962 fn parse(input: ParseStream) -> Result<Self> { 963 let head = input.fork(); 964 match input.parse() { 965 Ok(Lit::CStr(lit)) => Ok(lit), 966 _ => Err(head.error("expected C string literal")), 967 } 968 } 969 } 970 971 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 972 impl Parse for LitByte { 973 fn parse(input: ParseStream) -> Result<Self> { 974 let head = input.fork(); 975 match input.parse() { 976 Ok(Lit::Byte(lit)) => Ok(lit), 977 _ => Err(head.error("expected byte literal")), 978 } 979 } 980 } 981 982 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 983 impl Parse for LitChar { 984 fn parse(input: ParseStream) -> Result<Self> { 985 let head = input.fork(); 986 match input.parse() { 987 Ok(Lit::Char(lit)) => Ok(lit), 988 _ => Err(head.error("expected character literal")), 989 } 990 } 991 } 992 993 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 994 impl Parse for LitInt { 995 fn parse(input: ParseStream) -> Result<Self> { 996 let head = input.fork(); 997 match input.parse() { 998 Ok(Lit::Int(lit)) => Ok(lit), 999 _ => Err(head.error("expected integer literal")), 1000 } 1001 } 1002 } 1003 1004 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 1005 impl Parse for LitFloat { 1006 fn parse(input: ParseStream) -> Result<Self> { 1007 let head = input.fork(); 1008 match input.parse() { 1009 Ok(Lit::Float(lit)) => Ok(lit), 1010 _ => Err(head.error("expected floating point literal")), 1011 } 1012 } 1013 } 1014 1015 #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] 1016 impl Parse for LitBool { 1017 fn parse(input: ParseStream) -> Result<Self> { 1018 let head = input.fork(); 1019 match input.parse() { 1020 Ok(Lit::Bool(lit)) => Ok(lit), 1021 _ => Err(head.error("expected boolean literal")), 1022 } 1023 } 1024 } 1025 1026 fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool { 1027 let scope = Span::call_site(); 1028 let unexpected = Rc::new(Cell::new(Unexpected::None)); 1029 let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected); 1030 peek(&buffer) 1031 } 1032 1033 macro_rules! impl_token { 1034 ($display:literal $name:ty) => { 1035 impl Token for $name { 1036 fn peek(cursor: Cursor) -> bool { 1037 fn peek(input: ParseStream) -> bool { 1038 <$name as Parse>::parse(input).is_ok() 1039 } 1040 peek_impl(cursor, peek) 1041 } 1042 1043 fn display() -> &'static str { 1044 $display 1045 } 1046 } 1047 1048 impl token::private::Sealed for $name {} 1049 }; 1050 } 1051 1052 impl_token!("literal" Lit); 1053 impl_token!("string literal" LitStr); 1054 impl_token!("byte string literal" LitByteStr); 1055 impl_token!("C-string literal" LitCStr); 1056 impl_token!("byte literal" LitByte); 1057 impl_token!("character literal" LitChar); 1058 impl_token!("integer literal" LitInt); 1059 impl_token!("floating point literal" LitFloat); 1060 impl_token!("boolean literal" LitBool); 1061 } 1062 1063 #[cfg(feature = "printing")] 1064 mod printing { 1065 use crate::lit::{LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitInt, LitStr}; 1066 use proc_macro2::TokenStream; 1067 use quote::{ToTokens, TokenStreamExt}; 1068 1069 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1070 impl ToTokens for LitStr { 1071 fn to_tokens(&self, tokens: &mut TokenStream) { 1072 self.repr.token.to_tokens(tokens); 1073 } 1074 } 1075 1076 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1077 impl ToTokens for LitByteStr { 1078 fn to_tokens(&self, tokens: &mut TokenStream) { 1079 self.repr.token.to_tokens(tokens); 1080 } 1081 } 1082 1083 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1084 impl ToTokens for LitCStr { 1085 fn to_tokens(&self, tokens: &mut TokenStream) { 1086 self.repr.token.to_tokens(tokens); 1087 } 1088 } 1089 1090 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1091 impl ToTokens for LitByte { 1092 fn to_tokens(&self, tokens: &mut TokenStream) { 1093 self.repr.token.to_tokens(tokens); 1094 } 1095 } 1096 1097 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1098 impl ToTokens for LitChar { 1099 fn to_tokens(&self, tokens: &mut TokenStream) { 1100 self.repr.token.to_tokens(tokens); 1101 } 1102 } 1103 1104 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1105 impl ToTokens for LitInt { 1106 fn to_tokens(&self, tokens: &mut TokenStream) { 1107 self.repr.token.to_tokens(tokens); 1108 } 1109 } 1110 1111 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1112 impl ToTokens for LitFloat { 1113 fn to_tokens(&self, tokens: &mut TokenStream) { 1114 self.repr.token.to_tokens(tokens); 1115 } 1116 } 1117 1118 #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] 1119 impl ToTokens for LitBool { 1120 fn to_tokens(&self, tokens: &mut TokenStream) { 1121 tokens.append(self.token()); 1122 } 1123 } 1124 } 1125 1126 mod value { 1127 use crate::bigint::BigInt; 1128 use crate::lit::{ 1129 Lit, LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitFloatRepr, LitInt, 1130 LitIntRepr, LitRepr, LitStr, 1131 }; 1132 use proc_macro2::{Literal, Span}; 1133 use std::ascii; 1134 use std::char; 1135 use std::ffi::CString; 1136 use std::ops::{Index, RangeFrom}; 1137 1138 impl Lit { 1139 /// Interpret a Syn literal from a proc-macro2 literal. 1140 pub fn new(token: Literal) -> Self { 1141 let repr = token.to_string(); 1142 1143 match byte(&repr, 0) { 1144 // "...", r"...", r#"..."# 1145 b'"' | b'r' => { 1146 let (_, suffix) = parse_lit_str(&repr); 1147 return Lit::Str(LitStr { 1148 repr: Box::new(LitRepr { token, suffix }), 1149 }); 1150 } 1151 b'b' => match byte(&repr, 1) { 1152 // b"...", br"...", br#"...#" 1153 b'"' | b'r' => { 1154 let (_, suffix) = parse_lit_byte_str(&repr); 1155 return Lit::ByteStr(LitByteStr { 1156 repr: Box::new(LitRepr { token, suffix }), 1157 }); 1158 } 1159 // b'...' 1160 b'\'' => { 1161 let (_, suffix) = parse_lit_byte(&repr); 1162 return Lit::Byte(LitByte { 1163 repr: Box::new(LitRepr { token, suffix }), 1164 }); 1165 } 1166 _ => {} 1167 }, 1168 // c"...", cr"...", cr#"..."# 1169 b'c' => { 1170 let (_, suffix) = parse_lit_c_str(&repr); 1171 return Lit::CStr(LitCStr { 1172 repr: Box::new(LitRepr { token, suffix }), 1173 }); 1174 } 1175 // '...' 1176 b'\'' => { 1177 let (_, suffix) = parse_lit_char(&repr); 1178 return Lit::Char(LitChar { 1179 repr: Box::new(LitRepr { token, suffix }), 1180 }); 1181 } 1182 b'0'..=b'9' | b'-' => { 1183 // 0, 123, 0xFF, 0o77, 0b11 1184 if let Some((digits, suffix)) = parse_lit_int(&repr) { 1185 return Lit::Int(LitInt { 1186 repr: Box::new(LitIntRepr { 1187 token, 1188 digits, 1189 suffix, 1190 }), 1191 }); 1192 } 1193 // 1.0, 1e-1, 1e+1 1194 if let Some((digits, suffix)) = parse_lit_float(&repr) { 1195 return Lit::Float(LitFloat { 1196 repr: Box::new(LitFloatRepr { 1197 token, 1198 digits, 1199 suffix, 1200 }), 1201 }); 1202 } 1203 } 1204 // true, false 1205 b't' | b'f' => { 1206 if repr == "true" || repr == "false" { 1207 return Lit::Bool(LitBool { 1208 value: repr == "true", 1209 span: token.span(), 1210 }); 1211 } 1212 } 1213 b'(' if repr == "(/*ERROR*/)" => return Lit::Verbatim(token), 1214 _ => {} 1215 } 1216 1217 panic!("unrecognized literal: `{}`", repr); 1218 } 1219 1220 pub fn suffix(&self) -> &str { 1221 match self { 1222 Lit::Str(lit) => lit.suffix(), 1223 Lit::ByteStr(lit) => lit.suffix(), 1224 Lit::CStr(lit) => lit.suffix(), 1225 Lit::Byte(lit) => lit.suffix(), 1226 Lit::Char(lit) => lit.suffix(), 1227 Lit::Int(lit) => lit.suffix(), 1228 Lit::Float(lit) => lit.suffix(), 1229 Lit::Bool(_) | Lit::Verbatim(_) => "", 1230 } 1231 } 1232 1233 pub fn span(&self) -> Span { 1234 match self { 1235 Lit::Str(lit) => lit.span(), 1236 Lit::ByteStr(lit) => lit.span(), 1237 Lit::CStr(lit) => lit.span(), 1238 Lit::Byte(lit) => lit.span(), 1239 Lit::Char(lit) => lit.span(), 1240 Lit::Int(lit) => lit.span(), 1241 Lit::Float(lit) => lit.span(), 1242 Lit::Bool(lit) => lit.span, 1243 Lit::Verbatim(lit) => lit.span(), 1244 } 1245 } 1246 1247 pub fn set_span(&mut self, span: Span) { 1248 match self { 1249 Lit::Str(lit) => lit.set_span(span), 1250 Lit::ByteStr(lit) => lit.set_span(span), 1251 Lit::CStr(lit) => lit.set_span(span), 1252 Lit::Byte(lit) => lit.set_span(span), 1253 Lit::Char(lit) => lit.set_span(span), 1254 Lit::Int(lit) => lit.set_span(span), 1255 Lit::Float(lit) => lit.set_span(span), 1256 Lit::Bool(lit) => lit.span = span, 1257 Lit::Verbatim(lit) => lit.set_span(span), 1258 } 1259 } 1260 } 1261 1262 /// Get the byte at offset idx, or a default of `b'\0'` if we're looking 1263 /// past the end of the input buffer. 1264 pub(crate) fn byte<S: AsRef<[u8]> + ?Sized>(s: &S, idx: usize) -> u8 { 1265 let s = s.as_ref(); 1266 if idx < s.len() { 1267 s[idx] 1268 } else { 1269 0 1270 } 1271 } 1272 1273 fn next_chr(s: &str) -> char { 1274 s.chars().next().unwrap_or('\0') 1275 } 1276 1277 // Returns (content, suffix). 1278 pub(crate) fn parse_lit_str(s: &str) -> (Box<str>, Box<str>) { 1279 match byte(s, 0) { 1280 b'"' => parse_lit_str_cooked(s), 1281 b'r' => parse_lit_str_raw(s), 1282 _ => unreachable!(), 1283 } 1284 } 1285 1286 fn parse_lit_str_cooked(mut s: &str) -> (Box<str>, Box<str>) { 1287 assert_eq!(byte(s, 0), b'"'); 1288 s = &s[1..]; 1289 1290 let mut content = String::new(); 1291 'outer: loop { 1292 let ch = match byte(s, 0) { 1293 b'"' => break, 1294 b'\\' => { 1295 let b = byte(s, 1); 1296 s = &s[2..]; 1297 match b { 1298 b'x' => { 1299 let (byte, rest) = backslash_x(s); 1300 s = rest; 1301 assert!(byte <= 0x7F, "invalid \\x byte in string literal"); 1302 char::from_u32(u32::from(byte)).unwrap() 1303 } 1304 b'u' => { 1305 let (ch, rest) = backslash_u(s); 1306 s = rest; 1307 ch 1308 } 1309 b'n' => '\n', 1310 b'r' => '\r', 1311 b't' => '\t', 1312 b'\\' => '\\', 1313 b'0' => '\0', 1314 b'\'' => '\'', 1315 b'"' => '"', 1316 b'\r' | b'\n' => loop { 1317 let b = byte(s, 0); 1318 match b { 1319 b' ' | b'\t' | b'\n' | b'\r' => s = &s[1..], 1320 _ => continue 'outer, 1321 } 1322 }, 1323 b => panic!( 1324 "unexpected byte '{}' after \\ character in string literal", 1325 ascii::escape_default(b), 1326 ), 1327 } 1328 } 1329 b'\r' => { 1330 assert_eq!(byte(s, 1), b'\n', "bare CR not allowed in string"); 1331 s = &s[2..]; 1332 '\n' 1333 } 1334 _ => { 1335 let ch = next_chr(s); 1336 s = &s[ch.len_utf8()..]; 1337 ch 1338 } 1339 }; 1340 content.push(ch); 1341 } 1342 1343 assert!(s.starts_with('"')); 1344 let content = content.into_boxed_str(); 1345 let suffix = s[1..].to_owned().into_boxed_str(); 1346 (content, suffix) 1347 } 1348 1349 fn parse_lit_str_raw(mut s: &str) -> (Box<str>, Box<str>) { 1350 assert_eq!(byte(s, 0), b'r'); 1351 s = &s[1..]; 1352 1353 let mut pounds = 0; 1354 while byte(s, pounds) == b'#' { 1355 pounds += 1; 1356 } 1357 assert_eq!(byte(s, pounds), b'"'); 1358 let close = s.rfind('"').unwrap(); 1359 for end in s[close + 1..close + 1 + pounds].bytes() { 1360 assert_eq!(end, b'#'); 1361 } 1362 1363 let content = s[pounds + 1..close].to_owned().into_boxed_str(); 1364 let suffix = s[close + 1 + pounds..].to_owned().into_boxed_str(); 1365 (content, suffix) 1366 } 1367 1368 // Returns (content, suffix). 1369 pub(crate) fn parse_lit_byte_str(s: &str) -> (Vec<u8>, Box<str>) { 1370 assert_eq!(byte(s, 0), b'b'); 1371 match byte(s, 1) { 1372 b'"' => parse_lit_byte_str_cooked(s), 1373 b'r' => parse_lit_byte_str_raw(s), 1374 _ => unreachable!(), 1375 } 1376 } 1377 1378 fn parse_lit_byte_str_cooked(mut s: &str) -> (Vec<u8>, Box<str>) { 1379 assert_eq!(byte(s, 0), b'b'); 1380 assert_eq!(byte(s, 1), b'"'); 1381 s = &s[2..]; 1382 1383 // We're going to want to have slices which don't respect codepoint boundaries. 1384 let mut v = s.as_bytes(); 1385 1386 let mut out = Vec::new(); 1387 'outer: loop { 1388 let byte = match byte(v, 0) { 1389 b'"' => break, 1390 b'\\' => { 1391 let b = byte(v, 1); 1392 v = &v[2..]; 1393 match b { 1394 b'x' => { 1395 let (b, rest) = backslash_x(v); 1396 v = rest; 1397 b 1398 } 1399 b'n' => b'\n', 1400 b'r' => b'\r', 1401 b't' => b'\t', 1402 b'\\' => b'\\', 1403 b'0' => b'\0', 1404 b'\'' => b'\'', 1405 b'"' => b'"', 1406 b'\r' | b'\n' => loop { 1407 let byte = byte(v, 0); 1408 if matches!(byte, b' ' | b'\t' | b'\n' | b'\r') { 1409 v = &v[1..]; 1410 } else { 1411 continue 'outer; 1412 } 1413 }, 1414 b => panic!( 1415 "unexpected byte '{}' after \\ character in byte-string literal", 1416 ascii::escape_default(b), 1417 ), 1418 } 1419 } 1420 b'\r' => { 1421 assert_eq!(byte(v, 1), b'\n', "bare CR not allowed in string"); 1422 v = &v[2..]; 1423 b'\n' 1424 } 1425 b => { 1426 v = &v[1..]; 1427 b 1428 } 1429 }; 1430 out.push(byte); 1431 } 1432 1433 assert_eq!(byte(v, 0), b'"'); 1434 let suffix = s[s.len() - v.len() + 1..].to_owned().into_boxed_str(); 1435 (out, suffix) 1436 } 1437 1438 fn parse_lit_byte_str_raw(s: &str) -> (Vec<u8>, Box<str>) { 1439 assert_eq!(byte(s, 0), b'b'); 1440 let (value, suffix) = parse_lit_str_raw(&s[1..]); 1441 (String::from(value).into_bytes(), suffix) 1442 } 1443 1444 // Returns (content, suffix). 1445 pub(crate) fn parse_lit_c_str(s: &str) -> (CString, Box<str>) { 1446 assert_eq!(byte(s, 0), b'c'); 1447 match byte(s, 1) { 1448 b'"' => parse_lit_c_str_cooked(s), 1449 b'r' => parse_lit_c_str_raw(s), 1450 _ => unreachable!(), 1451 } 1452 } 1453 1454 fn parse_lit_c_str_cooked(mut s: &str) -> (CString, Box<str>) { 1455 assert_eq!(byte(s, 0), b'c'); 1456 assert_eq!(byte(s, 1), b'"'); 1457 s = &s[2..]; 1458 1459 // We're going to want to have slices which don't respect codepoint boundaries. 1460 let mut v = s.as_bytes(); 1461 1462 let mut out = Vec::new(); 1463 'outer: loop { 1464 let byte = match byte(v, 0) { 1465 b'"' => break, 1466 b'\\' => { 1467 let b = byte(v, 1); 1468 v = &v[2..]; 1469 match b { 1470 b'x' => { 1471 let (b, rest) = backslash_x(v); 1472 assert!(b != 0, "\\x00 is not allowed in C-string literal"); 1473 v = rest; 1474 b 1475 } 1476 b'u' => { 1477 let (ch, rest) = backslash_u(v); 1478 assert!(ch != '\0', "\\u{{0}} is not allowed in C-string literal"); 1479 v = rest; 1480 out.extend_from_slice(ch.encode_utf8(&mut [0u8; 4]).as_bytes()); 1481 continue 'outer; 1482 } 1483 b'n' => b'\n', 1484 b'r' => b'\r', 1485 b't' => b'\t', 1486 b'\\' => b'\\', 1487 b'\'' => b'\'', 1488 b'"' => b'"', 1489 b'\r' | b'\n' => loop { 1490 let byte = byte(v, 0); 1491 if matches!(byte, b' ' | b'\t' | b'\n' | b'\r') { 1492 v = &v[1..]; 1493 } else { 1494 continue 'outer; 1495 } 1496 }, 1497 b => panic!( 1498 "unexpected byte '{}' after \\ character in byte literal", 1499 ascii::escape_default(b), 1500 ), 1501 } 1502 } 1503 b'\r' => { 1504 assert_eq!(byte(v, 1), b'\n', "bare CR not allowed in string"); 1505 v = &v[2..]; 1506 b'\n' 1507 } 1508 b => { 1509 v = &v[1..]; 1510 b 1511 } 1512 }; 1513 out.push(byte); 1514 } 1515 1516 assert_eq!(byte(v, 0), b'"'); 1517 let suffix = s[s.len() - v.len() + 1..].to_owned().into_boxed_str(); 1518 (CString::new(out).unwrap(), suffix) 1519 } 1520 1521 fn parse_lit_c_str_raw(s: &str) -> (CString, Box<str>) { 1522 assert_eq!(byte(s, 0), b'c'); 1523 let (value, suffix) = parse_lit_str_raw(&s[1..]); 1524 (CString::new(String::from(value)).unwrap(), suffix) 1525 } 1526 1527 // Returns (value, suffix). 1528 pub(crate) fn parse_lit_byte(s: &str) -> (u8, Box<str>) { 1529 assert_eq!(byte(s, 0), b'b'); 1530 assert_eq!(byte(s, 1), b'\''); 1531 1532 // We're going to want to have slices which don't respect codepoint boundaries. 1533 let mut v = &s.as_bytes()[2..]; 1534 1535 let b = match byte(v, 0) { 1536 b'\\' => { 1537 let b = byte(v, 1); 1538 v = &v[2..]; 1539 match b { 1540 b'x' => { 1541 let (b, rest) = backslash_x(v); 1542 v = rest; 1543 b 1544 } 1545 b'n' => b'\n', 1546 b'r' => b'\r', 1547 b't' => b'\t', 1548 b'\\' => b'\\', 1549 b'0' => b'\0', 1550 b'\'' => b'\'', 1551 b'"' => b'"', 1552 b => panic!( 1553 "unexpected byte '{}' after \\ character in byte literal", 1554 ascii::escape_default(b), 1555 ), 1556 } 1557 } 1558 b => { 1559 v = &v[1..]; 1560 b 1561 } 1562 }; 1563 1564 assert_eq!(byte(v, 0), b'\''); 1565 let suffix = s[s.len() - v.len() + 1..].to_owned().into_boxed_str(); 1566 (b, suffix) 1567 } 1568 1569 // Returns (value, suffix). 1570 pub(crate) fn parse_lit_char(mut s: &str) -> (char, Box<str>) { 1571 assert_eq!(byte(s, 0), b'\''); 1572 s = &s[1..]; 1573 1574 let ch = match byte(s, 0) { 1575 b'\\' => { 1576 let b = byte(s, 1); 1577 s = &s[2..]; 1578 match b { 1579 b'x' => { 1580 let (byte, rest) = backslash_x(s); 1581 s = rest; 1582 assert!(byte <= 0x7F, "invalid \\x byte in character literal"); 1583 char::from_u32(u32::from(byte)).unwrap() 1584 } 1585 b'u' => { 1586 let (ch, rest) = backslash_u(s); 1587 s = rest; 1588 ch 1589 } 1590 b'n' => '\n', 1591 b'r' => '\r', 1592 b't' => '\t', 1593 b'\\' => '\\', 1594 b'0' => '\0', 1595 b'\'' => '\'', 1596 b'"' => '"', 1597 b => panic!( 1598 "unexpected byte '{}' after \\ character in character literal", 1599 ascii::escape_default(b), 1600 ), 1601 } 1602 } 1603 _ => { 1604 let ch = next_chr(s); 1605 s = &s[ch.len_utf8()..]; 1606 ch 1607 } 1608 }; 1609 assert_eq!(byte(s, 0), b'\''); 1610 let suffix = s[1..].to_owned().into_boxed_str(); 1611 (ch, suffix) 1612 } 1613 1614 fn backslash_x<S>(s: &S) -> (u8, &S) 1615 where 1616 S: Index<RangeFrom<usize>, Output = S> + AsRef<[u8]> + ?Sized, 1617 { 1618 let mut ch = 0; 1619 let b0 = byte(s, 0); 1620 let b1 = byte(s, 1); 1621 ch += 0x10 1622 * match b0 { 1623 b'0'..=b'9' => b0 - b'0', 1624 b'a'..=b'f' => 10 + (b0 - b'a'), 1625 b'A'..=b'F' => 10 + (b0 - b'A'), 1626 _ => panic!("unexpected non-hex character after \\x"), 1627 }; 1628 ch += match b1 { 1629 b'0'..=b'9' => b1 - b'0', 1630 b'a'..=b'f' => 10 + (b1 - b'a'), 1631 b'A'..=b'F' => 10 + (b1 - b'A'), 1632 _ => panic!("unexpected non-hex character after \\x"), 1633 }; 1634 (ch, &s[2..]) 1635 } 1636 1637 fn backslash_u<S>(mut s: &S) -> (char, &S) 1638 where 1639 S: Index<RangeFrom<usize>, Output = S> + AsRef<[u8]> + ?Sized, 1640 { 1641 if byte(s, 0) != b'{' { 1642 panic!("{}", "expected { after \\u"); 1643 } 1644 s = &s[1..]; 1645 1646 let mut ch = 0; 1647 let mut digits = 0; 1648 loop { 1649 let b = byte(s, 0); 1650 let digit = match b { 1651 b'0'..=b'9' => b - b'0', 1652 b'a'..=b'f' => 10 + b - b'a', 1653 b'A'..=b'F' => 10 + b - b'A', 1654 b'_' if digits > 0 => { 1655 s = &s[1..]; 1656 continue; 1657 } 1658 b'}' if digits == 0 => panic!("invalid empty unicode escape"), 1659 b'}' => break, 1660 _ => panic!("unexpected non-hex character after \\u"), 1661 }; 1662 if digits == 6 { 1663 panic!("overlong unicode escape (must have at most 6 hex digits)"); 1664 } 1665 ch *= 0x10; 1666 ch += u32::from(digit); 1667 digits += 1; 1668 s = &s[1..]; 1669 } 1670 assert!(byte(s, 0) == b'}'); 1671 s = &s[1..]; 1672 1673 if let Some(ch) = char::from_u32(ch) { 1674 (ch, s) 1675 } else { 1676 panic!("character code {:x} is not a valid unicode character", ch); 1677 } 1678 } 1679 1680 // Returns base 10 digits and suffix. 1681 pub(crate) fn parse_lit_int(mut s: &str) -> Option<(Box<str>, Box<str>)> { 1682 let negative = byte(s, 0) == b'-'; 1683 if negative { 1684 s = &s[1..]; 1685 } 1686 1687 let base = match (byte(s, 0), byte(s, 1)) { 1688 (b'0', b'x') => { 1689 s = &s[2..]; 1690 16 1691 } 1692 (b'0', b'o') => { 1693 s = &s[2..]; 1694 8 1695 } 1696 (b'0', b'b') => { 1697 s = &s[2..]; 1698 2 1699 } 1700 (b'0'..=b'9', _) => 10, 1701 _ => return None, 1702 }; 1703 1704 let mut value = BigInt::new(); 1705 let mut has_digit = false; 1706 'outer: loop { 1707 let b = byte(s, 0); 1708 let digit = match b { 1709 b'0'..=b'9' => b - b'0', 1710 b'a'..=b'f' if base > 10 => b - b'a' + 10, 1711 b'A'..=b'F' if base > 10 => b - b'A' + 10, 1712 b'_' => { 1713 s = &s[1..]; 1714 continue; 1715 } 1716 // If looking at a floating point literal, we don't want to 1717 // consider it an integer. 1718 b'.' if base == 10 => return None, 1719 b'e' | b'E' if base == 10 => { 1720 let mut has_exp = false; 1721 for (i, b) in s[1..].bytes().enumerate() { 1722 match b { 1723 b'_' => {} 1724 b'-' | b'+' => return None, 1725 b'0'..=b'9' => has_exp = true, 1726 _ => { 1727 let suffix = &s[1 + i..]; 1728 if has_exp && crate::ident::xid_ok(suffix) { 1729 return None; 1730 } else { 1731 break 'outer; 1732 } 1733 } 1734 } 1735 } 1736 if has_exp { 1737 return None; 1738 } else { 1739 break; 1740 } 1741 } 1742 _ => break, 1743 }; 1744 1745 if digit >= base { 1746 return None; 1747 } 1748 1749 has_digit = true; 1750 value *= base; 1751 value += digit; 1752 s = &s[1..]; 1753 } 1754 1755 if !has_digit { 1756 return None; 1757 } 1758 1759 let suffix = s; 1760 if suffix.is_empty() || crate::ident::xid_ok(suffix) { 1761 let mut repr = value.to_string(); 1762 if negative { 1763 repr.insert(0, '-'); 1764 } 1765 Some((repr.into_boxed_str(), suffix.to_owned().into_boxed_str())) 1766 } else { 1767 None 1768 } 1769 } 1770 1771 // Returns base 10 digits and suffix. 1772 pub(crate) fn parse_lit_float(input: &str) -> Option<(Box<str>, Box<str>)> { 1773 // Rust's floating point literals are very similar to the ones parsed by 1774 // the standard library, except that rust's literals can contain 1775 // ignorable underscores. Let's remove those underscores. 1776 1777 let mut bytes = input.to_owned().into_bytes(); 1778 1779 let start = (*bytes.first()? == b'-') as usize; 1780 match bytes.get(start)? { 1781 b'0'..=b'9' => {} 1782 _ => return None, 1783 } 1784 1785 let mut read = start; 1786 let mut write = start; 1787 let mut has_dot = false; 1788 let mut has_e = false; 1789 let mut has_sign = false; 1790 let mut has_exponent = false; 1791 while read < bytes.len() { 1792 match bytes[read] { 1793 b'_' => { 1794 // Don't increase write 1795 read += 1; 1796 continue; 1797 } 1798 b'0'..=b'9' => { 1799 if has_e { 1800 has_exponent = true; 1801 } 1802 bytes[write] = bytes[read]; 1803 } 1804 b'.' => { 1805 if has_e || has_dot { 1806 return None; 1807 } 1808 has_dot = true; 1809 bytes[write] = b'.'; 1810 } 1811 b'e' | b'E' => { 1812 match bytes[read + 1..] 1813 .iter() 1814 .find(|b| **b != b'_') 1815 .unwrap_or(&b'\0') 1816 { 1817 b'-' | b'+' | b'0'..=b'9' => {} 1818 _ => break, 1819 } 1820 if has_e { 1821 if has_exponent { 1822 break; 1823 } else { 1824 return None; 1825 } 1826 } 1827 has_e = true; 1828 bytes[write] = b'e'; 1829 } 1830 b'-' | b'+' => { 1831 if has_sign || has_exponent || !has_e { 1832 return None; 1833 } 1834 has_sign = true; 1835 if bytes[read] == b'-' { 1836 bytes[write] = bytes[read]; 1837 } else { 1838 // Omit '+' 1839 read += 1; 1840 continue; 1841 } 1842 } 1843 _ => break, 1844 } 1845 read += 1; 1846 write += 1; 1847 } 1848 1849 if has_e && !has_exponent { 1850 return None; 1851 } 1852 1853 let mut digits = String::from_utf8(bytes).unwrap(); 1854 let suffix = digits.split_off(read); 1855 digits.truncate(write); 1856 if suffix.is_empty() || crate::ident::xid_ok(&suffix) { 1857 Some((digits.into_boxed_str(), suffix.into_boxed_str())) 1858 } else { 1859 None 1860 } 1861 } 1862 } 1863