1 //===--- LiteralSupport.cpp - Code to parse and process literals ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the NumericLiteralParser, CharLiteralParser, and 10 // StringLiteralParser interfaces. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "clang/Lex/LiteralSupport.h" 15 #include "clang/Basic/CharInfo.h" 16 #include "clang/Basic/LangOptions.h" 17 #include "clang/Basic/SourceLocation.h" 18 #include "clang/Basic/TargetInfo.h" 19 #include "clang/Lex/LexDiagnostic.h" 20 #include "clang/Lex/Lexer.h" 21 #include "clang/Lex/Preprocessor.h" 22 #include "clang/Lex/Token.h" 23 #include "llvm/ADT/APInt.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/ADT/StringSwitch.h" 27 #include "llvm/Support/ConvertUTF.h" 28 #include "llvm/Support/Error.h" 29 #include "llvm/Support/ErrorHandling.h" 30 #include "llvm/Support/Unicode.h" 31 #include <algorithm> 32 #include <cassert> 33 #include <cstddef> 34 #include <cstdint> 35 #include <cstring> 36 #include <string> 37 38 using namespace clang; 39 40 static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) { 41 switch (kind) { 42 default: llvm_unreachable("Unknown token type!"); 43 case tok::char_constant: 44 case tok::string_literal: 45 case tok::utf8_char_constant: 46 case tok::utf8_string_literal: 47 return Target.getCharWidth(); 48 case tok::wide_char_constant: 49 case tok::wide_string_literal: 50 return Target.getWCharWidth(); 51 case tok::utf16_char_constant: 52 case tok::utf16_string_literal: 53 return Target.getChar16Width(); 54 case tok::utf32_char_constant: 55 case tok::utf32_string_literal: 56 return Target.getChar32Width(); 57 } 58 } 59 60 static CharSourceRange MakeCharSourceRange(const LangOptions &Features, 61 FullSourceLoc TokLoc, 62 const char *TokBegin, 63 const char *TokRangeBegin, 64 const char *TokRangeEnd) { 65 SourceLocation Begin = 66 Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin, 67 TokLoc.getManager(), Features); 68 SourceLocation End = 69 Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin, 70 TokLoc.getManager(), Features); 71 return CharSourceRange::getCharRange(Begin, End); 72 } 73 74 /// Produce a diagnostic highlighting some portion of a literal. 75 /// 76 /// Emits the diagnostic \p DiagID, highlighting the range of characters from 77 /// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be 78 /// a substring of a spelling buffer for the token beginning at \p TokBegin. 79 static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, 80 const LangOptions &Features, FullSourceLoc TokLoc, 81 const char *TokBegin, const char *TokRangeBegin, 82 const char *TokRangeEnd, unsigned DiagID) { 83 SourceLocation Begin = 84 Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin, 85 TokLoc.getManager(), Features); 86 return Diags->Report(Begin, DiagID) << 87 MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd); 88 } 89 90 /// ProcessCharEscape - Parse a standard C escape sequence, which can occur in 91 /// either a character or a string literal. 92 static unsigned ProcessCharEscape(const char *ThisTokBegin, 93 const char *&ThisTokBuf, 94 const char *ThisTokEnd, bool &HadError, 95 FullSourceLoc Loc, unsigned CharWidth, 96 DiagnosticsEngine *Diags, 97 const LangOptions &Features) { 98 const char *EscapeBegin = ThisTokBuf; 99 bool Delimited = false; 100 bool EndDelimiterFound = false; 101 102 // Skip the '\' char. 103 ++ThisTokBuf; 104 105 // We know that this character can't be off the end of the buffer, because 106 // that would have been \", which would not have been the end of string. 107 unsigned ResultChar = *ThisTokBuf++; 108 switch (ResultChar) { 109 // These map to themselves. 110 case '\\': case '\'': case '"': case '?': break; 111 112 // These have fixed mappings. 113 case 'a': 114 // TODO: K&R: the meaning of '\\a' is different in traditional C 115 ResultChar = 7; 116 break; 117 case 'b': 118 ResultChar = 8; 119 break; 120 case 'e': 121 if (Diags) 122 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 123 diag::ext_nonstandard_escape) << "e"; 124 ResultChar = 27; 125 break; 126 case 'E': 127 if (Diags) 128 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 129 diag::ext_nonstandard_escape) << "E"; 130 ResultChar = 27; 131 break; 132 case 'f': 133 ResultChar = 12; 134 break; 135 case 'n': 136 ResultChar = 10; 137 break; 138 case 'r': 139 ResultChar = 13; 140 break; 141 case 't': 142 ResultChar = 9; 143 break; 144 case 'v': 145 ResultChar = 11; 146 break; 147 case 'x': { // Hex escape. 148 ResultChar = 0; 149 if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') { 150 Delimited = true; 151 ThisTokBuf++; 152 if (*ThisTokBuf == '}') { 153 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 154 diag::err_delimited_escape_empty); 155 return ResultChar; 156 } 157 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) { 158 if (Diags) 159 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 160 diag::err_hex_escape_no_digits) << "x"; 161 return ResultChar; 162 } 163 164 // Hex escapes are a maximal series of hex digits. 165 bool Overflow = false; 166 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) { 167 if (Delimited && *ThisTokBuf == '}') { 168 ThisTokBuf++; 169 EndDelimiterFound = true; 170 break; 171 } 172 int CharVal = llvm::hexDigitValue(*ThisTokBuf); 173 if (CharVal == -1) { 174 // Non delimited hex escape sequences stop at the first non-hex digit. 175 if (!Delimited) 176 break; 177 HadError = true; 178 if (Diags) 179 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 180 diag::err_delimited_escape_invalid) 181 << StringRef(ThisTokBuf, 1); 182 continue; 183 } 184 // About to shift out a digit? 185 if (ResultChar & 0xF0000000) 186 Overflow = true; 187 ResultChar <<= 4; 188 ResultChar |= CharVal; 189 } 190 // See if any bits will be truncated when evaluated as a character. 191 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 192 Overflow = true; 193 ResultChar &= ~0U >> (32-CharWidth); 194 } 195 196 // Check for overflow. 197 if (!HadError && Overflow) { // Too many digits to fit in 198 HadError = true; 199 if (Diags) 200 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 201 diag::err_escape_too_large) 202 << 0; 203 } 204 break; 205 } 206 case '0': case '1': case '2': case '3': 207 case '4': case '5': case '6': case '7': { 208 // Octal escapes. 209 --ThisTokBuf; 210 ResultChar = 0; 211 212 // Octal escapes are a series of octal digits with maximum length 3. 213 // "\0123" is a two digit sequence equal to "\012" "3". 214 unsigned NumDigits = 0; 215 do { 216 ResultChar <<= 3; 217 ResultChar |= *ThisTokBuf++ - '0'; 218 ++NumDigits; 219 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 && 220 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7'); 221 222 // Check for overflow. Reject '\777', but not L'\777'. 223 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 224 if (Diags) 225 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 226 diag::err_escape_too_large) << 1; 227 ResultChar &= ~0U >> (32-CharWidth); 228 } 229 break; 230 } 231 case 'o': { 232 bool Overflow = false; 233 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') { 234 HadError = true; 235 if (Diags) 236 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 237 diag::err_delimited_escape_missing_brace) 238 << "o"; 239 240 break; 241 } 242 ResultChar = 0; 243 Delimited = true; 244 ++ThisTokBuf; 245 if (*ThisTokBuf == '}') { 246 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 247 diag::err_delimited_escape_empty); 248 return ResultChar; 249 } 250 251 while (ThisTokBuf != ThisTokEnd) { 252 if (*ThisTokBuf == '}') { 253 EndDelimiterFound = true; 254 ThisTokBuf++; 255 break; 256 } 257 if (*ThisTokBuf < '0' || *ThisTokBuf > '7') { 258 HadError = true; 259 if (Diags) 260 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 261 diag::err_delimited_escape_invalid) 262 << StringRef(ThisTokBuf, 1); 263 ThisTokBuf++; 264 continue; 265 } 266 if (ResultChar & 0x020000000) 267 Overflow = true; 268 269 ResultChar <<= 3; 270 ResultChar |= *ThisTokBuf++ - '0'; 271 } 272 // Check for overflow. Reject '\777', but not L'\777'. 273 if (!HadError && 274 (Overflow || (CharWidth != 32 && (ResultChar >> CharWidth) != 0))) { 275 HadError = true; 276 if (Diags) 277 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 278 diag::err_escape_too_large) 279 << 1; 280 ResultChar &= ~0U >> (32 - CharWidth); 281 } 282 break; 283 } 284 // Otherwise, these are not valid escapes. 285 case '(': case '{': case '[': case '%': 286 // GCC accepts these as extensions. We warn about them as such though. 287 if (Diags) 288 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 289 diag::ext_nonstandard_escape) 290 << std::string(1, ResultChar); 291 break; 292 default: 293 if (!Diags) 294 break; 295 296 if (isPrintable(ResultChar)) 297 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 298 diag::ext_unknown_escape) 299 << std::string(1, ResultChar); 300 else 301 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 302 diag::ext_unknown_escape) 303 << "x" + llvm::utohexstr(ResultChar); 304 break; 305 } 306 307 if (Delimited && Diags) { 308 if (!EndDelimiterFound) 309 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 310 diag::err_expected) 311 << tok::r_brace; 312 else if (!HadError) { 313 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 314 Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence 315 : diag::ext_delimited_escape_sequence) 316 << /*delimited*/ 0 << (Features.CPlusPlus ? 1 : 0); 317 } 318 } 319 320 return ResultChar; 321 } 322 323 static void appendCodePoint(unsigned Codepoint, 324 llvm::SmallVectorImpl<char> &Str) { 325 char ResultBuf[4]; 326 char *ResultPtr = ResultBuf; 327 if (llvm::ConvertCodePointToUTF8(Codepoint, ResultPtr)) 328 Str.append(ResultBuf, ResultPtr); 329 } 330 331 void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) { 332 for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) { 333 if (*I != '\\') { 334 Buf.push_back(*I); 335 continue; 336 } 337 338 ++I; 339 char Kind = *I; 340 ++I; 341 342 assert(Kind == 'u' || Kind == 'U' || Kind == 'N'); 343 uint32_t CodePoint = 0; 344 345 if (Kind == 'u' && *I == '{') { 346 for (++I; *I != '}'; ++I) { 347 unsigned Value = llvm::hexDigitValue(*I); 348 assert(Value != -1U); 349 CodePoint <<= 4; 350 CodePoint += Value; 351 } 352 appendCodePoint(CodePoint, Buf); 353 continue; 354 } 355 356 if (Kind == 'N') { 357 assert(*I == '{'); 358 ++I; 359 auto Delim = std::find(I, Input.end(), '}'); 360 assert(Delim != Input.end()); 361 std::optional<llvm::sys::unicode::LooseMatchingResult> Res = 362 llvm::sys::unicode::nameToCodepointLooseMatching( 363 StringRef(I, std::distance(I, Delim))); 364 assert(Res); 365 CodePoint = Res->CodePoint; 366 assert(CodePoint != 0xFFFFFFFF); 367 appendCodePoint(CodePoint, Buf); 368 I = Delim; 369 continue; 370 } 371 372 unsigned NumHexDigits; 373 if (Kind == 'u') 374 NumHexDigits = 4; 375 else 376 NumHexDigits = 8; 377 378 assert(I + NumHexDigits <= E); 379 380 for (; NumHexDigits != 0; ++I, --NumHexDigits) { 381 unsigned Value = llvm::hexDigitValue(*I); 382 assert(Value != -1U); 383 384 CodePoint <<= 4; 385 CodePoint += Value; 386 } 387 388 appendCodePoint(CodePoint, Buf); 389 --I; 390 } 391 } 392 393 static bool ProcessNumericUCNEscape(const char *ThisTokBegin, 394 const char *&ThisTokBuf, 395 const char *ThisTokEnd, uint32_t &UcnVal, 396 unsigned short &UcnLen, bool &Delimited, 397 FullSourceLoc Loc, DiagnosticsEngine *Diags, 398 const LangOptions &Features, 399 bool in_char_string_literal = false) { 400 const char *UcnBegin = ThisTokBuf; 401 bool HasError = false; 402 bool EndDelimiterFound = false; 403 404 // Skip the '\u' char's. 405 ThisTokBuf += 2; 406 Delimited = false; 407 if (UcnBegin[1] == 'u' && in_char_string_literal && 408 ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') { 409 Delimited = true; 410 ThisTokBuf++; 411 } else if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) { 412 if (Diags) 413 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 414 diag::err_hex_escape_no_digits) 415 << StringRef(&ThisTokBuf[-1], 1); 416 return false; 417 } 418 UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8); 419 420 bool Overflow = false; 421 unsigned short Count = 0; 422 for (; ThisTokBuf != ThisTokEnd && (Delimited || Count != UcnLen); 423 ++ThisTokBuf) { 424 if (Delimited && *ThisTokBuf == '}') { 425 ++ThisTokBuf; 426 EndDelimiterFound = true; 427 break; 428 } 429 int CharVal = llvm::hexDigitValue(*ThisTokBuf); 430 if (CharVal == -1) { 431 HasError = true; 432 if (!Delimited) 433 break; 434 if (Diags) { 435 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 436 diag::err_delimited_escape_invalid) 437 << StringRef(ThisTokBuf, 1); 438 } 439 Count++; 440 continue; 441 } 442 if (UcnVal & 0xF0000000) { 443 Overflow = true; 444 continue; 445 } 446 UcnVal <<= 4; 447 UcnVal |= CharVal; 448 Count++; 449 } 450 451 if (Overflow) { 452 if (Diags) 453 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 454 diag::err_escape_too_large) 455 << 0; 456 return false; 457 } 458 459 if (Delimited && !EndDelimiterFound) { 460 if (Diags) { 461 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 462 diag::err_expected) 463 << tok::r_brace; 464 } 465 return false; 466 } 467 468 // If we didn't consume the proper number of digits, there is a problem. 469 if (Count == 0 || (!Delimited && Count != UcnLen)) { 470 if (Diags) 471 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 472 Delimited ? diag::err_delimited_escape_empty 473 : diag::err_ucn_escape_incomplete); 474 return false; 475 } 476 return !HasError; 477 } 478 479 static void DiagnoseInvalidUnicodeCharacterName( 480 DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc Loc, 481 const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, 482 llvm::StringRef Name) { 483 484 Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd, 485 diag::err_invalid_ucn_name) 486 << Name; 487 488 namespace u = llvm::sys::unicode; 489 490 std::optional<u::LooseMatchingResult> Res = 491 u::nameToCodepointLooseMatching(Name); 492 if (Res) { 493 Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd, 494 diag::note_invalid_ucn_name_loose_matching) 495 << FixItHint::CreateReplacement( 496 MakeCharSourceRange(Features, Loc, TokBegin, TokRangeBegin, 497 TokRangeEnd), 498 Res->Name); 499 return; 500 } 501 502 unsigned Distance = 0; 503 SmallVector<u::MatchForCodepointName> Matches = 504 u::nearestMatchesForCodepointName(Name, 5); 505 assert(!Matches.empty() && "No unicode characters found"); 506 507 for (const auto &Match : Matches) { 508 if (Distance == 0) 509 Distance = Match.Distance; 510 if (std::max(Distance, Match.Distance) - 511 std::min(Distance, Match.Distance) > 512 3) 513 break; 514 Distance = Match.Distance; 515 516 std::string Str; 517 llvm::UTF32 V = Match.Value; 518 bool Converted = 519 llvm::convertUTF32ToUTF8String(llvm::ArrayRef<llvm::UTF32>(&V, 1), Str); 520 (void)Converted; 521 assert(Converted && "Found a match wich is not a unicode character"); 522 523 Diag(Diags, Features, Loc, TokBegin, TokRangeBegin, TokRangeEnd, 524 diag::note_invalid_ucn_name_candidate) 525 << Match.Name << llvm::utohexstr(Match.Value) 526 << Str // FIXME: Fix the rendering of non printable characters 527 << FixItHint::CreateReplacement( 528 MakeCharSourceRange(Features, Loc, TokBegin, TokRangeBegin, 529 TokRangeEnd), 530 Match.Name); 531 } 532 } 533 534 static bool ProcessNamedUCNEscape(const char *ThisTokBegin, 535 const char *&ThisTokBuf, 536 const char *ThisTokEnd, uint32_t &UcnVal, 537 unsigned short &UcnLen, FullSourceLoc Loc, 538 DiagnosticsEngine *Diags, 539 const LangOptions &Features) { 540 const char *UcnBegin = ThisTokBuf; 541 assert(UcnBegin[0] == '\\' && UcnBegin[1] == 'N'); 542 ThisTokBuf += 2; 543 if (ThisTokBuf == ThisTokEnd || *ThisTokBuf != '{') { 544 if (Diags) { 545 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 546 diag::err_delimited_escape_missing_brace) 547 << StringRef(&ThisTokBuf[-1], 1); 548 } 549 return false; 550 } 551 ThisTokBuf++; 552 const char *ClosingBrace = std::find_if(ThisTokBuf, ThisTokEnd, [](char C) { 553 return C == '}' || isVerticalWhitespace(C); 554 }); 555 bool Incomplete = ClosingBrace == ThisTokEnd; 556 bool Empty = ClosingBrace == ThisTokBuf; 557 if (Incomplete || Empty) { 558 if (Diags) { 559 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 560 Incomplete ? diag::err_ucn_escape_incomplete 561 : diag::err_delimited_escape_empty) 562 << StringRef(&UcnBegin[1], 1); 563 } 564 ThisTokBuf = ClosingBrace == ThisTokEnd ? ClosingBrace : ClosingBrace + 1; 565 return false; 566 } 567 StringRef Name(ThisTokBuf, ClosingBrace - ThisTokBuf); 568 ThisTokBuf = ClosingBrace + 1; 569 std::optional<char32_t> Res = llvm::sys::unicode::nameToCodepointStrict(Name); 570 if (!Res) { 571 if (Diags) 572 DiagnoseInvalidUnicodeCharacterName(Diags, Features, Loc, ThisTokBegin, 573 &UcnBegin[3], ClosingBrace, Name); 574 return false; 575 } 576 UcnVal = *Res; 577 UcnLen = UcnVal > 0xFFFF ? 8 : 4; 578 return true; 579 } 580 581 /// ProcessUCNEscape - Read the Universal Character Name, check constraints and 582 /// return the UTF32. 583 static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 584 const char *ThisTokEnd, uint32_t &UcnVal, 585 unsigned short &UcnLen, FullSourceLoc Loc, 586 DiagnosticsEngine *Diags, 587 const LangOptions &Features, 588 bool in_char_string_literal = false) { 589 590 bool HasError; 591 const char *UcnBegin = ThisTokBuf; 592 bool IsDelimitedEscapeSequence = false; 593 bool IsNamedEscapeSequence = false; 594 if (ThisTokBuf[1] == 'N') { 595 IsNamedEscapeSequence = true; 596 HasError = !ProcessNamedUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, 597 UcnVal, UcnLen, Loc, Diags, Features); 598 } else { 599 HasError = 600 !ProcessNumericUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, 601 UcnLen, IsDelimitedEscapeSequence, Loc, Diags, 602 Features, in_char_string_literal); 603 } 604 if (HasError) 605 return false; 606 607 // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2] 608 if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints 609 UcnVal > 0x10FFFF) { // maximum legal UTF32 value 610 if (Diags) 611 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 612 diag::err_ucn_escape_invalid); 613 return false; 614 } 615 616 // C++11 allows UCNs that refer to control characters and basic source 617 // characters inside character and string literals 618 if (UcnVal < 0xa0 && 619 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) { // $, @, ` 620 bool IsError = (!Features.CPlusPlus11 || !in_char_string_literal); 621 if (Diags) { 622 char BasicSCSChar = UcnVal; 623 if (UcnVal >= 0x20 && UcnVal < 0x7f) 624 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 625 IsError ? diag::err_ucn_escape_basic_scs : 626 diag::warn_cxx98_compat_literal_ucn_escape_basic_scs) 627 << StringRef(&BasicSCSChar, 1); 628 else 629 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 630 IsError ? diag::err_ucn_control_character : 631 diag::warn_cxx98_compat_literal_ucn_control_character); 632 } 633 if (IsError) 634 return false; 635 } 636 637 if (!Features.CPlusPlus && !Features.C99 && Diags) 638 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 639 diag::warn_ucn_not_valid_in_c89_literal); 640 641 if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags) 642 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 643 Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence 644 : diag::ext_delimited_escape_sequence) 645 << (IsNamedEscapeSequence ? 1 : 0) << (Features.CPlusPlus ? 1 : 0); 646 647 return true; 648 } 649 650 /// MeasureUCNEscape - Determine the number of bytes within the resulting string 651 /// which this UCN will occupy. 652 static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 653 const char *ThisTokEnd, unsigned CharByteWidth, 654 const LangOptions &Features, bool &HadError) { 655 // UTF-32: 4 bytes per escape. 656 if (CharByteWidth == 4) 657 return 4; 658 659 uint32_t UcnVal = 0; 660 unsigned short UcnLen = 0; 661 FullSourceLoc Loc; 662 663 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, 664 UcnLen, Loc, nullptr, Features, true)) { 665 HadError = true; 666 return 0; 667 } 668 669 // UTF-16: 2 bytes for BMP, 4 bytes otherwise. 670 if (CharByteWidth == 2) 671 return UcnVal <= 0xFFFF ? 2 : 4; 672 673 // UTF-8. 674 if (UcnVal < 0x80) 675 return 1; 676 if (UcnVal < 0x800) 677 return 2; 678 if (UcnVal < 0x10000) 679 return 3; 680 return 4; 681 } 682 683 /// EncodeUCNEscape - Read the Universal Character Name, check constraints and 684 /// convert the UTF32 to UTF8 or UTF16. This is a subroutine of 685 /// StringLiteralParser. When we decide to implement UCN's for identifiers, 686 /// we will likely rework our support for UCN's. 687 static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 688 const char *ThisTokEnd, 689 char *&ResultBuf, bool &HadError, 690 FullSourceLoc Loc, unsigned CharByteWidth, 691 DiagnosticsEngine *Diags, 692 const LangOptions &Features) { 693 typedef uint32_t UTF32; 694 UTF32 UcnVal = 0; 695 unsigned short UcnLen = 0; 696 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen, 697 Loc, Diags, Features, true)) { 698 HadError = true; 699 return; 700 } 701 702 assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) && 703 "only character widths of 1, 2, or 4 bytes supported"); 704 705 (void)UcnLen; 706 assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported"); 707 708 if (CharByteWidth == 4) { 709 // FIXME: Make the type of the result buffer correct instead of 710 // using reinterpret_cast. 711 llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf); 712 *ResultPtr = UcnVal; 713 ResultBuf += 4; 714 return; 715 } 716 717 if (CharByteWidth == 2) { 718 // FIXME: Make the type of the result buffer correct instead of 719 // using reinterpret_cast. 720 llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf); 721 722 if (UcnVal <= (UTF32)0xFFFF) { 723 *ResultPtr = UcnVal; 724 ResultBuf += 2; 725 return; 726 } 727 728 // Convert to UTF16. 729 UcnVal -= 0x10000; 730 *ResultPtr = 0xD800 + (UcnVal >> 10); 731 *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF); 732 ResultBuf += 4; 733 return; 734 } 735 736 assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters"); 737 738 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8. 739 // The conversion below was inspired by: 740 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c 741 // First, we determine how many bytes the result will require. 742 typedef uint8_t UTF8; 743 744 unsigned short bytesToWrite = 0; 745 if (UcnVal < (UTF32)0x80) 746 bytesToWrite = 1; 747 else if (UcnVal < (UTF32)0x800) 748 bytesToWrite = 2; 749 else if (UcnVal < (UTF32)0x10000) 750 bytesToWrite = 3; 751 else 752 bytesToWrite = 4; 753 754 const unsigned byteMask = 0xBF; 755 const unsigned byteMark = 0x80; 756 757 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed 758 // into the first byte, depending on how many bytes follow. 759 static const UTF8 firstByteMark[5] = { 760 0x00, 0x00, 0xC0, 0xE0, 0xF0 761 }; 762 // Finally, we write the bytes into ResultBuf. 763 ResultBuf += bytesToWrite; 764 switch (bytesToWrite) { // note: everything falls through. 765 case 4: 766 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 767 [[fallthrough]]; 768 case 3: 769 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 770 [[fallthrough]]; 771 case 2: 772 *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 773 [[fallthrough]]; 774 case 1: 775 *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]); 776 } 777 // Update the buffer. 778 ResultBuf += bytesToWrite; 779 } 780 781 /// integer-constant: [C99 6.4.4.1] 782 /// decimal-constant integer-suffix 783 /// octal-constant integer-suffix 784 /// hexadecimal-constant integer-suffix 785 /// binary-literal integer-suffix [GNU, C++1y] 786 /// user-defined-integer-literal: [C++11 lex.ext] 787 /// decimal-literal ud-suffix 788 /// octal-literal ud-suffix 789 /// hexadecimal-literal ud-suffix 790 /// binary-literal ud-suffix [GNU, C++1y] 791 /// decimal-constant: 792 /// nonzero-digit 793 /// decimal-constant digit 794 /// octal-constant: 795 /// 0 796 /// octal-constant octal-digit 797 /// hexadecimal-constant: 798 /// hexadecimal-prefix hexadecimal-digit 799 /// hexadecimal-constant hexadecimal-digit 800 /// hexadecimal-prefix: one of 801 /// 0x 0X 802 /// binary-literal: 803 /// 0b binary-digit 804 /// 0B binary-digit 805 /// binary-literal binary-digit 806 /// integer-suffix: 807 /// unsigned-suffix [long-suffix] 808 /// unsigned-suffix [long-long-suffix] 809 /// long-suffix [unsigned-suffix] 810 /// long-long-suffix [unsigned-sufix] 811 /// nonzero-digit: 812 /// 1 2 3 4 5 6 7 8 9 813 /// octal-digit: 814 /// 0 1 2 3 4 5 6 7 815 /// hexadecimal-digit: 816 /// 0 1 2 3 4 5 6 7 8 9 817 /// a b c d e f 818 /// A B C D E F 819 /// binary-digit: 820 /// 0 821 /// 1 822 /// unsigned-suffix: one of 823 /// u U 824 /// long-suffix: one of 825 /// l L 826 /// long-long-suffix: one of 827 /// ll LL 828 /// 829 /// floating-constant: [C99 6.4.4.2] 830 /// TODO: add rules... 831 /// 832 NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, 833 SourceLocation TokLoc, 834 const SourceManager &SM, 835 const LangOptions &LangOpts, 836 const TargetInfo &Target, 837 DiagnosticsEngine &Diags) 838 : SM(SM), LangOpts(LangOpts), Diags(Diags), 839 ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) { 840 841 s = DigitsBegin = ThisTokBegin; 842 saw_exponent = false; 843 saw_period = false; 844 saw_ud_suffix = false; 845 saw_fixed_point_suffix = false; 846 isLong = false; 847 isUnsigned = false; 848 isLongLong = false; 849 isSizeT = false; 850 isHalf = false; 851 isFloat = false; 852 isImaginary = false; 853 isFloat16 = false; 854 isFloat128 = false; 855 MicrosoftInteger = 0; 856 isFract = false; 857 isAccum = false; 858 hadError = false; 859 isBitInt = false; 860 861 // This routine assumes that the range begin/end matches the regex for integer 862 // and FP constants (specifically, the 'pp-number' regex), and assumes that 863 // the byte at "*end" is both valid and not part of the regex. Because of 864 // this, it doesn't have to check for 'overscan' in various places. 865 if (isPreprocessingNumberBody(*ThisTokEnd)) { 866 Diags.Report(TokLoc, diag::err_lexing_numeric); 867 hadError = true; 868 return; 869 } 870 871 if (*s == '0') { // parse radix 872 ParseNumberStartingWithZero(TokLoc); 873 if (hadError) 874 return; 875 } else { // the first digit is non-zero 876 radix = 10; 877 s = SkipDigits(s); 878 if (s == ThisTokEnd) { 879 // Done. 880 } else { 881 ParseDecimalOrOctalCommon(TokLoc); 882 if (hadError) 883 return; 884 } 885 } 886 887 SuffixBegin = s; 888 checkSeparator(TokLoc, s, CSK_AfterDigits); 889 890 // Initial scan to lookahead for fixed point suffix. 891 if (LangOpts.FixedPoint) { 892 for (const char *c = s; c != ThisTokEnd; ++c) { 893 if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') { 894 saw_fixed_point_suffix = true; 895 break; 896 } 897 } 898 } 899 900 // Parse the suffix. At this point we can classify whether we have an FP or 901 // integer constant. 902 bool isFixedPointConstant = isFixedPointLiteral(); 903 bool isFPConstant = isFloatingLiteral(); 904 bool HasSize = false; 905 906 // Loop over all of the characters of the suffix. If we see something bad, 907 // we break out of the loop. 908 for (; s != ThisTokEnd; ++s) { 909 switch (*s) { 910 case 'R': 911 case 'r': 912 if (!LangOpts.FixedPoint) 913 break; 914 if (isFract || isAccum) break; 915 if (!(saw_period || saw_exponent)) break; 916 isFract = true; 917 continue; 918 case 'K': 919 case 'k': 920 if (!LangOpts.FixedPoint) 921 break; 922 if (isFract || isAccum) break; 923 if (!(saw_period || saw_exponent)) break; 924 isAccum = true; 925 continue; 926 case 'h': // FP Suffix for "half". 927 case 'H': 928 // OpenCL Extension v1.2 s9.5 - h or H suffix for half type. 929 if (!(LangOpts.Half || LangOpts.FixedPoint)) 930 break; 931 if (isIntegerLiteral()) break; // Error for integer constant. 932 if (HasSize) 933 break; 934 HasSize = true; 935 isHalf = true; 936 continue; // Success. 937 case 'f': // FP Suffix for "float" 938 case 'F': 939 if (!isFPConstant) break; // Error for integer constant. 940 if (HasSize) 941 break; 942 HasSize = true; 943 944 // CUDA host and device may have different _Float16 support, therefore 945 // allows f16 literals to avoid false alarm. 946 // When we compile for OpenMP target offloading on NVPTX, f16 suffix 947 // should also be supported. 948 // ToDo: more precise check for CUDA. 949 // TODO: AMDGPU might also support it in the future. 950 if ((Target.hasFloat16Type() || LangOpts.CUDA || 951 (LangOpts.OpenMPIsDevice && Target.getTriple().isNVPTX())) && 952 s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') { 953 s += 2; // success, eat up 2 characters. 954 isFloat16 = true; 955 continue; 956 } 957 958 isFloat = true; 959 continue; // Success. 960 case 'q': // FP Suffix for "__float128" 961 case 'Q': 962 if (!isFPConstant) break; // Error for integer constant. 963 if (HasSize) 964 break; 965 HasSize = true; 966 isFloat128 = true; 967 continue; // Success. 968 case 'u': 969 case 'U': 970 if (isFPConstant) break; // Error for floating constant. 971 if (isUnsigned) break; // Cannot be repeated. 972 isUnsigned = true; 973 continue; // Success. 974 case 'l': 975 case 'L': 976 if (HasSize) 977 break; 978 HasSize = true; 979 980 // Check for long long. The L's need to be adjacent and the same case. 981 if (s[1] == s[0]) { 982 assert(s + 1 < ThisTokEnd && "didn't maximally munch?"); 983 if (isFPConstant) break; // long long invalid for floats. 984 isLongLong = true; 985 ++s; // Eat both of them. 986 } else { 987 isLong = true; 988 } 989 continue; // Success. 990 case 'z': 991 case 'Z': 992 if (isFPConstant) 993 break; // Invalid for floats. 994 if (HasSize) 995 break; 996 HasSize = true; 997 isSizeT = true; 998 continue; 999 case 'i': 1000 case 'I': 1001 if (LangOpts.MicrosoftExt && !isFPConstant) { 1002 // Allow i8, i16, i32, and i64. First, look ahead and check if 1003 // suffixes are Microsoft integers and not the imaginary unit. 1004 uint8_t Bits = 0; 1005 size_t ToSkip = 0; 1006 switch (s[1]) { 1007 case '8': // i8 suffix 1008 Bits = 8; 1009 ToSkip = 2; 1010 break; 1011 case '1': 1012 if (s[2] == '6') { // i16 suffix 1013 Bits = 16; 1014 ToSkip = 3; 1015 } 1016 break; 1017 case '3': 1018 if (s[2] == '2') { // i32 suffix 1019 Bits = 32; 1020 ToSkip = 3; 1021 } 1022 break; 1023 case '6': 1024 if (s[2] == '4') { // i64 suffix 1025 Bits = 64; 1026 ToSkip = 3; 1027 } 1028 break; 1029 default: 1030 break; 1031 } 1032 if (Bits) { 1033 if (HasSize) 1034 break; 1035 HasSize = true; 1036 MicrosoftInteger = Bits; 1037 s += ToSkip; 1038 assert(s <= ThisTokEnd && "didn't maximally munch?"); 1039 break; 1040 } 1041 } 1042 [[fallthrough]]; 1043 case 'j': 1044 case 'J': 1045 if (isImaginary) break; // Cannot be repeated. 1046 isImaginary = true; 1047 continue; // Success. 1048 case 'w': 1049 case 'W': 1050 if (isFPConstant) 1051 break; // Invalid for floats. 1052 if (HasSize) 1053 break; // Invalid if we already have a size for the literal. 1054 1055 // wb and WB are allowed, but a mixture of cases like Wb or wB is not. We 1056 // explicitly do not support the suffix in C++ as an extension because a 1057 // library-based UDL that resolves to a library type may be more 1058 // appropriate there. 1059 if (!LangOpts.CPlusPlus && ((s[0] == 'w' && s[1] == 'b') || 1060 (s[0] == 'W' && s[1] == 'B'))) { 1061 isBitInt = true; 1062 HasSize = true; 1063 ++s; // Skip both characters (2nd char skipped on continue). 1064 continue; // Success. 1065 } 1066 } 1067 // If we reached here, there was an error or a ud-suffix. 1068 break; 1069 } 1070 1071 // "i", "if", and "il" are user-defined suffixes in C++1y. 1072 if (s != ThisTokEnd || isImaginary) { 1073 // FIXME: Don't bother expanding UCNs if !tok.hasUCN(). 1074 expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)); 1075 if (isValidUDSuffix(LangOpts, UDSuffixBuf)) { 1076 if (!isImaginary) { 1077 // Any suffix pieces we might have parsed are actually part of the 1078 // ud-suffix. 1079 isLong = false; 1080 isUnsigned = false; 1081 isLongLong = false; 1082 isSizeT = false; 1083 isFloat = false; 1084 isFloat16 = false; 1085 isHalf = false; 1086 isImaginary = false; 1087 isBitInt = false; 1088 MicrosoftInteger = 0; 1089 saw_fixed_point_suffix = false; 1090 isFract = false; 1091 isAccum = false; 1092 } 1093 1094 saw_ud_suffix = true; 1095 return; 1096 } 1097 1098 if (s != ThisTokEnd) { 1099 // Report an error if there are any. 1100 Diags.Report(Lexer::AdvanceToTokenCharacter( 1101 TokLoc, SuffixBegin - ThisTokBegin, SM, LangOpts), 1102 diag::err_invalid_suffix_constant) 1103 << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin) 1104 << (isFixedPointConstant ? 2 : isFPConstant); 1105 hadError = true; 1106 } 1107 } 1108 1109 if (!hadError && saw_fixed_point_suffix) { 1110 assert(isFract || isAccum); 1111 } 1112 } 1113 1114 /// ParseDecimalOrOctalCommon - This method is called for decimal or octal 1115 /// numbers. It issues an error for illegal digits, and handles floating point 1116 /// parsing. If it detects a floating point number, the radix is set to 10. 1117 void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){ 1118 assert((radix == 8 || radix == 10) && "Unexpected radix"); 1119 1120 // If we have a hex digit other than 'e' (which denotes a FP exponent) then 1121 // the code is using an incorrect base. 1122 if (isHexDigit(*s) && *s != 'e' && *s != 'E' && 1123 !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) { 1124 Diags.Report( 1125 Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, LangOpts), 1126 diag::err_invalid_digit) 1127 << StringRef(s, 1) << (radix == 8 ? 1 : 0); 1128 hadError = true; 1129 return; 1130 } 1131 1132 if (*s == '.') { 1133 checkSeparator(TokLoc, s, CSK_AfterDigits); 1134 s++; 1135 radix = 10; 1136 saw_period = true; 1137 checkSeparator(TokLoc, s, CSK_BeforeDigits); 1138 s = SkipDigits(s); // Skip suffix. 1139 } 1140 if (*s == 'e' || *s == 'E') { // exponent 1141 checkSeparator(TokLoc, s, CSK_AfterDigits); 1142 const char *Exponent = s; 1143 s++; 1144 radix = 10; 1145 saw_exponent = true; 1146 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign 1147 const char *first_non_digit = SkipDigits(s); 1148 if (containsDigits(s, first_non_digit)) { 1149 checkSeparator(TokLoc, s, CSK_BeforeDigits); 1150 s = first_non_digit; 1151 } else { 1152 if (!hadError) { 1153 Diags.Report(Lexer::AdvanceToTokenCharacter( 1154 TokLoc, Exponent - ThisTokBegin, SM, LangOpts), 1155 diag::err_exponent_has_no_digits); 1156 hadError = true; 1157 } 1158 return; 1159 } 1160 } 1161 } 1162 1163 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved 1164 /// suffixes as ud-suffixes, because the diagnostic experience is better if we 1165 /// treat it as an invalid suffix. 1166 bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts, 1167 StringRef Suffix) { 1168 if (!LangOpts.CPlusPlus11 || Suffix.empty()) 1169 return false; 1170 1171 // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid. 1172 if (Suffix[0] == '_') 1173 return true; 1174 1175 // In C++11, there are no library suffixes. 1176 if (!LangOpts.CPlusPlus14) 1177 return false; 1178 1179 // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library. 1180 // Per tweaked N3660, "il", "i", and "if" are also used in the library. 1181 // In C++2a "d" and "y" are used in the library. 1182 return llvm::StringSwitch<bool>(Suffix) 1183 .Cases("h", "min", "s", true) 1184 .Cases("ms", "us", "ns", true) 1185 .Cases("il", "i", "if", true) 1186 .Cases("d", "y", LangOpts.CPlusPlus20) 1187 .Default(false); 1188 } 1189 1190 void NumericLiteralParser::checkSeparator(SourceLocation TokLoc, 1191 const char *Pos, 1192 CheckSeparatorKind IsAfterDigits) { 1193 if (IsAfterDigits == CSK_AfterDigits) { 1194 if (Pos == ThisTokBegin) 1195 return; 1196 --Pos; 1197 } else if (Pos == ThisTokEnd) 1198 return; 1199 1200 if (isDigitSeparator(*Pos)) { 1201 Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin, SM, 1202 LangOpts), 1203 diag::err_digit_separator_not_between_digits) 1204 << IsAfterDigits; 1205 hadError = true; 1206 } 1207 } 1208 1209 /// ParseNumberStartingWithZero - This method is called when the first character 1210 /// of the number is found to be a zero. This means it is either an octal 1211 /// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or 1212 /// a floating point number (01239.123e4). Eat the prefix, determining the 1213 /// radix etc. 1214 void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { 1215 assert(s[0] == '0' && "Invalid method call"); 1216 s++; 1217 1218 int c1 = s[0]; 1219 1220 // Handle a hex number like 0x1234. 1221 if ((c1 == 'x' || c1 == 'X') && (isHexDigit(s[1]) || s[1] == '.')) { 1222 s++; 1223 assert(s < ThisTokEnd && "didn't maximally munch?"); 1224 radix = 16; 1225 DigitsBegin = s; 1226 s = SkipHexDigits(s); 1227 bool HasSignificandDigits = containsDigits(DigitsBegin, s); 1228 if (s == ThisTokEnd) { 1229 // Done. 1230 } else if (*s == '.') { 1231 s++; 1232 saw_period = true; 1233 const char *floatDigitsBegin = s; 1234 s = SkipHexDigits(s); 1235 if (containsDigits(floatDigitsBegin, s)) 1236 HasSignificandDigits = true; 1237 if (HasSignificandDigits) 1238 checkSeparator(TokLoc, floatDigitsBegin, CSK_BeforeDigits); 1239 } 1240 1241 if (!HasSignificandDigits) { 1242 Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, 1243 LangOpts), 1244 diag::err_hex_constant_requires) 1245 << LangOpts.CPlusPlus << 1; 1246 hadError = true; 1247 return; 1248 } 1249 1250 // A binary exponent can appear with or with a '.'. If dotted, the 1251 // binary exponent is required. 1252 if (*s == 'p' || *s == 'P') { 1253 checkSeparator(TokLoc, s, CSK_AfterDigits); 1254 const char *Exponent = s; 1255 s++; 1256 saw_exponent = true; 1257 if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign 1258 const char *first_non_digit = SkipDigits(s); 1259 if (!containsDigits(s, first_non_digit)) { 1260 if (!hadError) { 1261 Diags.Report(Lexer::AdvanceToTokenCharacter( 1262 TokLoc, Exponent - ThisTokBegin, SM, LangOpts), 1263 diag::err_exponent_has_no_digits); 1264 hadError = true; 1265 } 1266 return; 1267 } 1268 checkSeparator(TokLoc, s, CSK_BeforeDigits); 1269 s = first_non_digit; 1270 1271 if (!LangOpts.HexFloats) 1272 Diags.Report(TokLoc, LangOpts.CPlusPlus 1273 ? diag::ext_hex_literal_invalid 1274 : diag::ext_hex_constant_invalid); 1275 else if (LangOpts.CPlusPlus17) 1276 Diags.Report(TokLoc, diag::warn_cxx17_hex_literal); 1277 } else if (saw_period) { 1278 Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, 1279 LangOpts), 1280 diag::err_hex_constant_requires) 1281 << LangOpts.CPlusPlus << 0; 1282 hadError = true; 1283 } 1284 return; 1285 } 1286 1287 // Handle simple binary numbers 0b01010 1288 if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) { 1289 // 0b101010 is a C++1y / GCC extension. 1290 Diags.Report(TokLoc, LangOpts.CPlusPlus14 1291 ? diag::warn_cxx11_compat_binary_literal 1292 : LangOpts.CPlusPlus ? diag::ext_binary_literal_cxx14 1293 : diag::ext_binary_literal); 1294 ++s; 1295 assert(s < ThisTokEnd && "didn't maximally munch?"); 1296 radix = 2; 1297 DigitsBegin = s; 1298 s = SkipBinaryDigits(s); 1299 if (s == ThisTokEnd) { 1300 // Done. 1301 } else if (isHexDigit(*s) && 1302 !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) { 1303 Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, 1304 LangOpts), 1305 diag::err_invalid_digit) 1306 << StringRef(s, 1) << 2; 1307 hadError = true; 1308 } 1309 // Other suffixes will be diagnosed by the caller. 1310 return; 1311 } 1312 1313 // For now, the radix is set to 8. If we discover that we have a 1314 // floating point constant, the radix will change to 10. Octal floating 1315 // point constants are not permitted (only decimal and hexadecimal). 1316 radix = 8; 1317 const char *PossibleNewDigitStart = s; 1318 s = SkipOctalDigits(s); 1319 // When the value is 0 followed by a suffix (like 0wb), we want to leave 0 1320 // as the start of the digits. So if skipping octal digits does not skip 1321 // anything, we leave the digit start where it was. 1322 if (s != PossibleNewDigitStart) 1323 DigitsBegin = PossibleNewDigitStart; 1324 1325 if (s == ThisTokEnd) 1326 return; // Done, simple octal number like 01234 1327 1328 // If we have some other non-octal digit that *is* a decimal digit, see if 1329 // this is part of a floating point number like 094.123 or 09e1. 1330 if (isDigit(*s)) { 1331 const char *EndDecimal = SkipDigits(s); 1332 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') { 1333 s = EndDecimal; 1334 radix = 10; 1335 } 1336 } 1337 1338 ParseDecimalOrOctalCommon(TokLoc); 1339 } 1340 1341 static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) { 1342 switch (Radix) { 1343 case 2: 1344 return NumDigits <= 64; 1345 case 8: 1346 return NumDigits <= 64 / 3; // Digits are groups of 3 bits. 1347 case 10: 1348 return NumDigits <= 19; // floor(log10(2^64)) 1349 case 16: 1350 return NumDigits <= 64 / 4; // Digits are groups of 4 bits. 1351 default: 1352 llvm_unreachable("impossible Radix"); 1353 } 1354 } 1355 1356 /// GetIntegerValue - Convert this numeric literal value to an APInt that 1357 /// matches Val's input width. If there is an overflow, set Val to the low bits 1358 /// of the result and return true. Otherwise, return false. 1359 bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) { 1360 // Fast path: Compute a conservative bound on the maximum number of 1361 // bits per digit in this radix. If we can't possibly overflow a 1362 // uint64 based on that bound then do the simple conversion to 1363 // integer. This avoids the expensive overflow checking below, and 1364 // handles the common cases that matter (small decimal integers and 1365 // hex/octal values which don't overflow). 1366 const unsigned NumDigits = SuffixBegin - DigitsBegin; 1367 if (alwaysFitsInto64Bits(radix, NumDigits)) { 1368 uint64_t N = 0; 1369 for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr) 1370 if (!isDigitSeparator(*Ptr)) 1371 N = N * radix + llvm::hexDigitValue(*Ptr); 1372 1373 // This will truncate the value to Val's input width. Simply check 1374 // for overflow by comparing. 1375 Val = N; 1376 return Val.getZExtValue() != N; 1377 } 1378 1379 Val = 0; 1380 const char *Ptr = DigitsBegin; 1381 1382 llvm::APInt RadixVal(Val.getBitWidth(), radix); 1383 llvm::APInt CharVal(Val.getBitWidth(), 0); 1384 llvm::APInt OldVal = Val; 1385 1386 bool OverflowOccurred = false; 1387 while (Ptr < SuffixBegin) { 1388 if (isDigitSeparator(*Ptr)) { 1389 ++Ptr; 1390 continue; 1391 } 1392 1393 unsigned C = llvm::hexDigitValue(*Ptr++); 1394 1395 // If this letter is out of bound for this radix, reject it. 1396 assert(C < radix && "NumericLiteralParser ctor should have rejected this"); 1397 1398 CharVal = C; 1399 1400 // Add the digit to the value in the appropriate radix. If adding in digits 1401 // made the value smaller, then this overflowed. 1402 OldVal = Val; 1403 1404 // Multiply by radix, did overflow occur on the multiply? 1405 Val *= RadixVal; 1406 OverflowOccurred |= Val.udiv(RadixVal) != OldVal; 1407 1408 // Add value, did overflow occur on the value? 1409 // (a + b) ult b <=> overflow 1410 Val += CharVal; 1411 OverflowOccurred |= Val.ult(CharVal); 1412 } 1413 return OverflowOccurred; 1414 } 1415 1416 llvm::APFloat::opStatus 1417 NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) { 1418 using llvm::APFloat; 1419 1420 unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin); 1421 1422 llvm::SmallString<16> Buffer; 1423 StringRef Str(ThisTokBegin, n); 1424 if (Str.contains('\'')) { 1425 Buffer.reserve(n); 1426 std::remove_copy_if(Str.begin(), Str.end(), std::back_inserter(Buffer), 1427 &isDigitSeparator); 1428 Str = Buffer; 1429 } 1430 1431 auto StatusOrErr = 1432 Result.convertFromString(Str, APFloat::rmNearestTiesToEven); 1433 assert(StatusOrErr && "Invalid floating point representation"); 1434 return !errorToBool(StatusOrErr.takeError()) ? *StatusOrErr 1435 : APFloat::opInvalidOp; 1436 } 1437 1438 static inline bool IsExponentPart(char c) { 1439 return c == 'p' || c == 'P' || c == 'e' || c == 'E'; 1440 } 1441 1442 bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) { 1443 assert(radix == 16 || radix == 10); 1444 1445 // Find how many digits are needed to store the whole literal. 1446 unsigned NumDigits = SuffixBegin - DigitsBegin; 1447 if (saw_period) --NumDigits; 1448 1449 // Initial scan of the exponent if it exists 1450 bool ExpOverflowOccurred = false; 1451 bool NegativeExponent = false; 1452 const char *ExponentBegin; 1453 uint64_t Exponent = 0; 1454 int64_t BaseShift = 0; 1455 if (saw_exponent) { 1456 const char *Ptr = DigitsBegin; 1457 1458 while (!IsExponentPart(*Ptr)) ++Ptr; 1459 ExponentBegin = Ptr; 1460 ++Ptr; 1461 NegativeExponent = *Ptr == '-'; 1462 if (NegativeExponent) ++Ptr; 1463 1464 unsigned NumExpDigits = SuffixBegin - Ptr; 1465 if (alwaysFitsInto64Bits(radix, NumExpDigits)) { 1466 llvm::StringRef ExpStr(Ptr, NumExpDigits); 1467 llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10); 1468 Exponent = ExpInt.getZExtValue(); 1469 } else { 1470 ExpOverflowOccurred = true; 1471 } 1472 1473 if (NegativeExponent) BaseShift -= Exponent; 1474 else BaseShift += Exponent; 1475 } 1476 1477 // Number of bits needed for decimal literal is 1478 // ceil(NumDigits * log2(10)) Integral part 1479 // + Scale Fractional part 1480 // + ceil(Exponent * log2(10)) Exponent 1481 // -------------------------------------------------- 1482 // ceil((NumDigits + Exponent) * log2(10)) + Scale 1483 // 1484 // But for simplicity in handling integers, we can round up log2(10) to 4, 1485 // making: 1486 // 4 * (NumDigits + Exponent) + Scale 1487 // 1488 // Number of digits needed for hexadecimal literal is 1489 // 4 * NumDigits Integral part 1490 // + Scale Fractional part 1491 // + Exponent Exponent 1492 // -------------------------------------------------- 1493 // (4 * NumDigits) + Scale + Exponent 1494 uint64_t NumBitsNeeded; 1495 if (radix == 10) 1496 NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale; 1497 else 1498 NumBitsNeeded = 4 * NumDigits + Exponent + Scale; 1499 1500 if (NumBitsNeeded > std::numeric_limits<unsigned>::max()) 1501 ExpOverflowOccurred = true; 1502 llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false); 1503 1504 bool FoundDecimal = false; 1505 1506 int64_t FractBaseShift = 0; 1507 const char *End = saw_exponent ? ExponentBegin : SuffixBegin; 1508 for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) { 1509 if (*Ptr == '.') { 1510 FoundDecimal = true; 1511 continue; 1512 } 1513 1514 // Normal reading of an integer 1515 unsigned C = llvm::hexDigitValue(*Ptr); 1516 assert(C < radix && "NumericLiteralParser ctor should have rejected this"); 1517 1518 Val *= radix; 1519 Val += C; 1520 1521 if (FoundDecimal) 1522 // Keep track of how much we will need to adjust this value by from the 1523 // number of digits past the radix point. 1524 --FractBaseShift; 1525 } 1526 1527 // For a radix of 16, we will be multiplying by 2 instead of 16. 1528 if (radix == 16) FractBaseShift *= 4; 1529 BaseShift += FractBaseShift; 1530 1531 Val <<= Scale; 1532 1533 uint64_t Base = (radix == 16) ? 2 : 10; 1534 if (BaseShift > 0) { 1535 for (int64_t i = 0; i < BaseShift; ++i) { 1536 Val *= Base; 1537 } 1538 } else if (BaseShift < 0) { 1539 for (int64_t i = BaseShift; i < 0 && !Val.isZero(); ++i) 1540 Val = Val.udiv(Base); 1541 } 1542 1543 bool IntOverflowOccurred = false; 1544 auto MaxVal = llvm::APInt::getMaxValue(StoreVal.getBitWidth()); 1545 if (Val.getBitWidth() > StoreVal.getBitWidth()) { 1546 IntOverflowOccurred |= Val.ugt(MaxVal.zext(Val.getBitWidth())); 1547 StoreVal = Val.trunc(StoreVal.getBitWidth()); 1548 } else if (Val.getBitWidth() < StoreVal.getBitWidth()) { 1549 IntOverflowOccurred |= Val.zext(MaxVal.getBitWidth()).ugt(MaxVal); 1550 StoreVal = Val.zext(StoreVal.getBitWidth()); 1551 } else { 1552 StoreVal = Val; 1553 } 1554 1555 return IntOverflowOccurred || ExpOverflowOccurred; 1556 } 1557 1558 /// \verbatim 1559 /// user-defined-character-literal: [C++11 lex.ext] 1560 /// character-literal ud-suffix 1561 /// ud-suffix: 1562 /// identifier 1563 /// character-literal: [C++11 lex.ccon] 1564 /// ' c-char-sequence ' 1565 /// u' c-char-sequence ' 1566 /// U' c-char-sequence ' 1567 /// L' c-char-sequence ' 1568 /// u8' c-char-sequence ' [C++1z lex.ccon] 1569 /// c-char-sequence: 1570 /// c-char 1571 /// c-char-sequence c-char 1572 /// c-char: 1573 /// any member of the source character set except the single-quote ', 1574 /// backslash \, or new-line character 1575 /// escape-sequence 1576 /// universal-character-name 1577 /// escape-sequence: 1578 /// simple-escape-sequence 1579 /// octal-escape-sequence 1580 /// hexadecimal-escape-sequence 1581 /// simple-escape-sequence: 1582 /// one of \' \" \? \\ \a \b \f \n \r \t \v 1583 /// octal-escape-sequence: 1584 /// \ octal-digit 1585 /// \ octal-digit octal-digit 1586 /// \ octal-digit octal-digit octal-digit 1587 /// hexadecimal-escape-sequence: 1588 /// \x hexadecimal-digit 1589 /// hexadecimal-escape-sequence hexadecimal-digit 1590 /// universal-character-name: [C++11 lex.charset] 1591 /// \u hex-quad 1592 /// \U hex-quad hex-quad 1593 /// hex-quad: 1594 /// hex-digit hex-digit hex-digit hex-digit 1595 /// \endverbatim 1596 /// 1597 CharLiteralParser::CharLiteralParser(const char *begin, const char *end, 1598 SourceLocation Loc, Preprocessor &PP, 1599 tok::TokenKind kind) { 1600 // At this point we know that the character matches the regex "(L|u|U)?'.*'". 1601 HadError = false; 1602 1603 Kind = kind; 1604 1605 const char *TokBegin = begin; 1606 1607 // Skip over wide character determinant. 1608 if (Kind != tok::char_constant) 1609 ++begin; 1610 if (Kind == tok::utf8_char_constant) 1611 ++begin; 1612 1613 // Skip over the entry quote. 1614 if (begin[0] != '\'') { 1615 PP.Diag(Loc, diag::err_lexing_char); 1616 HadError = true; 1617 return; 1618 } 1619 1620 ++begin; 1621 1622 // Remove an optional ud-suffix. 1623 if (end[-1] != '\'') { 1624 const char *UDSuffixEnd = end; 1625 do { 1626 --end; 1627 } while (end[-1] != '\''); 1628 // FIXME: Don't bother with this if !tok.hasUCN(). 1629 expandUCNs(UDSuffixBuf, StringRef(end, UDSuffixEnd - end)); 1630 UDSuffixOffset = end - TokBegin; 1631 } 1632 1633 // Trim the ending quote. 1634 assert(end != begin && "Invalid token lexed"); 1635 --end; 1636 1637 // FIXME: The "Value" is an uint64_t so we can handle char literals of 1638 // up to 64-bits. 1639 // FIXME: This extensively assumes that 'char' is 8-bits. 1640 assert(PP.getTargetInfo().getCharWidth() == 8 && 1641 "Assumes char is 8 bits"); 1642 assert(PP.getTargetInfo().getIntWidth() <= 64 && 1643 (PP.getTargetInfo().getIntWidth() & 7) == 0 && 1644 "Assumes sizeof(int) on target is <= 64 and a multiple of char"); 1645 assert(PP.getTargetInfo().getWCharWidth() <= 64 && 1646 "Assumes sizeof(wchar) on target is <= 64"); 1647 1648 SmallVector<uint32_t, 4> codepoint_buffer; 1649 codepoint_buffer.resize(end - begin); 1650 uint32_t *buffer_begin = &codepoint_buffer.front(); 1651 uint32_t *buffer_end = buffer_begin + codepoint_buffer.size(); 1652 1653 // Unicode escapes representing characters that cannot be correctly 1654 // represented in a single code unit are disallowed in character literals 1655 // by this implementation. 1656 uint32_t largest_character_for_kind; 1657 if (tok::wide_char_constant == Kind) { 1658 largest_character_for_kind = 1659 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth()); 1660 } else if (tok::utf8_char_constant == Kind) { 1661 largest_character_for_kind = 0x7F; 1662 } else if (tok::utf16_char_constant == Kind) { 1663 largest_character_for_kind = 0xFFFF; 1664 } else if (tok::utf32_char_constant == Kind) { 1665 largest_character_for_kind = 0x10FFFF; 1666 } else { 1667 largest_character_for_kind = 0x7Fu; 1668 } 1669 1670 while (begin != end) { 1671 // Is this a span of non-escape characters? 1672 if (begin[0] != '\\') { 1673 char const *start = begin; 1674 do { 1675 ++begin; 1676 } while (begin != end && *begin != '\\'); 1677 1678 char const *tmp_in_start = start; 1679 uint32_t *tmp_out_start = buffer_begin; 1680 llvm::ConversionResult res = 1681 llvm::ConvertUTF8toUTF32(reinterpret_cast<llvm::UTF8 const **>(&start), 1682 reinterpret_cast<llvm::UTF8 const *>(begin), 1683 &buffer_begin, buffer_end, llvm::strictConversion); 1684 if (res != llvm::conversionOK) { 1685 // If we see bad encoding for unprefixed character literals, warn and 1686 // simply copy the byte values, for compatibility with gcc and 1687 // older versions of clang. 1688 bool NoErrorOnBadEncoding = isOrdinary(); 1689 unsigned Msg = diag::err_bad_character_encoding; 1690 if (NoErrorOnBadEncoding) 1691 Msg = diag::warn_bad_character_encoding; 1692 PP.Diag(Loc, Msg); 1693 if (NoErrorOnBadEncoding) { 1694 start = tmp_in_start; 1695 buffer_begin = tmp_out_start; 1696 for (; start != begin; ++start, ++buffer_begin) 1697 *buffer_begin = static_cast<uint8_t>(*start); 1698 } else { 1699 HadError = true; 1700 } 1701 } else { 1702 for (; tmp_out_start < buffer_begin; ++tmp_out_start) { 1703 if (*tmp_out_start > largest_character_for_kind) { 1704 HadError = true; 1705 PP.Diag(Loc, diag::err_character_too_large); 1706 } 1707 } 1708 } 1709 1710 continue; 1711 } 1712 // Is this a Universal Character Name escape? 1713 if (begin[1] == 'u' || begin[1] == 'U' || begin[1] == 'N') { 1714 unsigned short UcnLen = 0; 1715 if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen, 1716 FullSourceLoc(Loc, PP.getSourceManager()), 1717 &PP.getDiagnostics(), PP.getLangOpts(), true)) { 1718 HadError = true; 1719 } else if (*buffer_begin > largest_character_for_kind) { 1720 HadError = true; 1721 PP.Diag(Loc, diag::err_character_too_large); 1722 } 1723 1724 ++buffer_begin; 1725 continue; 1726 } 1727 unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo()); 1728 uint64_t result = 1729 ProcessCharEscape(TokBegin, begin, end, HadError, 1730 FullSourceLoc(Loc,PP.getSourceManager()), 1731 CharWidth, &PP.getDiagnostics(), PP.getLangOpts()); 1732 *buffer_begin++ = result; 1733 } 1734 1735 unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front(); 1736 1737 if (NumCharsSoFar > 1) { 1738 if (isOrdinary() && NumCharsSoFar == 4) 1739 PP.Diag(Loc, diag::warn_four_char_character_literal); 1740 else if (isOrdinary()) 1741 PP.Diag(Loc, diag::warn_multichar_character_literal); 1742 else { 1743 PP.Diag(Loc, diag::err_multichar_character_literal) << (isWide() ? 0 : 1); 1744 HadError = true; 1745 } 1746 IsMultiChar = true; 1747 } else { 1748 IsMultiChar = false; 1749 } 1750 1751 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0); 1752 1753 // Narrow character literals act as though their value is concatenated 1754 // in this implementation, but warn on overflow. 1755 bool multi_char_too_long = false; 1756 if (isOrdinary() && isMultiChar()) { 1757 LitVal = 0; 1758 for (size_t i = 0; i < NumCharsSoFar; ++i) { 1759 // check for enough leading zeros to shift into 1760 multi_char_too_long |= (LitVal.countLeadingZeros() < 8); 1761 LitVal <<= 8; 1762 LitVal = LitVal + (codepoint_buffer[i] & 0xFF); 1763 } 1764 } else if (NumCharsSoFar > 0) { 1765 // otherwise just take the last character 1766 LitVal = buffer_begin[-1]; 1767 } 1768 1769 if (!HadError && multi_char_too_long) { 1770 PP.Diag(Loc, diag::warn_char_constant_too_large); 1771 } 1772 1773 // Transfer the value from APInt to uint64_t 1774 Value = LitVal.getZExtValue(); 1775 1776 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1") 1777 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple 1778 // character constants are not sign extended in the this implementation: 1779 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC. 1780 if (isOrdinary() && NumCharsSoFar == 1 && (Value & 128) && 1781 PP.getLangOpts().CharIsSigned) 1782 Value = (signed char)Value; 1783 } 1784 1785 /// \verbatim 1786 /// string-literal: [C++0x lex.string] 1787 /// encoding-prefix " [s-char-sequence] " 1788 /// encoding-prefix R raw-string 1789 /// encoding-prefix: 1790 /// u8 1791 /// u 1792 /// U 1793 /// L 1794 /// s-char-sequence: 1795 /// s-char 1796 /// s-char-sequence s-char 1797 /// s-char: 1798 /// any member of the source character set except the double-quote ", 1799 /// backslash \, or new-line character 1800 /// escape-sequence 1801 /// universal-character-name 1802 /// raw-string: 1803 /// " d-char-sequence ( r-char-sequence ) d-char-sequence " 1804 /// r-char-sequence: 1805 /// r-char 1806 /// r-char-sequence r-char 1807 /// r-char: 1808 /// any member of the source character set, except a right parenthesis ) 1809 /// followed by the initial d-char-sequence (which may be empty) 1810 /// followed by a double quote ". 1811 /// d-char-sequence: 1812 /// d-char 1813 /// d-char-sequence d-char 1814 /// d-char: 1815 /// any member of the basic source character set except: 1816 /// space, the left parenthesis (, the right parenthesis ), 1817 /// the backslash \, and the control characters representing horizontal 1818 /// tab, vertical tab, form feed, and newline. 1819 /// escape-sequence: [C++0x lex.ccon] 1820 /// simple-escape-sequence 1821 /// octal-escape-sequence 1822 /// hexadecimal-escape-sequence 1823 /// simple-escape-sequence: 1824 /// one of \' \" \? \\ \a \b \f \n \r \t \v 1825 /// octal-escape-sequence: 1826 /// \ octal-digit 1827 /// \ octal-digit octal-digit 1828 /// \ octal-digit octal-digit octal-digit 1829 /// hexadecimal-escape-sequence: 1830 /// \x hexadecimal-digit 1831 /// hexadecimal-escape-sequence hexadecimal-digit 1832 /// universal-character-name: 1833 /// \u hex-quad 1834 /// \U hex-quad hex-quad 1835 /// hex-quad: 1836 /// hex-digit hex-digit hex-digit hex-digit 1837 /// \endverbatim 1838 /// 1839 StringLiteralParser:: 1840 StringLiteralParser(ArrayRef<Token> StringToks, 1841 Preprocessor &PP) 1842 : SM(PP.getSourceManager()), Features(PP.getLangOpts()), 1843 Target(PP.getTargetInfo()), Diags(&PP.getDiagnostics()), 1844 MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown), 1845 ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) { 1846 init(StringToks); 1847 } 1848 1849 void StringLiteralParser::init(ArrayRef<Token> StringToks){ 1850 // The literal token may have come from an invalid source location (e.g. due 1851 // to a PCH error), in which case the token length will be 0. 1852 if (StringToks.empty() || StringToks[0].getLength() < 2) 1853 return DiagnoseLexingError(SourceLocation()); 1854 1855 // Scan all of the string portions, remember the max individual token length, 1856 // computing a bound on the concatenated string length, and see whether any 1857 // piece is a wide-string. If any of the string portions is a wide-string 1858 // literal, the result is a wide-string literal [C99 6.4.5p4]. 1859 assert(!StringToks.empty() && "expected at least one token"); 1860 MaxTokenLength = StringToks[0].getLength(); 1861 assert(StringToks[0].getLength() >= 2 && "literal token is invalid!"); 1862 SizeBound = StringToks[0].getLength()-2; // -2 for "". 1863 Kind = StringToks[0].getKind(); 1864 1865 hadError = false; 1866 1867 // Implement Translation Phase #6: concatenation of string literals 1868 /// (C99 5.1.1.2p1). The common case is only one string fragment. 1869 for (unsigned i = 1; i != StringToks.size(); ++i) { 1870 if (StringToks[i].getLength() < 2) 1871 return DiagnoseLexingError(StringToks[i].getLocation()); 1872 1873 // The string could be shorter than this if it needs cleaning, but this is a 1874 // reasonable bound, which is all we need. 1875 assert(StringToks[i].getLength() >= 2 && "literal token is invalid!"); 1876 SizeBound += StringToks[i].getLength()-2; // -2 for "". 1877 1878 // Remember maximum string piece length. 1879 if (StringToks[i].getLength() > MaxTokenLength) 1880 MaxTokenLength = StringToks[i].getLength(); 1881 1882 // Remember if we see any wide or utf-8/16/32 strings. 1883 // Also check for illegal concatenations. 1884 if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) { 1885 if (isOrdinary()) { 1886 Kind = StringToks[i].getKind(); 1887 } else { 1888 if (Diags) 1889 Diags->Report(StringToks[i].getLocation(), 1890 diag::err_unsupported_string_concat); 1891 hadError = true; 1892 } 1893 } 1894 } 1895 1896 // Include space for the null terminator. 1897 ++SizeBound; 1898 1899 // TODO: K&R warning: "traditional C rejects string constant concatenation" 1900 1901 // Get the width in bytes of char/wchar_t/char16_t/char32_t 1902 CharByteWidth = getCharWidth(Kind, Target); 1903 assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple"); 1904 CharByteWidth /= 8; 1905 1906 // The output buffer size needs to be large enough to hold wide characters. 1907 // This is a worst-case assumption which basically corresponds to L"" "long". 1908 SizeBound *= CharByteWidth; 1909 1910 // Size the temporary buffer to hold the result string data. 1911 ResultBuf.resize(SizeBound); 1912 1913 // Likewise, but for each string piece. 1914 SmallString<512> TokenBuf; 1915 TokenBuf.resize(MaxTokenLength); 1916 1917 // Loop over all the strings, getting their spelling, and expanding them to 1918 // wide strings as appropriate. 1919 ResultPtr = &ResultBuf[0]; // Next byte to fill in. 1920 1921 Pascal = false; 1922 1923 SourceLocation UDSuffixTokLoc; 1924 1925 for (unsigned i = 0, e = StringToks.size(); i != e; ++i) { 1926 const char *ThisTokBuf = &TokenBuf[0]; 1927 // Get the spelling of the token, which eliminates trigraphs, etc. We know 1928 // that ThisTokBuf points to a buffer that is big enough for the whole token 1929 // and 'spelled' tokens can only shrink. 1930 bool StringInvalid = false; 1931 unsigned ThisTokLen = 1932 Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features, 1933 &StringInvalid); 1934 if (StringInvalid) 1935 return DiagnoseLexingError(StringToks[i].getLocation()); 1936 1937 const char *ThisTokBegin = ThisTokBuf; 1938 const char *ThisTokEnd = ThisTokBuf+ThisTokLen; 1939 1940 // Remove an optional ud-suffix. 1941 if (ThisTokEnd[-1] != '"') { 1942 const char *UDSuffixEnd = ThisTokEnd; 1943 do { 1944 --ThisTokEnd; 1945 } while (ThisTokEnd[-1] != '"'); 1946 1947 StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd); 1948 1949 if (UDSuffixBuf.empty()) { 1950 if (StringToks[i].hasUCN()) 1951 expandUCNs(UDSuffixBuf, UDSuffix); 1952 else 1953 UDSuffixBuf.assign(UDSuffix); 1954 UDSuffixToken = i; 1955 UDSuffixOffset = ThisTokEnd - ThisTokBuf; 1956 UDSuffixTokLoc = StringToks[i].getLocation(); 1957 } else { 1958 SmallString<32> ExpandedUDSuffix; 1959 if (StringToks[i].hasUCN()) { 1960 expandUCNs(ExpandedUDSuffix, UDSuffix); 1961 UDSuffix = ExpandedUDSuffix; 1962 } 1963 1964 // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the 1965 // result of a concatenation involving at least one user-defined-string- 1966 // literal, all the participating user-defined-string-literals shall 1967 // have the same ud-suffix. 1968 if (UDSuffixBuf != UDSuffix) { 1969 if (Diags) { 1970 SourceLocation TokLoc = StringToks[i].getLocation(); 1971 Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix) 1972 << UDSuffixBuf << UDSuffix 1973 << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc) 1974 << SourceRange(TokLoc, TokLoc); 1975 } 1976 hadError = true; 1977 } 1978 } 1979 } 1980 1981 // Strip the end quote. 1982 --ThisTokEnd; 1983 1984 // TODO: Input character set mapping support. 1985 1986 // Skip marker for wide or unicode strings. 1987 if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') { 1988 ++ThisTokBuf; 1989 // Skip 8 of u8 marker for utf8 strings. 1990 if (ThisTokBuf[0] == '8') 1991 ++ThisTokBuf; 1992 } 1993 1994 // Check for raw string 1995 if (ThisTokBuf[0] == 'R') { 1996 if (ThisTokBuf[1] != '"') { 1997 // The file may have come from PCH and then changed after loading the 1998 // PCH; Fail gracefully. 1999 return DiagnoseLexingError(StringToks[i].getLocation()); 2000 } 2001 ThisTokBuf += 2; // skip R" 2002 2003 // C++11 [lex.string]p2: A `d-char-sequence` shall consist of at most 16 2004 // characters. 2005 constexpr unsigned MaxRawStrDelimLen = 16; 2006 2007 const char *Prefix = ThisTokBuf; 2008 while (static_cast<unsigned>(ThisTokBuf - Prefix) < MaxRawStrDelimLen && 2009 ThisTokBuf[0] != '(') 2010 ++ThisTokBuf; 2011 if (ThisTokBuf[0] != '(') 2012 return DiagnoseLexingError(StringToks[i].getLocation()); 2013 ++ThisTokBuf; // skip '(' 2014 2015 // Remove same number of characters from the end 2016 ThisTokEnd -= ThisTokBuf - Prefix; 2017 if (ThisTokEnd < ThisTokBuf) 2018 return DiagnoseLexingError(StringToks[i].getLocation()); 2019 2020 // C++14 [lex.string]p4: A source-file new-line in a raw string literal 2021 // results in a new-line in the resulting execution string-literal. 2022 StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf); 2023 while (!RemainingTokenSpan.empty()) { 2024 // Split the string literal on \r\n boundaries. 2025 size_t CRLFPos = RemainingTokenSpan.find("\r\n"); 2026 StringRef BeforeCRLF = RemainingTokenSpan.substr(0, CRLFPos); 2027 StringRef AfterCRLF = RemainingTokenSpan.substr(CRLFPos); 2028 2029 // Copy everything before the \r\n sequence into the string literal. 2030 if (CopyStringFragment(StringToks[i], ThisTokBegin, BeforeCRLF)) 2031 hadError = true; 2032 2033 // Point into the \n inside the \r\n sequence and operate on the 2034 // remaining portion of the literal. 2035 RemainingTokenSpan = AfterCRLF.substr(1); 2036 } 2037 } else { 2038 if (ThisTokBuf[0] != '"') { 2039 // The file may have come from PCH and then changed after loading the 2040 // PCH; Fail gracefully. 2041 return DiagnoseLexingError(StringToks[i].getLocation()); 2042 } 2043 ++ThisTokBuf; // skip " 2044 2045 // Check if this is a pascal string 2046 if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd && 2047 ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') { 2048 2049 // If the \p sequence is found in the first token, we have a pascal string 2050 // Otherwise, if we already have a pascal string, ignore the first \p 2051 if (i == 0) { 2052 ++ThisTokBuf; 2053 Pascal = true; 2054 } else if (Pascal) 2055 ThisTokBuf += 2; 2056 } 2057 2058 while (ThisTokBuf != ThisTokEnd) { 2059 // Is this a span of non-escape characters? 2060 if (ThisTokBuf[0] != '\\') { 2061 const char *InStart = ThisTokBuf; 2062 do { 2063 ++ThisTokBuf; 2064 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\'); 2065 2066 // Copy the character span over. 2067 if (CopyStringFragment(StringToks[i], ThisTokBegin, 2068 StringRef(InStart, ThisTokBuf - InStart))) 2069 hadError = true; 2070 continue; 2071 } 2072 // Is this a Universal Character Name escape? 2073 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U' || 2074 ThisTokBuf[1] == 'N') { 2075 EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, 2076 ResultPtr, hadError, 2077 FullSourceLoc(StringToks[i].getLocation(), SM), 2078 CharByteWidth, Diags, Features); 2079 continue; 2080 } 2081 // Otherwise, this is a non-UCN escape character. Process it. 2082 unsigned ResultChar = 2083 ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError, 2084 FullSourceLoc(StringToks[i].getLocation(), SM), 2085 CharByteWidth*8, Diags, Features); 2086 2087 if (CharByteWidth == 4) { 2088 // FIXME: Make the type of the result buffer correct instead of 2089 // using reinterpret_cast. 2090 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr); 2091 *ResultWidePtr = ResultChar; 2092 ResultPtr += 4; 2093 } else if (CharByteWidth == 2) { 2094 // FIXME: Make the type of the result buffer correct instead of 2095 // using reinterpret_cast. 2096 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr); 2097 *ResultWidePtr = ResultChar & 0xFFFF; 2098 ResultPtr += 2; 2099 } else { 2100 assert(CharByteWidth == 1 && "Unexpected char width"); 2101 *ResultPtr++ = ResultChar & 0xFF; 2102 } 2103 } 2104 } 2105 } 2106 2107 if (Pascal) { 2108 if (CharByteWidth == 4) { 2109 // FIXME: Make the type of the result buffer correct instead of 2110 // using reinterpret_cast. 2111 llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data()); 2112 ResultWidePtr[0] = GetNumStringChars() - 1; 2113 } else if (CharByteWidth == 2) { 2114 // FIXME: Make the type of the result buffer correct instead of 2115 // using reinterpret_cast. 2116 llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data()); 2117 ResultWidePtr[0] = GetNumStringChars() - 1; 2118 } else { 2119 assert(CharByteWidth == 1 && "Unexpected char width"); 2120 ResultBuf[0] = GetNumStringChars() - 1; 2121 } 2122 2123 // Verify that pascal strings aren't too large. 2124 if (GetStringLength() > 256) { 2125 if (Diags) 2126 Diags->Report(StringToks.front().getLocation(), 2127 diag::err_pascal_string_too_long) 2128 << SourceRange(StringToks.front().getLocation(), 2129 StringToks.back().getLocation()); 2130 hadError = true; 2131 return; 2132 } 2133 } else if (Diags) { 2134 // Complain if this string literal has too many characters. 2135 unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509; 2136 2137 if (GetNumStringChars() > MaxChars) 2138 Diags->Report(StringToks.front().getLocation(), 2139 diag::ext_string_too_long) 2140 << GetNumStringChars() << MaxChars 2141 << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0) 2142 << SourceRange(StringToks.front().getLocation(), 2143 StringToks.back().getLocation()); 2144 } 2145 } 2146 2147 static const char *resyncUTF8(const char *Err, const char *End) { 2148 if (Err == End) 2149 return End; 2150 End = Err + std::min<unsigned>(llvm::getNumBytesForUTF8(*Err), End-Err); 2151 while (++Err != End && (*Err & 0xC0) == 0x80) 2152 ; 2153 return Err; 2154 } 2155 2156 /// This function copies from Fragment, which is a sequence of bytes 2157 /// within Tok's contents (which begin at TokBegin) into ResultPtr. 2158 /// Performs widening for multi-byte characters. 2159 bool StringLiteralParser::CopyStringFragment(const Token &Tok, 2160 const char *TokBegin, 2161 StringRef Fragment) { 2162 const llvm::UTF8 *ErrorPtrTmp; 2163 if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp)) 2164 return false; 2165 2166 // If we see bad encoding for unprefixed string literals, warn and 2167 // simply copy the byte values, for compatibility with gcc and older 2168 // versions of clang. 2169 bool NoErrorOnBadEncoding = isOrdinary(); 2170 if (NoErrorOnBadEncoding) { 2171 memcpy(ResultPtr, Fragment.data(), Fragment.size()); 2172 ResultPtr += Fragment.size(); 2173 } 2174 2175 if (Diags) { 2176 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp); 2177 2178 FullSourceLoc SourceLoc(Tok.getLocation(), SM); 2179 const DiagnosticBuilder &Builder = 2180 Diag(Diags, Features, SourceLoc, TokBegin, 2181 ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()), 2182 NoErrorOnBadEncoding ? diag::warn_bad_string_encoding 2183 : diag::err_bad_string_encoding); 2184 2185 const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end()); 2186 StringRef NextFragment(NextStart, Fragment.end()-NextStart); 2187 2188 // Decode into a dummy buffer. 2189 SmallString<512> Dummy; 2190 Dummy.reserve(Fragment.size() * CharByteWidth); 2191 char *Ptr = Dummy.data(); 2192 2193 while (!ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) { 2194 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp); 2195 NextStart = resyncUTF8(ErrorPtr, Fragment.end()); 2196 Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin, 2197 ErrorPtr, NextStart); 2198 NextFragment = StringRef(NextStart, Fragment.end()-NextStart); 2199 } 2200 } 2201 return !NoErrorOnBadEncoding; 2202 } 2203 2204 void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) { 2205 hadError = true; 2206 if (Diags) 2207 Diags->Report(Loc, diag::err_lexing_string); 2208 } 2209 2210 /// getOffsetOfStringByte - This function returns the offset of the 2211 /// specified byte of the string data represented by Token. This handles 2212 /// advancing over escape sequences in the string. 2213 unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok, 2214 unsigned ByteNo) const { 2215 // Get the spelling of the token. 2216 SmallString<32> SpellingBuffer; 2217 SpellingBuffer.resize(Tok.getLength()); 2218 2219 bool StringInvalid = false; 2220 const char *SpellingPtr = &SpellingBuffer[0]; 2221 unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features, 2222 &StringInvalid); 2223 if (StringInvalid) 2224 return 0; 2225 2226 const char *SpellingStart = SpellingPtr; 2227 const char *SpellingEnd = SpellingPtr+TokLen; 2228 2229 // Handle UTF-8 strings just like narrow strings. 2230 if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8') 2231 SpellingPtr += 2; 2232 2233 assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' && 2234 SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet"); 2235 2236 // For raw string literals, this is easy. 2237 if (SpellingPtr[0] == 'R') { 2238 assert(SpellingPtr[1] == '"' && "Should be a raw string literal!"); 2239 // Skip 'R"'. 2240 SpellingPtr += 2; 2241 while (*SpellingPtr != '(') { 2242 ++SpellingPtr; 2243 assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal"); 2244 } 2245 // Skip '('. 2246 ++SpellingPtr; 2247 return SpellingPtr - SpellingStart + ByteNo; 2248 } 2249 2250 // Skip over the leading quote 2251 assert(SpellingPtr[0] == '"' && "Should be a string literal!"); 2252 ++SpellingPtr; 2253 2254 // Skip over bytes until we find the offset we're looking for. 2255 while (ByteNo) { 2256 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!"); 2257 2258 // Step over non-escapes simply. 2259 if (*SpellingPtr != '\\') { 2260 ++SpellingPtr; 2261 --ByteNo; 2262 continue; 2263 } 2264 2265 // Otherwise, this is an escape character. Advance over it. 2266 bool HadError = false; 2267 if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U' || 2268 SpellingPtr[1] == 'N') { 2269 const char *EscapePtr = SpellingPtr; 2270 unsigned Len = MeasureUCNEscape(SpellingStart, SpellingPtr, SpellingEnd, 2271 1, Features, HadError); 2272 if (Len > ByteNo) { 2273 // ByteNo is somewhere within the escape sequence. 2274 SpellingPtr = EscapePtr; 2275 break; 2276 } 2277 ByteNo -= Len; 2278 } else { 2279 ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError, 2280 FullSourceLoc(Tok.getLocation(), SM), 2281 CharByteWidth*8, Diags, Features); 2282 --ByteNo; 2283 } 2284 assert(!HadError && "This method isn't valid on erroneous strings"); 2285 } 2286 2287 return SpellingPtr-SpellingStart; 2288 } 2289 2290 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved 2291 /// suffixes as ud-suffixes, because the diagnostic experience is better if we 2292 /// treat it as an invalid suffix. 2293 bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts, 2294 StringRef Suffix) { 2295 return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) || 2296 Suffix == "sv"; 2297 } 2298