1 //===- Lexer.cpp - C Language Family Lexer --------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the Lexer and Token interfaces. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "clang/Lex/Lexer.h" 14 #include "UnicodeCharSets.h" 15 #include "clang/Basic/CharInfo.h" 16 #include "clang/Basic/Diagnostic.h" 17 #include "clang/Basic/IdentifierTable.h" 18 #include "clang/Basic/LLVM.h" 19 #include "clang/Basic/LangOptions.h" 20 #include "clang/Basic/SourceLocation.h" 21 #include "clang/Basic/SourceManager.h" 22 #include "clang/Basic/TokenKinds.h" 23 #include "clang/Lex/LexDiagnostic.h" 24 #include "clang/Lex/LiteralSupport.h" 25 #include "clang/Lex/MultipleIncludeOpt.h" 26 #include "clang/Lex/Preprocessor.h" 27 #include "clang/Lex/PreprocessorOptions.h" 28 #include "clang/Lex/Token.h" 29 #include "llvm/ADT/None.h" 30 #include "llvm/ADT/Optional.h" 31 #include "llvm/ADT/STLExtras.h" 32 #include "llvm/ADT/StringExtras.h" 33 #include "llvm/ADT/StringRef.h" 34 #include "llvm/ADT/StringSwitch.h" 35 #include "llvm/Support/Compiler.h" 36 #include "llvm/Support/ConvertUTF.h" 37 #include "llvm/Support/MathExtras.h" 38 #include "llvm/Support/MemoryBufferRef.h" 39 #include "llvm/Support/NativeFormatting.h" 40 #include "llvm/Support/UnicodeCharRanges.h" 41 #include <algorithm> 42 #include <cassert> 43 #include <cstddef> 44 #include <cstdint> 45 #include <cstring> 46 #include <string> 47 #include <tuple> 48 #include <utility> 49 50 using namespace clang; 51 52 //===----------------------------------------------------------------------===// 53 // Token Class Implementation 54 //===----------------------------------------------------------------------===// 55 56 /// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 57 bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 58 if (isAnnotation()) 59 return false; 60 if (IdentifierInfo *II = getIdentifierInfo()) 61 return II->getObjCKeywordID() == objcKey; 62 return false; 63 } 64 65 /// getObjCKeywordID - Return the ObjC keyword kind. 66 tok::ObjCKeywordKind Token::getObjCKeywordID() const { 67 if (isAnnotation()) 68 return tok::objc_not_keyword; 69 IdentifierInfo *specId = getIdentifierInfo(); 70 return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 71 } 72 73 //===----------------------------------------------------------------------===// 74 // Lexer Class Implementation 75 //===----------------------------------------------------------------------===// 76 77 void Lexer::anchor() {} 78 79 void Lexer::InitLexer(const char *BufStart, const char *BufPtr, 80 const char *BufEnd) { 81 BufferStart = BufStart; 82 BufferPtr = BufPtr; 83 BufferEnd = BufEnd; 84 85 assert(BufEnd[0] == 0 && 86 "We assume that the input buffer has a null character at the end" 87 " to simplify lexing!"); 88 89 // Check whether we have a BOM in the beginning of the buffer. If yes - act 90 // accordingly. Right now we support only UTF-8 with and without BOM, so, just 91 // skip the UTF-8 BOM if it's present. 92 if (BufferStart == BufferPtr) { 93 // Determine the size of the BOM. 94 StringRef Buf(BufferStart, BufferEnd - BufferStart); 95 size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 96 .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 97 .Default(0); 98 99 // Skip the BOM. 100 BufferPtr += BOMLength; 101 } 102 103 Is_PragmaLexer = false; 104 CurrentConflictMarkerState = CMK_None; 105 106 // Start of the file is a start of line. 107 IsAtStartOfLine = true; 108 IsAtPhysicalStartOfLine = true; 109 110 HasLeadingSpace = false; 111 HasLeadingEmptyMacro = false; 112 113 // We are not after parsing a #. 114 ParsingPreprocessorDirective = false; 115 116 // We are not after parsing #include. 117 ParsingFilename = false; 118 119 // We are not in raw mode. Raw mode disables diagnostics and interpretation 120 // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 121 // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 122 // or otherwise skipping over tokens. 123 LexingRawMode = false; 124 125 // Default to not keeping comments. 126 ExtendedTokenMode = 0; 127 128 NewLinePtr = nullptr; 129 } 130 131 /// Lexer constructor - Create a new lexer object for the specified buffer 132 /// with the specified preprocessor managing the lexing process. This lexer 133 /// assumes that the associated file buffer and Preprocessor objects will 134 /// outlive it, so it doesn't take ownership of either of them. 135 Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, 136 Preprocessor &PP) 137 : PreprocessorLexer(&PP, FID), 138 FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 139 LangOpts(PP.getLangOpts()) { 140 InitLexer(InputFile.getBufferStart(), InputFile.getBufferStart(), 141 InputFile.getBufferEnd()); 142 143 resetExtendedTokenMode(); 144 } 145 146 /// Lexer constructor - Create a new raw lexer object. This object is only 147 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 148 /// range will outlive it, so it doesn't take ownership of it. 149 Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 150 const char *BufStart, const char *BufPtr, const char *BufEnd) 151 : FileLoc(fileloc), LangOpts(langOpts) { 152 InitLexer(BufStart, BufPtr, BufEnd); 153 154 // We *are* in raw mode. 155 LexingRawMode = true; 156 } 157 158 /// Lexer constructor - Create a new raw lexer object. This object is only 159 /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 160 /// range will outlive it, so it doesn't take ownership of it. 161 Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile, 162 const SourceManager &SM, const LangOptions &langOpts) 163 : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(), 164 FromFile.getBufferStart(), FromFile.getBufferEnd()) {} 165 166 void Lexer::resetExtendedTokenMode() { 167 assert(PP && "Cannot reset token mode without a preprocessor"); 168 if (LangOpts.TraditionalCPP) 169 SetKeepWhitespaceMode(true); 170 else 171 SetCommentRetentionState(PP->getCommentRetentionState()); 172 } 173 174 /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 175 /// _Pragma expansion. This has a variety of magic semantics that this method 176 /// sets up. It returns a new'd Lexer that must be delete'd when done. 177 /// 178 /// On entrance to this routine, TokStartLoc is a macro location which has a 179 /// spelling loc that indicates the bytes to be lexed for the token and an 180 /// expansion location that indicates where all lexed tokens should be 181 /// "expanded from". 182 /// 183 /// TODO: It would really be nice to make _Pragma just be a wrapper around a 184 /// normal lexer that remaps tokens as they fly by. This would require making 185 /// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 186 /// interface that could handle this stuff. This would pull GetMappedTokenLoc 187 /// out of the critical path of the lexer! 188 /// 189 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 190 SourceLocation ExpansionLocStart, 191 SourceLocation ExpansionLocEnd, 192 unsigned TokLen, Preprocessor &PP) { 193 SourceManager &SM = PP.getSourceManager(); 194 195 // Create the lexer as if we were going to lex the file normally. 196 FileID SpellingFID = SM.getFileID(SpellingLoc); 197 llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(SpellingFID); 198 Lexer *L = new Lexer(SpellingFID, InputFile, PP); 199 200 // Now that the lexer is created, change the start/end locations so that we 201 // just lex the subsection of the file that we want. This is lexing from a 202 // scratch buffer. 203 const char *StrData = SM.getCharacterData(SpellingLoc); 204 205 L->BufferPtr = StrData; 206 L->BufferEnd = StrData+TokLen; 207 assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 208 209 // Set the SourceLocation with the remapping information. This ensures that 210 // GetMappedTokenLoc will remap the tokens as they are lexed. 211 L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 212 ExpansionLocStart, 213 ExpansionLocEnd, TokLen); 214 215 // Ensure that the lexer thinks it is inside a directive, so that end \n will 216 // return an EOD token. 217 L->ParsingPreprocessorDirective = true; 218 219 // This lexer really is for _Pragma. 220 L->Is_PragmaLexer = true; 221 return L; 222 } 223 224 bool Lexer::skipOver(unsigned NumBytes) { 225 IsAtPhysicalStartOfLine = true; 226 IsAtStartOfLine = true; 227 if ((BufferPtr + NumBytes) > BufferEnd) 228 return true; 229 BufferPtr += NumBytes; 230 return false; 231 } 232 233 template <typename T> static void StringifyImpl(T &Str, char Quote) { 234 typename T::size_type i = 0, e = Str.size(); 235 while (i < e) { 236 if (Str[i] == '\\' || Str[i] == Quote) { 237 Str.insert(Str.begin() + i, '\\'); 238 i += 2; 239 ++e; 240 } else if (Str[i] == '\n' || Str[i] == '\r') { 241 // Replace '\r\n' and '\n\r' to '\\' followed by 'n'. 242 if ((i < e - 1) && (Str[i + 1] == '\n' || Str[i + 1] == '\r') && 243 Str[i] != Str[i + 1]) { 244 Str[i] = '\\'; 245 Str[i + 1] = 'n'; 246 } else { 247 // Replace '\n' and '\r' to '\\' followed by 'n'. 248 Str[i] = '\\'; 249 Str.insert(Str.begin() + i + 1, 'n'); 250 ++e; 251 } 252 i += 2; 253 } else 254 ++i; 255 } 256 } 257 258 std::string Lexer::Stringify(StringRef Str, bool Charify) { 259 std::string Result = std::string(Str); 260 char Quote = Charify ? '\'' : '"'; 261 StringifyImpl(Result, Quote); 262 return Result; 263 } 264 265 void Lexer::Stringify(SmallVectorImpl<char> &Str) { StringifyImpl(Str, '"'); } 266 267 //===----------------------------------------------------------------------===// 268 // Token Spelling 269 //===----------------------------------------------------------------------===// 270 271 /// Slow case of getSpelling. Extract the characters comprising the 272 /// spelling of this token from the provided input buffer. 273 static size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 274 const LangOptions &LangOpts, char *Spelling) { 275 assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 276 277 size_t Length = 0; 278 const char *BufEnd = BufPtr + Tok.getLength(); 279 280 if (tok::isStringLiteral(Tok.getKind())) { 281 // Munch the encoding-prefix and opening double-quote. 282 while (BufPtr < BufEnd) { 283 unsigned Size; 284 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 285 BufPtr += Size; 286 287 if (Spelling[Length - 1] == '"') 288 break; 289 } 290 291 // Raw string literals need special handling; trigraph expansion and line 292 // splicing do not occur within their d-char-sequence nor within their 293 // r-char-sequence. 294 if (Length >= 2 && 295 Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 296 // Search backwards from the end of the token to find the matching closing 297 // quote. 298 const char *RawEnd = BufEnd; 299 do --RawEnd; while (*RawEnd != '"'); 300 size_t RawLength = RawEnd - BufPtr + 1; 301 302 // Everything between the quotes is included verbatim in the spelling. 303 memcpy(Spelling + Length, BufPtr, RawLength); 304 Length += RawLength; 305 BufPtr += RawLength; 306 307 // The rest of the token is lexed normally. 308 } 309 } 310 311 while (BufPtr < BufEnd) { 312 unsigned Size; 313 Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 314 BufPtr += Size; 315 } 316 317 assert(Length < Tok.getLength() && 318 "NeedsCleaning flag set on token that didn't need cleaning!"); 319 return Length; 320 } 321 322 /// getSpelling() - Return the 'spelling' of this token. The spelling of a 323 /// token are the characters used to represent the token in the source file 324 /// after trigraph expansion and escaped-newline folding. In particular, this 325 /// wants to get the true, uncanonicalized, spelling of things like digraphs 326 /// UCNs, etc. 327 StringRef Lexer::getSpelling(SourceLocation loc, 328 SmallVectorImpl<char> &buffer, 329 const SourceManager &SM, 330 const LangOptions &options, 331 bool *invalid) { 332 // Break down the source location. 333 std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 334 335 // Try to the load the file buffer. 336 bool invalidTemp = false; 337 StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 338 if (invalidTemp) { 339 if (invalid) *invalid = true; 340 return {}; 341 } 342 343 const char *tokenBegin = file.data() + locInfo.second; 344 345 // Lex from the start of the given location. 346 Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 347 file.begin(), tokenBegin, file.end()); 348 Token token; 349 lexer.LexFromRawLexer(token); 350 351 unsigned length = token.getLength(); 352 353 // Common case: no need for cleaning. 354 if (!token.needsCleaning()) 355 return StringRef(tokenBegin, length); 356 357 // Hard case, we need to relex the characters into the string. 358 buffer.resize(length); 359 buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 360 return StringRef(buffer.data(), buffer.size()); 361 } 362 363 /// getSpelling() - Return the 'spelling' of this token. The spelling of a 364 /// token are the characters used to represent the token in the source file 365 /// after trigraph expansion and escaped-newline folding. In particular, this 366 /// wants to get the true, uncanonicalized, spelling of things like digraphs 367 /// UCNs, etc. 368 std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 369 const LangOptions &LangOpts, bool *Invalid) { 370 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 371 372 bool CharDataInvalid = false; 373 const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 374 &CharDataInvalid); 375 if (Invalid) 376 *Invalid = CharDataInvalid; 377 if (CharDataInvalid) 378 return {}; 379 380 // If this token contains nothing interesting, return it directly. 381 if (!Tok.needsCleaning()) 382 return std::string(TokStart, TokStart + Tok.getLength()); 383 384 std::string Result; 385 Result.resize(Tok.getLength()); 386 Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 387 return Result; 388 } 389 390 /// getSpelling - This method is used to get the spelling of a token into a 391 /// preallocated buffer, instead of as an std::string. The caller is required 392 /// to allocate enough space for the token, which is guaranteed to be at least 393 /// Tok.getLength() bytes long. The actual length of the token is returned. 394 /// 395 /// Note that this method may do two possible things: it may either fill in 396 /// the buffer specified with characters, or it may *change the input pointer* 397 /// to point to a constant buffer with the data already in it (avoiding a 398 /// copy). The caller is not allowed to modify the returned buffer pointer 399 /// if an internal buffer is returned. 400 unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 401 const SourceManager &SourceMgr, 402 const LangOptions &LangOpts, bool *Invalid) { 403 assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 404 405 const char *TokStart = nullptr; 406 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 407 if (Tok.is(tok::raw_identifier)) 408 TokStart = Tok.getRawIdentifier().data(); 409 else if (!Tok.hasUCN()) { 410 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 411 // Just return the string from the identifier table, which is very quick. 412 Buffer = II->getNameStart(); 413 return II->getLength(); 414 } 415 } 416 417 // NOTE: this can be checked even after testing for an IdentifierInfo. 418 if (Tok.isLiteral()) 419 TokStart = Tok.getLiteralData(); 420 421 if (!TokStart) { 422 // Compute the start of the token in the input lexer buffer. 423 bool CharDataInvalid = false; 424 TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 425 if (Invalid) 426 *Invalid = CharDataInvalid; 427 if (CharDataInvalid) { 428 Buffer = ""; 429 return 0; 430 } 431 } 432 433 // If this token contains nothing interesting, return it directly. 434 if (!Tok.needsCleaning()) { 435 Buffer = TokStart; 436 return Tok.getLength(); 437 } 438 439 // Otherwise, hard case, relex the characters into the string. 440 return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 441 } 442 443 /// MeasureTokenLength - Relex the token at the specified location and return 444 /// its length in bytes in the input file. If the token needs cleaning (e.g. 445 /// includes a trigraph or an escaped newline) then this count includes bytes 446 /// that are part of that. 447 unsigned Lexer::MeasureTokenLength(SourceLocation Loc, 448 const SourceManager &SM, 449 const LangOptions &LangOpts) { 450 Token TheTok; 451 if (getRawToken(Loc, TheTok, SM, LangOpts)) 452 return 0; 453 return TheTok.getLength(); 454 } 455 456 /// Relex the token at the specified location. 457 /// \returns true if there was a failure, false on success. 458 bool Lexer::getRawToken(SourceLocation Loc, Token &Result, 459 const SourceManager &SM, 460 const LangOptions &LangOpts, 461 bool IgnoreWhiteSpace) { 462 // TODO: this could be special cased for common tokens like identifiers, ')', 463 // etc to make this faster, if it mattered. Just look at StrData[0] to handle 464 // all obviously single-char tokens. This could use 465 // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 466 // something. 467 468 // If this comes from a macro expansion, we really do want the macro name, not 469 // the token this macro expanded to. 470 Loc = SM.getExpansionLoc(Loc); 471 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 472 bool Invalid = false; 473 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 474 if (Invalid) 475 return true; 476 477 const char *StrData = Buffer.data()+LocInfo.second; 478 479 if (!IgnoreWhiteSpace && isWhitespace(StrData[0])) 480 return true; 481 482 // Create a lexer starting at the beginning of this token. 483 Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 484 Buffer.begin(), StrData, Buffer.end()); 485 TheLexer.SetCommentRetentionState(true); 486 TheLexer.LexFromRawLexer(Result); 487 return false; 488 } 489 490 /// Returns the pointer that points to the beginning of line that contains 491 /// the given offset, or null if the offset if invalid. 492 static const char *findBeginningOfLine(StringRef Buffer, unsigned Offset) { 493 const char *BufStart = Buffer.data(); 494 if (Offset >= Buffer.size()) 495 return nullptr; 496 497 const char *LexStart = BufStart + Offset; 498 for (; LexStart != BufStart; --LexStart) { 499 if (isVerticalWhitespace(LexStart[0]) && 500 !Lexer::isNewLineEscaped(BufStart, LexStart)) { 501 // LexStart should point at first character of logical line. 502 ++LexStart; 503 break; 504 } 505 } 506 return LexStart; 507 } 508 509 static SourceLocation getBeginningOfFileToken(SourceLocation Loc, 510 const SourceManager &SM, 511 const LangOptions &LangOpts) { 512 assert(Loc.isFileID()); 513 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 514 if (LocInfo.first.isInvalid()) 515 return Loc; 516 517 bool Invalid = false; 518 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 519 if (Invalid) 520 return Loc; 521 522 // Back up from the current location until we hit the beginning of a line 523 // (or the buffer). We'll relex from that point. 524 const char *StrData = Buffer.data() + LocInfo.second; 525 const char *LexStart = findBeginningOfLine(Buffer, LocInfo.second); 526 if (!LexStart || LexStart == StrData) 527 return Loc; 528 529 // Create a lexer starting at the beginning of this token. 530 SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 531 Lexer TheLexer(LexerStartLoc, LangOpts, Buffer.data(), LexStart, 532 Buffer.end()); 533 TheLexer.SetCommentRetentionState(true); 534 535 // Lex tokens until we find the token that contains the source location. 536 Token TheTok; 537 do { 538 TheLexer.LexFromRawLexer(TheTok); 539 540 if (TheLexer.getBufferLocation() > StrData) { 541 // Lexing this token has taken the lexer past the source location we're 542 // looking for. If the current token encompasses our source location, 543 // return the beginning of that token. 544 if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 545 return TheTok.getLocation(); 546 547 // We ended up skipping over the source location entirely, which means 548 // that it points into whitespace. We're done here. 549 break; 550 } 551 } while (TheTok.getKind() != tok::eof); 552 553 // We've passed our source location; just return the original source location. 554 return Loc; 555 } 556 557 SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 558 const SourceManager &SM, 559 const LangOptions &LangOpts) { 560 if (Loc.isFileID()) 561 return getBeginningOfFileToken(Loc, SM, LangOpts); 562 563 if (!SM.isMacroArgExpansion(Loc)) 564 return Loc; 565 566 SourceLocation FileLoc = SM.getSpellingLoc(Loc); 567 SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 568 std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 569 std::pair<FileID, unsigned> BeginFileLocInfo = 570 SM.getDecomposedLoc(BeginFileLoc); 571 assert(FileLocInfo.first == BeginFileLocInfo.first && 572 FileLocInfo.second >= BeginFileLocInfo.second); 573 return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 574 } 575 576 namespace { 577 578 enum PreambleDirectiveKind { 579 PDK_Skipped, 580 PDK_Unknown 581 }; 582 583 } // namespace 584 585 PreambleBounds Lexer::ComputePreamble(StringRef Buffer, 586 const LangOptions &LangOpts, 587 unsigned MaxLines) { 588 // Create a lexer starting at the beginning of the file. Note that we use a 589 // "fake" file source location at offset 1 so that the lexer will track our 590 // position within the file. 591 const unsigned StartOffset = 1; 592 SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 593 Lexer TheLexer(FileLoc, LangOpts, Buffer.begin(), Buffer.begin(), 594 Buffer.end()); 595 TheLexer.SetCommentRetentionState(true); 596 597 bool InPreprocessorDirective = false; 598 Token TheTok; 599 SourceLocation ActiveCommentLoc; 600 601 unsigned MaxLineOffset = 0; 602 if (MaxLines) { 603 const char *CurPtr = Buffer.begin(); 604 unsigned CurLine = 0; 605 while (CurPtr != Buffer.end()) { 606 char ch = *CurPtr++; 607 if (ch == '\n') { 608 ++CurLine; 609 if (CurLine == MaxLines) 610 break; 611 } 612 } 613 if (CurPtr != Buffer.end()) 614 MaxLineOffset = CurPtr - Buffer.begin(); 615 } 616 617 do { 618 TheLexer.LexFromRawLexer(TheTok); 619 620 if (InPreprocessorDirective) { 621 // If we've hit the end of the file, we're done. 622 if (TheTok.getKind() == tok::eof) { 623 break; 624 } 625 626 // If we haven't hit the end of the preprocessor directive, skip this 627 // token. 628 if (!TheTok.isAtStartOfLine()) 629 continue; 630 631 // We've passed the end of the preprocessor directive, and will look 632 // at this token again below. 633 InPreprocessorDirective = false; 634 } 635 636 // Keep track of the # of lines in the preamble. 637 if (TheTok.isAtStartOfLine()) { 638 unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 639 640 // If we were asked to limit the number of lines in the preamble, 641 // and we're about to exceed that limit, we're done. 642 if (MaxLineOffset && TokOffset >= MaxLineOffset) 643 break; 644 } 645 646 // Comments are okay; skip over them. 647 if (TheTok.getKind() == tok::comment) { 648 if (ActiveCommentLoc.isInvalid()) 649 ActiveCommentLoc = TheTok.getLocation(); 650 continue; 651 } 652 653 if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 654 // This is the start of a preprocessor directive. 655 Token HashTok = TheTok; 656 InPreprocessorDirective = true; 657 ActiveCommentLoc = SourceLocation(); 658 659 // Figure out which directive this is. Since we're lexing raw tokens, 660 // we don't have an identifier table available. Instead, just look at 661 // the raw identifier to recognize and categorize preprocessor directives. 662 TheLexer.LexFromRawLexer(TheTok); 663 if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 664 StringRef Keyword = TheTok.getRawIdentifier(); 665 PreambleDirectiveKind PDK 666 = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 667 .Case("include", PDK_Skipped) 668 .Case("__include_macros", PDK_Skipped) 669 .Case("define", PDK_Skipped) 670 .Case("undef", PDK_Skipped) 671 .Case("line", PDK_Skipped) 672 .Case("error", PDK_Skipped) 673 .Case("pragma", PDK_Skipped) 674 .Case("import", PDK_Skipped) 675 .Case("include_next", PDK_Skipped) 676 .Case("warning", PDK_Skipped) 677 .Case("ident", PDK_Skipped) 678 .Case("sccs", PDK_Skipped) 679 .Case("assert", PDK_Skipped) 680 .Case("unassert", PDK_Skipped) 681 .Case("if", PDK_Skipped) 682 .Case("ifdef", PDK_Skipped) 683 .Case("ifndef", PDK_Skipped) 684 .Case("elif", PDK_Skipped) 685 .Case("else", PDK_Skipped) 686 .Case("endif", PDK_Skipped) 687 .Default(PDK_Unknown); 688 689 switch (PDK) { 690 case PDK_Skipped: 691 continue; 692 693 case PDK_Unknown: 694 // We don't know what this directive is; stop at the '#'. 695 break; 696 } 697 } 698 699 // We only end up here if we didn't recognize the preprocessor 700 // directive or it was one that can't occur in the preamble at this 701 // point. Roll back the current token to the location of the '#'. 702 TheTok = HashTok; 703 } 704 705 // We hit a token that we don't recognize as being in the 706 // "preprocessing only" part of the file, so we're no longer in 707 // the preamble. 708 break; 709 } while (true); 710 711 SourceLocation End; 712 if (ActiveCommentLoc.isValid()) 713 End = ActiveCommentLoc; // don't truncate a decl comment. 714 else 715 End = TheTok.getLocation(); 716 717 return PreambleBounds(End.getRawEncoding() - FileLoc.getRawEncoding(), 718 TheTok.isAtStartOfLine()); 719 } 720 721 unsigned Lexer::getTokenPrefixLength(SourceLocation TokStart, unsigned CharNo, 722 const SourceManager &SM, 723 const LangOptions &LangOpts) { 724 // Figure out how many physical characters away the specified expansion 725 // character is. This needs to take into consideration newlines and 726 // trigraphs. 727 bool Invalid = false; 728 const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 729 730 // If they request the first char of the token, we're trivially done. 731 if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 732 return 0; 733 734 unsigned PhysOffset = 0; 735 736 // The usual case is that tokens don't contain anything interesting. Skip 737 // over the uninteresting characters. If a token only consists of simple 738 // chars, this method is extremely fast. 739 while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 740 if (CharNo == 0) 741 return PhysOffset; 742 ++TokPtr; 743 --CharNo; 744 ++PhysOffset; 745 } 746 747 // If we have a character that may be a trigraph or escaped newline, use a 748 // lexer to parse it correctly. 749 for (; CharNo; --CharNo) { 750 unsigned Size; 751 Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 752 TokPtr += Size; 753 PhysOffset += Size; 754 } 755 756 // Final detail: if we end up on an escaped newline, we want to return the 757 // location of the actual byte of the token. For example foo\<newline>bar 758 // advanced by 3 should return the location of b, not of \\. One compounding 759 // detail of this is that the escape may be made by a trigraph. 760 if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 761 PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 762 763 return PhysOffset; 764 } 765 766 /// Computes the source location just past the end of the 767 /// token at this source location. 768 /// 769 /// This routine can be used to produce a source location that 770 /// points just past the end of the token referenced by \p Loc, and 771 /// is generally used when a diagnostic needs to point just after a 772 /// token where it expected something different that it received. If 773 /// the returned source location would not be meaningful (e.g., if 774 /// it points into a macro), this routine returns an invalid 775 /// source location. 776 /// 777 /// \param Offset an offset from the end of the token, where the source 778 /// location should refer to. The default offset (0) produces a source 779 /// location pointing just past the end of the token; an offset of 1 produces 780 /// a source location pointing to the last character in the token, etc. 781 SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 782 const SourceManager &SM, 783 const LangOptions &LangOpts) { 784 if (Loc.isInvalid()) 785 return {}; 786 787 if (Loc.isMacroID()) { 788 if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 789 return {}; // Points inside the macro expansion. 790 } 791 792 unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 793 if (Len > Offset) 794 Len = Len - Offset; 795 else 796 return Loc; 797 798 return Loc.getLocWithOffset(Len); 799 } 800 801 /// Returns true if the given MacroID location points at the first 802 /// token of the macro expansion. 803 bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 804 const SourceManager &SM, 805 const LangOptions &LangOpts, 806 SourceLocation *MacroBegin) { 807 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 808 809 SourceLocation expansionLoc; 810 if (!SM.isAtStartOfImmediateMacroExpansion(loc, &expansionLoc)) 811 return false; 812 813 if (expansionLoc.isFileID()) { 814 // No other macro expansions, this is the first. 815 if (MacroBegin) 816 *MacroBegin = expansionLoc; 817 return true; 818 } 819 820 return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 821 } 822 823 /// Returns true if the given MacroID location points at the last 824 /// token of the macro expansion. 825 bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 826 const SourceManager &SM, 827 const LangOptions &LangOpts, 828 SourceLocation *MacroEnd) { 829 assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 830 831 SourceLocation spellLoc = SM.getSpellingLoc(loc); 832 unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 833 if (tokLen == 0) 834 return false; 835 836 SourceLocation afterLoc = loc.getLocWithOffset(tokLen); 837 SourceLocation expansionLoc; 838 if (!SM.isAtEndOfImmediateMacroExpansion(afterLoc, &expansionLoc)) 839 return false; 840 841 if (expansionLoc.isFileID()) { 842 // No other macro expansions. 843 if (MacroEnd) 844 *MacroEnd = expansionLoc; 845 return true; 846 } 847 848 return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 849 } 850 851 static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 852 const SourceManager &SM, 853 const LangOptions &LangOpts) { 854 SourceLocation Begin = Range.getBegin(); 855 SourceLocation End = Range.getEnd(); 856 assert(Begin.isFileID() && End.isFileID()); 857 if (Range.isTokenRange()) { 858 End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 859 if (End.isInvalid()) 860 return {}; 861 } 862 863 // Break down the source locations. 864 FileID FID; 865 unsigned BeginOffs; 866 std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 867 if (FID.isInvalid()) 868 return {}; 869 870 unsigned EndOffs; 871 if (!SM.isInFileID(End, FID, &EndOffs) || 872 BeginOffs > EndOffs) 873 return {}; 874 875 return CharSourceRange::getCharRange(Begin, End); 876 } 877 878 CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 879 const SourceManager &SM, 880 const LangOptions &LangOpts) { 881 SourceLocation Begin = Range.getBegin(); 882 SourceLocation End = Range.getEnd(); 883 if (Begin.isInvalid() || End.isInvalid()) 884 return {}; 885 886 if (Begin.isFileID() && End.isFileID()) 887 return makeRangeFromFileLocs(Range, SM, LangOpts); 888 889 if (Begin.isMacroID() && End.isFileID()) { 890 if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 891 return {}; 892 Range.setBegin(Begin); 893 return makeRangeFromFileLocs(Range, SM, LangOpts); 894 } 895 896 if (Begin.isFileID() && End.isMacroID()) { 897 if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 898 &End)) || 899 (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 900 &End))) 901 return {}; 902 Range.setEnd(End); 903 return makeRangeFromFileLocs(Range, SM, LangOpts); 904 } 905 906 assert(Begin.isMacroID() && End.isMacroID()); 907 SourceLocation MacroBegin, MacroEnd; 908 if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 909 ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 910 &MacroEnd)) || 911 (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 912 &MacroEnd)))) { 913 Range.setBegin(MacroBegin); 914 Range.setEnd(MacroEnd); 915 return makeRangeFromFileLocs(Range, SM, LangOpts); 916 } 917 918 bool Invalid = false; 919 const SrcMgr::SLocEntry &BeginEntry = SM.getSLocEntry(SM.getFileID(Begin), 920 &Invalid); 921 if (Invalid) 922 return {}; 923 924 if (BeginEntry.getExpansion().isMacroArgExpansion()) { 925 const SrcMgr::SLocEntry &EndEntry = SM.getSLocEntry(SM.getFileID(End), 926 &Invalid); 927 if (Invalid) 928 return {}; 929 930 if (EndEntry.getExpansion().isMacroArgExpansion() && 931 BeginEntry.getExpansion().getExpansionLocStart() == 932 EndEntry.getExpansion().getExpansionLocStart()) { 933 Range.setBegin(SM.getImmediateSpellingLoc(Begin)); 934 Range.setEnd(SM.getImmediateSpellingLoc(End)); 935 return makeFileCharRange(Range, SM, LangOpts); 936 } 937 } 938 939 return {}; 940 } 941 942 StringRef Lexer::getSourceText(CharSourceRange Range, 943 const SourceManager &SM, 944 const LangOptions &LangOpts, 945 bool *Invalid) { 946 Range = makeFileCharRange(Range, SM, LangOpts); 947 if (Range.isInvalid()) { 948 if (Invalid) *Invalid = true; 949 return {}; 950 } 951 952 // Break down the source location. 953 std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 954 if (beginInfo.first.isInvalid()) { 955 if (Invalid) *Invalid = true; 956 return {}; 957 } 958 959 unsigned EndOffs; 960 if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 961 beginInfo.second > EndOffs) { 962 if (Invalid) *Invalid = true; 963 return {}; 964 } 965 966 // Try to the load the file buffer. 967 bool invalidTemp = false; 968 StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 969 if (invalidTemp) { 970 if (Invalid) *Invalid = true; 971 return {}; 972 } 973 974 if (Invalid) *Invalid = false; 975 return file.substr(beginInfo.second, EndOffs - beginInfo.second); 976 } 977 978 StringRef Lexer::getImmediateMacroName(SourceLocation Loc, 979 const SourceManager &SM, 980 const LangOptions &LangOpts) { 981 assert(Loc.isMacroID() && "Only reasonable to call this on macros"); 982 983 // Find the location of the immediate macro expansion. 984 while (true) { 985 FileID FID = SM.getFileID(Loc); 986 const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 987 const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 988 Loc = Expansion.getExpansionLocStart(); 989 if (!Expansion.isMacroArgExpansion()) 990 break; 991 992 // For macro arguments we need to check that the argument did not come 993 // from an inner macro, e.g: "MAC1( MAC2(foo) )" 994 995 // Loc points to the argument id of the macro definition, move to the 996 // macro expansion. 997 Loc = SM.getImmediateExpansionRange(Loc).getBegin(); 998 SourceLocation SpellLoc = Expansion.getSpellingLoc(); 999 if (SpellLoc.isFileID()) 1000 break; // No inner macro. 1001 1002 // If spelling location resides in the same FileID as macro expansion 1003 // location, it means there is no inner macro. 1004 FileID MacroFID = SM.getFileID(Loc); 1005 if (SM.isInFileID(SpellLoc, MacroFID)) 1006 break; 1007 1008 // Argument came from inner macro. 1009 Loc = SpellLoc; 1010 } 1011 1012 // Find the spelling location of the start of the non-argument expansion 1013 // range. This is where the macro name was spelled in order to begin 1014 // expanding this macro. 1015 Loc = SM.getSpellingLoc(Loc); 1016 1017 // Dig out the buffer where the macro name was spelled and the extents of the 1018 // name so that we can render it into the expansion note. 1019 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1020 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1021 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1022 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1023 } 1024 1025 StringRef Lexer::getImmediateMacroNameForDiagnostics( 1026 SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts) { 1027 assert(Loc.isMacroID() && "Only reasonable to call this on macros"); 1028 // Walk past macro argument expansions. 1029 while (SM.isMacroArgExpansion(Loc)) 1030 Loc = SM.getImmediateExpansionRange(Loc).getBegin(); 1031 1032 // If the macro's spelling has no FileID, then it's actually a token paste 1033 // or stringization (or similar) and not a macro at all. 1034 if (!SM.getFileEntryForID(SM.getFileID(SM.getSpellingLoc(Loc)))) 1035 return {}; 1036 1037 // Find the spelling location of the start of the non-argument expansion 1038 // range. This is where the macro name was spelled in order to begin 1039 // expanding this macro. 1040 Loc = SM.getSpellingLoc(SM.getImmediateExpansionRange(Loc).getBegin()); 1041 1042 // Dig out the buffer where the macro name was spelled and the extents of the 1043 // name so that we can render it into the expansion note. 1044 std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1045 unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1046 StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1047 return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1048 } 1049 1050 bool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1051 return isIdentifierBody(c, LangOpts.DollarIdents); 1052 } 1053 1054 bool Lexer::isNewLineEscaped(const char *BufferStart, const char *Str) { 1055 assert(isVerticalWhitespace(Str[0])); 1056 if (Str - 1 < BufferStart) 1057 return false; 1058 1059 if ((Str[0] == '\n' && Str[-1] == '\r') || 1060 (Str[0] == '\r' && Str[-1] == '\n')) { 1061 if (Str - 2 < BufferStart) 1062 return false; 1063 --Str; 1064 } 1065 --Str; 1066 1067 // Rewind to first non-space character: 1068 while (Str > BufferStart && isHorizontalWhitespace(*Str)) 1069 --Str; 1070 1071 return *Str == '\\'; 1072 } 1073 1074 StringRef Lexer::getIndentationForLine(SourceLocation Loc, 1075 const SourceManager &SM) { 1076 if (Loc.isInvalid() || Loc.isMacroID()) 1077 return {}; 1078 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1079 if (LocInfo.first.isInvalid()) 1080 return {}; 1081 bool Invalid = false; 1082 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 1083 if (Invalid) 1084 return {}; 1085 const char *Line = findBeginningOfLine(Buffer, LocInfo.second); 1086 if (!Line) 1087 return {}; 1088 StringRef Rest = Buffer.substr(Line - Buffer.data()); 1089 size_t NumWhitespaceChars = Rest.find_first_not_of(" \t"); 1090 return NumWhitespaceChars == StringRef::npos 1091 ? "" 1092 : Rest.take_front(NumWhitespaceChars); 1093 } 1094 1095 //===----------------------------------------------------------------------===// 1096 // Diagnostics forwarding code. 1097 //===----------------------------------------------------------------------===// 1098 1099 /// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1100 /// lexer buffer was all expanded at a single point, perform the mapping. 1101 /// This is currently only used for _Pragma implementation, so it is the slow 1102 /// path of the hot getSourceLocation method. Do not allow it to be inlined. 1103 static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 1104 Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1105 static SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1106 SourceLocation FileLoc, 1107 unsigned CharNo, unsigned TokLen) { 1108 assert(FileLoc.isMacroID() && "Must be a macro expansion"); 1109 1110 // Otherwise, we're lexing "mapped tokens". This is used for things like 1111 // _Pragma handling. Combine the expansion location of FileLoc with the 1112 // spelling location. 1113 SourceManager &SM = PP.getSourceManager(); 1114 1115 // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1116 // characters come from spelling(FileLoc)+Offset. 1117 SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1118 SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 1119 1120 // Figure out the expansion loc range, which is the range covered by the 1121 // original _Pragma(...) sequence. 1122 CharSourceRange II = SM.getImmediateExpansionRange(FileLoc); 1123 1124 return SM.createExpansionLoc(SpellingLoc, II.getBegin(), II.getEnd(), TokLen); 1125 } 1126 1127 /// getSourceLocation - Return a source location identifier for the specified 1128 /// offset in the current file. 1129 SourceLocation Lexer::getSourceLocation(const char *Loc, 1130 unsigned TokLen) const { 1131 assert(Loc >= BufferStart && Loc <= BufferEnd && 1132 "Location out of range for this buffer!"); 1133 1134 // In the normal case, we're just lexing from a simple file buffer, return 1135 // the file id from FileLoc with the offset specified. 1136 unsigned CharNo = Loc-BufferStart; 1137 if (FileLoc.isFileID()) 1138 return FileLoc.getLocWithOffset(CharNo); 1139 1140 // Otherwise, this is the _Pragma lexer case, which pretends that all of the 1141 // tokens are lexed from where the _Pragma was defined. 1142 assert(PP && "This doesn't work on raw lexers"); 1143 return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 1144 } 1145 1146 /// Diag - Forwarding function for diagnostics. This translate a source 1147 /// position in the current buffer into a SourceLocation object for rendering. 1148 DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 1149 return PP->Diag(getSourceLocation(Loc), DiagID); 1150 } 1151 1152 //===----------------------------------------------------------------------===// 1153 // Trigraph and Escaped Newline Handling Code. 1154 //===----------------------------------------------------------------------===// 1155 1156 /// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 1157 /// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 1158 static char GetTrigraphCharForLetter(char Letter) { 1159 switch (Letter) { 1160 default: return 0; 1161 case '=': return '#'; 1162 case ')': return ']'; 1163 case '(': return '['; 1164 case '!': return '|'; 1165 case '\'': return '^'; 1166 case '>': return '}'; 1167 case '/': return '\\'; 1168 case '<': return '{'; 1169 case '-': return '~'; 1170 } 1171 } 1172 1173 /// DecodeTrigraphChar - If the specified character is a legal trigraph when 1174 /// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 1175 /// return the result character. Finally, emit a warning about trigraph use 1176 /// whether trigraphs are enabled or not. 1177 static char DecodeTrigraphChar(const char *CP, Lexer *L) { 1178 char Res = GetTrigraphCharForLetter(*CP); 1179 if (!Res || !L) return Res; 1180 1181 if (!L->getLangOpts().Trigraphs) { 1182 if (!L->isLexingRawMode()) 1183 L->Diag(CP-2, diag::trigraph_ignored); 1184 return 0; 1185 } 1186 1187 if (!L->isLexingRawMode()) 1188 L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 1189 return Res; 1190 } 1191 1192 /// getEscapedNewLineSize - Return the size of the specified escaped newline, 1193 /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 1194 /// trigraph equivalent on entry to this function. 1195 unsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 1196 unsigned Size = 0; 1197 while (isWhitespace(Ptr[Size])) { 1198 ++Size; 1199 1200 if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 1201 continue; 1202 1203 // If this is a \r\n or \n\r, skip the other half. 1204 if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 1205 Ptr[Size-1] != Ptr[Size]) 1206 ++Size; 1207 1208 return Size; 1209 } 1210 1211 // Not an escaped newline, must be a \t or something else. 1212 return 0; 1213 } 1214 1215 /// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1216 /// them), skip over them and return the first non-escaped-newline found, 1217 /// otherwise return P. 1218 const char *Lexer::SkipEscapedNewLines(const char *P) { 1219 while (true) { 1220 const char *AfterEscape; 1221 if (*P == '\\') { 1222 AfterEscape = P+1; 1223 } else if (*P == '?') { 1224 // If not a trigraph for escape, bail out. 1225 if (P[1] != '?' || P[2] != '/') 1226 return P; 1227 // FIXME: Take LangOpts into account; the language might not 1228 // support trigraphs. 1229 AfterEscape = P+3; 1230 } else { 1231 return P; 1232 } 1233 1234 unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1235 if (NewLineSize == 0) return P; 1236 P = AfterEscape+NewLineSize; 1237 } 1238 } 1239 1240 Optional<Token> Lexer::findNextToken(SourceLocation Loc, 1241 const SourceManager &SM, 1242 const LangOptions &LangOpts) { 1243 if (Loc.isMacroID()) { 1244 if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1245 return None; 1246 } 1247 Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1248 1249 // Break down the source location. 1250 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1251 1252 // Try to load the file buffer. 1253 bool InvalidTemp = false; 1254 StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1255 if (InvalidTemp) 1256 return None; 1257 1258 const char *TokenBegin = File.data() + LocInfo.second; 1259 1260 // Lex from the start of the given location. 1261 Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1262 TokenBegin, File.end()); 1263 // Find the token. 1264 Token Tok; 1265 lexer.LexFromRawLexer(Tok); 1266 return Tok; 1267 } 1268 1269 /// Checks that the given token is the first token that occurs after the 1270 /// given location (this excludes comments and whitespace). Returns the location 1271 /// immediately after the specified token. If the token is not found or the 1272 /// location is inside a macro, the returned source location will be invalid. 1273 SourceLocation Lexer::findLocationAfterToken( 1274 SourceLocation Loc, tok::TokenKind TKind, const SourceManager &SM, 1275 const LangOptions &LangOpts, bool SkipTrailingWhitespaceAndNewLine) { 1276 Optional<Token> Tok = findNextToken(Loc, SM, LangOpts); 1277 if (!Tok || Tok->isNot(TKind)) 1278 return {}; 1279 SourceLocation TokenLoc = Tok->getLocation(); 1280 1281 // Calculate how much whitespace needs to be skipped if any. 1282 unsigned NumWhitespaceChars = 0; 1283 if (SkipTrailingWhitespaceAndNewLine) { 1284 const char *TokenEnd = SM.getCharacterData(TokenLoc) + Tok->getLength(); 1285 unsigned char C = *TokenEnd; 1286 while (isHorizontalWhitespace(C)) { 1287 C = *(++TokenEnd); 1288 NumWhitespaceChars++; 1289 } 1290 1291 // Skip \r, \n, \r\n, or \n\r 1292 if (C == '\n' || C == '\r') { 1293 char PrevC = C; 1294 C = *(++TokenEnd); 1295 NumWhitespaceChars++; 1296 if ((C == '\n' || C == '\r') && C != PrevC) 1297 NumWhitespaceChars++; 1298 } 1299 } 1300 1301 return TokenLoc.getLocWithOffset(Tok->getLength() + NumWhitespaceChars); 1302 } 1303 1304 /// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 1305 /// get its size, and return it. This is tricky in several cases: 1306 /// 1. If currently at the start of a trigraph, we warn about the trigraph, 1307 /// then either return the trigraph (skipping 3 chars) or the '?', 1308 /// depending on whether trigraphs are enabled or not. 1309 /// 2. If this is an escaped newline (potentially with whitespace between 1310 /// the backslash and newline), implicitly skip the newline and return 1311 /// the char after it. 1312 /// 1313 /// This handles the slow/uncommon case of the getCharAndSize method. Here we 1314 /// know that we can accumulate into Size, and that we have already incremented 1315 /// Ptr by Size bytes. 1316 /// 1317 /// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 1318 /// be updated to match. 1319 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1320 Token *Tok) { 1321 // If we have a slash, look for an escaped newline. 1322 if (Ptr[0] == '\\') { 1323 ++Size; 1324 ++Ptr; 1325 Slash: 1326 // Common case, backslash-char where the char is not whitespace. 1327 if (!isWhitespace(Ptr[0])) return '\\'; 1328 1329 // See if we have optional whitespace characters between the slash and 1330 // newline. 1331 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1332 // Remember that this token needs to be cleaned. 1333 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1334 1335 // Warn if there was whitespace between the backslash and newline. 1336 if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 1337 Diag(Ptr, diag::backslash_newline_space); 1338 1339 // Found backslash<whitespace><newline>. Parse the char after it. 1340 Size += EscapedNewLineSize; 1341 Ptr += EscapedNewLineSize; 1342 1343 // Use slow version to accumulate a correct size field. 1344 return getCharAndSizeSlow(Ptr, Size, Tok); 1345 } 1346 1347 // Otherwise, this is not an escaped newline, just return the slash. 1348 return '\\'; 1349 } 1350 1351 // If this is a trigraph, process it. 1352 if (Ptr[0] == '?' && Ptr[1] == '?') { 1353 // If this is actually a legal trigraph (not something like "??x"), emit 1354 // a trigraph warning. If so, and if trigraphs are enabled, return it. 1355 if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : nullptr)) { 1356 // Remember that this token needs to be cleaned. 1357 if (Tok) Tok->setFlag(Token::NeedsCleaning); 1358 1359 Ptr += 3; 1360 Size += 3; 1361 if (C == '\\') goto Slash; 1362 return C; 1363 } 1364 } 1365 1366 // If this is neither, return a single character. 1367 ++Size; 1368 return *Ptr; 1369 } 1370 1371 /// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 1372 /// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 1373 /// and that we have already incremented Ptr by Size bytes. 1374 /// 1375 /// NOTE: When this method is updated, getCharAndSizeSlow (above) should 1376 /// be updated to match. 1377 char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 1378 const LangOptions &LangOpts) { 1379 // If we have a slash, look for an escaped newline. 1380 if (Ptr[0] == '\\') { 1381 ++Size; 1382 ++Ptr; 1383 Slash: 1384 // Common case, backslash-char where the char is not whitespace. 1385 if (!isWhitespace(Ptr[0])) return '\\'; 1386 1387 // See if we have optional whitespace characters followed by a newline. 1388 if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 1389 // Found backslash<whitespace><newline>. Parse the char after it. 1390 Size += EscapedNewLineSize; 1391 Ptr += EscapedNewLineSize; 1392 1393 // Use slow version to accumulate a correct size field. 1394 return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 1395 } 1396 1397 // Otherwise, this is not an escaped newline, just return the slash. 1398 return '\\'; 1399 } 1400 1401 // If this is a trigraph, process it. 1402 if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 1403 // If this is actually a legal trigraph (not something like "??x"), return 1404 // it. 1405 if (char C = GetTrigraphCharForLetter(Ptr[2])) { 1406 Ptr += 3; 1407 Size += 3; 1408 if (C == '\\') goto Slash; 1409 return C; 1410 } 1411 } 1412 1413 // If this is neither, return a single character. 1414 ++Size; 1415 return *Ptr; 1416 } 1417 1418 //===----------------------------------------------------------------------===// 1419 // Helper methods for lexing. 1420 //===----------------------------------------------------------------------===// 1421 1422 /// Routine that indiscriminately sets the offset into the source file. 1423 void Lexer::SetByteOffset(unsigned Offset, bool StartOfLine) { 1424 BufferPtr = BufferStart + Offset; 1425 if (BufferPtr > BufferEnd) 1426 BufferPtr = BufferEnd; 1427 // FIXME: What exactly does the StartOfLine bit mean? There are two 1428 // possible meanings for the "start" of the line: the first token on the 1429 // unexpanded line, or the first token on the expanded line. 1430 IsAtStartOfLine = StartOfLine; 1431 IsAtPhysicalStartOfLine = StartOfLine; 1432 } 1433 1434 static bool isAllowedIDChar(uint32_t C, const LangOptions &LangOpts) { 1435 if (LangOpts.AsmPreprocessor) { 1436 return false; 1437 } else if (LangOpts.DollarIdents && '$' == C) { 1438 return true; 1439 } else if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1440 static const llvm::sys::UnicodeCharSet C11AllowedIDChars( 1441 C11AllowedIDCharRanges); 1442 return C11AllowedIDChars.contains(C); 1443 } else if (LangOpts.CPlusPlus) { 1444 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1445 CXX03AllowedIDCharRanges); 1446 return CXX03AllowedIDChars.contains(C); 1447 } else { 1448 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1449 C99AllowedIDCharRanges); 1450 return C99AllowedIDChars.contains(C); 1451 } 1452 } 1453 1454 static bool isAllowedInitiallyIDChar(uint32_t C, const LangOptions &LangOpts) { 1455 assert(isAllowedIDChar(C, LangOpts)); 1456 if (LangOpts.AsmPreprocessor) { 1457 return false; 1458 } else if (LangOpts.CPlusPlus11 || LangOpts.C11) { 1459 static const llvm::sys::UnicodeCharSet C11DisallowedInitialIDChars( 1460 C11DisallowedInitialIDCharRanges); 1461 return !C11DisallowedInitialIDChars.contains(C); 1462 } else if (LangOpts.CPlusPlus) { 1463 return true; 1464 } else { 1465 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1466 C99DisallowedInitialIDCharRanges); 1467 return !C99DisallowedInitialIDChars.contains(C); 1468 } 1469 } 1470 1471 static inline CharSourceRange makeCharRange(Lexer &L, const char *Begin, 1472 const char *End) { 1473 return CharSourceRange::getCharRange(L.getSourceLocation(Begin), 1474 L.getSourceLocation(End)); 1475 } 1476 1477 static void maybeDiagnoseIDCharCompat(DiagnosticsEngine &Diags, uint32_t C, 1478 CharSourceRange Range, bool IsFirst) { 1479 // Check C99 compatibility. 1480 if (!Diags.isIgnored(diag::warn_c99_compat_unicode_id, Range.getBegin())) { 1481 enum { 1482 CannotAppearInIdentifier = 0, 1483 CannotStartIdentifier 1484 }; 1485 1486 static const llvm::sys::UnicodeCharSet C99AllowedIDChars( 1487 C99AllowedIDCharRanges); 1488 static const llvm::sys::UnicodeCharSet C99DisallowedInitialIDChars( 1489 C99DisallowedInitialIDCharRanges); 1490 if (!C99AllowedIDChars.contains(C)) { 1491 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1492 << Range 1493 << CannotAppearInIdentifier; 1494 } else if (IsFirst && C99DisallowedInitialIDChars.contains(C)) { 1495 Diags.Report(Range.getBegin(), diag::warn_c99_compat_unicode_id) 1496 << Range 1497 << CannotStartIdentifier; 1498 } 1499 } 1500 1501 // Check C++98 compatibility. 1502 if (!Diags.isIgnored(diag::warn_cxx98_compat_unicode_id, Range.getBegin())) { 1503 static const llvm::sys::UnicodeCharSet CXX03AllowedIDChars( 1504 CXX03AllowedIDCharRanges); 1505 if (!CXX03AllowedIDChars.contains(C)) { 1506 Diags.Report(Range.getBegin(), diag::warn_cxx98_compat_unicode_id) 1507 << Range; 1508 } 1509 } 1510 } 1511 1512 /// After encountering UTF-8 character C and interpreting it as an identifier 1513 /// character, check whether it's a homoglyph for a common non-identifier 1514 /// source character that is unlikely to be an intentional identifier 1515 /// character and warn if so. 1516 static void maybeDiagnoseUTF8Homoglyph(DiagnosticsEngine &Diags, uint32_t C, 1517 CharSourceRange Range) { 1518 // FIXME: Handle Unicode quotation marks (smart quotes, fullwidth quotes). 1519 struct HomoglyphPair { 1520 uint32_t Character; 1521 char LooksLike; 1522 bool operator<(HomoglyphPair R) const { return Character < R.Character; } 1523 }; 1524 static constexpr HomoglyphPair SortedHomoglyphs[] = { 1525 {U'\u00ad', 0}, // SOFT HYPHEN 1526 {U'\u01c3', '!'}, // LATIN LETTER RETROFLEX CLICK 1527 {U'\u037e', ';'}, // GREEK QUESTION MARK 1528 {U'\u200b', 0}, // ZERO WIDTH SPACE 1529 {U'\u200c', 0}, // ZERO WIDTH NON-JOINER 1530 {U'\u200d', 0}, // ZERO WIDTH JOINER 1531 {U'\u2060', 0}, // WORD JOINER 1532 {U'\u2061', 0}, // FUNCTION APPLICATION 1533 {U'\u2062', 0}, // INVISIBLE TIMES 1534 {U'\u2063', 0}, // INVISIBLE SEPARATOR 1535 {U'\u2064', 0}, // INVISIBLE PLUS 1536 {U'\u2212', '-'}, // MINUS SIGN 1537 {U'\u2215', '/'}, // DIVISION SLASH 1538 {U'\u2216', '\\'}, // SET MINUS 1539 {U'\u2217', '*'}, // ASTERISK OPERATOR 1540 {U'\u2223', '|'}, // DIVIDES 1541 {U'\u2227', '^'}, // LOGICAL AND 1542 {U'\u2236', ':'}, // RATIO 1543 {U'\u223c', '~'}, // TILDE OPERATOR 1544 {U'\ua789', ':'}, // MODIFIER LETTER COLON 1545 {U'\ufeff', 0}, // ZERO WIDTH NO-BREAK SPACE 1546 {U'\uff01', '!'}, // FULLWIDTH EXCLAMATION MARK 1547 {U'\uff03', '#'}, // FULLWIDTH NUMBER SIGN 1548 {U'\uff04', '$'}, // FULLWIDTH DOLLAR SIGN 1549 {U'\uff05', '%'}, // FULLWIDTH PERCENT SIGN 1550 {U'\uff06', '&'}, // FULLWIDTH AMPERSAND 1551 {U'\uff08', '('}, // FULLWIDTH LEFT PARENTHESIS 1552 {U'\uff09', ')'}, // FULLWIDTH RIGHT PARENTHESIS 1553 {U'\uff0a', '*'}, // FULLWIDTH ASTERISK 1554 {U'\uff0b', '+'}, // FULLWIDTH ASTERISK 1555 {U'\uff0c', ','}, // FULLWIDTH COMMA 1556 {U'\uff0d', '-'}, // FULLWIDTH HYPHEN-MINUS 1557 {U'\uff0e', '.'}, // FULLWIDTH FULL STOP 1558 {U'\uff0f', '/'}, // FULLWIDTH SOLIDUS 1559 {U'\uff1a', ':'}, // FULLWIDTH COLON 1560 {U'\uff1b', ';'}, // FULLWIDTH SEMICOLON 1561 {U'\uff1c', '<'}, // FULLWIDTH LESS-THAN SIGN 1562 {U'\uff1d', '='}, // FULLWIDTH EQUALS SIGN 1563 {U'\uff1e', '>'}, // FULLWIDTH GREATER-THAN SIGN 1564 {U'\uff1f', '?'}, // FULLWIDTH QUESTION MARK 1565 {U'\uff20', '@'}, // FULLWIDTH COMMERCIAL AT 1566 {U'\uff3b', '['}, // FULLWIDTH LEFT SQUARE BRACKET 1567 {U'\uff3c', '\\'}, // FULLWIDTH REVERSE SOLIDUS 1568 {U'\uff3d', ']'}, // FULLWIDTH RIGHT SQUARE BRACKET 1569 {U'\uff3e', '^'}, // FULLWIDTH CIRCUMFLEX ACCENT 1570 {U'\uff5b', '{'}, // FULLWIDTH LEFT CURLY BRACKET 1571 {U'\uff5c', '|'}, // FULLWIDTH VERTICAL LINE 1572 {U'\uff5d', '}'}, // FULLWIDTH RIGHT CURLY BRACKET 1573 {U'\uff5e', '~'}, // FULLWIDTH TILDE 1574 {0, 0} 1575 }; 1576 auto Homoglyph = 1577 std::lower_bound(std::begin(SortedHomoglyphs), 1578 std::end(SortedHomoglyphs) - 1, HomoglyphPair{C, '\0'}); 1579 if (Homoglyph->Character == C) { 1580 llvm::SmallString<5> CharBuf; 1581 { 1582 llvm::raw_svector_ostream CharOS(CharBuf); 1583 llvm::write_hex(CharOS, C, llvm::HexPrintStyle::Upper, 4); 1584 } 1585 if (Homoglyph->LooksLike) { 1586 const char LooksLikeStr[] = {Homoglyph->LooksLike, 0}; 1587 Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_homoglyph) 1588 << Range << CharBuf << LooksLikeStr; 1589 } else { 1590 Diags.Report(Range.getBegin(), diag::warn_utf8_symbol_zero_width) 1591 << Range << CharBuf; 1592 } 1593 } 1594 } 1595 1596 bool Lexer::tryConsumeIdentifierUCN(const char *&CurPtr, unsigned Size, 1597 Token &Result) { 1598 const char *UCNPtr = CurPtr + Size; 1599 uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/nullptr); 1600 if (CodePoint == 0 || !isAllowedIDChar(CodePoint, LangOpts)) 1601 return false; 1602 1603 if (!isLexingRawMode()) 1604 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1605 makeCharRange(*this, CurPtr, UCNPtr), 1606 /*IsFirst=*/false); 1607 1608 Result.setFlag(Token::HasUCN); 1609 if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1610 (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1611 CurPtr = UCNPtr; 1612 else 1613 while (CurPtr != UCNPtr) 1614 (void)getAndAdvanceChar(CurPtr, Result); 1615 return true; 1616 } 1617 1618 bool Lexer::tryConsumeIdentifierUTF8Char(const char *&CurPtr) { 1619 const char *UnicodePtr = CurPtr; 1620 llvm::UTF32 CodePoint; 1621 llvm::ConversionResult Result = 1622 llvm::convertUTF8Sequence((const llvm::UTF8 **)&UnicodePtr, 1623 (const llvm::UTF8 *)BufferEnd, 1624 &CodePoint, 1625 llvm::strictConversion); 1626 if (Result != llvm::conversionOK || 1627 !isAllowedIDChar(static_cast<uint32_t>(CodePoint), LangOpts)) 1628 return false; 1629 1630 if (!isLexingRawMode()) { 1631 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), CodePoint, 1632 makeCharRange(*this, CurPtr, UnicodePtr), 1633 /*IsFirst=*/false); 1634 maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), CodePoint, 1635 makeCharRange(*this, CurPtr, UnicodePtr)); 1636 } 1637 1638 CurPtr = UnicodePtr; 1639 return true; 1640 } 1641 1642 bool Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 1643 // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 1644 unsigned Size; 1645 unsigned char C = *CurPtr++; 1646 while (isIdentifierBody(C)) 1647 C = *CurPtr++; 1648 1649 --CurPtr; // Back up over the skipped character. 1650 1651 // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 1652 // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1653 // 1654 // TODO: Could merge these checks into an InfoTable flag to make the 1655 // comparison cheaper 1656 if (isASCII(C) && C != '\\' && C != '?' && 1657 (C != '$' || !LangOpts.DollarIdents)) { 1658 FinishIdentifier: 1659 const char *IdStart = BufferPtr; 1660 FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1661 Result.setRawIdentifierData(IdStart); 1662 1663 // If we are in raw mode, return this identifier raw. There is no need to 1664 // look up identifier information or attempt to macro expand it. 1665 if (LexingRawMode) 1666 return true; 1667 1668 // Fill in Result.IdentifierInfo and update the token kind, 1669 // looking up the identifier in the identifier table. 1670 IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 1671 // Note that we have to call PP->LookUpIdentifierInfo() even for code 1672 // completion, it writes IdentifierInfo into Result, and callers rely on it. 1673 1674 // If the completion point is at the end of an identifier, we want to treat 1675 // the identifier as incomplete even if it resolves to a macro or a keyword. 1676 // This allows e.g. 'class^' to complete to 'classifier'. 1677 if (isCodeCompletionPoint(CurPtr)) { 1678 // Return the code-completion token. 1679 Result.setKind(tok::code_completion); 1680 // Skip the code-completion char and all immediate identifier characters. 1681 // This ensures we get consistent behavior when completing at any point in 1682 // an identifier (i.e. at the start, in the middle, at the end). Note that 1683 // only simple cases (i.e. [a-zA-Z0-9_]) are supported to keep the code 1684 // simpler. 1685 assert(*CurPtr == 0 && "Completion character must be 0"); 1686 ++CurPtr; 1687 // Note that code completion token is not added as a separate character 1688 // when the completion point is at the end of the buffer. Therefore, we need 1689 // to check if the buffer has ended. 1690 if (CurPtr < BufferEnd) { 1691 while (isIdentifierBody(*CurPtr)) 1692 ++CurPtr; 1693 } 1694 BufferPtr = CurPtr; 1695 return true; 1696 } 1697 1698 // Finally, now that we know we have an identifier, pass this off to the 1699 // preprocessor, which may macro expand it or something. 1700 if (II->isHandleIdentifierCase()) 1701 return PP->HandleIdentifier(Result); 1702 1703 return true; 1704 } 1705 1706 // Otherwise, $,\,? in identifier found. Enter slower path. 1707 1708 C = getCharAndSize(CurPtr, Size); 1709 while (true) { 1710 if (C == '$') { 1711 // If we hit a $ and they are not supported in identifiers, we are done. 1712 if (!LangOpts.DollarIdents) goto FinishIdentifier; 1713 1714 // Otherwise, emit a diagnostic and continue. 1715 if (!isLexingRawMode()) 1716 Diag(CurPtr, diag::ext_dollar_in_identifier); 1717 CurPtr = ConsumeChar(CurPtr, Size, Result); 1718 C = getCharAndSize(CurPtr, Size); 1719 continue; 1720 } else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) { 1721 C = getCharAndSize(CurPtr, Size); 1722 continue; 1723 } else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) { 1724 C = getCharAndSize(CurPtr, Size); 1725 continue; 1726 } else if (!isIdentifierBody(C)) { 1727 goto FinishIdentifier; 1728 } 1729 1730 // Otherwise, this character is good, consume it. 1731 CurPtr = ConsumeChar(CurPtr, Size, Result); 1732 1733 C = getCharAndSize(CurPtr, Size); 1734 while (isIdentifierBody(C)) { 1735 CurPtr = ConsumeChar(CurPtr, Size, Result); 1736 C = getCharAndSize(CurPtr, Size); 1737 } 1738 } 1739 } 1740 1741 /// isHexaLiteral - Return true if Start points to a hex constant. 1742 /// in microsoft mode (where this is supposed to be several different tokens). 1743 bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 1744 unsigned Size; 1745 char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 1746 if (C1 != '0') 1747 return false; 1748 char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 1749 return (C2 == 'x' || C2 == 'X'); 1750 } 1751 1752 /// LexNumericConstant - Lex the remainder of a integer or floating point 1753 /// constant. From[-1] is the first character lexed. Return the end of the 1754 /// constant. 1755 bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 1756 unsigned Size; 1757 char C = getCharAndSize(CurPtr, Size); 1758 char PrevCh = 0; 1759 while (isPreprocessingNumberBody(C)) { 1760 CurPtr = ConsumeChar(CurPtr, Size, Result); 1761 PrevCh = C; 1762 C = getCharAndSize(CurPtr, Size); 1763 } 1764 1765 // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1766 if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1767 // If we are in Microsoft mode, don't continue if the constant is hex. 1768 // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 1769 if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1770 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1771 } 1772 1773 // If we have a hex FP constant, continue. 1774 if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1775 // Outside C99 and C++17, we accept hexadecimal floating point numbers as a 1776 // not-quite-conforming extension. Only do so if this looks like it's 1777 // actually meant to be a hexfloat, and not if it has a ud-suffix. 1778 bool IsHexFloat = true; 1779 if (!LangOpts.C99) { 1780 if (!isHexaLiteral(BufferPtr, LangOpts)) 1781 IsHexFloat = false; 1782 else if (!getLangOpts().CPlusPlus17 && 1783 std::find(BufferPtr, CurPtr, '_') != CurPtr) 1784 IsHexFloat = false; 1785 } 1786 if (IsHexFloat) 1787 return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1788 } 1789 1790 // If we have a digit separator, continue. 1791 if (C == '\'' && getLangOpts().CPlusPlus14) { 1792 unsigned NextSize; 1793 char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts()); 1794 if (isIdentifierBody(Next)) { 1795 if (!isLexingRawMode()) 1796 Diag(CurPtr, diag::warn_cxx11_compat_digit_separator); 1797 CurPtr = ConsumeChar(CurPtr, Size, Result); 1798 CurPtr = ConsumeChar(CurPtr, NextSize, Result); 1799 return LexNumericConstant(Result, CurPtr); 1800 } 1801 } 1802 1803 // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue. 1804 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) 1805 return LexNumericConstant(Result, CurPtr); 1806 if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) 1807 return LexNumericConstant(Result, CurPtr); 1808 1809 // Update the location of token as well as BufferPtr. 1810 const char *TokStart = BufferPtr; 1811 FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 1812 Result.setLiteralData(TokStart); 1813 return true; 1814 } 1815 1816 /// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1817 /// in C++11, or warn on a ud-suffix in C++98. 1818 const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr, 1819 bool IsStringLiteral) { 1820 assert(getLangOpts().CPlusPlus); 1821 1822 // Maximally munch an identifier. 1823 unsigned Size; 1824 char C = getCharAndSize(CurPtr, Size); 1825 bool Consumed = false; 1826 1827 if (!isIdentifierHead(C)) { 1828 if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) 1829 Consumed = true; 1830 else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) 1831 Consumed = true; 1832 else 1833 return CurPtr; 1834 } 1835 1836 if (!getLangOpts().CPlusPlus11) { 1837 if (!isLexingRawMode()) 1838 Diag(CurPtr, 1839 C == '_' ? diag::warn_cxx11_compat_user_defined_literal 1840 : diag::warn_cxx11_compat_reserved_user_defined_literal) 1841 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1842 return CurPtr; 1843 } 1844 1845 // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 1846 // that does not start with an underscore is ill-formed. As a conforming 1847 // extension, we treat all such suffixes as if they had whitespace before 1848 // them. We assume a suffix beginning with a UCN or UTF-8 character is more 1849 // likely to be a ud-suffix than a macro, however, and accept that. 1850 if (!Consumed) { 1851 bool IsUDSuffix = false; 1852 if (C == '_') 1853 IsUDSuffix = true; 1854 else if (IsStringLiteral && getLangOpts().CPlusPlus14) { 1855 // In C++1y, we need to look ahead a few characters to see if this is a 1856 // valid suffix for a string literal or a numeric literal (this could be 1857 // the 'operator""if' defining a numeric literal operator). 1858 const unsigned MaxStandardSuffixLength = 3; 1859 char Buffer[MaxStandardSuffixLength] = { C }; 1860 unsigned Consumed = Size; 1861 unsigned Chars = 1; 1862 while (true) { 1863 unsigned NextSize; 1864 char Next = getCharAndSizeNoWarn(CurPtr + Consumed, NextSize, 1865 getLangOpts()); 1866 if (!isIdentifierBody(Next)) { 1867 // End of suffix. Check whether this is on the allowed list. 1868 const StringRef CompleteSuffix(Buffer, Chars); 1869 IsUDSuffix = StringLiteralParser::isValidUDSuffix(getLangOpts(), 1870 CompleteSuffix); 1871 break; 1872 } 1873 1874 if (Chars == MaxStandardSuffixLength) 1875 // Too long: can't be a standard suffix. 1876 break; 1877 1878 Buffer[Chars++] = Next; 1879 Consumed += NextSize; 1880 } 1881 } 1882 1883 if (!IsUDSuffix) { 1884 if (!isLexingRawMode()) 1885 Diag(CurPtr, getLangOpts().MSVCCompat 1886 ? diag::ext_ms_reserved_user_defined_literal 1887 : diag::ext_reserved_user_defined_literal) 1888 << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1889 return CurPtr; 1890 } 1891 1892 CurPtr = ConsumeChar(CurPtr, Size, Result); 1893 } 1894 1895 Result.setFlag(Token::HasUDSuffix); 1896 while (true) { 1897 C = getCharAndSize(CurPtr, Size); 1898 if (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); } 1899 else if (C == '\\' && tryConsumeIdentifierUCN(CurPtr, Size, Result)) {} 1900 else if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurPtr)) {} 1901 else break; 1902 } 1903 1904 return CurPtr; 1905 } 1906 1907 /// LexStringLiteral - Lex the remainder of a string literal, after having lexed 1908 /// either " or L" or u8" or u" or U". 1909 bool Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 1910 tok::TokenKind Kind) { 1911 const char *AfterQuote = CurPtr; 1912 // Does this string contain the \0 character? 1913 const char *NulCharacter = nullptr; 1914 1915 if (!isLexingRawMode() && 1916 (Kind == tok::utf8_string_literal || 1917 Kind == tok::utf16_string_literal || 1918 Kind == tok::utf32_string_literal)) 1919 Diag(BufferPtr, getLangOpts().CPlusPlus 1920 ? diag::warn_cxx98_compat_unicode_literal 1921 : diag::warn_c99_compat_unicode_literal); 1922 1923 char C = getAndAdvanceChar(CurPtr, Result); 1924 while (C != '"') { 1925 // Skip escaped characters. Escaped newlines will already be processed by 1926 // getAndAdvanceChar. 1927 if (C == '\\') 1928 C = getAndAdvanceChar(CurPtr, Result); 1929 1930 if (C == '\n' || C == '\r' || // Newline. 1931 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 1932 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1933 Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 1; 1934 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1935 return true; 1936 } 1937 1938 if (C == 0) { 1939 if (isCodeCompletionPoint(CurPtr-1)) { 1940 if (ParsingFilename) 1941 codeCompleteIncludedFile(AfterQuote, CurPtr - 1, /*IsAngled=*/false); 1942 else 1943 PP->CodeCompleteNaturalLanguage(); 1944 FormTokenWithChars(Result, CurPtr - 1, tok::unknown); 1945 cutOffLexing(); 1946 return true; 1947 } 1948 1949 NulCharacter = CurPtr-1; 1950 } 1951 C = getAndAdvanceChar(CurPtr, Result); 1952 } 1953 1954 // If we are in C++11, lex the optional ud-suffix. 1955 if (getLangOpts().CPlusPlus) 1956 CurPtr = LexUDSuffix(Result, CurPtr, true); 1957 1958 // If a nul character existed in the string, warn about it. 1959 if (NulCharacter && !isLexingRawMode()) 1960 Diag(NulCharacter, diag::null_in_char_or_string) << 1; 1961 1962 // Update the location of the token as well as the BufferPtr instance var. 1963 const char *TokStart = BufferPtr; 1964 FormTokenWithChars(Result, CurPtr, Kind); 1965 Result.setLiteralData(TokStart); 1966 return true; 1967 } 1968 1969 /// LexRawStringLiteral - Lex the remainder of a raw string literal, after 1970 /// having lexed R", LR", u8R", uR", or UR". 1971 bool Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 1972 tok::TokenKind Kind) { 1973 // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 1974 // Between the initial and final double quote characters of the raw string, 1975 // any transformations performed in phases 1 and 2 (trigraphs, 1976 // universal-character-names, and line splicing) are reverted. 1977 1978 if (!isLexingRawMode()) 1979 Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1980 1981 unsigned PrefixLen = 0; 1982 1983 while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 1984 ++PrefixLen; 1985 1986 // If the last character was not a '(', then we didn't lex a valid delimiter. 1987 if (CurPtr[PrefixLen] != '(') { 1988 if (!isLexingRawMode()) { 1989 const char *PrefixEnd = &CurPtr[PrefixLen]; 1990 if (PrefixLen == 16) { 1991 Diag(PrefixEnd, diag::err_raw_delim_too_long); 1992 } else { 1993 Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 1994 << StringRef(PrefixEnd, 1); 1995 } 1996 } 1997 1998 // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 1999 // it's possible the '"' was intended to be part of the raw string, but 2000 // there's not much we can do about that. 2001 while (true) { 2002 char C = *CurPtr++; 2003 2004 if (C == '"') 2005 break; 2006 if (C == 0 && CurPtr-1 == BufferEnd) { 2007 --CurPtr; 2008 break; 2009 } 2010 } 2011 2012 FormTokenWithChars(Result, CurPtr, tok::unknown); 2013 return true; 2014 } 2015 2016 // Save prefix and move CurPtr past it 2017 const char *Prefix = CurPtr; 2018 CurPtr += PrefixLen + 1; // skip over prefix and '(' 2019 2020 while (true) { 2021 char C = *CurPtr++; 2022 2023 if (C == ')') { 2024 // Check for prefix match and closing quote. 2025 if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 2026 CurPtr += PrefixLen + 1; // skip over prefix and '"' 2027 break; 2028 } 2029 } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 2030 if (!isLexingRawMode()) 2031 Diag(BufferPtr, diag::err_unterminated_raw_string) 2032 << StringRef(Prefix, PrefixLen); 2033 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 2034 return true; 2035 } 2036 } 2037 2038 // If we are in C++11, lex the optional ud-suffix. 2039 if (getLangOpts().CPlusPlus) 2040 CurPtr = LexUDSuffix(Result, CurPtr, true); 2041 2042 // Update the location of token as well as BufferPtr. 2043 const char *TokStart = BufferPtr; 2044 FormTokenWithChars(Result, CurPtr, Kind); 2045 Result.setLiteralData(TokStart); 2046 return true; 2047 } 2048 2049 /// LexAngledStringLiteral - Lex the remainder of an angled string literal, 2050 /// after having lexed the '<' character. This is used for #include filenames. 2051 bool Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 2052 // Does this string contain the \0 character? 2053 const char *NulCharacter = nullptr; 2054 const char *AfterLessPos = CurPtr; 2055 char C = getAndAdvanceChar(CurPtr, Result); 2056 while (C != '>') { 2057 // Skip escaped characters. Escaped newlines will already be processed by 2058 // getAndAdvanceChar. 2059 if (C == '\\') 2060 C = getAndAdvanceChar(CurPtr, Result); 2061 2062 if (C == '\n' || C == '\r' || // Newline. 2063 (C == 0 && (CurPtr - 1 == BufferEnd))) { // End of file. 2064 // If the filename is unterminated, then it must just be a lone < 2065 // character. Return this as such. 2066 FormTokenWithChars(Result, AfterLessPos, tok::less); 2067 return true; 2068 } 2069 2070 if (C == 0) { 2071 if (isCodeCompletionPoint(CurPtr - 1)) { 2072 codeCompleteIncludedFile(AfterLessPos, CurPtr - 1, /*IsAngled=*/true); 2073 cutOffLexing(); 2074 FormTokenWithChars(Result, CurPtr - 1, tok::unknown); 2075 return true; 2076 } 2077 NulCharacter = CurPtr-1; 2078 } 2079 C = getAndAdvanceChar(CurPtr, Result); 2080 } 2081 2082 // If a nul character existed in the string, warn about it. 2083 if (NulCharacter && !isLexingRawMode()) 2084 Diag(NulCharacter, diag::null_in_char_or_string) << 1; 2085 2086 // Update the location of token as well as BufferPtr. 2087 const char *TokStart = BufferPtr; 2088 FormTokenWithChars(Result, CurPtr, tok::header_name); 2089 Result.setLiteralData(TokStart); 2090 return true; 2091 } 2092 2093 void Lexer::codeCompleteIncludedFile(const char *PathStart, 2094 const char *CompletionPoint, 2095 bool IsAngled) { 2096 // Completion only applies to the filename, after the last slash. 2097 StringRef PartialPath(PathStart, CompletionPoint - PathStart); 2098 llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/"; 2099 auto Slash = PartialPath.find_last_of(SlashChars); 2100 StringRef Dir = 2101 (Slash == StringRef::npos) ? "" : PartialPath.take_front(Slash); 2102 const char *StartOfFilename = 2103 (Slash == StringRef::npos) ? PathStart : PathStart + Slash + 1; 2104 // Code completion filter range is the filename only, up to completion point. 2105 PP->setCodeCompletionIdentifierInfo(&PP->getIdentifierTable().get( 2106 StringRef(StartOfFilename, CompletionPoint - StartOfFilename))); 2107 // We should replace the characters up to the closing quote or closest slash, 2108 // if any. 2109 while (CompletionPoint < BufferEnd) { 2110 char Next = *(CompletionPoint + 1); 2111 if (Next == 0 || Next == '\r' || Next == '\n') 2112 break; 2113 ++CompletionPoint; 2114 if (Next == (IsAngled ? '>' : '"')) 2115 break; 2116 if (llvm::is_contained(SlashChars, Next)) 2117 break; 2118 } 2119 2120 PP->setCodeCompletionTokenRange( 2121 FileLoc.getLocWithOffset(StartOfFilename - BufferStart), 2122 FileLoc.getLocWithOffset(CompletionPoint - BufferStart)); 2123 PP->CodeCompleteIncludedFile(Dir, IsAngled); 2124 } 2125 2126 /// LexCharConstant - Lex the remainder of a character constant, after having 2127 /// lexed either ' or L' or u8' or u' or U'. 2128 bool Lexer::LexCharConstant(Token &Result, const char *CurPtr, 2129 tok::TokenKind Kind) { 2130 // Does this character contain the \0 character? 2131 const char *NulCharacter = nullptr; 2132 2133 if (!isLexingRawMode()) { 2134 if (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant) 2135 Diag(BufferPtr, getLangOpts().CPlusPlus 2136 ? diag::warn_cxx98_compat_unicode_literal 2137 : diag::warn_c99_compat_unicode_literal); 2138 else if (Kind == tok::utf8_char_constant) 2139 Diag(BufferPtr, diag::warn_cxx14_compat_u8_character_literal); 2140 } 2141 2142 char C = getAndAdvanceChar(CurPtr, Result); 2143 if (C == '\'') { 2144 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 2145 Diag(BufferPtr, diag::ext_empty_character); 2146 FormTokenWithChars(Result, CurPtr, tok::unknown); 2147 return true; 2148 } 2149 2150 while (C != '\'') { 2151 // Skip escaped characters. 2152 if (C == '\\') 2153 C = getAndAdvanceChar(CurPtr, Result); 2154 2155 if (C == '\n' || C == '\r' || // Newline. 2156 (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 2157 if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 2158 Diag(BufferPtr, diag::ext_unterminated_char_or_string) << 0; 2159 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 2160 return true; 2161 } 2162 2163 if (C == 0) { 2164 if (isCodeCompletionPoint(CurPtr-1)) { 2165 PP->CodeCompleteNaturalLanguage(); 2166 FormTokenWithChars(Result, CurPtr-1, tok::unknown); 2167 cutOffLexing(); 2168 return true; 2169 } 2170 2171 NulCharacter = CurPtr-1; 2172 } 2173 C = getAndAdvanceChar(CurPtr, Result); 2174 } 2175 2176 // If we are in C++11, lex the optional ud-suffix. 2177 if (getLangOpts().CPlusPlus) 2178 CurPtr = LexUDSuffix(Result, CurPtr, false); 2179 2180 // If a nul character existed in the character, warn about it. 2181 if (NulCharacter && !isLexingRawMode()) 2182 Diag(NulCharacter, diag::null_in_char_or_string) << 0; 2183 2184 // Update the location of token as well as BufferPtr. 2185 const char *TokStart = BufferPtr; 2186 FormTokenWithChars(Result, CurPtr, Kind); 2187 Result.setLiteralData(TokStart); 2188 return true; 2189 } 2190 2191 /// SkipWhitespace - Efficiently skip over a series of whitespace characters. 2192 /// Update BufferPtr to point to the next non-whitespace character and return. 2193 /// 2194 /// This method forms a token and returns true if KeepWhitespaceMode is enabled. 2195 bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr, 2196 bool &TokAtPhysicalStartOfLine) { 2197 // Whitespace - Skip it, then return the token after the whitespace. 2198 bool SawNewline = isVerticalWhitespace(CurPtr[-1]); 2199 2200 unsigned char Char = *CurPtr; 2201 2202 const char *lastNewLine = nullptr; 2203 auto setLastNewLine = [&](const char *Ptr) { 2204 lastNewLine = Ptr; 2205 if (!NewLinePtr) 2206 NewLinePtr = Ptr; 2207 }; 2208 if (SawNewline) 2209 setLastNewLine(CurPtr - 1); 2210 2211 // Skip consecutive spaces efficiently. 2212 while (true) { 2213 // Skip horizontal whitespace very aggressively. 2214 while (isHorizontalWhitespace(Char)) 2215 Char = *++CurPtr; 2216 2217 // Otherwise if we have something other than whitespace, we're done. 2218 if (!isVerticalWhitespace(Char)) 2219 break; 2220 2221 if (ParsingPreprocessorDirective) { 2222 // End of preprocessor directive line, let LexTokenInternal handle this. 2223 BufferPtr = CurPtr; 2224 return false; 2225 } 2226 2227 // OK, but handle newline. 2228 if (*CurPtr == '\n') 2229 setLastNewLine(CurPtr); 2230 SawNewline = true; 2231 Char = *++CurPtr; 2232 } 2233 2234 // If the client wants us to return whitespace, return it now. 2235 if (isKeepWhitespaceMode()) { 2236 FormTokenWithChars(Result, CurPtr, tok::unknown); 2237 if (SawNewline) { 2238 IsAtStartOfLine = true; 2239 IsAtPhysicalStartOfLine = true; 2240 } 2241 // FIXME: The next token will not have LeadingSpace set. 2242 return true; 2243 } 2244 2245 // If this isn't immediately after a newline, there is leading space. 2246 char PrevChar = CurPtr[-1]; 2247 bool HasLeadingSpace = !isVerticalWhitespace(PrevChar); 2248 2249 Result.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 2250 if (SawNewline) { 2251 Result.setFlag(Token::StartOfLine); 2252 TokAtPhysicalStartOfLine = true; 2253 2254 if (NewLinePtr && lastNewLine && NewLinePtr != lastNewLine && PP) { 2255 if (auto *Handler = PP->getEmptylineHandler()) 2256 Handler->HandleEmptyline(SourceRange(getSourceLocation(NewLinePtr + 1), 2257 getSourceLocation(lastNewLine))); 2258 } 2259 } 2260 2261 BufferPtr = CurPtr; 2262 return false; 2263 } 2264 2265 /// We have just read the // characters from input. Skip until we find the 2266 /// newline character that terminates the comment. Then update BufferPtr and 2267 /// return. 2268 /// 2269 /// If we're in KeepCommentMode or any CommentHandler has inserted 2270 /// some tokens, this will store the first token and return true. 2271 bool Lexer::SkipLineComment(Token &Result, const char *CurPtr, 2272 bool &TokAtPhysicalStartOfLine) { 2273 // If Line comments aren't explicitly enabled for this language, emit an 2274 // extension warning. 2275 if (!LangOpts.LineComment && !isLexingRawMode()) { 2276 Diag(BufferPtr, diag::ext_line_comment); 2277 2278 // Mark them enabled so we only emit one warning for this translation 2279 // unit. 2280 LangOpts.LineComment = true; 2281 } 2282 2283 // Scan over the body of the comment. The common case, when scanning, is that 2284 // the comment contains normal ascii characters with nothing interesting in 2285 // them. As such, optimize for this case with the inner loop. 2286 // 2287 // This loop terminates with CurPtr pointing at the newline (or end of buffer) 2288 // character that ends the line comment. 2289 char C; 2290 while (true) { 2291 C = *CurPtr; 2292 // Skip over characters in the fast loop. 2293 while (C != 0 && // Potentially EOF. 2294 C != '\n' && C != '\r') // Newline or DOS-style newline. 2295 C = *++CurPtr; 2296 2297 const char *NextLine = CurPtr; 2298 if (C != 0) { 2299 // We found a newline, see if it's escaped. 2300 const char *EscapePtr = CurPtr-1; 2301 bool HasSpace = false; 2302 while (isHorizontalWhitespace(*EscapePtr)) { // Skip whitespace. 2303 --EscapePtr; 2304 HasSpace = true; 2305 } 2306 2307 if (*EscapePtr == '\\') 2308 // Escaped newline. 2309 CurPtr = EscapePtr; 2310 else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 2311 EscapePtr[-2] == '?' && LangOpts.Trigraphs) 2312 // Trigraph-escaped newline. 2313 CurPtr = EscapePtr-2; 2314 else 2315 break; // This is a newline, we're done. 2316 2317 // If there was space between the backslash and newline, warn about it. 2318 if (HasSpace && !isLexingRawMode()) 2319 Diag(EscapePtr, diag::backslash_newline_space); 2320 } 2321 2322 // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 2323 // properly decode the character. Read it in raw mode to avoid emitting 2324 // diagnostics about things like trigraphs. If we see an escaped newline, 2325 // we'll handle it below. 2326 const char *OldPtr = CurPtr; 2327 bool OldRawMode = isLexingRawMode(); 2328 LexingRawMode = true; 2329 C = getAndAdvanceChar(CurPtr, Result); 2330 LexingRawMode = OldRawMode; 2331 2332 // If we only read only one character, then no special handling is needed. 2333 // We're done and can skip forward to the newline. 2334 if (C != 0 && CurPtr == OldPtr+1) { 2335 CurPtr = NextLine; 2336 break; 2337 } 2338 2339 // If we read multiple characters, and one of those characters was a \r or 2340 // \n, then we had an escaped newline within the comment. Emit diagnostic 2341 // unless the next line is also a // comment. 2342 if (CurPtr != OldPtr + 1 && C != '/' && 2343 (CurPtr == BufferEnd + 1 || CurPtr[0] != '/')) { 2344 for (; OldPtr != CurPtr; ++OldPtr) 2345 if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 2346 // Okay, we found a // comment that ends in a newline, if the next 2347 // line is also a // comment, but has spaces, don't emit a diagnostic. 2348 if (isWhitespace(C)) { 2349 const char *ForwardPtr = CurPtr; 2350 while (isWhitespace(*ForwardPtr)) // Skip whitespace. 2351 ++ForwardPtr; 2352 if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 2353 break; 2354 } 2355 2356 if (!isLexingRawMode()) 2357 Diag(OldPtr-1, diag::ext_multi_line_line_comment); 2358 break; 2359 } 2360 } 2361 2362 if (C == '\r' || C == '\n' || CurPtr == BufferEnd + 1) { 2363 --CurPtr; 2364 break; 2365 } 2366 2367 if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2368 PP->CodeCompleteNaturalLanguage(); 2369 cutOffLexing(); 2370 return false; 2371 } 2372 } 2373 2374 // Found but did not consume the newline. Notify comment handlers about the 2375 // comment unless we're in a #if 0 block. 2376 if (PP && !isLexingRawMode() && 2377 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2378 getSourceLocation(CurPtr)))) { 2379 BufferPtr = CurPtr; 2380 return true; // A token has to be returned. 2381 } 2382 2383 // If we are returning comments as tokens, return this comment as a token. 2384 if (inKeepCommentMode()) 2385 return SaveLineComment(Result, CurPtr); 2386 2387 // If we are inside a preprocessor directive and we see the end of line, 2388 // return immediately, so that the lexer can return this as an EOD token. 2389 if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 2390 BufferPtr = CurPtr; 2391 return false; 2392 } 2393 2394 // Otherwise, eat the \n character. We don't care if this is a \n\r or 2395 // \r\n sequence. This is an efficiency hack (because we know the \n can't 2396 // contribute to another token), it isn't needed for correctness. Note that 2397 // this is ok even in KeepWhitespaceMode, because we would have returned the 2398 /// comment above in that mode. 2399 NewLinePtr = CurPtr++; 2400 2401 // The next returned token is at the start of the line. 2402 Result.setFlag(Token::StartOfLine); 2403 TokAtPhysicalStartOfLine = true; 2404 // No leading whitespace seen so far. 2405 Result.clearFlag(Token::LeadingSpace); 2406 BufferPtr = CurPtr; 2407 return false; 2408 } 2409 2410 /// If in save-comment mode, package up this Line comment in an appropriate 2411 /// way and return it. 2412 bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 2413 // If we're not in a preprocessor directive, just return the // comment 2414 // directly. 2415 FormTokenWithChars(Result, CurPtr, tok::comment); 2416 2417 if (!ParsingPreprocessorDirective || LexingRawMode) 2418 return true; 2419 2420 // If this Line-style comment is in a macro definition, transmogrify it into 2421 // a C-style block comment. 2422 bool Invalid = false; 2423 std::string Spelling = PP->getSpelling(Result, &Invalid); 2424 if (Invalid) 2425 return true; 2426 2427 assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 2428 Spelling[1] = '*'; // Change prefix to "/*". 2429 Spelling += "*/"; // add suffix. 2430 2431 Result.setKind(tok::comment); 2432 PP->CreateString(Spelling, Result, 2433 Result.getLocation(), Result.getLocation()); 2434 return true; 2435 } 2436 2437 /// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 2438 /// character (either \\n or \\r) is part of an escaped newline sequence. Issue 2439 /// a diagnostic if so. We know that the newline is inside of a block comment. 2440 static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 2441 Lexer *L) { 2442 assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 2443 2444 // Back up off the newline. 2445 --CurPtr; 2446 2447 // If this is a two-character newline sequence, skip the other character. 2448 if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 2449 // \n\n or \r\r -> not escaped newline. 2450 if (CurPtr[0] == CurPtr[1]) 2451 return false; 2452 // \n\r or \r\n -> skip the newline. 2453 --CurPtr; 2454 } 2455 2456 // If we have horizontal whitespace, skip over it. We allow whitespace 2457 // between the slash and newline. 2458 bool HasSpace = false; 2459 while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 2460 --CurPtr; 2461 HasSpace = true; 2462 } 2463 2464 // If we have a slash, we know this is an escaped newline. 2465 if (*CurPtr == '\\') { 2466 if (CurPtr[-1] != '*') return false; 2467 } else { 2468 // It isn't a slash, is it the ?? / trigraph? 2469 if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 2470 CurPtr[-3] != '*') 2471 return false; 2472 2473 // This is the trigraph ending the comment. Emit a stern warning! 2474 CurPtr -= 2; 2475 2476 // If no trigraphs are enabled, warn that we ignored this trigraph and 2477 // ignore this * character. 2478 if (!L->getLangOpts().Trigraphs) { 2479 if (!L->isLexingRawMode()) 2480 L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 2481 return false; 2482 } 2483 if (!L->isLexingRawMode()) 2484 L->Diag(CurPtr, diag::trigraph_ends_block_comment); 2485 } 2486 2487 // Warn about having an escaped newline between the */ characters. 2488 if (!L->isLexingRawMode()) 2489 L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 2490 2491 // If there was space between the backslash and newline, warn about it. 2492 if (HasSpace && !L->isLexingRawMode()) 2493 L->Diag(CurPtr, diag::backslash_newline_space); 2494 2495 return true; 2496 } 2497 2498 #ifdef __SSE2__ 2499 #include <emmintrin.h> 2500 #elif __ALTIVEC__ 2501 #include <altivec.h> 2502 #undef bool 2503 #endif 2504 2505 /// We have just read from input the / and * characters that started a comment. 2506 /// Read until we find the * and / characters that terminate the comment. 2507 /// Note that we don't bother decoding trigraphs or escaped newlines in block 2508 /// comments, because they cannot cause the comment to end. The only thing 2509 /// that can happen is the comment could end with an escaped newline between 2510 /// the terminating * and /. 2511 /// 2512 /// If we're in KeepCommentMode or any CommentHandler has inserted 2513 /// some tokens, this will store the first token and return true. 2514 bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr, 2515 bool &TokAtPhysicalStartOfLine) { 2516 // Scan one character past where we should, looking for a '/' character. Once 2517 // we find it, check to see if it was preceded by a *. This common 2518 // optimization helps people who like to put a lot of * characters in their 2519 // comments. 2520 2521 // The first character we get with newlines and trigraphs skipped to handle 2522 // the degenerate /*/ case below correctly if the * has an escaped newline 2523 // after it. 2524 unsigned CharSize; 2525 unsigned char C = getCharAndSize(CurPtr, CharSize); 2526 CurPtr += CharSize; 2527 if (C == 0 && CurPtr == BufferEnd+1) { 2528 if (!isLexingRawMode()) 2529 Diag(BufferPtr, diag::err_unterminated_block_comment); 2530 --CurPtr; 2531 2532 // KeepWhitespaceMode should return this broken comment as a token. Since 2533 // it isn't a well formed comment, just return it as an 'unknown' token. 2534 if (isKeepWhitespaceMode()) { 2535 FormTokenWithChars(Result, CurPtr, tok::unknown); 2536 return true; 2537 } 2538 2539 BufferPtr = CurPtr; 2540 return false; 2541 } 2542 2543 // Check to see if the first character after the '/*' is another /. If so, 2544 // then this slash does not end the block comment, it is part of it. 2545 if (C == '/') 2546 C = *CurPtr++; 2547 2548 while (true) { 2549 // Skip over all non-interesting characters until we find end of buffer or a 2550 // (probably ending) '/' character. 2551 if (CurPtr + 24 < BufferEnd && 2552 // If there is a code-completion point avoid the fast scan because it 2553 // doesn't check for '\0'. 2554 !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 2555 // While not aligned to a 16-byte boundary. 2556 while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 2557 C = *CurPtr++; 2558 2559 if (C == '/') goto FoundSlash; 2560 2561 #ifdef __SSE2__ 2562 __m128i Slashes = _mm_set1_epi8('/'); 2563 while (CurPtr+16 <= BufferEnd) { 2564 int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 2565 Slashes)); 2566 if (cmp != 0) { 2567 // Adjust the pointer to point directly after the first slash. It's 2568 // not necessary to set C here, it will be overwritten at the end of 2569 // the outer loop. 2570 CurPtr += llvm::countTrailingZeros<unsigned>(cmp) + 1; 2571 goto FoundSlash; 2572 } 2573 CurPtr += 16; 2574 } 2575 #elif __ALTIVEC__ 2576 __vector unsigned char Slashes = { 2577 '/', '/', '/', '/', '/', '/', '/', '/', 2578 '/', '/', '/', '/', '/', '/', '/', '/' 2579 }; 2580 while (CurPtr + 16 <= BufferEnd && 2581 !vec_any_eq(*(const __vector unsigned char *)CurPtr, Slashes)) 2582 CurPtr += 16; 2583 #else 2584 // Scan for '/' quickly. Many block comments are very large. 2585 while (CurPtr[0] != '/' && 2586 CurPtr[1] != '/' && 2587 CurPtr[2] != '/' && 2588 CurPtr[3] != '/' && 2589 CurPtr+4 < BufferEnd) { 2590 CurPtr += 4; 2591 } 2592 #endif 2593 2594 // It has to be one of the bytes scanned, increment to it and read one. 2595 C = *CurPtr++; 2596 } 2597 2598 // Loop to scan the remainder. 2599 while (C != '/' && C != '\0') 2600 C = *CurPtr++; 2601 2602 if (C == '/') { 2603 FoundSlash: 2604 if (CurPtr[-2] == '*') // We found the final */. We're done! 2605 break; 2606 2607 if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 2608 if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 2609 // We found the final */, though it had an escaped newline between the 2610 // * and /. We're done! 2611 break; 2612 } 2613 } 2614 if (CurPtr[0] == '*' && CurPtr[1] != '/') { 2615 // If this is a /* inside of the comment, emit a warning. Don't do this 2616 // if this is a /*/, which will end the comment. This misses cases with 2617 // embedded escaped newlines, but oh well. 2618 if (!isLexingRawMode()) 2619 Diag(CurPtr-1, diag::warn_nested_block_comment); 2620 } 2621 } else if (C == 0 && CurPtr == BufferEnd+1) { 2622 if (!isLexingRawMode()) 2623 Diag(BufferPtr, diag::err_unterminated_block_comment); 2624 // Note: the user probably forgot a */. We could continue immediately 2625 // after the /*, but this would involve lexing a lot of what really is the 2626 // comment, which surely would confuse the parser. 2627 --CurPtr; 2628 2629 // KeepWhitespaceMode should return this broken comment as a token. Since 2630 // it isn't a well formed comment, just return it as an 'unknown' token. 2631 if (isKeepWhitespaceMode()) { 2632 FormTokenWithChars(Result, CurPtr, tok::unknown); 2633 return true; 2634 } 2635 2636 BufferPtr = CurPtr; 2637 return false; 2638 } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 2639 PP->CodeCompleteNaturalLanguage(); 2640 cutOffLexing(); 2641 return false; 2642 } 2643 2644 C = *CurPtr++; 2645 } 2646 2647 // Notify comment handlers about the comment unless we're in a #if 0 block. 2648 if (PP && !isLexingRawMode() && 2649 PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 2650 getSourceLocation(CurPtr)))) { 2651 BufferPtr = CurPtr; 2652 return true; // A token has to be returned. 2653 } 2654 2655 // If we are returning comments as tokens, return this comment as a token. 2656 if (inKeepCommentMode()) { 2657 FormTokenWithChars(Result, CurPtr, tok::comment); 2658 return true; 2659 } 2660 2661 // It is common for the tokens immediately after a /**/ comment to be 2662 // whitespace. Instead of going through the big switch, handle it 2663 // efficiently now. This is safe even in KeepWhitespaceMode because we would 2664 // have already returned above with the comment as a token. 2665 if (isHorizontalWhitespace(*CurPtr)) { 2666 SkipWhitespace(Result, CurPtr+1, TokAtPhysicalStartOfLine); 2667 return false; 2668 } 2669 2670 // Otherwise, just return so that the next character will be lexed as a token. 2671 BufferPtr = CurPtr; 2672 Result.setFlag(Token::LeadingSpace); 2673 return false; 2674 } 2675 2676 //===----------------------------------------------------------------------===// 2677 // Primary Lexing Entry Points 2678 //===----------------------------------------------------------------------===// 2679 2680 /// ReadToEndOfLine - Read the rest of the current preprocessor line as an 2681 /// uninterpreted string. This switches the lexer out of directive mode. 2682 void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 2683 assert(ParsingPreprocessorDirective && ParsingFilename == false && 2684 "Must be in a preprocessing directive!"); 2685 Token Tmp; 2686 Tmp.startToken(); 2687 2688 // CurPtr - Cache BufferPtr in an automatic variable. 2689 const char *CurPtr = BufferPtr; 2690 while (true) { 2691 char Char = getAndAdvanceChar(CurPtr, Tmp); 2692 switch (Char) { 2693 default: 2694 if (Result) 2695 Result->push_back(Char); 2696 break; 2697 case 0: // Null. 2698 // Found end of file? 2699 if (CurPtr-1 != BufferEnd) { 2700 if (isCodeCompletionPoint(CurPtr-1)) { 2701 PP->CodeCompleteNaturalLanguage(); 2702 cutOffLexing(); 2703 return; 2704 } 2705 2706 // Nope, normal character, continue. 2707 if (Result) 2708 Result->push_back(Char); 2709 break; 2710 } 2711 // FALL THROUGH. 2712 LLVM_FALLTHROUGH; 2713 case '\r': 2714 case '\n': 2715 // Okay, we found the end of the line. First, back up past the \0, \r, \n. 2716 assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 2717 BufferPtr = CurPtr-1; 2718 2719 // Next, lex the character, which should handle the EOD transition. 2720 Lex(Tmp); 2721 if (Tmp.is(tok::code_completion)) { 2722 if (PP) 2723 PP->CodeCompleteNaturalLanguage(); 2724 Lex(Tmp); 2725 } 2726 assert(Tmp.is(tok::eod) && "Unexpected token!"); 2727 2728 // Finally, we're done; 2729 return; 2730 } 2731 } 2732 } 2733 2734 /// LexEndOfFile - CurPtr points to the end of this file. Handle this 2735 /// condition, reporting diagnostics and handling other edge cases as required. 2736 /// This returns true if Result contains a token, false if PP.Lex should be 2737 /// called again. 2738 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 2739 // If we hit the end of the file while parsing a preprocessor directive, 2740 // end the preprocessor directive first. The next token returned will 2741 // then be the end of file. 2742 if (ParsingPreprocessorDirective) { 2743 // Done parsing the "line". 2744 ParsingPreprocessorDirective = false; 2745 // Update the location of token as well as BufferPtr. 2746 FormTokenWithChars(Result, CurPtr, tok::eod); 2747 2748 // Restore comment saving mode, in case it was disabled for directive. 2749 if (PP) 2750 resetExtendedTokenMode(); 2751 return true; // Have a token. 2752 } 2753 2754 // If we are in raw mode, return this event as an EOF token. Let the caller 2755 // that put us in raw mode handle the event. 2756 if (isLexingRawMode()) { 2757 Result.startToken(); 2758 BufferPtr = BufferEnd; 2759 FormTokenWithChars(Result, BufferEnd, tok::eof); 2760 return true; 2761 } 2762 2763 if (PP->isRecordingPreamble() && PP->isInPrimaryFile()) { 2764 PP->setRecordedPreambleConditionalStack(ConditionalStack); 2765 ConditionalStack.clear(); 2766 } 2767 2768 // Issue diagnostics for unterminated #if and missing newline. 2769 2770 // If we are in a #if directive, emit an error. 2771 while (!ConditionalStack.empty()) { 2772 if (PP->getCodeCompletionFileLoc() != FileLoc) 2773 PP->Diag(ConditionalStack.back().IfLoc, 2774 diag::err_pp_unterminated_conditional); 2775 ConditionalStack.pop_back(); 2776 } 2777 2778 // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2779 // a pedwarn. 2780 if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) { 2781 DiagnosticsEngine &Diags = PP->getDiagnostics(); 2782 SourceLocation EndLoc = getSourceLocation(BufferEnd); 2783 unsigned DiagID; 2784 2785 if (LangOpts.CPlusPlus11) { 2786 // C++11 [lex.phases] 2.2 p2 2787 // Prefer the C++98 pedantic compatibility warning over the generic, 2788 // non-extension, user-requested "missing newline at EOF" warning. 2789 if (!Diags.isIgnored(diag::warn_cxx98_compat_no_newline_eof, EndLoc)) { 2790 DiagID = diag::warn_cxx98_compat_no_newline_eof; 2791 } else { 2792 DiagID = diag::warn_no_newline_eof; 2793 } 2794 } else { 2795 DiagID = diag::ext_no_newline_eof; 2796 } 2797 2798 Diag(BufferEnd, DiagID) 2799 << FixItHint::CreateInsertion(EndLoc, "\n"); 2800 } 2801 2802 BufferPtr = CurPtr; 2803 2804 // Finally, let the preprocessor handle this. 2805 return PP->HandleEndOfFile(Result, isPragmaLexer()); 2806 } 2807 2808 /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 2809 /// the specified lexer will return a tok::l_paren token, 0 if it is something 2810 /// else and 2 if there are no more tokens in the buffer controlled by the 2811 /// lexer. 2812 unsigned Lexer::isNextPPTokenLParen() { 2813 assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 2814 2815 // Switch to 'skipping' mode. This will ensure that we can lex a token 2816 // without emitting diagnostics, disables macro expansion, and will cause EOF 2817 // to return an EOF token instead of popping the include stack. 2818 LexingRawMode = true; 2819 2820 // Save state that can be changed while lexing so that we can restore it. 2821 const char *TmpBufferPtr = BufferPtr; 2822 bool inPPDirectiveMode = ParsingPreprocessorDirective; 2823 bool atStartOfLine = IsAtStartOfLine; 2824 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; 2825 bool leadingSpace = HasLeadingSpace; 2826 2827 Token Tok; 2828 Lex(Tok); 2829 2830 // Restore state that may have changed. 2831 BufferPtr = TmpBufferPtr; 2832 ParsingPreprocessorDirective = inPPDirectiveMode; 2833 HasLeadingSpace = leadingSpace; 2834 IsAtStartOfLine = atStartOfLine; 2835 IsAtPhysicalStartOfLine = atPhysicalStartOfLine; 2836 2837 // Restore the lexer back to non-skipping mode. 2838 LexingRawMode = false; 2839 2840 if (Tok.is(tok::eof)) 2841 return 2; 2842 return Tok.is(tok::l_paren); 2843 } 2844 2845 /// Find the end of a version control conflict marker. 2846 static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2847 ConflictMarkerKind CMK) { 2848 const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2849 size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2850 auto RestOfBuffer = StringRef(CurPtr, BufferEnd - CurPtr).substr(TermLen); 2851 size_t Pos = RestOfBuffer.find(Terminator); 2852 while (Pos != StringRef::npos) { 2853 // Must occur at start of line. 2854 if (Pos == 0 || 2855 (RestOfBuffer[Pos - 1] != '\r' && RestOfBuffer[Pos - 1] != '\n')) { 2856 RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2857 Pos = RestOfBuffer.find(Terminator); 2858 continue; 2859 } 2860 return RestOfBuffer.data()+Pos; 2861 } 2862 return nullptr; 2863 } 2864 2865 /// IsStartOfConflictMarker - If the specified pointer is the start of a version 2866 /// control conflict marker like '<<<<<<<', recognize it as such, emit an error 2867 /// and recover nicely. This returns true if it is a conflict marker and false 2868 /// if not. 2869 bool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 2870 // Only a conflict marker if it starts at the beginning of a line. 2871 if (CurPtr != BufferStart && 2872 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2873 return false; 2874 2875 // Check to see if we have <<<<<<< or >>>>. 2876 if (!StringRef(CurPtr, BufferEnd - CurPtr).startswith("<<<<<<<") && 2877 !StringRef(CurPtr, BufferEnd - CurPtr).startswith(">>>> ")) 2878 return false; 2879 2880 // If we have a situation where we don't care about conflict markers, ignore 2881 // it. 2882 if (CurrentConflictMarkerState || isLexingRawMode()) 2883 return false; 2884 2885 ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2886 2887 // Check to see if there is an ending marker somewhere in the buffer at the 2888 // start of a line to terminate this conflict marker. 2889 if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 2890 // We found a match. We are really in a conflict marker. 2891 // Diagnose this, and ignore to the end of line. 2892 Diag(CurPtr, diag::err_conflict_marker); 2893 CurrentConflictMarkerState = Kind; 2894 2895 // Skip ahead to the end of line. We know this exists because the 2896 // end-of-conflict marker starts with \r or \n. 2897 while (*CurPtr != '\r' && *CurPtr != '\n') { 2898 assert(CurPtr != BufferEnd && "Didn't find end of line"); 2899 ++CurPtr; 2900 } 2901 BufferPtr = CurPtr; 2902 return true; 2903 } 2904 2905 // No end of conflict marker found. 2906 return false; 2907 } 2908 2909 /// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2910 /// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2911 /// is the end of a conflict marker. Handle it by ignoring up until the end of 2912 /// the line. This returns true if it is a conflict marker and false if not. 2913 bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 2914 // Only a conflict marker if it starts at the beginning of a line. 2915 if (CurPtr != BufferStart && 2916 CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 2917 return false; 2918 2919 // If we have a situation where we don't care about conflict markers, ignore 2920 // it. 2921 if (!CurrentConflictMarkerState || isLexingRawMode()) 2922 return false; 2923 2924 // Check to see if we have the marker (4 characters in a row). 2925 for (unsigned i = 1; i != 4; ++i) 2926 if (CurPtr[i] != CurPtr[0]) 2927 return false; 2928 2929 // If we do have it, search for the end of the conflict marker. This could 2930 // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 2931 // be the end of conflict marker. 2932 if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2933 CurrentConflictMarkerState)) { 2934 CurPtr = End; 2935 2936 // Skip ahead to the end of line. 2937 while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 2938 ++CurPtr; 2939 2940 BufferPtr = CurPtr; 2941 2942 // No longer in the conflict marker. 2943 CurrentConflictMarkerState = CMK_None; 2944 return true; 2945 } 2946 2947 return false; 2948 } 2949 2950 static const char *findPlaceholderEnd(const char *CurPtr, 2951 const char *BufferEnd) { 2952 if (CurPtr == BufferEnd) 2953 return nullptr; 2954 BufferEnd -= 1; // Scan until the second last character. 2955 for (; CurPtr != BufferEnd; ++CurPtr) { 2956 if (CurPtr[0] == '#' && CurPtr[1] == '>') 2957 return CurPtr + 2; 2958 } 2959 return nullptr; 2960 } 2961 2962 bool Lexer::lexEditorPlaceholder(Token &Result, const char *CurPtr) { 2963 assert(CurPtr[-1] == '<' && CurPtr[0] == '#' && "Not a placeholder!"); 2964 if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode) 2965 return false; 2966 const char *End = findPlaceholderEnd(CurPtr + 1, BufferEnd); 2967 if (!End) 2968 return false; 2969 const char *Start = CurPtr - 1; 2970 if (!LangOpts.AllowEditorPlaceholders) 2971 Diag(Start, diag::err_placeholder_in_source); 2972 Result.startToken(); 2973 FormTokenWithChars(Result, End, tok::raw_identifier); 2974 Result.setRawIdentifierData(Start); 2975 PP->LookUpIdentifierInfo(Result); 2976 Result.setFlag(Token::IsEditorPlaceholder); 2977 BufferPtr = End; 2978 return true; 2979 } 2980 2981 bool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 2982 if (PP && PP->isCodeCompletionEnabled()) { 2983 SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 2984 return Loc == PP->getCodeCompletionLoc(); 2985 } 2986 2987 return false; 2988 } 2989 2990 uint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2991 Token *Result) { 2992 unsigned CharSize; 2993 char Kind = getCharAndSize(StartPtr, CharSize); 2994 2995 unsigned NumHexDigits; 2996 if (Kind == 'u') 2997 NumHexDigits = 4; 2998 else if (Kind == 'U') 2999 NumHexDigits = 8; 3000 else 3001 return 0; 3002 3003 if (!LangOpts.CPlusPlus && !LangOpts.C99) { 3004 if (Result && !isLexingRawMode()) 3005 Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 3006 return 0; 3007 } 3008 3009 const char *CurPtr = StartPtr + CharSize; 3010 const char *KindLoc = &CurPtr[-1]; 3011 3012 uint32_t CodePoint = 0; 3013 for (unsigned i = 0; i < NumHexDigits; ++i) { 3014 char C = getCharAndSize(CurPtr, CharSize); 3015 3016 unsigned Value = llvm::hexDigitValue(C); 3017 if (Value == -1U) { 3018 if (Result && !isLexingRawMode()) { 3019 if (i == 0) { 3020 Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 3021 << StringRef(KindLoc, 1); 3022 } else { 3023 Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 3024 3025 // If the user wrote \U1234, suggest a fixit to \u. 3026 if (i == 4 && NumHexDigits == 8) { 3027 CharSourceRange URange = makeCharRange(*this, KindLoc, KindLoc + 1); 3028 Diag(KindLoc, diag::note_ucn_four_not_eight) 3029 << FixItHint::CreateReplacement(URange, "u"); 3030 } 3031 } 3032 } 3033 3034 return 0; 3035 } 3036 3037 CodePoint <<= 4; 3038 CodePoint += Value; 3039 3040 CurPtr += CharSize; 3041 } 3042 3043 if (Result) { 3044 Result->setFlag(Token::HasUCN); 3045 if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 3046 StartPtr = CurPtr; 3047 else 3048 while (StartPtr != CurPtr) 3049 (void)getAndAdvanceChar(StartPtr, *Result); 3050 } else { 3051 StartPtr = CurPtr; 3052 } 3053 3054 // Don't apply C family restrictions to UCNs in assembly mode 3055 if (LangOpts.AsmPreprocessor) 3056 return CodePoint; 3057 3058 // C99 6.4.3p2: A universal character name shall not specify a character whose 3059 // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 3060 // 0060 (`), nor one in the range D800 through DFFF inclusive.) 3061 // C++11 [lex.charset]p2: If the hexadecimal value for a 3062 // universal-character-name corresponds to a surrogate code point (in the 3063 // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 3064 // if the hexadecimal value for a universal-character-name outside the 3065 // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 3066 // string literal corresponds to a control character (in either of the 3067 // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 3068 // basic source character set, the program is ill-formed. 3069 if (CodePoint < 0xA0) { 3070 if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 3071 return CodePoint; 3072 3073 // We don't use isLexingRawMode() here because we need to warn about bad 3074 // UCNs even when skipping preprocessing tokens in a #if block. 3075 if (Result && PP) { 3076 if (CodePoint < 0x20 || CodePoint >= 0x7F) 3077 Diag(BufferPtr, diag::err_ucn_control_character); 3078 else { 3079 char C = static_cast<char>(CodePoint); 3080 Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 3081 } 3082 } 3083 3084 return 0; 3085 } else if (CodePoint >= 0xD800 && CodePoint <= 0xDFFF) { 3086 // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 3087 // We don't use isLexingRawMode() here because we need to diagnose bad 3088 // UCNs even when skipping preprocessing tokens in a #if block. 3089 if (Result && PP) { 3090 if (LangOpts.CPlusPlus && !LangOpts.CPlusPlus11) 3091 Diag(BufferPtr, diag::warn_ucn_escape_surrogate); 3092 else 3093 Diag(BufferPtr, diag::err_ucn_escape_invalid); 3094 } 3095 return 0; 3096 } 3097 3098 return CodePoint; 3099 } 3100 3101 bool Lexer::CheckUnicodeWhitespace(Token &Result, uint32_t C, 3102 const char *CurPtr) { 3103 static const llvm::sys::UnicodeCharSet UnicodeWhitespaceChars( 3104 UnicodeWhitespaceCharRanges); 3105 if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 3106 UnicodeWhitespaceChars.contains(C)) { 3107 Diag(BufferPtr, diag::ext_unicode_whitespace) 3108 << makeCharRange(*this, BufferPtr, CurPtr); 3109 3110 Result.setFlag(Token::LeadingSpace); 3111 return true; 3112 } 3113 return false; 3114 } 3115 3116 bool Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 3117 if (isAllowedIDChar(C, LangOpts) && isAllowedInitiallyIDChar(C, LangOpts)) { 3118 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 3119 !PP->isPreprocessedOutput()) { 3120 maybeDiagnoseIDCharCompat(PP->getDiagnostics(), C, 3121 makeCharRange(*this, BufferPtr, CurPtr), 3122 /*IsFirst=*/true); 3123 maybeDiagnoseUTF8Homoglyph(PP->getDiagnostics(), C, 3124 makeCharRange(*this, BufferPtr, CurPtr)); 3125 } 3126 3127 MIOpt.ReadToken(); 3128 return LexIdentifier(Result, CurPtr); 3129 } 3130 3131 if (!isLexingRawMode() && !ParsingPreprocessorDirective && 3132 !PP->isPreprocessedOutput() && 3133 !isASCII(*BufferPtr) && !isAllowedIDChar(C, LangOpts)) { 3134 // Non-ASCII characters tend to creep into source code unintentionally. 3135 // Instead of letting the parser complain about the unknown token, 3136 // just drop the character. 3137 // Note that we can /only/ do this when the non-ASCII character is actually 3138 // spelled as Unicode, not written as a UCN. The standard requires that 3139 // we not throw away any possible preprocessor tokens, but there's a 3140 // loophole in the mapping of Unicode characters to basic character set 3141 // characters that allows us to map these particular characters to, say, 3142 // whitespace. 3143 Diag(BufferPtr, diag::err_non_ascii) 3144 << FixItHint::CreateRemoval(makeCharRange(*this, BufferPtr, CurPtr)); 3145 3146 BufferPtr = CurPtr; 3147 return false; 3148 } 3149 3150 // Otherwise, we have an explicit UCN or a character that's unlikely to show 3151 // up by accident. 3152 MIOpt.ReadToken(); 3153 FormTokenWithChars(Result, CurPtr, tok::unknown); 3154 return true; 3155 } 3156 3157 void Lexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { 3158 IsAtStartOfLine = Result.isAtStartOfLine(); 3159 HasLeadingSpace = Result.hasLeadingSpace(); 3160 HasLeadingEmptyMacro = Result.hasLeadingEmptyMacro(); 3161 // Note that this doesn't affect IsAtPhysicalStartOfLine. 3162 } 3163 3164 bool Lexer::Lex(Token &Result) { 3165 // Start a new token. 3166 Result.startToken(); 3167 3168 // Set up misc whitespace flags for LexTokenInternal. 3169 if (IsAtStartOfLine) { 3170 Result.setFlag(Token::StartOfLine); 3171 IsAtStartOfLine = false; 3172 } 3173 3174 if (HasLeadingSpace) { 3175 Result.setFlag(Token::LeadingSpace); 3176 HasLeadingSpace = false; 3177 } 3178 3179 if (HasLeadingEmptyMacro) { 3180 Result.setFlag(Token::LeadingEmptyMacro); 3181 HasLeadingEmptyMacro = false; 3182 } 3183 3184 bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine; 3185 IsAtPhysicalStartOfLine = false; 3186 bool isRawLex = isLexingRawMode(); 3187 (void) isRawLex; 3188 bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine); 3189 // (After the LexTokenInternal call, the lexer might be destroyed.) 3190 assert((returnedToken || !isRawLex) && "Raw lex must succeed"); 3191 return returnedToken; 3192 } 3193 3194 /// LexTokenInternal - This implements a simple C family lexer. It is an 3195 /// extremely performance critical piece of code. This assumes that the buffer 3196 /// has a null character at the end of the file. This returns a preprocessing 3197 /// token, not a normal token, as such, it is an internal interface. It assumes 3198 /// that the Flags of result have been cleared before calling this. 3199 bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) { 3200 LexNextToken: 3201 // New token, can't need cleaning yet. 3202 Result.clearFlag(Token::NeedsCleaning); 3203 Result.setIdentifierInfo(nullptr); 3204 3205 // CurPtr - Cache BufferPtr in an automatic variable. 3206 const char *CurPtr = BufferPtr; 3207 3208 // Small amounts of horizontal whitespace is very common between tokens. 3209 if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 3210 ++CurPtr; 3211 while ((*CurPtr == ' ') || (*CurPtr == '\t')) 3212 ++CurPtr; 3213 3214 // If we are keeping whitespace and other tokens, just return what we just 3215 // skipped. The next lexer invocation will return the token after the 3216 // whitespace. 3217 if (isKeepWhitespaceMode()) { 3218 FormTokenWithChars(Result, CurPtr, tok::unknown); 3219 // FIXME: The next token will not have LeadingSpace set. 3220 return true; 3221 } 3222 3223 BufferPtr = CurPtr; 3224 Result.setFlag(Token::LeadingSpace); 3225 } 3226 3227 unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 3228 3229 // Read a character, advancing over it. 3230 char Char = getAndAdvanceChar(CurPtr, Result); 3231 tok::TokenKind Kind; 3232 3233 if (!isVerticalWhitespace(Char)) 3234 NewLinePtr = nullptr; 3235 3236 switch (Char) { 3237 case 0: // Null. 3238 // Found end of file? 3239 if (CurPtr-1 == BufferEnd) 3240 return LexEndOfFile(Result, CurPtr-1); 3241 3242 // Check if we are performing code completion. 3243 if (isCodeCompletionPoint(CurPtr-1)) { 3244 // Return the code-completion token. 3245 Result.startToken(); 3246 FormTokenWithChars(Result, CurPtr, tok::code_completion); 3247 return true; 3248 } 3249 3250 if (!isLexingRawMode()) 3251 Diag(CurPtr-1, diag::null_in_file); 3252 Result.setFlag(Token::LeadingSpace); 3253 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3254 return true; // KeepWhitespaceMode 3255 3256 // We know the lexer hasn't changed, so just try again with this lexer. 3257 // (We manually eliminate the tail call to avoid recursion.) 3258 goto LexNextToken; 3259 3260 case 26: // DOS & CP/M EOF: "^Z". 3261 // If we're in Microsoft extensions mode, treat this as end of file. 3262 if (LangOpts.MicrosoftExt) { 3263 if (!isLexingRawMode()) 3264 Diag(CurPtr-1, diag::ext_ctrl_z_eof_microsoft); 3265 return LexEndOfFile(Result, CurPtr-1); 3266 } 3267 3268 // If Microsoft extensions are disabled, this is just random garbage. 3269 Kind = tok::unknown; 3270 break; 3271 3272 case '\r': 3273 if (CurPtr[0] == '\n') 3274 (void)getAndAdvanceChar(CurPtr, Result); 3275 LLVM_FALLTHROUGH; 3276 case '\n': 3277 // If we are inside a preprocessor directive and we see the end of line, 3278 // we know we are done with the directive, so return an EOD token. 3279 if (ParsingPreprocessorDirective) { 3280 // Done parsing the "line". 3281 ParsingPreprocessorDirective = false; 3282 3283 // Restore comment saving mode, in case it was disabled for directive. 3284 if (PP) 3285 resetExtendedTokenMode(); 3286 3287 // Since we consumed a newline, we are back at the start of a line. 3288 IsAtStartOfLine = true; 3289 IsAtPhysicalStartOfLine = true; 3290 NewLinePtr = CurPtr - 1; 3291 3292 Kind = tok::eod; 3293 break; 3294 } 3295 3296 // No leading whitespace seen so far. 3297 Result.clearFlag(Token::LeadingSpace); 3298 3299 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3300 return true; // KeepWhitespaceMode 3301 3302 // We only saw whitespace, so just try again with this lexer. 3303 // (We manually eliminate the tail call to avoid recursion.) 3304 goto LexNextToken; 3305 case ' ': 3306 case '\t': 3307 case '\f': 3308 case '\v': 3309 SkipHorizontalWhitespace: 3310 Result.setFlag(Token::LeadingSpace); 3311 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3312 return true; // KeepWhitespaceMode 3313 3314 SkipIgnoredUnits: 3315 CurPtr = BufferPtr; 3316 3317 // If the next token is obviously a // or /* */ comment, skip it efficiently 3318 // too (without going through the big switch stmt). 3319 if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 3320 LangOpts.LineComment && 3321 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP)) { 3322 if (SkipLineComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) 3323 return true; // There is a token to return. 3324 goto SkipIgnoredUnits; 3325 } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 3326 if (SkipBlockComment(Result, CurPtr+2, TokAtPhysicalStartOfLine)) 3327 return true; // There is a token to return. 3328 goto SkipIgnoredUnits; 3329 } else if (isHorizontalWhitespace(*CurPtr)) { 3330 goto SkipHorizontalWhitespace; 3331 } 3332 // We only saw whitespace, so just try again with this lexer. 3333 // (We manually eliminate the tail call to avoid recursion.) 3334 goto LexNextToken; 3335 3336 // C99 6.4.4.1: Integer Constants. 3337 // C99 6.4.4.2: Floating Constants. 3338 case '0': case '1': case '2': case '3': case '4': 3339 case '5': case '6': case '7': case '8': case '9': 3340 // Notify MIOpt that we read a non-whitespace/non-comment token. 3341 MIOpt.ReadToken(); 3342 return LexNumericConstant(Result, CurPtr); 3343 3344 case 'u': // Identifier (uber) or C11/C++11 UTF-8 or UTF-16 string literal 3345 // Notify MIOpt that we read a non-whitespace/non-comment token. 3346 MIOpt.ReadToken(); 3347 3348 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 3349 Char = getCharAndSize(CurPtr, SizeTmp); 3350 3351 // UTF-16 string literal 3352 if (Char == '"') 3353 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3354 tok::utf16_string_literal); 3355 3356 // UTF-16 character constant 3357 if (Char == '\'') 3358 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3359 tok::utf16_char_constant); 3360 3361 // UTF-16 raw string literal 3362 if (Char == 'R' && LangOpts.CPlusPlus11 && 3363 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3364 return LexRawStringLiteral(Result, 3365 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3366 SizeTmp2, Result), 3367 tok::utf16_string_literal); 3368 3369 if (Char == '8') { 3370 char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 3371 3372 // UTF-8 string literal 3373 if (Char2 == '"') 3374 return LexStringLiteral(Result, 3375 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3376 SizeTmp2, Result), 3377 tok::utf8_string_literal); 3378 if (Char2 == '\'' && LangOpts.CPlusPlus17) 3379 return LexCharConstant( 3380 Result, ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3381 SizeTmp2, Result), 3382 tok::utf8_char_constant); 3383 3384 if (Char2 == 'R' && LangOpts.CPlusPlus11) { 3385 unsigned SizeTmp3; 3386 char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3387 // UTF-8 raw string literal 3388 if (Char3 == '"') { 3389 return LexRawStringLiteral(Result, 3390 ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3391 SizeTmp2, Result), 3392 SizeTmp3, Result), 3393 tok::utf8_string_literal); 3394 } 3395 } 3396 } 3397 } 3398 3399 // treat u like the start of an identifier. 3400 return LexIdentifier(Result, CurPtr); 3401 3402 case 'U': // Identifier (Uber) or C11/C++11 UTF-32 string literal 3403 // Notify MIOpt that we read a non-whitespace/non-comment token. 3404 MIOpt.ReadToken(); 3405 3406 if (LangOpts.CPlusPlus11 || LangOpts.C11) { 3407 Char = getCharAndSize(CurPtr, SizeTmp); 3408 3409 // UTF-32 string literal 3410 if (Char == '"') 3411 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3412 tok::utf32_string_literal); 3413 3414 // UTF-32 character constant 3415 if (Char == '\'') 3416 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3417 tok::utf32_char_constant); 3418 3419 // UTF-32 raw string literal 3420 if (Char == 'R' && LangOpts.CPlusPlus11 && 3421 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3422 return LexRawStringLiteral(Result, 3423 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3424 SizeTmp2, Result), 3425 tok::utf32_string_literal); 3426 } 3427 3428 // treat U like the start of an identifier. 3429 return LexIdentifier(Result, CurPtr); 3430 3431 case 'R': // Identifier or C++0x raw string literal 3432 // Notify MIOpt that we read a non-whitespace/non-comment token. 3433 MIOpt.ReadToken(); 3434 3435 if (LangOpts.CPlusPlus11) { 3436 Char = getCharAndSize(CurPtr, SizeTmp); 3437 3438 if (Char == '"') 3439 return LexRawStringLiteral(Result, 3440 ConsumeChar(CurPtr, SizeTmp, Result), 3441 tok::string_literal); 3442 } 3443 3444 // treat R like the start of an identifier. 3445 return LexIdentifier(Result, CurPtr); 3446 3447 case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 3448 // Notify MIOpt that we read a non-whitespace/non-comment token. 3449 MIOpt.ReadToken(); 3450 Char = getCharAndSize(CurPtr, SizeTmp); 3451 3452 // Wide string literal. 3453 if (Char == '"') 3454 return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3455 tok::wide_string_literal); 3456 3457 // Wide raw string literal. 3458 if (LangOpts.CPlusPlus11 && Char == 'R' && 3459 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 3460 return LexRawStringLiteral(Result, 3461 ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3462 SizeTmp2, Result), 3463 tok::wide_string_literal); 3464 3465 // Wide character constant. 3466 if (Char == '\'') 3467 return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3468 tok::wide_char_constant); 3469 // FALL THROUGH, treating L like the start of an identifier. 3470 LLVM_FALLTHROUGH; 3471 3472 // C99 6.4.2: Identifiers. 3473 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 3474 case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 3475 case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 3476 case 'V': case 'W': case 'X': case 'Y': case 'Z': 3477 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 3478 case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 3479 case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 3480 case 'v': case 'w': case 'x': case 'y': case 'z': 3481 case '_': 3482 // Notify MIOpt that we read a non-whitespace/non-comment token. 3483 MIOpt.ReadToken(); 3484 return LexIdentifier(Result, CurPtr); 3485 3486 case '$': // $ in identifiers. 3487 if (LangOpts.DollarIdents) { 3488 if (!isLexingRawMode()) 3489 Diag(CurPtr-1, diag::ext_dollar_in_identifier); 3490 // Notify MIOpt that we read a non-whitespace/non-comment token. 3491 MIOpt.ReadToken(); 3492 return LexIdentifier(Result, CurPtr); 3493 } 3494 3495 Kind = tok::unknown; 3496 break; 3497 3498 // C99 6.4.4: Character Constants. 3499 case '\'': 3500 // Notify MIOpt that we read a non-whitespace/non-comment token. 3501 MIOpt.ReadToken(); 3502 return LexCharConstant(Result, CurPtr, tok::char_constant); 3503 3504 // C99 6.4.5: String Literals. 3505 case '"': 3506 // Notify MIOpt that we read a non-whitespace/non-comment token. 3507 MIOpt.ReadToken(); 3508 return LexStringLiteral(Result, CurPtr, 3509 ParsingFilename ? tok::header_name 3510 : tok::string_literal); 3511 3512 // C99 6.4.6: Punctuators. 3513 case '?': 3514 Kind = tok::question; 3515 break; 3516 case '[': 3517 Kind = tok::l_square; 3518 break; 3519 case ']': 3520 Kind = tok::r_square; 3521 break; 3522 case '(': 3523 Kind = tok::l_paren; 3524 break; 3525 case ')': 3526 Kind = tok::r_paren; 3527 break; 3528 case '{': 3529 Kind = tok::l_brace; 3530 break; 3531 case '}': 3532 Kind = tok::r_brace; 3533 break; 3534 case '.': 3535 Char = getCharAndSize(CurPtr, SizeTmp); 3536 if (Char >= '0' && Char <= '9') { 3537 // Notify MIOpt that we read a non-whitespace/non-comment token. 3538 MIOpt.ReadToken(); 3539 3540 return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 3541 } else if (LangOpts.CPlusPlus && Char == '*') { 3542 Kind = tok::periodstar; 3543 CurPtr += SizeTmp; 3544 } else if (Char == '.' && 3545 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 3546 Kind = tok::ellipsis; 3547 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3548 SizeTmp2, Result); 3549 } else { 3550 Kind = tok::period; 3551 } 3552 break; 3553 case '&': 3554 Char = getCharAndSize(CurPtr, SizeTmp); 3555 if (Char == '&') { 3556 Kind = tok::ampamp; 3557 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3558 } else if (Char == '=') { 3559 Kind = tok::ampequal; 3560 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3561 } else { 3562 Kind = tok::amp; 3563 } 3564 break; 3565 case '*': 3566 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3567 Kind = tok::starequal; 3568 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3569 } else { 3570 Kind = tok::star; 3571 } 3572 break; 3573 case '+': 3574 Char = getCharAndSize(CurPtr, SizeTmp); 3575 if (Char == '+') { 3576 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3577 Kind = tok::plusplus; 3578 } else if (Char == '=') { 3579 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3580 Kind = tok::plusequal; 3581 } else { 3582 Kind = tok::plus; 3583 } 3584 break; 3585 case '-': 3586 Char = getCharAndSize(CurPtr, SizeTmp); 3587 if (Char == '-') { // -- 3588 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3589 Kind = tok::minusminus; 3590 } else if (Char == '>' && LangOpts.CPlusPlus && 3591 getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 3592 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3593 SizeTmp2, Result); 3594 Kind = tok::arrowstar; 3595 } else if (Char == '>') { // -> 3596 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3597 Kind = tok::arrow; 3598 } else if (Char == '=') { // -= 3599 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3600 Kind = tok::minusequal; 3601 } else { 3602 Kind = tok::minus; 3603 } 3604 break; 3605 case '~': 3606 Kind = tok::tilde; 3607 break; 3608 case '!': 3609 if (getCharAndSize(CurPtr, SizeTmp) == '=') { 3610 Kind = tok::exclaimequal; 3611 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3612 } else { 3613 Kind = tok::exclaim; 3614 } 3615 break; 3616 case '/': 3617 // 6.4.9: Comments 3618 Char = getCharAndSize(CurPtr, SizeTmp); 3619 if (Char == '/') { // Line comment. 3620 // Even if Line comments are disabled (e.g. in C89 mode), we generally 3621 // want to lex this as a comment. There is one problem with this though, 3622 // that in one particular corner case, this can change the behavior of the 3623 // resultant program. For example, In "foo //**/ bar", C89 would lex 3624 // this as "foo / bar" and languages with Line comments would lex it as 3625 // "foo". Check to see if the character after the second slash is a '*'. 3626 // If so, we will lex that as a "/" instead of the start of a comment. 3627 // However, we never do this if we are just preprocessing. 3628 bool TreatAsComment = LangOpts.LineComment && 3629 (LangOpts.CPlusPlus || !LangOpts.TraditionalCPP); 3630 if (!TreatAsComment) 3631 if (!(PP && PP->isPreprocessedOutput())) 3632 TreatAsComment = getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*'; 3633 3634 if (TreatAsComment) { 3635 if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3636 TokAtPhysicalStartOfLine)) 3637 return true; // There is a token to return. 3638 3639 // It is common for the tokens immediately after a // comment to be 3640 // whitespace (indentation for the next line). Instead of going through 3641 // the big switch, handle it efficiently now. 3642 goto SkipIgnoredUnits; 3643 } 3644 } 3645 3646 if (Char == '*') { // /**/ comment. 3647 if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result), 3648 TokAtPhysicalStartOfLine)) 3649 return true; // There is a token to return. 3650 3651 // We only saw whitespace, so just try again with this lexer. 3652 // (We manually eliminate the tail call to avoid recursion.) 3653 goto LexNextToken; 3654 } 3655 3656 if (Char == '=') { 3657 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3658 Kind = tok::slashequal; 3659 } else { 3660 Kind = tok::slash; 3661 } 3662 break; 3663 case '%': 3664 Char = getCharAndSize(CurPtr, SizeTmp); 3665 if (Char == '=') { 3666 Kind = tok::percentequal; 3667 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3668 } else if (LangOpts.Digraphs && Char == '>') { 3669 Kind = tok::r_brace; // '%>' -> '}' 3670 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3671 } else if (LangOpts.Digraphs && Char == ':') { 3672 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3673 Char = getCharAndSize(CurPtr, SizeTmp); 3674 if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 3675 Kind = tok::hashhash; // '%:%:' -> '##' 3676 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3677 SizeTmp2, Result); 3678 } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 3679 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3680 if (!isLexingRawMode()) 3681 Diag(BufferPtr, diag::ext_charize_microsoft); 3682 Kind = tok::hashat; 3683 } else { // '%:' -> '#' 3684 // We parsed a # character. If this occurs at the start of the line, 3685 // it's actually the start of a preprocessing directive. Callback to 3686 // the preprocessor to handle it. 3687 // TODO: -fpreprocessed mode?? 3688 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) 3689 goto HandleDirective; 3690 3691 Kind = tok::hash; 3692 } 3693 } else { 3694 Kind = tok::percent; 3695 } 3696 break; 3697 case '<': 3698 Char = getCharAndSize(CurPtr, SizeTmp); 3699 if (ParsingFilename) { 3700 return LexAngledStringLiteral(Result, CurPtr); 3701 } else if (Char == '<') { 3702 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3703 if (After == '=') { 3704 Kind = tok::lesslessequal; 3705 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3706 SizeTmp2, Result); 3707 } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 3708 // If this is actually a '<<<<<<<' version control conflict marker, 3709 // recognize it as such and recover nicely. 3710 goto LexNextToken; 3711 } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3712 // If this is '<<<<' and we're in a Perforce-style conflict marker, 3713 // ignore it. 3714 goto LexNextToken; 3715 } else if (LangOpts.CUDA && After == '<') { 3716 Kind = tok::lesslessless; 3717 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3718 SizeTmp2, Result); 3719 } else { 3720 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3721 Kind = tok::lessless; 3722 } 3723 } else if (Char == '=') { 3724 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3725 if (After == '>') { 3726 if (getLangOpts().CPlusPlus20) { 3727 if (!isLexingRawMode()) 3728 Diag(BufferPtr, diag::warn_cxx17_compat_spaceship); 3729 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3730 SizeTmp2, Result); 3731 Kind = tok::spaceship; 3732 break; 3733 } 3734 // Suggest adding a space between the '<=' and the '>' to avoid a 3735 // change in semantics if this turns up in C++ <=17 mode. 3736 if (getLangOpts().CPlusPlus && !isLexingRawMode()) { 3737 Diag(BufferPtr, diag::warn_cxx20_compat_spaceship) 3738 << FixItHint::CreateInsertion( 3739 getSourceLocation(CurPtr + SizeTmp, SizeTmp2), " "); 3740 } 3741 } 3742 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3743 Kind = tok::lessequal; 3744 } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 3745 if (LangOpts.CPlusPlus11 && 3746 getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 3747 // C++0x [lex.pptoken]p3: 3748 // Otherwise, if the next three characters are <:: and the subsequent 3749 // character is neither : nor >, the < is treated as a preprocessor 3750 // token by itself and not as the first character of the alternative 3751 // token <:. 3752 unsigned SizeTmp3; 3753 char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 3754 if (After != ':' && After != '>') { 3755 Kind = tok::less; 3756 if (!isLexingRawMode()) 3757 Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 3758 break; 3759 } 3760 } 3761 3762 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3763 Kind = tok::l_square; 3764 } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 3765 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3766 Kind = tok::l_brace; 3767 } else if (Char == '#' && /*Not a trigraph*/ SizeTmp == 1 && 3768 lexEditorPlaceholder(Result, CurPtr)) { 3769 return true; 3770 } else { 3771 Kind = tok::less; 3772 } 3773 break; 3774 case '>': 3775 Char = getCharAndSize(CurPtr, SizeTmp); 3776 if (Char == '=') { 3777 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3778 Kind = tok::greaterequal; 3779 } else if (Char == '>') { 3780 char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 3781 if (After == '=') { 3782 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3783 SizeTmp2, Result); 3784 Kind = tok::greatergreaterequal; 3785 } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3786 // If this is actually a '>>>>' conflict marker, recognize it as such 3787 // and recover nicely. 3788 goto LexNextToken; 3789 } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 3790 // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 3791 goto LexNextToken; 3792 } else if (LangOpts.CUDA && After == '>') { 3793 Kind = tok::greatergreatergreater; 3794 CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 3795 SizeTmp2, Result); 3796 } else { 3797 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3798 Kind = tok::greatergreater; 3799 } 3800 } else { 3801 Kind = tok::greater; 3802 } 3803 break; 3804 case '^': 3805 Char = getCharAndSize(CurPtr, SizeTmp); 3806 if (Char == '=') { 3807 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3808 Kind = tok::caretequal; 3809 } else if (LangOpts.OpenCL && Char == '^') { 3810 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3811 Kind = tok::caretcaret; 3812 } else { 3813 Kind = tok::caret; 3814 } 3815 break; 3816 case '|': 3817 Char = getCharAndSize(CurPtr, SizeTmp); 3818 if (Char == '=') { 3819 Kind = tok::pipeequal; 3820 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3821 } else if (Char == '|') { 3822 // If this is '|||||||' and we're in a conflict marker, ignore it. 3823 if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 3824 goto LexNextToken; 3825 Kind = tok::pipepipe; 3826 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3827 } else { 3828 Kind = tok::pipe; 3829 } 3830 break; 3831 case ':': 3832 Char = getCharAndSize(CurPtr, SizeTmp); 3833 if (LangOpts.Digraphs && Char == '>') { 3834 Kind = tok::r_square; // ':>' -> ']' 3835 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3836 } else if ((LangOpts.CPlusPlus || 3837 LangOpts.DoubleSquareBracketAttributes) && 3838 Char == ':') { 3839 Kind = tok::coloncolon; 3840 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3841 } else { 3842 Kind = tok::colon; 3843 } 3844 break; 3845 case ';': 3846 Kind = tok::semi; 3847 break; 3848 case '=': 3849 Char = getCharAndSize(CurPtr, SizeTmp); 3850 if (Char == '=') { 3851 // If this is '====' and we're in a conflict marker, ignore it. 3852 if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 3853 goto LexNextToken; 3854 3855 Kind = tok::equalequal; 3856 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3857 } else { 3858 Kind = tok::equal; 3859 } 3860 break; 3861 case ',': 3862 Kind = tok::comma; 3863 break; 3864 case '#': 3865 Char = getCharAndSize(CurPtr, SizeTmp); 3866 if (Char == '#') { 3867 Kind = tok::hashhash; 3868 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3869 } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 3870 Kind = tok::hashat; 3871 if (!isLexingRawMode()) 3872 Diag(BufferPtr, diag::ext_charize_microsoft); 3873 CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 3874 } else { 3875 // We parsed a # character. If this occurs at the start of the line, 3876 // it's actually the start of a preprocessing directive. Callback to 3877 // the preprocessor to handle it. 3878 // TODO: -fpreprocessed mode?? 3879 if (TokAtPhysicalStartOfLine && !LexingRawMode && !Is_PragmaLexer) 3880 goto HandleDirective; 3881 3882 Kind = tok::hash; 3883 } 3884 break; 3885 3886 case '@': 3887 // Objective C support. 3888 if (CurPtr[-1] == '@' && LangOpts.ObjC) 3889 Kind = tok::at; 3890 else 3891 Kind = tok::unknown; 3892 break; 3893 3894 // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 3895 case '\\': 3896 if (!LangOpts.AsmPreprocessor) { 3897 if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) { 3898 if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { 3899 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3900 return true; // KeepWhitespaceMode 3901 3902 // We only saw whitespace, so just try again with this lexer. 3903 // (We manually eliminate the tail call to avoid recursion.) 3904 goto LexNextToken; 3905 } 3906 3907 return LexUnicode(Result, CodePoint, CurPtr); 3908 } 3909 } 3910 3911 Kind = tok::unknown; 3912 break; 3913 3914 default: { 3915 if (isASCII(Char)) { 3916 Kind = tok::unknown; 3917 break; 3918 } 3919 3920 llvm::UTF32 CodePoint; 3921 3922 // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3923 // an escaped newline. 3924 --CurPtr; 3925 llvm::ConversionResult Status = 3926 llvm::convertUTF8Sequence((const llvm::UTF8 **)&CurPtr, 3927 (const llvm::UTF8 *)BufferEnd, 3928 &CodePoint, 3929 llvm::strictConversion); 3930 if (Status == llvm::conversionOK) { 3931 if (CheckUnicodeWhitespace(Result, CodePoint, CurPtr)) { 3932 if (SkipWhitespace(Result, CurPtr, TokAtPhysicalStartOfLine)) 3933 return true; // KeepWhitespaceMode 3934 3935 // We only saw whitespace, so just try again with this lexer. 3936 // (We manually eliminate the tail call to avoid recursion.) 3937 goto LexNextToken; 3938 } 3939 return LexUnicode(Result, CodePoint, CurPtr); 3940 } 3941 3942 if (isLexingRawMode() || ParsingPreprocessorDirective || 3943 PP->isPreprocessedOutput()) { 3944 ++CurPtr; 3945 Kind = tok::unknown; 3946 break; 3947 } 3948 3949 // Non-ASCII characters tend to creep into source code unintentionally. 3950 // Instead of letting the parser complain about the unknown token, 3951 // just diagnose the invalid UTF-8, then drop the character. 3952 Diag(CurPtr, diag::err_invalid_utf8); 3953 3954 BufferPtr = CurPtr+1; 3955 // We're pretending the character didn't exist, so just try again with 3956 // this lexer. 3957 // (We manually eliminate the tail call to avoid recursion.) 3958 goto LexNextToken; 3959 } 3960 } 3961 3962 // Notify MIOpt that we read a non-whitespace/non-comment token. 3963 MIOpt.ReadToken(); 3964 3965 // Update the location of token as well as BufferPtr. 3966 FormTokenWithChars(Result, CurPtr, Kind); 3967 return true; 3968 3969 HandleDirective: 3970 // We parsed a # character and it's the start of a preprocessing directive. 3971 3972 FormTokenWithChars(Result, CurPtr, tok::hash); 3973 PP->HandleDirective(Result); 3974 3975 if (PP->hadModuleLoaderFatalFailure()) { 3976 // With a fatal failure in the module loader, we abort parsing. 3977 assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof"); 3978 return true; 3979 } 3980 3981 // We parsed the directive; lex a token with the new state. 3982 return false; 3983 } 3984