1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the Preprocessor interface. 10 // 11 //===----------------------------------------------------------------------===// 12 // 13 // Options to support: 14 // -H - Print the name of each header file used. 15 // -d[DNI] - Dump various things. 16 // -fworking-directory - #line's with preprocessor's working dir. 17 // -fpreprocessed 18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD 19 // -W* 20 // -w 21 // 22 // Messages to emit: 23 // "Multiple include guards may be useful for:\n" 24 // 25 //===----------------------------------------------------------------------===// 26 27 #include "clang/Lex/Preprocessor.h" 28 #include "clang/Basic/Builtins.h" 29 #include "clang/Basic/FileManager.h" 30 #include "clang/Basic/FileSystemStatCache.h" 31 #include "clang/Basic/IdentifierTable.h" 32 #include "clang/Basic/LLVM.h" 33 #include "clang/Basic/LangOptions.h" 34 #include "clang/Basic/Module.h" 35 #include "clang/Basic/SourceLocation.h" 36 #include "clang/Basic/SourceManager.h" 37 #include "clang/Basic/TargetInfo.h" 38 #include "clang/Lex/CodeCompletionHandler.h" 39 #include "clang/Lex/ExternalPreprocessorSource.h" 40 #include "clang/Lex/HeaderSearch.h" 41 #include "clang/Lex/LexDiagnostic.h" 42 #include "clang/Lex/Lexer.h" 43 #include "clang/Lex/LiteralSupport.h" 44 #include "clang/Lex/MacroArgs.h" 45 #include "clang/Lex/MacroInfo.h" 46 #include "clang/Lex/ModuleLoader.h" 47 #include "clang/Lex/Pragma.h" 48 #include "clang/Lex/PreprocessingRecord.h" 49 #include "clang/Lex/PreprocessorLexer.h" 50 #include "clang/Lex/PreprocessorOptions.h" 51 #include "clang/Lex/ScratchBuffer.h" 52 #include "clang/Lex/Token.h" 53 #include "clang/Lex/TokenLexer.h" 54 #include "llvm/ADT/APInt.h" 55 #include "llvm/ADT/ArrayRef.h" 56 #include "llvm/ADT/DenseMap.h" 57 #include "llvm/ADT/STLExtras.h" 58 #include "llvm/ADT/SmallString.h" 59 #include "llvm/ADT/SmallVector.h" 60 #include "llvm/ADT/StringRef.h" 61 #include "llvm/ADT/StringSwitch.h" 62 #include "llvm/Support/Capacity.h" 63 #include "llvm/Support/ErrorHandling.h" 64 #include "llvm/Support/MemoryBuffer.h" 65 #include "llvm/Support/raw_ostream.h" 66 #include <algorithm> 67 #include <cassert> 68 #include <memory> 69 #include <string> 70 #include <utility> 71 #include <vector> 72 73 using namespace clang; 74 75 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry) 76 77 ExternalPreprocessorSource::~ExternalPreprocessorSource() = default; 78 79 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, 80 DiagnosticsEngine &diags, LangOptions &opts, 81 SourceManager &SM, HeaderSearch &Headers, 82 ModuleLoader &TheModuleLoader, 83 IdentifierInfoLookup *IILookup, bool OwnsHeaders, 84 TranslationUnitKind TUKind) 85 : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), 86 FileMgr(Headers.getFileMgr()), SourceMgr(SM), 87 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), 88 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), 89 // As the language options may have not been loaded yet (when 90 // deserializing an ASTUnit), adding keywords to the identifier table is 91 // deferred to Preprocessor::Initialize(). 92 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), 93 TUKind(TUKind), SkipMainFilePreamble(0, true), 94 CurSubmoduleState(&NullSubmoduleState) { 95 OwnsHeaderSearch = OwnsHeaders; 96 97 // Default to discarding comments. 98 KeepComments = false; 99 KeepMacroComments = false; 100 SuppressIncludeNotFoundError = false; 101 102 // Macro expansion is enabled. 103 DisableMacroExpansion = false; 104 MacroExpansionInDirectivesOverride = false; 105 InMacroArgs = false; 106 ArgMacro = nullptr; 107 InMacroArgPreExpansion = false; 108 NumCachedTokenLexers = 0; 109 PragmasEnabled = true; 110 ParsingIfOrElifDirective = false; 111 PreprocessedOutput = false; 112 113 // We haven't read anything from the external source. 114 ReadMacrosFromExternalSource = false; 115 116 BuiltinInfo = std::make_unique<Builtin::Context>(); 117 118 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of 119 // a macro. They get unpoisoned where it is allowed. 120 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); 121 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); 122 (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned(); 123 SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use); 124 125 // Initialize the pragma handlers. 126 RegisterBuiltinPragmas(); 127 128 // Initialize builtin macros like __LINE__ and friends. 129 RegisterBuiltinMacros(); 130 131 if(LangOpts.Borland) { 132 Ident__exception_info = getIdentifierInfo("_exception_info"); 133 Ident___exception_info = getIdentifierInfo("__exception_info"); 134 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation"); 135 Ident__exception_code = getIdentifierInfo("_exception_code"); 136 Ident___exception_code = getIdentifierInfo("__exception_code"); 137 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode"); 138 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination"); 139 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination"); 140 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination"); 141 } else { 142 Ident__exception_info = Ident__exception_code = nullptr; 143 Ident__abnormal_termination = Ident___exception_info = nullptr; 144 Ident___exception_code = Ident___abnormal_termination = nullptr; 145 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr; 146 Ident_AbnormalTermination = nullptr; 147 } 148 149 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens. 150 if (usingPCHWithPragmaHdrStop()) 151 SkippingUntilPragmaHdrStop = true; 152 153 // If using a PCH with a through header, start skipping tokens. 154 if (!this->PPOpts->PCHThroughHeader.empty() && 155 !this->PPOpts->ImplicitPCHInclude.empty()) 156 SkippingUntilPCHThroughHeader = true; 157 158 if (this->PPOpts->GeneratePreamble) 159 PreambleConditionalStack.startRecording(); 160 161 MaxTokens = LangOpts.MaxTokens; 162 } 163 164 Preprocessor::~Preprocessor() { 165 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!"); 166 167 IncludeMacroStack.clear(); 168 169 // Destroy any macro definitions. 170 while (MacroInfoChain *I = MIChainHead) { 171 MIChainHead = I->Next; 172 I->~MacroInfoChain(); 173 } 174 175 // Free any cached macro expanders. 176 // This populates MacroArgCache, so all TokenLexers need to be destroyed 177 // before the code below that frees up the MacroArgCache list. 178 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr); 179 CurTokenLexer.reset(); 180 181 // Free any cached MacroArgs. 182 for (MacroArgs *ArgList = MacroArgCache; ArgList;) 183 ArgList = ArgList->deallocate(); 184 185 // Delete the header search info, if we own it. 186 if (OwnsHeaderSearch) 187 delete &HeaderInfo; 188 } 189 190 void Preprocessor::Initialize(const TargetInfo &Target, 191 const TargetInfo *AuxTarget) { 192 assert((!this->Target || this->Target == &Target) && 193 "Invalid override of target information"); 194 this->Target = &Target; 195 196 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) && 197 "Invalid override of aux target information."); 198 this->AuxTarget = AuxTarget; 199 200 // Initialize information about built-ins. 201 BuiltinInfo->InitializeTarget(Target, AuxTarget); 202 HeaderInfo.setTarget(Target); 203 204 // Populate the identifier table with info about keywords for the current language. 205 Identifiers.AddKeywords(LangOpts); 206 207 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo. 208 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod()); 209 210 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine) 211 // Use setting from TargetInfo. 212 setCurrentFPEvalMethod(SourceLocation(), Target.getFPEvalMethod()); 213 else 214 // Set initial value of __FLT_EVAL_METHOD__ from the command line. 215 setCurrentFPEvalMethod(SourceLocation(), getLangOpts().getFPEvalMethod()); 216 // When `-ffast-math` option is enabled, it triggers several driver math 217 // options to be enabled. Among those, only one the following two modes 218 // affect the eval-method: reciprocal or reassociate. 219 if (getLangOpts().AllowFPReassoc || getLangOpts().AllowRecip) 220 setCurrentFPEvalMethod(SourceLocation(), LangOptions::FEM_Indeterminable); 221 } 222 223 void Preprocessor::InitializeForModelFile() { 224 NumEnteredSourceFiles = 0; 225 226 // Reset pragmas 227 PragmaHandlersBackup = std::move(PragmaHandlers); 228 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef()); 229 RegisterBuiltinPragmas(); 230 231 // Reset PredefinesFileID 232 PredefinesFileID = FileID(); 233 } 234 235 void Preprocessor::FinalizeForModelFile() { 236 NumEnteredSourceFiles = 1; 237 238 PragmaHandlers = std::move(PragmaHandlersBackup); 239 } 240 241 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { 242 llvm::errs() << tok::getTokenName(Tok.getKind()); 243 244 if (!Tok.isAnnotation()) 245 llvm::errs() << " '" << getSpelling(Tok) << "'"; 246 247 if (!DumpFlags) return; 248 249 llvm::errs() << "\t"; 250 if (Tok.isAtStartOfLine()) 251 llvm::errs() << " [StartOfLine]"; 252 if (Tok.hasLeadingSpace()) 253 llvm::errs() << " [LeadingSpace]"; 254 if (Tok.isExpandDisabled()) 255 llvm::errs() << " [ExpandDisabled]"; 256 if (Tok.needsCleaning()) { 257 const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); 258 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) 259 << "']"; 260 } 261 262 llvm::errs() << "\tLoc=<"; 263 DumpLocation(Tok.getLocation()); 264 llvm::errs() << ">"; 265 } 266 267 void Preprocessor::DumpLocation(SourceLocation Loc) const { 268 Loc.print(llvm::errs(), SourceMgr); 269 } 270 271 void Preprocessor::DumpMacro(const MacroInfo &MI) const { 272 llvm::errs() << "MACRO: "; 273 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { 274 DumpToken(MI.getReplacementToken(i)); 275 llvm::errs() << " "; 276 } 277 llvm::errs() << "\n"; 278 } 279 280 void Preprocessor::PrintStats() { 281 llvm::errs() << "\n*** Preprocessor Stats:\n"; 282 llvm::errs() << NumDirectives << " directives found:\n"; 283 llvm::errs() << " " << NumDefined << " #define.\n"; 284 llvm::errs() << " " << NumUndefined << " #undef.\n"; 285 llvm::errs() << " #include/#include_next/#import:\n"; 286 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n"; 287 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n"; 288 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n"; 289 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n"; 290 llvm::errs() << " " << NumEndif << " #endif.\n"; 291 llvm::errs() << " " << NumPragma << " #pragma.\n"; 292 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; 293 294 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/" 295 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " 296 << NumFastMacroExpanded << " on the fast path.\n"; 297 llvm::errs() << (NumFastTokenPaste+NumTokenPaste) 298 << " token paste (##) operations performed, " 299 << NumFastTokenPaste << " on the fast path.\n"; 300 301 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total"; 302 303 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory(); 304 llvm::errs() << "\n Macro Expanded Tokens: " 305 << llvm::capacity_in_bytes(MacroExpandedTokens); 306 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity(); 307 // FIXME: List information for all submodules. 308 llvm::errs() << "\n Macros: " 309 << llvm::capacity_in_bytes(CurSubmoduleState->Macros); 310 llvm::errs() << "\n #pragma push_macro Info: " 311 << llvm::capacity_in_bytes(PragmaPushMacroInfo); 312 llvm::errs() << "\n Poison Reasons: " 313 << llvm::capacity_in_bytes(PoisonReasons); 314 llvm::errs() << "\n Comment Handlers: " 315 << llvm::capacity_in_bytes(CommentHandlers) << "\n"; 316 } 317 318 Preprocessor::macro_iterator 319 Preprocessor::macro_begin(bool IncludeExternalMacros) const { 320 if (IncludeExternalMacros && ExternalSource && 321 !ReadMacrosFromExternalSource) { 322 ReadMacrosFromExternalSource = true; 323 ExternalSource->ReadDefinedMacros(); 324 } 325 326 // Make sure we cover all macros in visible modules. 327 for (const ModuleMacro &Macro : ModuleMacros) 328 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState())); 329 330 return CurSubmoduleState->Macros.begin(); 331 } 332 333 size_t Preprocessor::getTotalMemory() const { 334 return BP.getTotalMemory() 335 + llvm::capacity_in_bytes(MacroExpandedTokens) 336 + Predefines.capacity() /* Predefines buffer. */ 337 // FIXME: Include sizes from all submodules, and include MacroInfo sizes, 338 // and ModuleMacros. 339 + llvm::capacity_in_bytes(CurSubmoduleState->Macros) 340 + llvm::capacity_in_bytes(PragmaPushMacroInfo) 341 + llvm::capacity_in_bytes(PoisonReasons) 342 + llvm::capacity_in_bytes(CommentHandlers); 343 } 344 345 Preprocessor::macro_iterator 346 Preprocessor::macro_end(bool IncludeExternalMacros) const { 347 if (IncludeExternalMacros && ExternalSource && 348 !ReadMacrosFromExternalSource) { 349 ReadMacrosFromExternalSource = true; 350 ExternalSource->ReadDefinedMacros(); 351 } 352 353 return CurSubmoduleState->Macros.end(); 354 } 355 356 /// Compares macro tokens with a specified token value sequence. 357 static bool MacroDefinitionEquals(const MacroInfo *MI, 358 ArrayRef<TokenValue> Tokens) { 359 return Tokens.size() == MI->getNumTokens() && 360 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin()); 361 } 362 363 StringRef Preprocessor::getLastMacroWithSpelling( 364 SourceLocation Loc, 365 ArrayRef<TokenValue> Tokens) const { 366 SourceLocation BestLocation; 367 StringRef BestSpelling; 368 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end(); 369 I != E; ++I) { 370 const MacroDirective::DefInfo 371 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr); 372 if (!Def || !Def.getMacroInfo()) 373 continue; 374 if (!Def.getMacroInfo()->isObjectLike()) 375 continue; 376 if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens)) 377 continue; 378 SourceLocation Location = Def.getLocation(); 379 // Choose the macro defined latest. 380 if (BestLocation.isInvalid() || 381 (Location.isValid() && 382 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) { 383 BestLocation = Location; 384 BestSpelling = I->first->getName(); 385 } 386 } 387 return BestSpelling; 388 } 389 390 void Preprocessor::recomputeCurLexerKind() { 391 if (CurLexer) 392 CurLexerKind = CurLexer->isDependencyDirectivesLexer() 393 ? CLK_DependencyDirectivesLexer 394 : CLK_Lexer; 395 else if (CurTokenLexer) 396 CurLexerKind = CLK_TokenLexer; 397 else 398 CurLexerKind = CLK_CachingLexer; 399 } 400 401 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 402 unsigned CompleteLine, 403 unsigned CompleteColumn) { 404 assert(File); 405 assert(CompleteLine && CompleteColumn && "Starts from 1:1"); 406 assert(!CodeCompletionFile && "Already set"); 407 408 // Load the actual file's contents. 409 Optional<llvm::MemoryBufferRef> Buffer = 410 SourceMgr.getMemoryBufferForFileOrNone(File); 411 if (!Buffer) 412 return true; 413 414 // Find the byte position of the truncation point. 415 const char *Position = Buffer->getBufferStart(); 416 for (unsigned Line = 1; Line < CompleteLine; ++Line) { 417 for (; *Position; ++Position) { 418 if (*Position != '\r' && *Position != '\n') 419 continue; 420 421 // Eat \r\n or \n\r as a single line. 422 if ((Position[1] == '\r' || Position[1] == '\n') && 423 Position[0] != Position[1]) 424 ++Position; 425 ++Position; 426 break; 427 } 428 } 429 430 Position += CompleteColumn - 1; 431 432 // If pointing inside the preamble, adjust the position at the beginning of 433 // the file after the preamble. 434 if (SkipMainFilePreamble.first && 435 SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) { 436 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first) 437 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first; 438 } 439 440 if (Position > Buffer->getBufferEnd()) 441 Position = Buffer->getBufferEnd(); 442 443 CodeCompletionFile = File; 444 CodeCompletionOffset = Position - Buffer->getBufferStart(); 445 446 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer( 447 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier()); 448 char *NewBuf = NewBuffer->getBufferStart(); 449 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf); 450 *NewPos = '\0'; 451 std::copy(Position, Buffer->getBufferEnd(), NewPos+1); 452 SourceMgr.overrideFileContents(File, std::move(NewBuffer)); 453 454 return false; 455 } 456 457 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir, 458 bool IsAngled) { 459 setCodeCompletionReached(); 460 if (CodeComplete) 461 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled); 462 } 463 464 void Preprocessor::CodeCompleteNaturalLanguage() { 465 setCodeCompletionReached(); 466 if (CodeComplete) 467 CodeComplete->CodeCompleteNaturalLanguage(); 468 } 469 470 /// getSpelling - This method is used to get the spelling of a token into a 471 /// SmallVector. Note that the returned StringRef may not point to the 472 /// supplied buffer if a copy can be avoided. 473 StringRef Preprocessor::getSpelling(const Token &Tok, 474 SmallVectorImpl<char> &Buffer, 475 bool *Invalid) const { 476 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 477 if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) { 478 // Try the fast path. 479 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) 480 return II->getName(); 481 } 482 483 // Resize the buffer if we need to copy into it. 484 if (Tok.needsCleaning()) 485 Buffer.resize(Tok.getLength()); 486 487 const char *Ptr = Buffer.data(); 488 unsigned Len = getSpelling(Tok, Ptr, Invalid); 489 return StringRef(Ptr, Len); 490 } 491 492 /// CreateString - Plop the specified string into a scratch buffer and return a 493 /// location for it. If specified, the source location provides a source 494 /// location for the token. 495 void Preprocessor::CreateString(StringRef Str, Token &Tok, 496 SourceLocation ExpansionLocStart, 497 SourceLocation ExpansionLocEnd) { 498 Tok.setLength(Str.size()); 499 500 const char *DestPtr; 501 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr); 502 503 if (ExpansionLocStart.isValid()) 504 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart, 505 ExpansionLocEnd, Str.size()); 506 Tok.setLocation(Loc); 507 508 // If this is a raw identifier or a literal token, set the pointer data. 509 if (Tok.is(tok::raw_identifier)) 510 Tok.setRawIdentifierData(DestPtr); 511 else if (Tok.isLiteral()) 512 Tok.setLiteralData(DestPtr); 513 } 514 515 SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) { 516 auto &SM = getSourceManager(); 517 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc); 518 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc); 519 bool Invalid = false; 520 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 521 if (Invalid) 522 return SourceLocation(); 523 524 // FIXME: We could consider re-using spelling for tokens we see repeatedly. 525 const char *DestPtr; 526 SourceLocation Spelling = 527 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr); 528 return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length)); 529 } 530 531 Module *Preprocessor::getCurrentModule() { 532 if (!getLangOpts().isCompilingModule()) 533 return nullptr; 534 535 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule); 536 } 537 538 //===----------------------------------------------------------------------===// 539 // Preprocessor Initialization Methods 540 //===----------------------------------------------------------------------===// 541 542 /// EnterMainSourceFile - Enter the specified FileID as the main source file, 543 /// which implicitly adds the builtin defines etc. 544 void Preprocessor::EnterMainSourceFile() { 545 // We do not allow the preprocessor to reenter the main file. Doing so will 546 // cause FileID's to accumulate information from both runs (e.g. #line 547 // information) and predefined macros aren't guaranteed to be set properly. 548 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); 549 FileID MainFileID = SourceMgr.getMainFileID(); 550 551 // If MainFileID is loaded it means we loaded an AST file, no need to enter 552 // a main file. 553 if (!SourceMgr.isLoadedFileID(MainFileID)) { 554 // Enter the main file source buffer. 555 EnterSourceFile(MainFileID, nullptr, SourceLocation()); 556 557 // If we've been asked to skip bytes in the main file (e.g., as part of a 558 // precompiled preamble), do so now. 559 if (SkipMainFilePreamble.first > 0) 560 CurLexer->SetByteOffset(SkipMainFilePreamble.first, 561 SkipMainFilePreamble.second); 562 563 // Tell the header info that the main file was entered. If the file is later 564 // #imported, it won't be re-entered. 565 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) 566 markIncluded(FE); 567 } 568 569 // Preprocess Predefines to populate the initial preprocessor state. 570 std::unique_ptr<llvm::MemoryBuffer> SB = 571 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>"); 572 assert(SB && "Cannot create predefined source buffer"); 573 FileID FID = SourceMgr.createFileID(std::move(SB)); 574 assert(FID.isValid() && "Could not create FileID for predefines?"); 575 setPredefinesFileID(FID); 576 577 // Start parsing the predefines. 578 EnterSourceFile(FID, nullptr, SourceLocation()); 579 580 if (!PPOpts->PCHThroughHeader.empty()) { 581 // Lookup and save the FileID for the through header. If it isn't found 582 // in the search path, it's a fatal error. 583 Optional<FileEntryRef> File = LookupFile( 584 SourceLocation(), PPOpts->PCHThroughHeader, 585 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, 586 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr, 587 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr, 588 /*IsFrameworkFound=*/nullptr); 589 if (!File) { 590 Diag(SourceLocation(), diag::err_pp_through_header_not_found) 591 << PPOpts->PCHThroughHeader; 592 return; 593 } 594 setPCHThroughHeaderFileID( 595 SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User)); 596 } 597 598 // Skip tokens from the Predefines and if needed the main file. 599 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) || 600 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop)) 601 SkipTokensWhileUsingPCH(); 602 } 603 604 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) { 605 assert(PCHThroughHeaderFileID.isInvalid() && 606 "PCHThroughHeaderFileID already set!"); 607 PCHThroughHeaderFileID = FID; 608 } 609 610 bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) { 611 assert(PCHThroughHeaderFileID.isValid() && 612 "Invalid PCH through header FileID"); 613 return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID); 614 } 615 616 bool Preprocessor::creatingPCHWithThroughHeader() { 617 return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() && 618 PCHThroughHeaderFileID.isValid(); 619 } 620 621 bool Preprocessor::usingPCHWithThroughHeader() { 622 return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() && 623 PCHThroughHeaderFileID.isValid(); 624 } 625 626 bool Preprocessor::creatingPCHWithPragmaHdrStop() { 627 return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop; 628 } 629 630 bool Preprocessor::usingPCHWithPragmaHdrStop() { 631 return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop; 632 } 633 634 /// Skip tokens until after the #include of the through header or 635 /// until after a #pragma hdrstop is seen. Tokens in the predefines file 636 /// and the main file may be skipped. If the end of the predefines file 637 /// is reached, skipping continues into the main file. If the end of the 638 /// main file is reached, it's a fatal error. 639 void Preprocessor::SkipTokensWhileUsingPCH() { 640 bool ReachedMainFileEOF = false; 641 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader; 642 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop; 643 Token Tok; 644 while (true) { 645 bool InPredefines = 646 (CurLexer && CurLexer->getFileID() == getPredefinesFileID()); 647 switch (CurLexerKind) { 648 case CLK_Lexer: 649 CurLexer->Lex(Tok); 650 break; 651 case CLK_TokenLexer: 652 CurTokenLexer->Lex(Tok); 653 break; 654 case CLK_CachingLexer: 655 CachingLex(Tok); 656 break; 657 case CLK_DependencyDirectivesLexer: 658 CurLexer->LexDependencyDirectiveToken(Tok); 659 break; 660 case CLK_LexAfterModuleImport: 661 LexAfterModuleImport(Tok); 662 break; 663 } 664 if (Tok.is(tok::eof) && !InPredefines) { 665 ReachedMainFileEOF = true; 666 break; 667 } 668 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader) 669 break; 670 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop) 671 break; 672 } 673 if (ReachedMainFileEOF) { 674 if (UsingPCHThroughHeader) 675 Diag(SourceLocation(), diag::err_pp_through_header_not_seen) 676 << PPOpts->PCHThroughHeader << 1; 677 else if (!PPOpts->PCHWithHdrStopCreate) 678 Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen); 679 } 680 } 681 682 void Preprocessor::replayPreambleConditionalStack() { 683 // Restore the conditional stack from the preamble, if there is one. 684 if (PreambleConditionalStack.isReplaying()) { 685 assert(CurPPLexer && 686 "CurPPLexer is null when calling replayPreambleConditionalStack."); 687 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); 688 PreambleConditionalStack.doneReplaying(); 689 if (PreambleConditionalStack.reachedEOFWhileSkipping()) 690 SkipExcludedConditionalBlock( 691 PreambleConditionalStack.SkipInfo->HashTokenLoc, 692 PreambleConditionalStack.SkipInfo->IfTokenLoc, 693 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion, 694 PreambleConditionalStack.SkipInfo->FoundElse, 695 PreambleConditionalStack.SkipInfo->ElseLoc); 696 } 697 } 698 699 void Preprocessor::EndSourceFile() { 700 // Notify the client that we reached the end of the source file. 701 if (Callbacks) 702 Callbacks->EndOfMainFile(); 703 } 704 705 //===----------------------------------------------------------------------===// 706 // Lexer Event Handling. 707 //===----------------------------------------------------------------------===// 708 709 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the 710 /// identifier information for the token and install it into the token, 711 /// updating the token kind accordingly. 712 IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { 713 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!"); 714 715 // Look up this token, see if it is a macro, or if it is a language keyword. 716 IdentifierInfo *II; 717 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) { 718 // No cleaning needed, just use the characters from the lexed buffer. 719 II = getIdentifierInfo(Identifier.getRawIdentifier()); 720 } else { 721 // Cleaning needed, alloca a buffer, clean into it, then use the buffer. 722 SmallString<64> IdentifierBuffer; 723 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer); 724 725 if (Identifier.hasUCN()) { 726 SmallString<64> UCNIdentifierBuffer; 727 expandUCNs(UCNIdentifierBuffer, CleanedStr); 728 II = getIdentifierInfo(UCNIdentifierBuffer); 729 } else { 730 II = getIdentifierInfo(CleanedStr); 731 } 732 } 733 734 // Update the token info (identifier info and appropriate token kind). 735 // FIXME: the raw_identifier may contain leading whitespace which is removed 736 // from the cleaned identifier token. The SourceLocation should be updated to 737 // refer to the non-whitespace character. For instance, the text "\\\nB" (a 738 // line continuation before 'B') is parsed as a single tok::raw_identifier and 739 // is cleaned to tok::identifier "B". After cleaning the token's length is 740 // still 3 and the SourceLocation refers to the location of the backslash. 741 Identifier.setIdentifierInfo(II); 742 Identifier.setKind(II->getTokenID()); 743 744 return II; 745 } 746 747 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) { 748 PoisonReasons[II] = DiagID; 749 } 750 751 void Preprocessor::PoisonSEHIdentifiers(bool Poison) { 752 assert(Ident__exception_code && Ident__exception_info); 753 assert(Ident___exception_code && Ident___exception_info); 754 Ident__exception_code->setIsPoisoned(Poison); 755 Ident___exception_code->setIsPoisoned(Poison); 756 Ident_GetExceptionCode->setIsPoisoned(Poison); 757 Ident__exception_info->setIsPoisoned(Poison); 758 Ident___exception_info->setIsPoisoned(Poison); 759 Ident_GetExceptionInfo->setIsPoisoned(Poison); 760 Ident__abnormal_termination->setIsPoisoned(Poison); 761 Ident___abnormal_termination->setIsPoisoned(Poison); 762 Ident_AbnormalTermination->setIsPoisoned(Poison); 763 } 764 765 void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) { 766 assert(Identifier.getIdentifierInfo() && 767 "Can't handle identifiers without identifier info!"); 768 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it = 769 PoisonReasons.find(Identifier.getIdentifierInfo()); 770 if(it == PoisonReasons.end()) 771 Diag(Identifier, diag::err_pp_used_poisoned_id); 772 else 773 Diag(Identifier,it->second) << Identifier.getIdentifierInfo(); 774 } 775 776 /// Returns a diagnostic message kind for reporting a future keyword as 777 /// appropriate for the identifier and specified language. 778 static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, 779 const LangOptions &LangOpts) { 780 assert(II.isFutureCompatKeyword() && "diagnostic should not be needed"); 781 782 if (LangOpts.CPlusPlus) 783 return llvm::StringSwitch<diag::kind>(II.getName()) 784 #define CXX11_KEYWORD(NAME, FLAGS) \ 785 .Case(#NAME, diag::warn_cxx11_keyword) 786 #define CXX20_KEYWORD(NAME, FLAGS) \ 787 .Case(#NAME, diag::warn_cxx20_keyword) 788 #include "clang/Basic/TokenKinds.def" 789 // char8_t is not modeled as a CXX20_KEYWORD because it's not 790 // unconditionally enabled in C++20 mode. (It can be disabled 791 // by -fno-char8_t.) 792 .Case("char8_t", diag::warn_cxx20_keyword) 793 ; 794 795 llvm_unreachable( 796 "Keyword not known to come from a newer Standard or proposed Standard"); 797 } 798 799 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const { 800 assert(II.isOutOfDate() && "not out of date"); 801 getExternalSource()->updateOutOfDateIdentifier(II); 802 } 803 804 /// HandleIdentifier - This callback is invoked when the lexer reads an 805 /// identifier. This callback looks up the identifier in the map and/or 806 /// potentially macro expands it or turns it into a named token (like 'for'). 807 /// 808 /// Note that callers of this method are guarded by checking the 809 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the 810 /// IdentifierInfo methods that compute these properties will need to change to 811 /// match. 812 bool Preprocessor::HandleIdentifier(Token &Identifier) { 813 assert(Identifier.getIdentifierInfo() && 814 "Can't handle identifiers without identifier info!"); 815 816 IdentifierInfo &II = *Identifier.getIdentifierInfo(); 817 818 // If the information about this identifier is out of date, update it from 819 // the external source. 820 // We have to treat __VA_ARGS__ in a special way, since it gets 821 // serialized with isPoisoned = true, but our preprocessor may have 822 // unpoisoned it if we're defining a C99 macro. 823 if (II.isOutOfDate()) { 824 bool CurrentIsPoisoned = false; 825 const bool IsSpecialVariadicMacro = 826 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__; 827 if (IsSpecialVariadicMacro) 828 CurrentIsPoisoned = II.isPoisoned(); 829 830 updateOutOfDateIdentifier(II); 831 Identifier.setKind(II.getTokenID()); 832 833 if (IsSpecialVariadicMacro) 834 II.setIsPoisoned(CurrentIsPoisoned); 835 } 836 837 // If this identifier was poisoned, and if it was not produced from a macro 838 // expansion, emit an error. 839 if (II.isPoisoned() && CurPPLexer) { 840 HandlePoisonedIdentifier(Identifier); 841 } 842 843 // If this is a macro to be expanded, do it. 844 if (MacroDefinition MD = getMacroDefinition(&II)) { 845 auto *MI = MD.getMacroInfo(); 846 assert(MI && "macro definition with no macro info?"); 847 if (!DisableMacroExpansion) { 848 if (!Identifier.isExpandDisabled() && MI->isEnabled()) { 849 // C99 6.10.3p10: If the preprocessing token immediately after the 850 // macro name isn't a '(', this macro should not be expanded. 851 if (!MI->isFunctionLike() || isNextPPTokenLParen()) 852 return HandleMacroExpandedIdentifier(Identifier, MD); 853 } else { 854 // C99 6.10.3.4p2 says that a disabled macro may never again be 855 // expanded, even if it's in a context where it could be expanded in the 856 // future. 857 Identifier.setFlag(Token::DisableExpand); 858 if (MI->isObjectLike() || isNextPPTokenLParen()) 859 Diag(Identifier, diag::pp_disabled_macro_expansion); 860 } 861 } 862 } 863 864 // If this identifier is a keyword in a newer Standard or proposed Standard, 865 // produce a warning. Don't warn if we're not considering macro expansion, 866 // since this identifier might be the name of a macro. 867 // FIXME: This warning is disabled in cases where it shouldn't be, like 868 // "#define constexpr constexpr", "int constexpr;" 869 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) { 870 Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts())) 871 << II.getName(); 872 // Don't diagnose this keyword again in this translation unit. 873 II.setIsFutureCompatKeyword(false); 874 } 875 876 // If this is an extension token, diagnose its use. 877 // We avoid diagnosing tokens that originate from macro definitions. 878 // FIXME: This warning is disabled in cases where it shouldn't be, 879 // like "#define TY typeof", "TY(1) x". 880 if (II.isExtensionToken() && !DisableMacroExpansion) 881 Diag(Identifier, diag::ext_token_used); 882 883 // If this is the 'import' contextual keyword following an '@', note 884 // that the next token indicates a module name. 885 // 886 // Note that we do not treat 'import' as a contextual 887 // keyword when we're in a caching lexer, because caching lexers only get 888 // used in contexts where import declarations are disallowed. 889 // 890 // Likewise if this is the C++ Modules TS import keyword. 891 if (((LastTokenWasAt && II.isModulesImport()) || 892 Identifier.is(tok::kw_import)) && 893 !InMacroArgs && !DisableMacroExpansion && 894 (getLangOpts().Modules || getLangOpts().DebuggerSupport) && 895 CurLexerKind != CLK_CachingLexer) { 896 ModuleImportLoc = Identifier.getLocation(); 897 ModuleImportPath.clear(); 898 ModuleImportExpectsIdentifier = true; 899 CurLexerKind = CLK_LexAfterModuleImport; 900 } 901 return true; 902 } 903 904 void Preprocessor::Lex(Token &Result) { 905 ++LexLevel; 906 907 // We loop here until a lex function returns a token; this avoids recursion. 908 bool ReturnedToken; 909 do { 910 switch (CurLexerKind) { 911 case CLK_Lexer: 912 ReturnedToken = CurLexer->Lex(Result); 913 break; 914 case CLK_TokenLexer: 915 ReturnedToken = CurTokenLexer->Lex(Result); 916 break; 917 case CLK_CachingLexer: 918 CachingLex(Result); 919 ReturnedToken = true; 920 break; 921 case CLK_DependencyDirectivesLexer: 922 ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result); 923 break; 924 case CLK_LexAfterModuleImport: 925 ReturnedToken = LexAfterModuleImport(Result); 926 break; 927 } 928 } while (!ReturnedToken); 929 930 if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure) 931 return; 932 933 if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) { 934 // Remember the identifier before code completion token. 935 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo()); 936 setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc()); 937 // Set IdenfitierInfo to null to avoid confusing code that handles both 938 // identifiers and completion tokens. 939 Result.setIdentifierInfo(nullptr); 940 } 941 942 // Update ImportSeqState to track our position within a C++20 import-seq 943 // if this token is being produced as a result of phase 4 of translation. 944 // Update TrackGMFState to decide if we are currently in a Global Module 945 // Fragment. GMF state updates should precede ImportSeq ones, since GMF state 946 // depends on the prevailing ImportSeq state in two cases. 947 if (getLangOpts().CPlusPlusModules && LexLevel == 1 && 948 !Result.getFlag(Token::IsReinjected)) { 949 switch (Result.getKind()) { 950 case tok::l_paren: case tok::l_square: case tok::l_brace: 951 ImportSeqState.handleOpenBracket(); 952 break; 953 case tok::r_paren: case tok::r_square: 954 ImportSeqState.handleCloseBracket(); 955 break; 956 case tok::r_brace: 957 ImportSeqState.handleCloseBrace(); 958 break; 959 // This token is injected to represent the translation of '#include "a.h"' 960 // into "import a.h;". Mimic the notional ';'. 961 case tok::annot_module_include: 962 case tok::semi: 963 TrackGMFState.handleSemi(); 964 ImportSeqState.handleSemi(); 965 break; 966 case tok::header_name: 967 case tok::annot_header_unit: 968 ImportSeqState.handleHeaderName(); 969 break; 970 case tok::kw_export: 971 TrackGMFState.handleExport(); 972 ImportSeqState.handleExport(); 973 break; 974 case tok::identifier: 975 if (Result.getIdentifierInfo()->isModulesImport()) { 976 TrackGMFState.handleImport(ImportSeqState.afterTopLevelSeq()); 977 ImportSeqState.handleImport(); 978 if (ImportSeqState.afterImportSeq()) { 979 ModuleImportLoc = Result.getLocation(); 980 ModuleImportPath.clear(); 981 ModuleImportExpectsIdentifier = true; 982 CurLexerKind = CLK_LexAfterModuleImport; 983 } 984 break; 985 } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) { 986 TrackGMFState.handleModule(ImportSeqState.afterTopLevelSeq()); 987 break; 988 } 989 LLVM_FALLTHROUGH; 990 default: 991 TrackGMFState.handleMisc(); 992 ImportSeqState.handleMisc(); 993 break; 994 } 995 } 996 997 LastTokenWasAt = Result.is(tok::at); 998 --LexLevel; 999 1000 if ((LexLevel == 0 || PreprocessToken) && 1001 !Result.getFlag(Token::IsReinjected)) { 1002 if (LexLevel == 0) 1003 ++TokenCount; 1004 if (OnToken) 1005 OnToken(Result); 1006 } 1007 } 1008 1009 /// Lex a header-name token (including one formed from header-name-tokens if 1010 /// \p AllowConcatenation is \c true). 1011 /// 1012 /// \param FilenameTok Filled in with the next token. On success, this will 1013 /// be either a header_name token. On failure, it will be whatever other 1014 /// token was found instead. 1015 /// \param AllowMacroExpansion If \c true, allow the header name to be formed 1016 /// by macro expansion (concatenating tokens as necessary if the first 1017 /// token is a '<'). 1018 /// \return \c true if we reached EOD or EOF while looking for a > token in 1019 /// a concatenated header name and diagnosed it. \c false otherwise. 1020 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { 1021 // Lex using header-name tokenization rules if tokens are being lexed from 1022 // a file. Just grab a token normally if we're in a macro expansion. 1023 if (CurPPLexer) 1024 CurPPLexer->LexIncludeFilename(FilenameTok); 1025 else 1026 Lex(FilenameTok); 1027 1028 // This could be a <foo/bar.h> file coming from a macro expansion. In this 1029 // case, glue the tokens together into an angle_string_literal token. 1030 SmallString<128> FilenameBuffer; 1031 if (FilenameTok.is(tok::less) && AllowMacroExpansion) { 1032 bool StartOfLine = FilenameTok.isAtStartOfLine(); 1033 bool LeadingSpace = FilenameTok.hasLeadingSpace(); 1034 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro(); 1035 1036 SourceLocation Start = FilenameTok.getLocation(); 1037 SourceLocation End; 1038 FilenameBuffer.push_back('<'); 1039 1040 // Consume tokens until we find a '>'. 1041 // FIXME: A header-name could be formed starting or ending with an 1042 // alternative token. It's not clear whether that's ill-formed in all 1043 // cases. 1044 while (FilenameTok.isNot(tok::greater)) { 1045 Lex(FilenameTok); 1046 if (FilenameTok.isOneOf(tok::eod, tok::eof)) { 1047 Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater; 1048 Diag(Start, diag::note_matching) << tok::less; 1049 return true; 1050 } 1051 1052 End = FilenameTok.getLocation(); 1053 1054 // FIXME: Provide code completion for #includes. 1055 if (FilenameTok.is(tok::code_completion)) { 1056 setCodeCompletionReached(); 1057 Lex(FilenameTok); 1058 continue; 1059 } 1060 1061 // Append the spelling of this token to the buffer. If there was a space 1062 // before it, add it now. 1063 if (FilenameTok.hasLeadingSpace()) 1064 FilenameBuffer.push_back(' '); 1065 1066 // Get the spelling of the token, directly into FilenameBuffer if 1067 // possible. 1068 size_t PreAppendSize = FilenameBuffer.size(); 1069 FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength()); 1070 1071 const char *BufPtr = &FilenameBuffer[PreAppendSize]; 1072 unsigned ActualLen = getSpelling(FilenameTok, BufPtr); 1073 1074 // If the token was spelled somewhere else, copy it into FilenameBuffer. 1075 if (BufPtr != &FilenameBuffer[PreAppendSize]) 1076 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen); 1077 1078 // Resize FilenameBuffer to the correct size. 1079 if (FilenameTok.getLength() != ActualLen) 1080 FilenameBuffer.resize(PreAppendSize + ActualLen); 1081 } 1082 1083 FilenameTok.startToken(); 1084 FilenameTok.setKind(tok::header_name); 1085 FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine); 1086 FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace); 1087 FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro); 1088 CreateString(FilenameBuffer, FilenameTok, Start, End); 1089 } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) { 1090 // Convert a string-literal token of the form " h-char-sequence " 1091 // (produced by macro expansion) into a header-name token. 1092 // 1093 // The rules for header-names don't quite match the rules for 1094 // string-literals, but all the places where they differ result in 1095 // undefined behavior, so we can and do treat them the same. 1096 // 1097 // A string-literal with a prefix or suffix is not translated into a 1098 // header-name. This could theoretically be observable via the C++20 1099 // context-sensitive header-name formation rules. 1100 StringRef Str = getSpelling(FilenameTok, FilenameBuffer); 1101 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"') 1102 FilenameTok.setKind(tok::header_name); 1103 } 1104 1105 return false; 1106 } 1107 1108 /// Collect the tokens of a C++20 pp-import-suffix. 1109 void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) { 1110 // FIXME: For error recovery, consider recognizing attribute syntax here 1111 // and terminating / diagnosing a missing semicolon if we find anything 1112 // else? (Can we leave that to the parser?) 1113 unsigned BracketDepth = 0; 1114 while (true) { 1115 Toks.emplace_back(); 1116 Lex(Toks.back()); 1117 1118 switch (Toks.back().getKind()) { 1119 case tok::l_paren: case tok::l_square: case tok::l_brace: 1120 ++BracketDepth; 1121 break; 1122 1123 case tok::r_paren: case tok::r_square: case tok::r_brace: 1124 if (BracketDepth == 0) 1125 return; 1126 --BracketDepth; 1127 break; 1128 1129 case tok::semi: 1130 if (BracketDepth == 0) 1131 return; 1132 break; 1133 1134 case tok::eof: 1135 return; 1136 1137 default: 1138 break; 1139 } 1140 } 1141 } 1142 1143 1144 /// Lex a token following the 'import' contextual keyword. 1145 /// 1146 /// pp-import: [C++20] 1147 /// import header-name pp-import-suffix[opt] ; 1148 /// import header-name-tokens pp-import-suffix[opt] ; 1149 /// [ObjC] @ import module-name ; 1150 /// [Clang] import module-name ; 1151 /// 1152 /// header-name-tokens: 1153 /// string-literal 1154 /// < [any sequence of preprocessing-tokens other than >] > 1155 /// 1156 /// module-name: 1157 /// module-name-qualifier[opt] identifier 1158 /// 1159 /// module-name-qualifier 1160 /// module-name-qualifier[opt] identifier . 1161 /// 1162 /// We respond to a pp-import by importing macros from the named module. 1163 bool Preprocessor::LexAfterModuleImport(Token &Result) { 1164 // Figure out what kind of lexer we actually have. 1165 recomputeCurLexerKind(); 1166 1167 // Lex the next token. The header-name lexing rules are used at the start of 1168 // a pp-import. 1169 // 1170 // For now, we only support header-name imports in C++20 mode. 1171 // FIXME: Should we allow this in all language modes that support an import 1172 // declaration as an extension? 1173 if (ModuleImportPath.empty() && getLangOpts().CPlusPlusModules) { 1174 if (LexHeaderName(Result)) 1175 return true; 1176 } else { 1177 Lex(Result); 1178 } 1179 1180 // Allocate a holding buffer for a sequence of tokens and introduce it into 1181 // the token stream. 1182 auto EnterTokens = [this](ArrayRef<Token> Toks) { 1183 auto ToksCopy = std::make_unique<Token[]>(Toks.size()); 1184 std::copy(Toks.begin(), Toks.end(), ToksCopy.get()); 1185 EnterTokenStream(std::move(ToksCopy), Toks.size(), 1186 /*DisableMacroExpansion*/ true, /*IsReinject*/ false); 1187 }; 1188 1189 // Check for a header-name. 1190 SmallVector<Token, 32> Suffix; 1191 if (Result.is(tok::header_name)) { 1192 // Enter the header-name token into the token stream; a Lex action cannot 1193 // both return a token and cache tokens (doing so would corrupt the token 1194 // cache if the call to Lex comes from CachingLex / PeekAhead). 1195 Suffix.push_back(Result); 1196 1197 // Consume the pp-import-suffix and expand any macros in it now. We'll add 1198 // it back into the token stream later. 1199 CollectPpImportSuffix(Suffix); 1200 if (Suffix.back().isNot(tok::semi)) { 1201 // This is not a pp-import after all. 1202 EnterTokens(Suffix); 1203 return false; 1204 } 1205 1206 // C++2a [cpp.module]p1: 1207 // The ';' preprocessing-token terminating a pp-import shall not have 1208 // been produced by macro replacement. 1209 SourceLocation SemiLoc = Suffix.back().getLocation(); 1210 if (SemiLoc.isMacroID()) 1211 Diag(SemiLoc, diag::err_header_import_semi_in_macro); 1212 1213 // Reconstitute the import token. 1214 Token ImportTok; 1215 ImportTok.startToken(); 1216 ImportTok.setKind(tok::kw_import); 1217 ImportTok.setLocation(ModuleImportLoc); 1218 ImportTok.setIdentifierInfo(getIdentifierInfo("import")); 1219 ImportTok.setLength(6); 1220 1221 auto Action = HandleHeaderIncludeOrImport( 1222 /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc); 1223 switch (Action.Kind) { 1224 case ImportAction::None: 1225 break; 1226 1227 case ImportAction::ModuleBegin: 1228 // Let the parser know we're textually entering the module. 1229 Suffix.emplace_back(); 1230 Suffix.back().startToken(); 1231 Suffix.back().setKind(tok::annot_module_begin); 1232 Suffix.back().setLocation(SemiLoc); 1233 Suffix.back().setAnnotationEndLoc(SemiLoc); 1234 Suffix.back().setAnnotationValue(Action.ModuleForHeader); 1235 LLVM_FALLTHROUGH; 1236 1237 case ImportAction::ModuleImport: 1238 case ImportAction::HeaderUnitImport: 1239 case ImportAction::SkippedModuleImport: 1240 // We chose to import (or textually enter) the file. Convert the 1241 // header-name token into a header unit annotation token. 1242 Suffix[0].setKind(tok::annot_header_unit); 1243 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation()); 1244 Suffix[0].setAnnotationValue(Action.ModuleForHeader); 1245 // FIXME: Call the moduleImport callback? 1246 break; 1247 case ImportAction::Failure: 1248 assert(TheModuleLoader.HadFatalFailure && 1249 "This should be an early exit only to a fatal error"); 1250 Result.setKind(tok::eof); 1251 CurLexer->cutOffLexing(); 1252 EnterTokens(Suffix); 1253 return true; 1254 } 1255 1256 EnterTokens(Suffix); 1257 return false; 1258 } 1259 1260 // The token sequence 1261 // 1262 // import identifier (. identifier)* 1263 // 1264 // indicates a module import directive. We already saw the 'import' 1265 // contextual keyword, so now we're looking for the identifiers. 1266 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) { 1267 // We expected to see an identifier here, and we did; continue handling 1268 // identifiers. 1269 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(), 1270 Result.getLocation())); 1271 ModuleImportExpectsIdentifier = false; 1272 CurLexerKind = CLK_LexAfterModuleImport; 1273 return true; 1274 } 1275 1276 // If we're expecting a '.' or a ';', and we got a '.', then wait until we 1277 // see the next identifier. (We can also see a '[[' that begins an 1278 // attribute-specifier-seq here under the C++ Modules TS.) 1279 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) { 1280 ModuleImportExpectsIdentifier = true; 1281 CurLexerKind = CLK_LexAfterModuleImport; 1282 return true; 1283 } 1284 1285 // If we didn't recognize a module name at all, this is not a (valid) import. 1286 if (ModuleImportPath.empty() || Result.is(tok::eof)) 1287 return true; 1288 1289 // Consume the pp-import-suffix and expand any macros in it now, if we're not 1290 // at the semicolon already. 1291 SourceLocation SemiLoc = Result.getLocation(); 1292 if (Result.isNot(tok::semi)) { 1293 Suffix.push_back(Result); 1294 CollectPpImportSuffix(Suffix); 1295 if (Suffix.back().isNot(tok::semi)) { 1296 // This is not an import after all. 1297 EnterTokens(Suffix); 1298 return false; 1299 } 1300 SemiLoc = Suffix.back().getLocation(); 1301 } 1302 1303 // Under the Modules TS, the dot is just part of the module name, and not 1304 // a real hierarchy separator. Flatten such module names now. 1305 // 1306 // FIXME: Is this the right level to be performing this transformation? 1307 std::string FlatModuleName; 1308 if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) { 1309 for (auto &Piece : ModuleImportPath) { 1310 if (!FlatModuleName.empty()) 1311 FlatModuleName += "."; 1312 FlatModuleName += Piece.first->getName(); 1313 } 1314 SourceLocation FirstPathLoc = ModuleImportPath[0].second; 1315 ModuleImportPath.clear(); 1316 ModuleImportPath.push_back( 1317 std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc)); 1318 } 1319 1320 Module *Imported = nullptr; 1321 if (getLangOpts().Modules) { 1322 Imported = TheModuleLoader.loadModule(ModuleImportLoc, 1323 ModuleImportPath, 1324 Module::Hidden, 1325 /*IsInclusionDirective=*/false); 1326 if (Imported) 1327 makeModuleVisible(Imported, SemiLoc); 1328 } 1329 if (Callbacks) 1330 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported); 1331 1332 if (!Suffix.empty()) { 1333 EnterTokens(Suffix); 1334 return false; 1335 } 1336 return true; 1337 } 1338 1339 void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) { 1340 CurSubmoduleState->VisibleModules.setVisible( 1341 M, Loc, [](Module *) {}, 1342 [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) { 1343 // FIXME: Include the path in the diagnostic. 1344 // FIXME: Include the import location for the conflicting module. 1345 Diag(ModuleImportLoc, diag::warn_module_conflict) 1346 << Path[0]->getFullModuleName() 1347 << Conflict->getFullModuleName() 1348 << Message; 1349 }); 1350 1351 // Add this module to the imports list of the currently-built submodule. 1352 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M) 1353 BuildingSubmoduleStack.back().M->Imports.insert(M); 1354 } 1355 1356 bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String, 1357 const char *DiagnosticTag, 1358 bool AllowMacroExpansion) { 1359 // We need at least one string literal. 1360 if (Result.isNot(tok::string_literal)) { 1361 Diag(Result, diag::err_expected_string_literal) 1362 << /*Source='in...'*/0 << DiagnosticTag; 1363 return false; 1364 } 1365 1366 // Lex string literal tokens, optionally with macro expansion. 1367 SmallVector<Token, 4> StrToks; 1368 do { 1369 StrToks.push_back(Result); 1370 1371 if (Result.hasUDSuffix()) 1372 Diag(Result, diag::err_invalid_string_udl); 1373 1374 if (AllowMacroExpansion) 1375 Lex(Result); 1376 else 1377 LexUnexpandedToken(Result); 1378 } while (Result.is(tok::string_literal)); 1379 1380 // Concatenate and parse the strings. 1381 StringLiteralParser Literal(StrToks, *this); 1382 assert(Literal.isOrdinary() && "Didn't allow wide strings in"); 1383 1384 if (Literal.hadError) 1385 return false; 1386 1387 if (Literal.Pascal) { 1388 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal) 1389 << /*Source='in...'*/0 << DiagnosticTag; 1390 return false; 1391 } 1392 1393 String = std::string(Literal.GetString()); 1394 return true; 1395 } 1396 1397 bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) { 1398 assert(Tok.is(tok::numeric_constant)); 1399 SmallString<8> IntegerBuffer; 1400 bool NumberInvalid = false; 1401 StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid); 1402 if (NumberInvalid) 1403 return false; 1404 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(), 1405 getLangOpts(), getTargetInfo(), 1406 getDiagnostics()); 1407 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix()) 1408 return false; 1409 llvm::APInt APVal(64, 0); 1410 if (Literal.GetIntegerValue(APVal)) 1411 return false; 1412 Lex(Tok); 1413 Value = APVal.getLimitedValue(); 1414 return true; 1415 } 1416 1417 void Preprocessor::addCommentHandler(CommentHandler *Handler) { 1418 assert(Handler && "NULL comment handler"); 1419 assert(!llvm::is_contained(CommentHandlers, Handler) && 1420 "Comment handler already registered"); 1421 CommentHandlers.push_back(Handler); 1422 } 1423 1424 void Preprocessor::removeCommentHandler(CommentHandler *Handler) { 1425 std::vector<CommentHandler *>::iterator Pos = 1426 llvm::find(CommentHandlers, Handler); 1427 assert(Pos != CommentHandlers.end() && "Comment handler not registered"); 1428 CommentHandlers.erase(Pos); 1429 } 1430 1431 bool Preprocessor::HandleComment(Token &result, SourceRange Comment) { 1432 bool AnyPendingTokens = false; 1433 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), 1434 HEnd = CommentHandlers.end(); 1435 H != HEnd; ++H) { 1436 if ((*H)->HandleComment(*this, Comment)) 1437 AnyPendingTokens = true; 1438 } 1439 if (!AnyPendingTokens || getCommentRetentionState()) 1440 return false; 1441 Lex(result); 1442 return true; 1443 } 1444 1445 void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const { 1446 const MacroAnnotations &A = 1447 getMacroAnnotations(Identifier.getIdentifierInfo()); 1448 assert(A.DeprecationInfo && 1449 "Macro deprecation warning without recorded annotation!"); 1450 const MacroAnnotationInfo &Info = *A.DeprecationInfo; 1451 if (Info.Message.empty()) 1452 Diag(Identifier, diag::warn_pragma_deprecated_macro_use) 1453 << Identifier.getIdentifierInfo() << 0; 1454 else 1455 Diag(Identifier, diag::warn_pragma_deprecated_macro_use) 1456 << Identifier.getIdentifierInfo() << 1 << Info.Message; 1457 Diag(Info.Location, diag::note_pp_macro_annotation) << 0; 1458 } 1459 1460 void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const { 1461 const MacroAnnotations &A = 1462 getMacroAnnotations(Identifier.getIdentifierInfo()); 1463 assert(A.RestrictExpansionInfo && 1464 "Macro restricted expansion warning without recorded annotation!"); 1465 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo; 1466 if (Info.Message.empty()) 1467 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use) 1468 << Identifier.getIdentifierInfo() << 0; 1469 else 1470 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use) 1471 << Identifier.getIdentifierInfo() << 1 << Info.Message; 1472 Diag(Info.Location, diag::note_pp_macro_annotation) << 1; 1473 } 1474 1475 void Preprocessor::emitFinalMacroWarning(const Token &Identifier, 1476 bool IsUndef) const { 1477 const MacroAnnotations &A = 1478 getMacroAnnotations(Identifier.getIdentifierInfo()); 1479 assert(A.FinalAnnotationLoc && 1480 "Final macro warning without recorded annotation!"); 1481 1482 Diag(Identifier, diag::warn_pragma_final_macro) 1483 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1); 1484 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2; 1485 } 1486 1487 ModuleLoader::~ModuleLoader() = default; 1488 1489 CommentHandler::~CommentHandler() = default; 1490 1491 EmptylineHandler::~EmptylineHandler() = default; 1492 1493 CodeCompletionHandler::~CodeCompletionHandler() = default; 1494 1495 void Preprocessor::createPreprocessingRecord() { 1496 if (Record) 1497 return; 1498 1499 Record = new PreprocessingRecord(getSourceManager()); 1500 addPPCallbacks(std::unique_ptr<PPCallbacks>(Record)); 1501 } 1502