1 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements pieces of the Preprocessor interface that manage the 10 // current lexer stack. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "clang/Basic/FileManager.h" 15 #include "clang/Basic/SourceLocation.h" 16 #include "clang/Basic/SourceManager.h" 17 #include "clang/Lex/HeaderSearch.h" 18 #include "clang/Lex/LexDiagnostic.h" 19 #include "clang/Lex/MacroInfo.h" 20 #include "clang/Lex/Preprocessor.h" 21 #include "clang/Lex/PreprocessorOptions.h" 22 #include "llvm/ADT/StringSwitch.h" 23 #include "llvm/Support/FileSystem.h" 24 #include "llvm/Support/MemoryBufferRef.h" 25 #include "llvm/Support/Path.h" 26 27 using namespace clang; 28 29 //===----------------------------------------------------------------------===// 30 // Miscellaneous Methods. 31 //===----------------------------------------------------------------------===// 32 33 /// isInPrimaryFile - Return true if we're in the top-level file, not in a 34 /// \#include. This looks through macro expansions and active _Pragma lexers. 35 bool Preprocessor::isInPrimaryFile() const { 36 if (IsFileLexer()) 37 return IncludeMacroStack.empty(); 38 39 // If there are any stacked lexers, we're in a #include. 40 assert(IsFileLexer(IncludeMacroStack[0]) && 41 "Top level include stack isn't our primary lexer?"); 42 return std::none_of( 43 IncludeMacroStack.begin() + 1, IncludeMacroStack.end(), 44 [&](const IncludeStackInfo &ISI) -> bool { return IsFileLexer(ISI); }); 45 } 46 47 /// getCurrentLexer - Return the current file lexer being lexed from. Note 48 /// that this ignores any potentially active macro expansions and _Pragma 49 /// expansions going on at the time. 50 PreprocessorLexer *Preprocessor::getCurrentFileLexer() const { 51 if (IsFileLexer()) 52 return CurPPLexer; 53 54 // Look for a stacked lexer. 55 for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) { 56 if (IsFileLexer(ISI)) 57 return ISI.ThePPLexer; 58 } 59 return nullptr; 60 } 61 62 63 //===----------------------------------------------------------------------===// 64 // Methods for Entering and Callbacks for leaving various contexts 65 //===----------------------------------------------------------------------===// 66 67 /// EnterSourceFile - Add a source file to the top of the include stack and 68 /// start lexing tokens from it instead of the current buffer. 69 bool Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir, 70 SourceLocation Loc) { 71 assert(!CurTokenLexer && "Cannot #include a file inside a macro!"); 72 ++NumEnteredSourceFiles; 73 74 if (MaxIncludeStackDepth < IncludeMacroStack.size()) 75 MaxIncludeStackDepth = IncludeMacroStack.size(); 76 77 // Get the MemoryBuffer for this FID, if it fails, we fail. 78 llvm::Optional<llvm::MemoryBufferRef> InputFile = 79 getSourceManager().getBufferOrNone(FID, Loc); 80 if (!InputFile) { 81 SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID); 82 Diag(Loc, diag::err_pp_error_opening_file) 83 << std::string(SourceMgr.getBufferName(FileStart)) << ""; 84 return true; 85 } 86 87 if (isCodeCompletionEnabled() && 88 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) { 89 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID); 90 CodeCompletionLoc = 91 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset); 92 } 93 94 EnterSourceFileWithLexer(new Lexer(FID, *InputFile, *this), CurDir); 95 return false; 96 } 97 98 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack 99 /// and start lexing tokens from it instead of the current buffer. 100 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer, 101 const DirectoryLookup *CurDir) { 102 103 // Add the current lexer to the include stack. 104 if (CurPPLexer || CurTokenLexer) 105 PushIncludeMacroStack(); 106 107 CurLexer.reset(TheLexer); 108 CurPPLexer = TheLexer; 109 CurDirLookup = CurDir; 110 CurLexerSubmodule = nullptr; 111 if (CurLexerKind != CLK_LexAfterModuleImport) 112 CurLexerKind = CLK_Lexer; 113 114 // Notify the client, if desired, that we are in a new source file. 115 if (Callbacks && !CurLexer->Is_PragmaLexer) { 116 SrcMgr::CharacteristicKind FileType = 117 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc()); 118 119 Callbacks->FileChanged(CurLexer->getFileLoc(), 120 PPCallbacks::EnterFile, FileType); 121 } 122 } 123 124 /// EnterMacro - Add a Macro to the top of the include stack and start lexing 125 /// tokens from it instead of the current buffer. 126 void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd, 127 MacroInfo *Macro, MacroArgs *Args) { 128 std::unique_ptr<TokenLexer> TokLexer; 129 if (NumCachedTokenLexers == 0) { 130 TokLexer = std::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this); 131 } else { 132 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]); 133 TokLexer->Init(Tok, ILEnd, Macro, Args); 134 } 135 136 PushIncludeMacroStack(); 137 CurDirLookup = nullptr; 138 CurTokenLexer = std::move(TokLexer); 139 if (CurLexerKind != CLK_LexAfterModuleImport) 140 CurLexerKind = CLK_TokenLexer; 141 } 142 143 /// EnterTokenStream - Add a "macro" context to the top of the include stack, 144 /// which will cause the lexer to start returning the specified tokens. 145 /// 146 /// If DisableMacroExpansion is true, tokens lexed from the token stream will 147 /// not be subject to further macro expansion. Otherwise, these tokens will 148 /// be re-macro-expanded when/if expansion is enabled. 149 /// 150 /// If OwnsTokens is false, this method assumes that the specified stream of 151 /// tokens has a permanent owner somewhere, so they do not need to be copied. 152 /// If it is true, it assumes the array of tokens is allocated with new[] and 153 /// must be freed. 154 /// 155 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks, 156 bool DisableMacroExpansion, bool OwnsTokens, 157 bool IsReinject) { 158 if (CurLexerKind == CLK_CachingLexer) { 159 if (CachedLexPos < CachedTokens.size()) { 160 assert(IsReinject && "new tokens in the middle of cached stream"); 161 // We're entering tokens into the middle of our cached token stream. We 162 // can't represent that, so just insert the tokens into the buffer. 163 CachedTokens.insert(CachedTokens.begin() + CachedLexPos, 164 Toks, Toks + NumToks); 165 if (OwnsTokens) 166 delete [] Toks; 167 return; 168 } 169 170 // New tokens are at the end of the cached token sequnece; insert the 171 // token stream underneath the caching lexer. 172 ExitCachingLexMode(); 173 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens, 174 IsReinject); 175 EnterCachingLexMode(); 176 return; 177 } 178 179 // Create a macro expander to expand from the specified token stream. 180 std::unique_ptr<TokenLexer> TokLexer; 181 if (NumCachedTokenLexers == 0) { 182 TokLexer = std::make_unique<TokenLexer>( 183 Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this); 184 } else { 185 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]); 186 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens, 187 IsReinject); 188 } 189 190 // Save our current state. 191 PushIncludeMacroStack(); 192 CurDirLookup = nullptr; 193 CurTokenLexer = std::move(TokLexer); 194 if (CurLexerKind != CLK_LexAfterModuleImport) 195 CurLexerKind = CLK_TokenLexer; 196 } 197 198 /// Compute the relative path that names the given file relative to 199 /// the given directory. 200 static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, 201 const FileEntry *File, 202 SmallString<128> &Result) { 203 Result.clear(); 204 205 StringRef FilePath = File->getDir()->getName(); 206 StringRef Path = FilePath; 207 while (!Path.empty()) { 208 if (auto CurDir = FM.getDirectory(Path)) { 209 if (*CurDir == Dir) { 210 Result = FilePath.substr(Path.size()); 211 llvm::sys::path::append(Result, 212 llvm::sys::path::filename(File->getName())); 213 return; 214 } 215 } 216 217 Path = llvm::sys::path::parent_path(Path); 218 } 219 220 Result = File->getName(); 221 } 222 223 void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) { 224 if (CurTokenLexer) { 225 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result); 226 return; 227 } 228 if (CurLexer) { 229 CurLexer->PropagateLineStartLeadingSpaceInfo(Result); 230 return; 231 } 232 // FIXME: Handle other kinds of lexers? It generally shouldn't matter, 233 // but it might if they're empty? 234 } 235 236 /// Determine the location to use as the end of the buffer for a lexer. 237 /// 238 /// If the file ends with a newline, form the EOF token on the newline itself, 239 /// rather than "on the line following it", which doesn't exist. This makes 240 /// diagnostics relating to the end of file include the last file that the user 241 /// actually typed, which is goodness. 242 const char *Preprocessor::getCurLexerEndPos() { 243 const char *EndPos = CurLexer->BufferEnd; 244 if (EndPos != CurLexer->BufferStart && 245 (EndPos[-1] == '\n' || EndPos[-1] == '\r')) { 246 --EndPos; 247 248 // Handle \n\r and \r\n: 249 if (EndPos != CurLexer->BufferStart && 250 (EndPos[-1] == '\n' || EndPos[-1] == '\r') && 251 EndPos[-1] != EndPos[0]) 252 --EndPos; 253 } 254 255 return EndPos; 256 } 257 258 static void collectAllSubModulesWithUmbrellaHeader( 259 const Module &Mod, SmallVectorImpl<const Module *> &SubMods) { 260 if (Mod.getUmbrellaHeader()) 261 SubMods.push_back(&Mod); 262 for (auto *M : Mod.submodules()) 263 collectAllSubModulesWithUmbrellaHeader(*M, SubMods); 264 } 265 266 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) { 267 const Module::Header &UmbrellaHeader = Mod.getUmbrellaHeader(); 268 assert(UmbrellaHeader.Entry && "Module must use umbrella header"); 269 const FileID &File = SourceMgr.translateFile(UmbrellaHeader.Entry); 270 SourceLocation ExpectedHeadersLoc = SourceMgr.getLocForEndOfFile(File); 271 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, 272 ExpectedHeadersLoc)) 273 return; 274 275 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap(); 276 const DirectoryEntry *Dir = Mod.getUmbrellaDir().Entry; 277 llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem(); 278 std::error_code EC; 279 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), 280 End; 281 Entry != End && !EC; Entry.increment(EC)) { 282 using llvm::StringSwitch; 283 284 // Check whether this entry has an extension typically associated with 285 // headers. 286 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path())) 287 .Cases(".h", ".H", ".hh", ".hpp", true) 288 .Default(false)) 289 continue; 290 291 if (auto Header = getFileManager().getFile(Entry->path())) 292 if (!getSourceManager().hasFileInfo(*Header)) { 293 if (!ModMap.isHeaderInUnavailableModule(*Header)) { 294 // Find the relative path that would access this header. 295 SmallString<128> RelativePath; 296 computeRelativePath(FileMgr, Dir, *Header, RelativePath); 297 Diag(ExpectedHeadersLoc, diag::warn_uncovered_module_header) 298 << Mod.getFullModuleName() << RelativePath; 299 } 300 } 301 } 302 } 303 304 void Preprocessor::ResolvePragmaIncludeInstead( 305 const SourceLocation Location) const { 306 assert(Location.isValid()); 307 if (CurLexer == nullptr) 308 return; 309 310 if (SourceMgr.isInSystemHeader(Location)) 311 return; 312 313 for (const auto &Include : CurLexer->getIncludeHistory()) { 314 StringRef Filename = Include.getKey(); 315 const PreprocessorLexer::IncludeInfo &Info = Include.getValue(); 316 ArrayRef<SmallString<32>> Aliases = 317 HeaderInfo.getFileInfo(Info.File).Aliases.getArrayRef(); 318 319 if (Aliases.empty()) 320 continue; 321 322 switch (Aliases.size()) { 323 case 1: 324 Diag(Info.Location, diag::err_pragma_include_instead_system_reserved) 325 << Filename << 0 << Aliases[0]; 326 continue; 327 case 2: 328 Diag(Info.Location, diag::err_pragma_include_instead_system_reserved) 329 << Filename << 1 << Aliases[0] << Aliases[1]; 330 continue; 331 default: { 332 Diag(Info.Location, diag::err_pragma_include_instead_system_reserved) 333 << Filename << 2 << ("{'" + llvm::join(Aliases, "', '") + "'}"); 334 } 335 } 336 } 337 } 338 339 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of 340 /// the current file. This either returns the EOF token or pops a level off 341 /// the include stack and keeps going. 342 bool Preprocessor::HandleEndOfFile(Token &Result, SourceLocation EndLoc, 343 bool isEndOfMacro) { 344 assert(!CurTokenLexer && 345 "Ending a file when currently in a macro!"); 346 347 // If we have an unclosed module region from a pragma at the end of a 348 // module, complain and close it now. 349 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule; 350 if ((LeavingSubmodule || IncludeMacroStack.empty()) && 351 !BuildingSubmoduleStack.empty() && 352 BuildingSubmoduleStack.back().IsPragma) { 353 Diag(BuildingSubmoduleStack.back().ImportLoc, 354 diag::err_pp_module_begin_without_module_end); 355 Module *M = LeaveSubmodule(/*ForPragma*/true); 356 357 Result.startToken(); 358 const char *EndPos = getCurLexerEndPos(); 359 CurLexer->BufferPtr = EndPos; 360 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end); 361 Result.setAnnotationEndLoc(Result.getLocation()); 362 Result.setAnnotationValue(M); 363 return true; 364 } 365 366 // See if this file had a controlling macro. 367 if (CurPPLexer) { // Not ending a macro, ignore it. 368 if (const IdentifierInfo *ControllingMacro = 369 CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) { 370 // Okay, this has a controlling macro, remember in HeaderFileInfo. 371 if (const FileEntry *FE = CurPPLexer->getFileEntry()) { 372 HeaderInfo.SetFileControllingMacro(FE, ControllingMacro); 373 if (MacroInfo *MI = 374 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro))) 375 MI->setUsedForHeaderGuard(true); 376 if (const IdentifierInfo *DefinedMacro = 377 CurPPLexer->MIOpt.GetDefinedMacro()) { 378 if (!isMacroDefined(ControllingMacro) && 379 DefinedMacro != ControllingMacro && 380 HeaderInfo.FirstTimeLexingFile(FE)) { 381 382 // If the edit distance between the two macros is more than 50%, 383 // DefinedMacro may not be header guard, or can be header guard of 384 // another header file. Therefore, it maybe defining something 385 // completely different. This can be observed in the wild when 386 // handling feature macros or header guards in different files. 387 388 const StringRef ControllingMacroName = ControllingMacro->getName(); 389 const StringRef DefinedMacroName = DefinedMacro->getName(); 390 const size_t MaxHalfLength = std::max(ControllingMacroName.size(), 391 DefinedMacroName.size()) / 2; 392 const unsigned ED = ControllingMacroName.edit_distance( 393 DefinedMacroName, true, MaxHalfLength); 394 if (ED <= MaxHalfLength) { 395 // Emit a warning for a bad header guard. 396 Diag(CurPPLexer->MIOpt.GetMacroLocation(), 397 diag::warn_header_guard) 398 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro; 399 Diag(CurPPLexer->MIOpt.GetDefinedLocation(), 400 diag::note_header_guard) 401 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro 402 << ControllingMacro 403 << FixItHint::CreateReplacement( 404 CurPPLexer->MIOpt.GetDefinedLocation(), 405 ControllingMacro->getName()); 406 } 407 } 408 } 409 } 410 } 411 } 412 413 if (EndLoc.isValid()) 414 ResolvePragmaIncludeInstead(EndLoc); 415 416 // Complain about reaching a true EOF within arc_cf_code_audited. 417 // We don't want to complain about reaching the end of a macro 418 // instantiation or a _Pragma. 419 if (PragmaARCCFCodeAuditedInfo.second.isValid() && !isEndOfMacro && 420 !(CurLexer && CurLexer->Is_PragmaLexer)) { 421 Diag(PragmaARCCFCodeAuditedInfo.second, 422 diag::err_pp_eof_in_arc_cf_code_audited); 423 424 // Recover by leaving immediately. 425 PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()}; 426 } 427 428 // Complain about reaching a true EOF within assume_nonnull. 429 // We don't want to complain about reaching the end of a macro 430 // instantiation or a _Pragma. 431 if (PragmaAssumeNonNullLoc.isValid() && 432 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) { 433 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull); 434 435 // Recover by leaving immediately. 436 PragmaAssumeNonNullLoc = SourceLocation(); 437 } 438 439 bool LeavingPCHThroughHeader = false; 440 441 // If this is a #include'd file, pop it off the include stack and continue 442 // lexing the #includer file. 443 if (!IncludeMacroStack.empty()) { 444 445 // If we lexed the code-completion file, act as if we reached EOF. 446 if (isCodeCompletionEnabled() && CurPPLexer && 447 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) == 448 CodeCompletionFileLoc) { 449 assert(CurLexer && "Got EOF but no current lexer set!"); 450 Result.startToken(); 451 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof); 452 CurLexer.reset(); 453 454 CurPPLexer = nullptr; 455 recomputeCurLexerKind(); 456 return true; 457 } 458 459 if (!isEndOfMacro && CurPPLexer && 460 (SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid() || 461 // Predefines file doesn't have a valid include location. 462 (PredefinesFileID.isValid() && 463 CurPPLexer->getFileID() == PredefinesFileID))) { 464 // Notify SourceManager to record the number of FileIDs that were created 465 // during lexing of the #include'd file. 466 unsigned NumFIDs = 467 SourceMgr.local_sloc_entry_size() - 468 CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/; 469 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs); 470 } 471 472 bool ExitedFromPredefinesFile = false; 473 FileID ExitedFID; 474 if (!isEndOfMacro && CurPPLexer) { 475 ExitedFID = CurPPLexer->getFileID(); 476 477 assert(PredefinesFileID.isValid() && 478 "HandleEndOfFile is called before PredefinesFileId is set"); 479 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID); 480 } 481 482 if (LeavingSubmodule) { 483 // We're done with this submodule. 484 Module *M = LeaveSubmodule(/*ForPragma*/false); 485 486 // Notify the parser that we've left the module. 487 const char *EndPos = getCurLexerEndPos(); 488 Result.startToken(); 489 CurLexer->BufferPtr = EndPos; 490 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end); 491 Result.setAnnotationEndLoc(Result.getLocation()); 492 Result.setAnnotationValue(M); 493 } 494 495 bool FoundPCHThroughHeader = false; 496 if (CurPPLexer && creatingPCHWithThroughHeader() && 497 isPCHThroughHeader( 498 SourceMgr.getFileEntryForID(CurPPLexer->getFileID()))) 499 FoundPCHThroughHeader = true; 500 501 // We're done with the #included file. 502 RemoveTopOfLexerStack(); 503 504 // Propagate info about start-of-line/leading white-space/etc. 505 PropagateLineStartLeadingSpaceInfo(Result); 506 507 // Notify the client, if desired, that we are in a new source file. 508 if (Callbacks && !isEndOfMacro && CurPPLexer) { 509 SrcMgr::CharacteristicKind FileType = 510 SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation()); 511 Callbacks->FileChanged(CurPPLexer->getSourceLocation(), 512 PPCallbacks::ExitFile, FileType, ExitedFID); 513 } 514 515 // Restore conditional stack from the preamble right after exiting from the 516 // predefines file. 517 if (ExitedFromPredefinesFile) 518 replayPreambleConditionalStack(); 519 520 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader && 521 (isInPrimaryFile() || 522 CurPPLexer->getFileID() == getPredefinesFileID())) { 523 // Leaving the through header. Continue directly to end of main file 524 // processing. 525 LeavingPCHThroughHeader = true; 526 } else { 527 // Client should lex another token unless we generated an EOM. 528 return LeavingSubmodule; 529 } 530 } 531 532 // If this is the end of the main file, form an EOF token. 533 assert(CurLexer && "Got EOF but no current lexer set!"); 534 const char *EndPos = getCurLexerEndPos(); 535 Result.startToken(); 536 CurLexer->BufferPtr = EndPos; 537 CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); 538 539 if (isCodeCompletionEnabled()) { 540 // Inserting the code-completion point increases the source buffer by 1, 541 // but the main FileID was created before inserting the point. 542 // Compensate by reducing the EOF location by 1, otherwise the location 543 // will point to the next FileID. 544 // FIXME: This is hacky, the code-completion point should probably be 545 // inserted before the main FileID is created. 546 if (CurLexer->getFileLoc() == CodeCompletionFileLoc) 547 Result.setLocation(Result.getLocation().getLocWithOffset(-1)); 548 } 549 550 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) { 551 // Reached the end of the compilation without finding the through header. 552 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen) 553 << PPOpts->PCHThroughHeader << 0; 554 } 555 556 if (!isIncrementalProcessingEnabled()) 557 // We're done with lexing. 558 CurLexer.reset(); 559 560 if (!isIncrementalProcessingEnabled()) 561 CurPPLexer = nullptr; 562 563 if (TUKind == TU_Complete) { 564 // This is the end of the top-level file. 'WarnUnusedMacroLocs' has 565 // collected all macro locations that we need to warn because they are not 566 // used. 567 for (WarnUnusedMacroLocsTy::iterator 568 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end(); 569 I!=E; ++I) 570 Diag(*I, diag::pp_macro_not_used); 571 } 572 573 // If we are building a module that has an umbrella header, make sure that 574 // each of the headers within the directory, including all submodules, is 575 // covered by the umbrella header was actually included by the umbrella 576 // header. 577 if (Module *Mod = getCurrentModule()) { 578 llvm::SmallVector<const Module *, 4> AllMods; 579 collectAllSubModulesWithUmbrellaHeader(*Mod, AllMods); 580 for (auto *M : AllMods) 581 diagnoseMissingHeaderInUmbrellaDir(*M); 582 } 583 584 return true; 585 } 586 587 /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer 588 /// hits the end of its token stream. 589 bool Preprocessor::HandleEndOfTokenLexer(Token &Result) { 590 assert(CurTokenLexer && !CurPPLexer && 591 "Ending a macro when currently in a #include file!"); 592 593 if (!MacroExpandingLexersStack.empty() && 594 MacroExpandingLexersStack.back().first == CurTokenLexer.get()) 595 removeCachedMacroExpandedTokensOfLastLexer(); 596 597 // Delete or cache the now-dead macro expander. 598 if (NumCachedTokenLexers == TokenLexerCacheSize) 599 CurTokenLexer.reset(); 600 else 601 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer); 602 603 // Handle this like a #include file being popped off the stack. 604 return HandleEndOfFile(Result, {}, true); 605 } 606 607 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the 608 /// lexer stack. This should only be used in situations where the current 609 /// state of the top-of-stack lexer is unknown. 610 void Preprocessor::RemoveTopOfLexerStack() { 611 assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load"); 612 613 if (CurTokenLexer) { 614 // Delete or cache the now-dead macro expander. 615 if (NumCachedTokenLexers == TokenLexerCacheSize) 616 CurTokenLexer.reset(); 617 else 618 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer); 619 } 620 621 PopIncludeMacroStack(); 622 } 623 624 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a 625 /// comment (/##/) in microsoft mode, this method handles updating the current 626 /// state, returning the token on the next source line. 627 void Preprocessor::HandleMicrosoftCommentPaste(Token &Tok) { 628 assert(CurTokenLexer && !CurPPLexer && 629 "Pasted comment can only be formed from macro"); 630 // We handle this by scanning for the closest real lexer, switching it to 631 // raw mode and preprocessor mode. This will cause it to return \n as an 632 // explicit EOD token. 633 PreprocessorLexer *FoundLexer = nullptr; 634 bool LexerWasInPPMode = false; 635 for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) { 636 if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer. 637 638 // Once we find a real lexer, mark it as raw mode (disabling macro 639 // expansions) and preprocessor mode (return EOD). We know that the lexer 640 // was *not* in raw mode before, because the macro that the comment came 641 // from was expanded. However, it could have already been in preprocessor 642 // mode (#if COMMENT) in which case we have to return it to that mode and 643 // return EOD. 644 FoundLexer = ISI.ThePPLexer; 645 FoundLexer->LexingRawMode = true; 646 LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective; 647 FoundLexer->ParsingPreprocessorDirective = true; 648 break; 649 } 650 651 // Okay, we either found and switched over the lexer, or we didn't find a 652 // lexer. In either case, finish off the macro the comment came from, getting 653 // the next token. 654 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok); 655 656 // Discarding comments as long as we don't have EOF or EOD. This 'comments 657 // out' the rest of the line, including any tokens that came from other macros 658 // that were active, as in: 659 // #define submacro a COMMENT b 660 // submacro c 661 // which should lex to 'a' only: 'b' and 'c' should be removed. 662 while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof)) 663 Lex(Tok); 664 665 // If we got an eod token, then we successfully found the end of the line. 666 if (Tok.is(tok::eod)) { 667 assert(FoundLexer && "Can't get end of line without an active lexer"); 668 // Restore the lexer back to normal mode instead of raw mode. 669 FoundLexer->LexingRawMode = false; 670 671 // If the lexer was already in preprocessor mode, just return the EOD token 672 // to finish the preprocessor line. 673 if (LexerWasInPPMode) return; 674 675 // Otherwise, switch out of PP mode and return the next lexed token. 676 FoundLexer->ParsingPreprocessorDirective = false; 677 return Lex(Tok); 678 } 679 680 // If we got an EOF token, then we reached the end of the token stream but 681 // didn't find an explicit \n. This can only happen if there was no lexer 682 // active (an active lexer would return EOD at EOF if there was no \n in 683 // preprocessor directive mode), so just return EOF as our token. 684 assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode"); 685 } 686 687 void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc, 688 bool ForPragma) { 689 if (!getLangOpts().ModulesLocalVisibility) { 690 // Just track that we entered this submodule. 691 BuildingSubmoduleStack.push_back( 692 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState, 693 PendingModuleMacroNames.size())); 694 if (Callbacks) 695 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma); 696 return; 697 } 698 699 // Resolve as much of the module definition as we can now, before we enter 700 // one of its headers. 701 // FIXME: Can we enable Complain here? 702 // FIXME: Can we do this when local visibility is disabled? 703 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap(); 704 ModMap.resolveExports(M, /*Complain=*/false); 705 ModMap.resolveUses(M, /*Complain=*/false); 706 ModMap.resolveConflicts(M, /*Complain=*/false); 707 708 // If this is the first time we've entered this module, set up its state. 709 auto R = Submodules.insert(std::make_pair(M, SubmoduleState())); 710 auto &State = R.first->second; 711 bool FirstTime = R.second; 712 if (FirstTime) { 713 // Determine the set of starting macros for this submodule; take these 714 // from the "null" module (the predefines buffer). 715 // 716 // FIXME: If we have local visibility but not modules enabled, the 717 // NullSubmoduleState is polluted by #defines in the top-level source 718 // file. 719 auto &StartingMacros = NullSubmoduleState.Macros; 720 721 // Restore to the starting state. 722 // FIXME: Do this lazily, when each macro name is first referenced. 723 for (auto &Macro : StartingMacros) { 724 // Skip uninteresting macros. 725 if (!Macro.second.getLatest() && 726 Macro.second.getOverriddenMacros().empty()) 727 continue; 728 729 MacroState MS(Macro.second.getLatest()); 730 MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros()); 731 State.Macros.insert(std::make_pair(Macro.first, std::move(MS))); 732 } 733 } 734 735 // Track that we entered this module. 736 BuildingSubmoduleStack.push_back( 737 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState, 738 PendingModuleMacroNames.size())); 739 740 if (Callbacks) 741 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma); 742 743 // Switch to this submodule as the current submodule. 744 CurSubmoduleState = &State; 745 746 // This module is visible to itself. 747 if (FirstTime) 748 makeModuleVisible(M, ImportLoc); 749 } 750 751 bool Preprocessor::needModuleMacros() const { 752 // If we're not within a submodule, we never need to create ModuleMacros. 753 if (BuildingSubmoduleStack.empty()) 754 return false; 755 // If we are tracking module macro visibility even for textually-included 756 // headers, we need ModuleMacros. 757 if (getLangOpts().ModulesLocalVisibility) 758 return true; 759 // Otherwise, we only need module macros if we're actually compiling a module 760 // interface. 761 return getLangOpts().isCompilingModule(); 762 } 763 764 Module *Preprocessor::LeaveSubmodule(bool ForPragma) { 765 if (BuildingSubmoduleStack.empty() || 766 BuildingSubmoduleStack.back().IsPragma != ForPragma) { 767 assert(ForPragma && "non-pragma module enter/leave mismatch"); 768 return nullptr; 769 } 770 771 auto &Info = BuildingSubmoduleStack.back(); 772 773 Module *LeavingMod = Info.M; 774 SourceLocation ImportLoc = Info.ImportLoc; 775 776 if (!needModuleMacros() || 777 (!getLangOpts().ModulesLocalVisibility && 778 LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) { 779 // If we don't need module macros, or this is not a module for which we 780 // are tracking macro visibility, don't build any, and preserve the list 781 // of pending names for the surrounding submodule. 782 BuildingSubmoduleStack.pop_back(); 783 784 if (Callbacks) 785 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma); 786 787 makeModuleVisible(LeavingMod, ImportLoc); 788 return LeavingMod; 789 } 790 791 // Create ModuleMacros for any macros defined in this submodule. 792 llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros; 793 for (unsigned I = Info.OuterPendingModuleMacroNames; 794 I != PendingModuleMacroNames.size(); ++I) { 795 auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]); 796 if (!VisitedMacros.insert(II).second) 797 continue; 798 799 auto MacroIt = CurSubmoduleState->Macros.find(II); 800 if (MacroIt == CurSubmoduleState->Macros.end()) 801 continue; 802 auto &Macro = MacroIt->second; 803 804 // Find the starting point for the MacroDirective chain in this submodule. 805 MacroDirective *OldMD = nullptr; 806 auto *OldState = Info.OuterSubmoduleState; 807 if (getLangOpts().ModulesLocalVisibility) 808 OldState = &NullSubmoduleState; 809 if (OldState && OldState != CurSubmoduleState) { 810 // FIXME: It'd be better to start at the state from when we most recently 811 // entered this submodule, but it doesn't really matter. 812 auto &OldMacros = OldState->Macros; 813 auto OldMacroIt = OldMacros.find(II); 814 if (OldMacroIt == OldMacros.end()) 815 OldMD = nullptr; 816 else 817 OldMD = OldMacroIt->second.getLatest(); 818 } 819 820 // This module may have exported a new macro. If so, create a ModuleMacro 821 // representing that fact. 822 bool ExplicitlyPublic = false; 823 for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) { 824 assert(MD && "broken macro directive chain"); 825 826 if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) { 827 // The latest visibility directive for a name in a submodule affects 828 // all the directives that come before it. 829 if (VisMD->isPublic()) 830 ExplicitlyPublic = true; 831 else if (!ExplicitlyPublic) 832 // Private with no following public directive: not exported. 833 break; 834 } else { 835 MacroInfo *Def = nullptr; 836 if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD)) 837 Def = DefMD->getInfo(); 838 839 // FIXME: Issue a warning if multiple headers for the same submodule 840 // define a macro, rather than silently ignoring all but the first. 841 bool IsNew; 842 // Don't bother creating a module macro if it would represent a #undef 843 // that doesn't override anything. 844 if (Def || !Macro.getOverriddenMacros().empty()) 845 addModuleMacro(LeavingMod, II, Def, 846 Macro.getOverriddenMacros(), IsNew); 847 848 if (!getLangOpts().ModulesLocalVisibility) { 849 // This macro is exposed to the rest of this compilation as a 850 // ModuleMacro; we don't need to track its MacroDirective any more. 851 Macro.setLatest(nullptr); 852 Macro.setOverriddenMacros(*this, {}); 853 } 854 break; 855 } 856 } 857 } 858 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames); 859 860 // FIXME: Before we leave this submodule, we should parse all the other 861 // headers within it. Otherwise, we're left with an inconsistent state 862 // where we've made the module visible but don't yet have its complete 863 // contents. 864 865 // Put back the outer module's state, if we're tracking it. 866 if (getLangOpts().ModulesLocalVisibility) 867 CurSubmoduleState = Info.OuterSubmoduleState; 868 869 BuildingSubmoduleStack.pop_back(); 870 871 if (Callbacks) 872 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma); 873 874 // A nested #include makes the included submodule visible. 875 makeModuleVisible(LeavingMod, ImportLoc); 876 return LeavingMod; 877 } 878