1 //===-- HeaderIncludeGen.cpp - Generate Header Includes -------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Frontend/DependencyOutputOptions.h" 10 #include "clang/Frontend/Utils.h" 11 #include "clang/Basic/SourceManager.h" 12 #include "clang/Frontend/FrontendDiagnostic.h" 13 #include "clang/Lex/Preprocessor.h" 14 #include "llvm/ADT/SmallString.h" 15 #include "llvm/Support/JSON.h" 16 #include "llvm/Support/raw_ostream.h" 17 using namespace clang; 18 19 namespace { 20 class HeaderIncludesCallback : public PPCallbacks { 21 SourceManager &SM; 22 raw_ostream *OutputFile; 23 const DependencyOutputOptions &DepOpts; 24 unsigned CurrentIncludeDepth; 25 bool HasProcessedPredefines; 26 bool OwnsOutputFile; 27 bool ShowAllHeaders; 28 bool ShowDepth; 29 bool MSStyle; 30 31 public: 32 HeaderIncludesCallback(const Preprocessor *PP, bool ShowAllHeaders_, 33 raw_ostream *OutputFile_, 34 const DependencyOutputOptions &DepOpts, 35 bool OwnsOutputFile_, bool ShowDepth_, bool MSStyle_) 36 : SM(PP->getSourceManager()), OutputFile(OutputFile_), DepOpts(DepOpts), 37 CurrentIncludeDepth(0), HasProcessedPredefines(false), 38 OwnsOutputFile(OwnsOutputFile_), ShowAllHeaders(ShowAllHeaders_), 39 ShowDepth(ShowDepth_), MSStyle(MSStyle_) {} 40 41 ~HeaderIncludesCallback() override { 42 if (OwnsOutputFile) 43 delete OutputFile; 44 } 45 46 HeaderIncludesCallback(const HeaderIncludesCallback &) = delete; 47 HeaderIncludesCallback &operator=(const HeaderIncludesCallback &) = delete; 48 49 void FileChanged(SourceLocation Loc, FileChangeReason Reason, 50 SrcMgr::CharacteristicKind FileType, 51 FileID PrevFID) override; 52 53 void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok, 54 SrcMgr::CharacteristicKind FileType) override; 55 56 private: 57 bool ShouldShowHeader(SrcMgr::CharacteristicKind HeaderType) { 58 if (!DepOpts.IncludeSystemHeaders && isSystem(HeaderType)) 59 return false; 60 61 // Show the current header if we are (a) past the predefines, or (b) showing 62 // all headers and in the predefines at a depth past the initial file and 63 // command line buffers. 64 return (HasProcessedPredefines || 65 (ShowAllHeaders && CurrentIncludeDepth > 2)); 66 } 67 }; 68 69 /// A callback for emitting header usage information to a file in JSON. Each 70 /// line in the file is a JSON object that includes the source file name and 71 /// the list of headers directly or indirectly included from it. For example: 72 /// 73 /// {"source":"/tmp/foo.c", 74 /// "includes":["/usr/include/stdio.h", "/usr/include/stdlib.h"]} 75 /// 76 /// To reduce the amount of data written to the file, we only record system 77 /// headers that are directly included from a file that isn't in the system 78 /// directory. 79 class HeaderIncludesJSONCallback : public PPCallbacks { 80 SourceManager &SM; 81 raw_ostream *OutputFile; 82 bool OwnsOutputFile; 83 SmallVector<std::string, 16> IncludedHeaders; 84 85 public: 86 HeaderIncludesJSONCallback(const Preprocessor *PP, raw_ostream *OutputFile_, 87 bool OwnsOutputFile_) 88 : SM(PP->getSourceManager()), OutputFile(OutputFile_), 89 OwnsOutputFile(OwnsOutputFile_) {} 90 91 ~HeaderIncludesJSONCallback() override { 92 if (OwnsOutputFile) 93 delete OutputFile; 94 } 95 96 HeaderIncludesJSONCallback(const HeaderIncludesJSONCallback &) = delete; 97 HeaderIncludesJSONCallback & 98 operator=(const HeaderIncludesJSONCallback &) = delete; 99 100 void EndOfMainFile() override; 101 102 void FileChanged(SourceLocation Loc, FileChangeReason Reason, 103 SrcMgr::CharacteristicKind FileType, 104 FileID PrevFID) override; 105 106 void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok, 107 SrcMgr::CharacteristicKind FileType) override; 108 }; 109 110 /// A callback for emitting direct header and module usage information to a 111 /// file in JSON. The output format is like HeaderIncludesJSONCallback but has 112 /// an array of separate entries, one for each non-system source file used in 113 /// the compilation showing only the direct includes and imports from that file. 114 class HeaderIncludesDirectPerFileCallback : public PPCallbacks { 115 SourceManager &SM; 116 HeaderSearch &HSI; 117 raw_ostream *OutputFile; 118 bool OwnsOutputFile; 119 using DependencyMap = llvm::DenseMap<FileEntryRef, SmallVector<FileEntryRef>>; 120 DependencyMap Dependencies; 121 122 public: 123 HeaderIncludesDirectPerFileCallback(const Preprocessor *PP, 124 raw_ostream *OutputFile_, 125 bool OwnsOutputFile_) 126 : SM(PP->getSourceManager()), HSI(PP->getHeaderSearchInfo()), 127 OutputFile(OutputFile_), OwnsOutputFile(OwnsOutputFile_) {} 128 129 ~HeaderIncludesDirectPerFileCallback() override { 130 if (OwnsOutputFile) 131 delete OutputFile; 132 } 133 134 HeaderIncludesDirectPerFileCallback( 135 const HeaderIncludesDirectPerFileCallback &) = delete; 136 HeaderIncludesDirectPerFileCallback & 137 operator=(const HeaderIncludesDirectPerFileCallback &) = delete; 138 139 void EndOfMainFile() override; 140 141 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, 142 StringRef FileName, bool IsAngled, 143 CharSourceRange FilenameRange, 144 OptionalFileEntryRef File, StringRef SearchPath, 145 StringRef RelativePath, const Module *SuggestedModule, 146 bool ModuleImported, 147 SrcMgr::CharacteristicKind FileType) override; 148 149 void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path, 150 const Module *Imported) override; 151 }; 152 } 153 154 static void PrintHeaderInfo(raw_ostream *OutputFile, StringRef Filename, 155 bool ShowDepth, unsigned CurrentIncludeDepth, 156 bool MSStyle) { 157 // Write to a temporary string to avoid unnecessary flushing on errs(). 158 SmallString<512> Pathname(Filename); 159 if (!MSStyle) 160 Lexer::Stringify(Pathname); 161 162 SmallString<256> Msg; 163 if (MSStyle) 164 Msg += "Note: including file:"; 165 166 if (ShowDepth) { 167 // The main source file is at depth 1, so skip one dot. 168 for (unsigned i = 1; i != CurrentIncludeDepth; ++i) 169 Msg += MSStyle ? ' ' : '.'; 170 171 if (!MSStyle) 172 Msg += ' '; 173 } 174 Msg += Pathname; 175 Msg += '\n'; 176 177 *OutputFile << Msg; 178 OutputFile->flush(); 179 } 180 181 void clang::AttachHeaderIncludeGen(Preprocessor &PP, 182 const DependencyOutputOptions &DepOpts, 183 bool ShowAllHeaders, StringRef OutputPath, 184 bool ShowDepth, bool MSStyle) { 185 raw_ostream *OutputFile = &llvm::errs(); 186 bool OwnsOutputFile = false; 187 188 // Choose output stream, when printing in cl.exe /showIncludes style. 189 if (MSStyle) { 190 switch (DepOpts.ShowIncludesDest) { 191 default: 192 llvm_unreachable("Invalid destination for /showIncludes output!"); 193 case ShowIncludesDestination::Stderr: 194 OutputFile = &llvm::errs(); 195 break; 196 case ShowIncludesDestination::Stdout: 197 OutputFile = &llvm::outs(); 198 break; 199 } 200 } 201 202 // Open the output file, if used. 203 if (!OutputPath.empty()) { 204 std::error_code EC; 205 llvm::raw_fd_ostream *OS = new llvm::raw_fd_ostream( 206 OutputPath.str(), EC, 207 llvm::sys::fs::OF_Append | llvm::sys::fs::OF_TextWithCRLF); 208 if (EC) { 209 PP.getDiagnostics().Report(clang::diag::warn_fe_cc_print_header_failure) 210 << EC.message(); 211 delete OS; 212 } else { 213 OS->SetUnbuffered(); 214 OutputFile = OS; 215 OwnsOutputFile = true; 216 } 217 } 218 219 switch (DepOpts.HeaderIncludeFormat) { 220 case HIFMT_None: 221 llvm_unreachable("unexpected header format kind"); 222 case HIFMT_Textual: { 223 assert(DepOpts.HeaderIncludeFiltering == HIFIL_None && 224 "header filtering is currently always disabled when output format is" 225 "textual"); 226 // Print header info for extra headers, pretending they were discovered by 227 // the regular preprocessor. The primary use case is to support proper 228 // generation of Make / Ninja file dependencies for implicit includes, such 229 // as sanitizer ignorelists. It's only important for cl.exe compatibility, 230 // the GNU way to generate rules is -M / -MM / -MD / -MMD. 231 for (const auto &Header : DepOpts.ExtraDeps) 232 PrintHeaderInfo(OutputFile, Header.first, ShowDepth, 2, MSStyle); 233 PP.addPPCallbacks(std::make_unique<HeaderIncludesCallback>( 234 &PP, ShowAllHeaders, OutputFile, DepOpts, OwnsOutputFile, ShowDepth, 235 MSStyle)); 236 break; 237 } 238 case HIFMT_JSON: 239 switch (DepOpts.HeaderIncludeFiltering) { 240 default: 241 llvm_unreachable("Unknown HeaderIncludeFilteringKind enum"); 242 case HIFIL_Only_Direct_System: 243 PP.addPPCallbacks(std::make_unique<HeaderIncludesJSONCallback>( 244 &PP, OutputFile, OwnsOutputFile)); 245 break; 246 case HIFIL_Direct_Per_File: 247 PP.addPPCallbacks(std::make_unique<HeaderIncludesDirectPerFileCallback>( 248 &PP, OutputFile, OwnsOutputFile)); 249 break; 250 } 251 break; 252 } 253 } 254 255 void HeaderIncludesCallback::FileChanged(SourceLocation Loc, 256 FileChangeReason Reason, 257 SrcMgr::CharacteristicKind NewFileType, 258 FileID PrevFID) { 259 // Unless we are exiting a #include, make sure to skip ahead to the line the 260 // #include directive was at. 261 PresumedLoc UserLoc = SM.getPresumedLoc(Loc); 262 if (UserLoc.isInvalid()) 263 return; 264 265 // Adjust the current include depth. 266 if (Reason == PPCallbacks::EnterFile) { 267 ++CurrentIncludeDepth; 268 } else if (Reason == PPCallbacks::ExitFile) { 269 if (CurrentIncludeDepth) 270 --CurrentIncludeDepth; 271 272 // We track when we are done with the predefines by watching for the first 273 // place where we drop back to a nesting depth of 1. 274 if (CurrentIncludeDepth == 1 && !HasProcessedPredefines) 275 HasProcessedPredefines = true; 276 277 return; 278 } else { 279 return; 280 } 281 282 if (!ShouldShowHeader(NewFileType)) 283 return; 284 285 unsigned IncludeDepth = CurrentIncludeDepth; 286 if (!HasProcessedPredefines) 287 --IncludeDepth; // Ignore indent from <built-in>. 288 289 // FIXME: Identify headers in a more robust way than comparing their name to 290 // "<command line>" and "<built-in>" in a bunch of places. 291 if (Reason == PPCallbacks::EnterFile && 292 UserLoc.getFilename() != StringRef("<command line>")) { 293 PrintHeaderInfo(OutputFile, UserLoc.getFilename(), ShowDepth, IncludeDepth, 294 MSStyle); 295 } 296 } 297 298 void HeaderIncludesCallback::FileSkipped(const FileEntryRef &SkippedFile, const 299 Token &FilenameTok, 300 SrcMgr::CharacteristicKind FileType) { 301 if (!DepOpts.ShowSkippedHeaderIncludes) 302 return; 303 304 if (!ShouldShowHeader(FileType)) 305 return; 306 307 PrintHeaderInfo(OutputFile, SkippedFile.getName(), ShowDepth, 308 CurrentIncludeDepth + 1, MSStyle); 309 } 310 311 void HeaderIncludesJSONCallback::EndOfMainFile() { 312 OptionalFileEntryRef FE = SM.getFileEntryRefForID(SM.getMainFileID()); 313 SmallString<256> MainFile; 314 if (FE) { 315 MainFile += FE->getName(); 316 SM.getFileManager().makeAbsolutePath(MainFile); 317 } 318 319 std::string Str; 320 llvm::raw_string_ostream OS(Str); 321 llvm::json::OStream JOS(OS); 322 JOS.object([&] { 323 JOS.attribute("source", MainFile.c_str()); 324 JOS.attributeArray("includes", [&] { 325 llvm::StringSet<> SeenHeaders; 326 for (const std::string &H : IncludedHeaders) 327 if (SeenHeaders.insert(H).second) 328 JOS.value(H); 329 }); 330 }); 331 OS << "\n"; 332 333 if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) { 334 llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile); 335 if (auto L = FDS->lock()) 336 *OutputFile << Str; 337 } else 338 *OutputFile << Str; 339 } 340 341 /// Determine whether the header file should be recorded. The header file should 342 /// be recorded only if the header file is a system header and the current file 343 /// isn't a system header. 344 static bool shouldRecordNewFile(SrcMgr::CharacteristicKind NewFileType, 345 SourceLocation PrevLoc, SourceManager &SM) { 346 return SrcMgr::isSystem(NewFileType) && !SM.isInSystemHeader(PrevLoc); 347 } 348 349 void HeaderIncludesJSONCallback::FileChanged( 350 SourceLocation Loc, FileChangeReason Reason, 351 SrcMgr::CharacteristicKind NewFileType, FileID PrevFID) { 352 if (PrevFID.isInvalid() || 353 !shouldRecordNewFile(NewFileType, SM.getLocForStartOfFile(PrevFID), SM)) 354 return; 355 356 // Unless we are exiting a #include, make sure to skip ahead to the line the 357 // #include directive was at. 358 PresumedLoc UserLoc = SM.getPresumedLoc(Loc); 359 if (UserLoc.isInvalid()) 360 return; 361 362 if (Reason == PPCallbacks::EnterFile && 363 UserLoc.getFilename() != StringRef("<command line>")) 364 IncludedHeaders.push_back(UserLoc.getFilename()); 365 } 366 367 void HeaderIncludesJSONCallback::FileSkipped( 368 const FileEntryRef &SkippedFile, const Token &FilenameTok, 369 SrcMgr::CharacteristicKind FileType) { 370 if (!shouldRecordNewFile(FileType, FilenameTok.getLocation(), SM)) 371 return; 372 373 IncludedHeaders.push_back(SkippedFile.getName().str()); 374 } 375 376 void HeaderIncludesDirectPerFileCallback::EndOfMainFile() { 377 if (Dependencies.empty()) 378 return; 379 380 // Sort the files so that the output does not depend on the DenseMap order. 381 SmallVector<FileEntryRef> SourceFiles; 382 for (auto F = Dependencies.begin(), FEnd = Dependencies.end(); F != FEnd; 383 ++F) { 384 SourceFiles.push_back(F->first); 385 } 386 llvm::sort(SourceFiles, [](const FileEntryRef &LHS, const FileEntryRef &RHS) { 387 return LHS.getUID() < RHS.getUID(); 388 }); 389 390 std::string Str; 391 llvm::raw_string_ostream OS(Str); 392 llvm::json::OStream JOS(OS); 393 JOS.array([&] { 394 for (auto S = SourceFiles.begin(), SE = SourceFiles.end(); S != SE; ++S) { 395 JOS.object([&] { 396 SmallVector<FileEntryRef> &Deps = Dependencies[*S]; 397 JOS.attribute("source", S->getName().str()); 398 JOS.attributeArray("includes", [&] { 399 for (unsigned I = 0, N = Deps.size(); I != N; ++I) 400 JOS.value(Deps[I].getName().str()); 401 }); 402 }); 403 } 404 }); 405 OS << "\n"; 406 407 if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) { 408 llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile); 409 if (auto L = FDS->lock()) 410 *OutputFile << Str; 411 } else 412 *OutputFile << Str; 413 } 414 415 void HeaderIncludesDirectPerFileCallback::InclusionDirective( 416 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName, 417 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File, 418 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule, 419 bool ModuleImported, SrcMgr::CharacteristicKind FileType) { 420 if (!File) 421 return; 422 423 SourceLocation Loc = SM.getExpansionLoc(HashLoc); 424 if (SM.isInSystemHeader(Loc)) 425 return; 426 OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(SM.getFileID(Loc)); 427 if (!FromFile) 428 return; 429 430 Dependencies[*FromFile].push_back(*File); 431 } 432 433 void HeaderIncludesDirectPerFileCallback::moduleImport(SourceLocation ImportLoc, 434 ModuleIdPath Path, 435 const Module *Imported) { 436 if (!Imported) 437 return; 438 439 SourceLocation Loc = SM.getExpansionLoc(ImportLoc); 440 if (SM.isInSystemHeader(Loc)) 441 return; 442 OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(SM.getFileID(Loc)); 443 if (!FromFile) 444 return; 445 446 OptionalFileEntryRef ModuleMapFile = 447 HSI.getModuleMap().getModuleMapFileForUniquing(Imported); 448 if (!ModuleMapFile) 449 return; 450 451 Dependencies[*FromFile].push_back(*ModuleMapFile); 452 } 453