1 //===-- HeaderIncludeGen.cpp - Generate Header Includes -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "clang/Frontend/DependencyOutputOptions.h"
10 #include "clang/Frontend/Utils.h"
11 #include "clang/Basic/SourceManager.h"
12 #include "clang/Frontend/FrontendDiagnostic.h"
13 #include "clang/Lex/Preprocessor.h"
14 #include "llvm/ADT/SmallString.h"
15 #include "llvm/Support/JSON.h"
16 #include "llvm/Support/raw_ostream.h"
17 using namespace clang;
18
19 namespace {
20 class HeaderIncludesCallback : public PPCallbacks {
21 SourceManager &SM;
22 raw_ostream *OutputFile;
23 const DependencyOutputOptions &DepOpts;
24 unsigned CurrentIncludeDepth;
25 bool HasProcessedPredefines;
26 bool OwnsOutputFile;
27 bool ShowAllHeaders;
28 bool ShowDepth;
29 bool MSStyle;
30
31 public:
HeaderIncludesCallback(const Preprocessor * PP,bool ShowAllHeaders_,raw_ostream * OutputFile_,const DependencyOutputOptions & DepOpts,bool OwnsOutputFile_,bool ShowDepth_,bool MSStyle_)32 HeaderIncludesCallback(const Preprocessor *PP, bool ShowAllHeaders_,
33 raw_ostream *OutputFile_,
34 const DependencyOutputOptions &DepOpts,
35 bool OwnsOutputFile_, bool ShowDepth_, bool MSStyle_)
36 : SM(PP->getSourceManager()), OutputFile(OutputFile_), DepOpts(DepOpts),
37 CurrentIncludeDepth(0), HasProcessedPredefines(false),
38 OwnsOutputFile(OwnsOutputFile_), ShowAllHeaders(ShowAllHeaders_),
39 ShowDepth(ShowDepth_), MSStyle(MSStyle_) {}
40
~HeaderIncludesCallback()41 ~HeaderIncludesCallback() override {
42 if (OwnsOutputFile)
43 delete OutputFile;
44 }
45
46 HeaderIncludesCallback(const HeaderIncludesCallback &) = delete;
47 HeaderIncludesCallback &operator=(const HeaderIncludesCallback &) = delete;
48
49 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
50 SrcMgr::CharacteristicKind FileType,
51 FileID PrevFID) override;
52
53 void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
54 SrcMgr::CharacteristicKind FileType) override;
55
56 private:
ShouldShowHeader(SrcMgr::CharacteristicKind HeaderType)57 bool ShouldShowHeader(SrcMgr::CharacteristicKind HeaderType) {
58 if (!DepOpts.IncludeSystemHeaders && isSystem(HeaderType))
59 return false;
60
61 // Show the current header if we are (a) past the predefines, or (b) showing
62 // all headers and in the predefines at a depth past the initial file and
63 // command line buffers.
64 return (HasProcessedPredefines ||
65 (ShowAllHeaders && CurrentIncludeDepth > 2));
66 }
67 };
68
69 /// A callback for emitting header usage information to a file in JSON. Each
70 /// line in the file is a JSON object that includes the source file name and
71 /// the list of headers directly or indirectly included from it. For example:
72 ///
73 /// {"source":"/tmp/foo.c",
74 /// "includes":["/usr/include/stdio.h", "/usr/include/stdlib.h"]}
75 ///
76 /// To reduce the amount of data written to the file, we only record system
77 /// headers that are directly included from a file that isn't in the system
78 /// directory.
79 class HeaderIncludesJSONCallback : public PPCallbacks {
80 SourceManager &SM;
81 raw_ostream *OutputFile;
82 bool OwnsOutputFile;
83 SmallVector<std::string, 16> IncludedHeaders;
84
85 public:
HeaderIncludesJSONCallback(const Preprocessor * PP,raw_ostream * OutputFile_,bool OwnsOutputFile_)86 HeaderIncludesJSONCallback(const Preprocessor *PP, raw_ostream *OutputFile_,
87 bool OwnsOutputFile_)
88 : SM(PP->getSourceManager()), OutputFile(OutputFile_),
89 OwnsOutputFile(OwnsOutputFile_) {}
90
~HeaderIncludesJSONCallback()91 ~HeaderIncludesJSONCallback() override {
92 if (OwnsOutputFile)
93 delete OutputFile;
94 }
95
96 HeaderIncludesJSONCallback(const HeaderIncludesJSONCallback &) = delete;
97 HeaderIncludesJSONCallback &
98 operator=(const HeaderIncludesJSONCallback &) = delete;
99
100 void EndOfMainFile() override;
101
102 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
103 SrcMgr::CharacteristicKind FileType,
104 FileID PrevFID) override;
105
106 void FileSkipped(const FileEntryRef &SkippedFile, const Token &FilenameTok,
107 SrcMgr::CharacteristicKind FileType) override;
108 };
109
110 /// A callback for emitting direct header and module usage information to a
111 /// file in JSON. The output format is like HeaderIncludesJSONCallback but has
112 /// an array of separate entries, one for each non-system source file used in
113 /// the compilation showing only the direct includes and imports from that file.
114 class HeaderIncludesDirectPerFileCallback : public PPCallbacks {
115 SourceManager &SM;
116 HeaderSearch &HSI;
117 raw_ostream *OutputFile;
118 bool OwnsOutputFile;
119 using DependencyMap = llvm::DenseMap<FileEntryRef, SmallVector<FileEntryRef>>;
120 DependencyMap Dependencies;
121
122 public:
HeaderIncludesDirectPerFileCallback(const Preprocessor * PP,raw_ostream * OutputFile_,bool OwnsOutputFile_)123 HeaderIncludesDirectPerFileCallback(const Preprocessor *PP,
124 raw_ostream *OutputFile_,
125 bool OwnsOutputFile_)
126 : SM(PP->getSourceManager()), HSI(PP->getHeaderSearchInfo()),
127 OutputFile(OutputFile_), OwnsOutputFile(OwnsOutputFile_) {}
128
~HeaderIncludesDirectPerFileCallback()129 ~HeaderIncludesDirectPerFileCallback() override {
130 if (OwnsOutputFile)
131 delete OutputFile;
132 }
133
134 HeaderIncludesDirectPerFileCallback(
135 const HeaderIncludesDirectPerFileCallback &) = delete;
136 HeaderIncludesDirectPerFileCallback &
137 operator=(const HeaderIncludesDirectPerFileCallback &) = delete;
138
139 void EndOfMainFile() override;
140
141 void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
142 StringRef FileName, bool IsAngled,
143 CharSourceRange FilenameRange,
144 OptionalFileEntryRef File, StringRef SearchPath,
145 StringRef RelativePath, const Module *SuggestedModule,
146 bool ModuleImported,
147 SrcMgr::CharacteristicKind FileType) override;
148
149 void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
150 const Module *Imported) override;
151 };
152 }
153
PrintHeaderInfo(raw_ostream * OutputFile,StringRef Filename,bool ShowDepth,unsigned CurrentIncludeDepth,bool MSStyle)154 static void PrintHeaderInfo(raw_ostream *OutputFile, StringRef Filename,
155 bool ShowDepth, unsigned CurrentIncludeDepth,
156 bool MSStyle) {
157 // Write to a temporary string to avoid unnecessary flushing on errs().
158 SmallString<512> Pathname(Filename);
159 if (!MSStyle)
160 Lexer::Stringify(Pathname);
161
162 SmallString<256> Msg;
163 if (MSStyle)
164 Msg += "Note: including file:";
165
166 if (ShowDepth) {
167 // The main source file is at depth 1, so skip one dot.
168 for (unsigned i = 1; i != CurrentIncludeDepth; ++i)
169 Msg += MSStyle ? ' ' : '.';
170
171 if (!MSStyle)
172 Msg += ' ';
173 }
174 Msg += Pathname;
175 Msg += '\n';
176
177 *OutputFile << Msg;
178 OutputFile->flush();
179 }
180
AttachHeaderIncludeGen(Preprocessor & PP,const DependencyOutputOptions & DepOpts,bool ShowAllHeaders,StringRef OutputPath,bool ShowDepth,bool MSStyle)181 void clang::AttachHeaderIncludeGen(Preprocessor &PP,
182 const DependencyOutputOptions &DepOpts,
183 bool ShowAllHeaders, StringRef OutputPath,
184 bool ShowDepth, bool MSStyle) {
185 raw_ostream *OutputFile = &llvm::errs();
186 bool OwnsOutputFile = false;
187
188 // Choose output stream, when printing in cl.exe /showIncludes style.
189 if (MSStyle) {
190 switch (DepOpts.ShowIncludesDest) {
191 default:
192 llvm_unreachable("Invalid destination for /showIncludes output!");
193 case ShowIncludesDestination::Stderr:
194 OutputFile = &llvm::errs();
195 break;
196 case ShowIncludesDestination::Stdout:
197 OutputFile = &llvm::outs();
198 break;
199 }
200 }
201
202 // Open the output file, if used.
203 if (!OutputPath.empty()) {
204 std::error_code EC;
205 llvm::raw_fd_ostream *OS = new llvm::raw_fd_ostream(
206 OutputPath.str(), EC,
207 llvm::sys::fs::OF_Append | llvm::sys::fs::OF_TextWithCRLF);
208 if (EC) {
209 PP.getDiagnostics().Report(clang::diag::warn_fe_cc_print_header_failure)
210 << EC.message();
211 delete OS;
212 } else {
213 OS->SetUnbuffered();
214 OutputFile = OS;
215 OwnsOutputFile = true;
216 }
217 }
218
219 switch (DepOpts.HeaderIncludeFormat) {
220 case HIFMT_None:
221 llvm_unreachable("unexpected header format kind");
222 case HIFMT_Textual: {
223 assert(DepOpts.HeaderIncludeFiltering == HIFIL_None &&
224 "header filtering is currently always disabled when output format is"
225 "textual");
226 // Print header info for extra headers, pretending they were discovered by
227 // the regular preprocessor. The primary use case is to support proper
228 // generation of Make / Ninja file dependencies for implicit includes, such
229 // as sanitizer ignorelists. It's only important for cl.exe compatibility,
230 // the GNU way to generate rules is -M / -MM / -MD / -MMD.
231 for (const auto &Header : DepOpts.ExtraDeps)
232 PrintHeaderInfo(OutputFile, Header.first, ShowDepth, 2, MSStyle);
233 PP.addPPCallbacks(std::make_unique<HeaderIncludesCallback>(
234 &PP, ShowAllHeaders, OutputFile, DepOpts, OwnsOutputFile, ShowDepth,
235 MSStyle));
236 break;
237 }
238 case HIFMT_JSON:
239 switch (DepOpts.HeaderIncludeFiltering) {
240 default:
241 llvm_unreachable("Unknown HeaderIncludeFilteringKind enum");
242 case HIFIL_Only_Direct_System:
243 PP.addPPCallbacks(std::make_unique<HeaderIncludesJSONCallback>(
244 &PP, OutputFile, OwnsOutputFile));
245 break;
246 case HIFIL_Direct_Per_File:
247 PP.addPPCallbacks(std::make_unique<HeaderIncludesDirectPerFileCallback>(
248 &PP, OutputFile, OwnsOutputFile));
249 break;
250 }
251 break;
252 }
253 }
254
FileChanged(SourceLocation Loc,FileChangeReason Reason,SrcMgr::CharacteristicKind NewFileType,FileID PrevFID)255 void HeaderIncludesCallback::FileChanged(SourceLocation Loc,
256 FileChangeReason Reason,
257 SrcMgr::CharacteristicKind NewFileType,
258 FileID PrevFID) {
259 // Unless we are exiting a #include, make sure to skip ahead to the line the
260 // #include directive was at.
261 PresumedLoc UserLoc = SM.getPresumedLoc(Loc);
262 if (UserLoc.isInvalid())
263 return;
264
265 // Adjust the current include depth.
266 if (Reason == PPCallbacks::EnterFile) {
267 ++CurrentIncludeDepth;
268 } else if (Reason == PPCallbacks::ExitFile) {
269 if (CurrentIncludeDepth)
270 --CurrentIncludeDepth;
271
272 // We track when we are done with the predefines by watching for the first
273 // place where we drop back to a nesting depth of 1.
274 if (CurrentIncludeDepth == 1 && !HasProcessedPredefines)
275 HasProcessedPredefines = true;
276
277 return;
278 } else {
279 return;
280 }
281
282 if (!ShouldShowHeader(NewFileType))
283 return;
284
285 unsigned IncludeDepth = CurrentIncludeDepth;
286 if (!HasProcessedPredefines)
287 --IncludeDepth; // Ignore indent from <built-in>.
288
289 // FIXME: Identify headers in a more robust way than comparing their name to
290 // "<command line>" and "<built-in>" in a bunch of places.
291 if (Reason == PPCallbacks::EnterFile &&
292 UserLoc.getFilename() != StringRef("<command line>")) {
293 PrintHeaderInfo(OutputFile, UserLoc.getFilename(), ShowDepth, IncludeDepth,
294 MSStyle);
295 }
296 }
297
FileSkipped(const FileEntryRef & SkippedFile,const Token & FilenameTok,SrcMgr::CharacteristicKind FileType)298 void HeaderIncludesCallback::FileSkipped(const FileEntryRef &SkippedFile, const
299 Token &FilenameTok,
300 SrcMgr::CharacteristicKind FileType) {
301 if (!DepOpts.ShowSkippedHeaderIncludes)
302 return;
303
304 if (!ShouldShowHeader(FileType))
305 return;
306
307 PrintHeaderInfo(OutputFile, SkippedFile.getName(), ShowDepth,
308 CurrentIncludeDepth + 1, MSStyle);
309 }
310
EndOfMainFile()311 void HeaderIncludesJSONCallback::EndOfMainFile() {
312 OptionalFileEntryRef FE = SM.getFileEntryRefForID(SM.getMainFileID());
313 SmallString<256> MainFile;
314 if (FE) {
315 MainFile += FE->getName();
316 SM.getFileManager().makeAbsolutePath(MainFile);
317 }
318
319 std::string Str;
320 llvm::raw_string_ostream OS(Str);
321 llvm::json::OStream JOS(OS);
322 JOS.object([&] {
323 JOS.attribute("source", MainFile.c_str());
324 JOS.attributeArray("includes", [&] {
325 llvm::StringSet<> SeenHeaders;
326 for (const std::string &H : IncludedHeaders)
327 if (SeenHeaders.insert(H).second)
328 JOS.value(H);
329 });
330 });
331 OS << "\n";
332
333 if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) {
334 llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile);
335 if (auto L = FDS->lock())
336 *OutputFile << Str;
337 } else
338 *OutputFile << Str;
339 }
340
341 /// Determine whether the header file should be recorded. The header file should
342 /// be recorded only if the header file is a system header and the current file
343 /// isn't a system header.
shouldRecordNewFile(SrcMgr::CharacteristicKind NewFileType,SourceLocation PrevLoc,SourceManager & SM)344 static bool shouldRecordNewFile(SrcMgr::CharacteristicKind NewFileType,
345 SourceLocation PrevLoc, SourceManager &SM) {
346 return SrcMgr::isSystem(NewFileType) && !SM.isInSystemHeader(PrevLoc);
347 }
348
FileChanged(SourceLocation Loc,FileChangeReason Reason,SrcMgr::CharacteristicKind NewFileType,FileID PrevFID)349 void HeaderIncludesJSONCallback::FileChanged(
350 SourceLocation Loc, FileChangeReason Reason,
351 SrcMgr::CharacteristicKind NewFileType, FileID PrevFID) {
352 if (PrevFID.isInvalid() ||
353 !shouldRecordNewFile(NewFileType, SM.getLocForStartOfFile(PrevFID), SM))
354 return;
355
356 // Unless we are exiting a #include, make sure to skip ahead to the line the
357 // #include directive was at.
358 PresumedLoc UserLoc = SM.getPresumedLoc(Loc);
359 if (UserLoc.isInvalid())
360 return;
361
362 if (Reason == PPCallbacks::EnterFile &&
363 UserLoc.getFilename() != StringRef("<command line>"))
364 IncludedHeaders.push_back(UserLoc.getFilename());
365 }
366
FileSkipped(const FileEntryRef & SkippedFile,const Token & FilenameTok,SrcMgr::CharacteristicKind FileType)367 void HeaderIncludesJSONCallback::FileSkipped(
368 const FileEntryRef &SkippedFile, const Token &FilenameTok,
369 SrcMgr::CharacteristicKind FileType) {
370 if (!shouldRecordNewFile(FileType, FilenameTok.getLocation(), SM))
371 return;
372
373 IncludedHeaders.push_back(SkippedFile.getName().str());
374 }
375
EndOfMainFile()376 void HeaderIncludesDirectPerFileCallback::EndOfMainFile() {
377 if (Dependencies.empty())
378 return;
379
380 // Sort the files so that the output does not depend on the DenseMap order.
381 SmallVector<FileEntryRef> SourceFiles;
382 for (auto F = Dependencies.begin(), FEnd = Dependencies.end(); F != FEnd;
383 ++F) {
384 SourceFiles.push_back(F->first);
385 }
386 llvm::sort(SourceFiles, [](const FileEntryRef &LHS, const FileEntryRef &RHS) {
387 return LHS.getUID() < RHS.getUID();
388 });
389
390 std::string Str;
391 llvm::raw_string_ostream OS(Str);
392 llvm::json::OStream JOS(OS);
393 JOS.array([&] {
394 for (auto S = SourceFiles.begin(), SE = SourceFiles.end(); S != SE; ++S) {
395 JOS.object([&] {
396 SmallVector<FileEntryRef> &Deps = Dependencies[*S];
397 JOS.attribute("source", S->getName().str());
398 JOS.attributeArray("includes", [&] {
399 for (unsigned I = 0, N = Deps.size(); I != N; ++I)
400 JOS.value(Deps[I].getName().str());
401 });
402 });
403 }
404 });
405 OS << "\n";
406
407 if (OutputFile->get_kind() == raw_ostream::OStreamKind::OK_FDStream) {
408 llvm::raw_fd_ostream *FDS = static_cast<llvm::raw_fd_ostream *>(OutputFile);
409 if (auto L = FDS->lock())
410 *OutputFile << Str;
411 } else
412 *OutputFile << Str;
413 }
414
InclusionDirective(SourceLocation HashLoc,const Token & IncludeTok,StringRef FileName,bool IsAngled,CharSourceRange FilenameRange,OptionalFileEntryRef File,StringRef SearchPath,StringRef RelativePath,const Module * SuggestedModule,bool ModuleImported,SrcMgr::CharacteristicKind FileType)415 void HeaderIncludesDirectPerFileCallback::InclusionDirective(
416 SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
417 bool IsAngled, CharSourceRange FilenameRange, OptionalFileEntryRef File,
418 StringRef SearchPath, StringRef RelativePath, const Module *SuggestedModule,
419 bool ModuleImported, SrcMgr::CharacteristicKind FileType) {
420 if (!File)
421 return;
422
423 SourceLocation Loc = SM.getExpansionLoc(HashLoc);
424 if (SM.isInSystemHeader(Loc))
425 return;
426 OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(SM.getFileID(Loc));
427 if (!FromFile)
428 return;
429
430 Dependencies[*FromFile].push_back(*File);
431 }
432
moduleImport(SourceLocation ImportLoc,ModuleIdPath Path,const Module * Imported)433 void HeaderIncludesDirectPerFileCallback::moduleImport(SourceLocation ImportLoc,
434 ModuleIdPath Path,
435 const Module *Imported) {
436 if (!Imported)
437 return;
438
439 SourceLocation Loc = SM.getExpansionLoc(ImportLoc);
440 if (SM.isInSystemHeader(Loc))
441 return;
442 OptionalFileEntryRef FromFile = SM.getFileEntryRefForID(SM.getFileID(Loc));
443 if (!FromFile)
444 return;
445
446 OptionalFileEntryRef ModuleMapFile =
447 HSI.getModuleMap().getModuleMapFileForUniquing(Imported);
448 if (!ModuleMapFile)
449 return;
450
451 Dependencies[*FromFile].push_back(*ModuleMapFile);
452 }
453