1 //===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements the ExtractAPIAction, and ASTConsumer to collect API
11 /// information.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #include "clang/AST/ASTConcept.h"
16 #include "clang/AST/ASTConsumer.h"
17 #include "clang/AST/ASTContext.h"
18 #include "clang/AST/DeclObjC.h"
19 #include "clang/Basic/DiagnosticFrontend.h"
20 #include "clang/Basic/FileEntry.h"
21 #include "clang/Basic/SourceLocation.h"
22 #include "clang/Basic/SourceManager.h"
23 #include "clang/Basic/TargetInfo.h"
24 #include "clang/ExtractAPI/API.h"
25 #include "clang/ExtractAPI/APIIgnoresList.h"
26 #include "clang/ExtractAPI/ExtractAPIVisitor.h"
27 #include "clang/ExtractAPI/FrontendActions.h"
28 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
29 #include "clang/Frontend/ASTConsumers.h"
30 #include "clang/Frontend/CompilerInstance.h"
31 #include "clang/Frontend/FrontendOptions.h"
32 #include "clang/Frontend/MultiplexConsumer.h"
33 #include "clang/Index/USRGeneration.h"
34 #include "clang/InstallAPI/HeaderFile.h"
35 #include "clang/Lex/MacroInfo.h"
36 #include "clang/Lex/PPCallbacks.h"
37 #include "clang/Lex/Preprocessor.h"
38 #include "clang/Lex/PreprocessorOptions.h"
39 #include "llvm/ADT/DenseSet.h"
40 #include "llvm/ADT/STLExtras.h"
41 #include "llvm/ADT/SmallString.h"
42 #include "llvm/ADT/SmallVector.h"
43 #include "llvm/ADT/StringRef.h"
44 #include "llvm/Support/Casting.h"
45 #include "llvm/Support/Error.h"
46 #include "llvm/Support/MemoryBuffer.h"
47 #include "llvm/Support/Path.h"
48 #include "llvm/Support/Regex.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include <memory>
51 #include <optional>
52 #include <utility>
53
54 using namespace clang;
55 using namespace extractapi;
56
57 namespace {
58
getRelativeIncludeName(const CompilerInstance & CI,StringRef File,bool * IsQuoted=nullptr)59 std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
60 StringRef File,
61 bool *IsQuoted = nullptr) {
62 assert(CI.hasFileManager() &&
63 "CompilerInstance does not have a FileNamager!");
64
65 using namespace llvm::sys;
66 const auto &FS = CI.getVirtualFileSystem();
67
68 SmallString<128> FilePath(File.begin(), File.end());
69 FS.makeAbsolute(FilePath);
70 path::remove_dots(FilePath, true);
71 FilePath = path::convert_to_slash(FilePath);
72 File = FilePath;
73
74 // Checks whether `Dir` is a strict path prefix of `File`. If so returns
75 // the prefix length. Otherwise return 0.
76 auto CheckDir = [&](llvm::StringRef Dir) -> unsigned {
77 llvm::SmallString<32> DirPath(Dir.begin(), Dir.end());
78 FS.makeAbsolute(DirPath);
79 path::remove_dots(DirPath, true);
80 Dir = DirPath;
81 for (auto NI = path::begin(File), NE = path::end(File),
82 DI = path::begin(Dir), DE = path::end(Dir);
83 /*termination condition in loop*/; ++NI, ++DI) {
84 // '.' components in File are ignored.
85 while (NI != NE && *NI == ".")
86 ++NI;
87 if (NI == NE)
88 break;
89
90 // '.' components in Dir are ignored.
91 while (DI != DE && *DI == ".")
92 ++DI;
93
94 // Dir is a prefix of File, up to '.' components and choice of path
95 // separators.
96 if (DI == DE)
97 return NI - path::begin(File);
98
99 // Consider all path separators equal.
100 if (NI->size() == 1 && DI->size() == 1 &&
101 path::is_separator(NI->front()) && path::is_separator(DI->front()))
102 continue;
103
104 // Special case Apple .sdk folders since the search path is typically a
105 // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
106 // located in `iPhoneSimulator.sdk` (the real folder).
107 if (NI->ends_with(".sdk") && DI->ends_with(".sdk")) {
108 StringRef NBasename = path::stem(*NI);
109 StringRef DBasename = path::stem(*DI);
110 if (DBasename.starts_with(NBasename))
111 continue;
112 }
113
114 if (*NI != *DI)
115 break;
116 }
117 return 0;
118 };
119
120 unsigned PrefixLength = 0;
121
122 // Go through the search paths and find the first one that is a prefix of
123 // the header.
124 for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) {
125 // Note whether the match is found in a quoted entry.
126 if (IsQuoted)
127 *IsQuoted = Entry.Group == frontend::Quoted;
128
129 if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Entry.Path)) {
130 if (auto HMap = HeaderMap::Create(*EntryFile, CI.getFileManager())) {
131 // If this is a headermap entry, try to reverse lookup the full path
132 // for a spelled name before mapping.
133 StringRef SpelledFilename = HMap->reverseLookupFilename(File);
134 if (!SpelledFilename.empty())
135 return SpelledFilename.str();
136
137 // No matching mapping in this headermap, try next search entry.
138 continue;
139 }
140 }
141
142 // Entry is a directory search entry, try to check if it's a prefix of File.
143 PrefixLength = CheckDir(Entry.Path);
144 if (PrefixLength > 0) {
145 // The header is found in a framework path, construct the framework-style
146 // include name `<Framework/Header.h>`
147 if (Entry.IsFramework) {
148 SmallVector<StringRef, 4> Matches;
149 clang::installapi::HeaderFile::getFrameworkIncludeRule().match(
150 File, &Matches);
151 // Returned matches are always in stable order.
152 if (Matches.size() != 4)
153 return std::nullopt;
154
155 return path::convert_to_slash(
156 (Matches[1].drop_front(Matches[1].rfind('/') + 1) + "/" +
157 Matches[3])
158 .str());
159 }
160
161 // The header is found in a normal search path, strip the search path
162 // prefix to get an include name.
163 return path::convert_to_slash(File.drop_front(PrefixLength));
164 }
165 }
166
167 // Couldn't determine a include name, use full path instead.
168 return std::nullopt;
169 }
170
getRelativeIncludeName(const CompilerInstance & CI,FileEntryRef FE,bool * IsQuoted=nullptr)171 std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
172 FileEntryRef FE,
173 bool *IsQuoted = nullptr) {
174 return getRelativeIncludeName(CI, FE.getNameAsRequested(), IsQuoted);
175 }
176
177 struct LocationFileChecker {
operator ()__anon9569e29f0111::LocationFileChecker178 bool operator()(SourceLocation Loc) {
179 // If the loc refers to a macro expansion we need to first get the file
180 // location of the expansion.
181 auto &SM = CI.getSourceManager();
182 auto FileLoc = SM.getFileLoc(Loc);
183 FileID FID = SM.getFileID(FileLoc);
184 if (FID.isInvalid())
185 return false;
186
187 OptionalFileEntryRef File = SM.getFileEntryRefForID(FID);
188 if (!File)
189 return false;
190
191 if (KnownFileEntries.count(*File))
192 return true;
193
194 if (ExternalFileEntries.count(*File))
195 return false;
196
197 // Try to reduce the include name the same way we tried to include it.
198 bool IsQuoted = false;
199 if (auto IncludeName = getRelativeIncludeName(CI, *File, &IsQuoted))
200 if (llvm::any_of(KnownFiles,
201 [&IsQuoted, &IncludeName](const auto &KnownFile) {
202 return KnownFile.first.equals(*IncludeName) &&
203 KnownFile.second == IsQuoted;
204 })) {
205 KnownFileEntries.insert(*File);
206 return true;
207 }
208
209 // Record that the file was not found to avoid future reverse lookup for
210 // the same file.
211 ExternalFileEntries.insert(*File);
212 return false;
213 }
214
LocationFileChecker__anon9569e29f0111::LocationFileChecker215 LocationFileChecker(const CompilerInstance &CI,
216 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles)
217 : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() {
218 for (const auto &KnownFile : KnownFiles)
219 if (auto FE = CI.getFileManager().getOptionalFileRef(KnownFile.first))
220 KnownFileEntries.insert(*FE);
221 }
222
223 private:
224 const CompilerInstance &CI;
225 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles;
226 llvm::DenseSet<const FileEntry *> KnownFileEntries;
227 llvm::DenseSet<const FileEntry *> ExternalFileEntries;
228 };
229
230 struct BatchExtractAPIVisitor : ExtractAPIVisitor<BatchExtractAPIVisitor> {
shouldDeclBeIncluded__anon9569e29f0111::BatchExtractAPIVisitor231 bool shouldDeclBeIncluded(const Decl *D) const {
232 bool ShouldBeIncluded = true;
233 // Check that we have the definition for redeclarable types.
234 if (auto *TD = llvm::dyn_cast<TagDecl>(D))
235 ShouldBeIncluded = TD->isThisDeclarationADefinition();
236 else if (auto *Interface = llvm::dyn_cast<ObjCInterfaceDecl>(D))
237 ShouldBeIncluded = Interface->isThisDeclarationADefinition();
238 else if (auto *Protocol = llvm::dyn_cast<ObjCProtocolDecl>(D))
239 ShouldBeIncluded = Protocol->isThisDeclarationADefinition();
240
241 ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation());
242 return ShouldBeIncluded;
243 }
244
BatchExtractAPIVisitor__anon9569e29f0111::BatchExtractAPIVisitor245 BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context,
246 APISet &API)
247 : ExtractAPIVisitor<BatchExtractAPIVisitor>(Context, API), LCF(LCF) {}
248
249 private:
250 LocationFileChecker &LCF;
251 };
252
253 class WrappingExtractAPIConsumer : public ASTConsumer {
254 public:
WrappingExtractAPIConsumer(ASTContext & Context,APISet & API)255 WrappingExtractAPIConsumer(ASTContext &Context, APISet &API)
256 : Visitor(Context, API) {}
257
HandleTranslationUnit(ASTContext & Context)258 void HandleTranslationUnit(ASTContext &Context) override {
259 // Use ExtractAPIVisitor to traverse symbol declarations in the context.
260 Visitor.TraverseDecl(Context.getTranslationUnitDecl());
261 }
262
263 private:
264 ExtractAPIVisitor<> Visitor;
265 };
266
267 class ExtractAPIConsumer : public ASTConsumer {
268 public:
ExtractAPIConsumer(ASTContext & Context,std::unique_ptr<LocationFileChecker> LCF,APISet & API)269 ExtractAPIConsumer(ASTContext &Context,
270 std::unique_ptr<LocationFileChecker> LCF, APISet &API)
271 : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {}
272
HandleTranslationUnit(ASTContext & Context)273 void HandleTranslationUnit(ASTContext &Context) override {
274 // Use ExtractAPIVisitor to traverse symbol declarations in the context.
275 Visitor.TraverseDecl(Context.getTranslationUnitDecl());
276 }
277
278 private:
279 BatchExtractAPIVisitor Visitor;
280 std::unique_ptr<LocationFileChecker> LCF;
281 };
282
283 class MacroCallback : public PPCallbacks {
284 public:
MacroCallback(const SourceManager & SM,APISet & API,Preprocessor & PP)285 MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP)
286 : SM(SM), API(API), PP(PP) {}
287
EndOfMainFile()288 void EndOfMainFile() override {
289 for (const auto &M : PP.macros()) {
290 auto *II = M.getFirst();
291 auto MD = PP.getMacroDefinition(II);
292 auto *MI = MD.getMacroInfo();
293
294 if (!MI)
295 continue;
296
297 // Ignore header guard macros
298 if (MI->isUsedForHeaderGuard())
299 continue;
300
301 // Ignore builtin macros and ones defined via the command line.
302 if (MI->isBuiltinMacro())
303 continue;
304
305 auto DefLoc = MI->getDefinitionLoc();
306
307 if (SM.isInPredefinedFile(DefLoc))
308 continue;
309
310 auto AssociatedModuleMacros = MD.getModuleMacros();
311 StringRef OwningModuleName;
312 if (!AssociatedModuleMacros.empty())
313 OwningModuleName = AssociatedModuleMacros.back()
314 ->getOwningModule()
315 ->getTopLevelModuleName();
316
317 if (!shouldMacroBeIncluded(DefLoc, OwningModuleName))
318 continue;
319
320 StringRef Name = II->getName();
321 PresumedLoc Loc = SM.getPresumedLoc(DefLoc);
322 SmallString<128> USR;
323 index::generateUSRForMacro(Name, DefLoc, SM, USR);
324 API.createRecord<extractapi::MacroDefinitionRecord>(
325 USR, Name, SymbolReference(), Loc,
326 DeclarationFragmentsBuilder::getFragmentsForMacro(Name, MI),
327 DeclarationFragmentsBuilder::getSubHeadingForMacro(Name),
328 SM.isInSystemHeader(DefLoc));
329 }
330 }
331
shouldMacroBeIncluded(const SourceLocation & MacroLoc,StringRef ModuleName)332 virtual bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
333 StringRef ModuleName) {
334 return true;
335 }
336
337 const SourceManager &SM;
338 APISet &API;
339 Preprocessor &PP;
340 };
341
342 class APIMacroCallback : public MacroCallback {
343 public:
APIMacroCallback(const SourceManager & SM,APISet & API,Preprocessor & PP,LocationFileChecker & LCF)344 APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP,
345 LocationFileChecker &LCF)
346 : MacroCallback(SM, API, PP), LCF(LCF) {}
347
shouldMacroBeIncluded(const SourceLocation & MacroLoc,StringRef ModuleName)348 bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
349 StringRef ModuleName) override {
350 // Do not include macros from external files
351 return LCF(MacroLoc);
352 }
353
354 private:
355 LocationFileChecker &LCF;
356 };
357
358 std::unique_ptr<llvm::raw_pwrite_stream>
createAdditionalSymbolGraphFile(CompilerInstance & CI,Twine BaseName)359 createAdditionalSymbolGraphFile(CompilerInstance &CI, Twine BaseName) {
360 auto OutputDirectory = CI.getFrontendOpts().SymbolGraphOutputDir;
361
362 SmallString<256> FileName;
363 llvm::sys::path::append(FileName, OutputDirectory,
364 BaseName + ".symbols.json");
365 return CI.createOutputFile(
366 FileName, /*Binary*/ false, /*RemoveFileOnSignal*/ false,
367 /*UseTemporary*/ true, /*CreateMissingDirectories*/ true);
368 }
369
370 } // namespace
371
ImplEndSourceFileAction(CompilerInstance & CI)372 void ExtractAPIActionBase::ImplEndSourceFileAction(CompilerInstance &CI) {
373 SymbolGraphSerializerOption SerializationOptions;
374 SerializationOptions.Compact = !CI.getFrontendOpts().EmitPrettySymbolGraphs;
375 SerializationOptions.EmitSymbolLabelsForTesting =
376 CI.getFrontendOpts().EmitSymbolGraphSymbolLabelsForTesting;
377
378 if (CI.getFrontendOpts().EmitExtensionSymbolGraphs) {
379 auto ConstructOutputFile = [&CI](Twine BaseName) {
380 return createAdditionalSymbolGraphFile(CI, BaseName);
381 };
382
383 SymbolGraphSerializer::serializeWithExtensionGraphs(
384 *OS, *API, IgnoresList, ConstructOutputFile, SerializationOptions);
385 } else {
386 SymbolGraphSerializer::serializeMainSymbolGraph(*OS, *API, IgnoresList,
387 SerializationOptions);
388 }
389
390 // Flush the stream and close the main output stream.
391 OS.reset();
392 }
393
394 std::unique_ptr<ASTConsumer>
CreateASTConsumer(CompilerInstance & CI,StringRef InFile)395 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
396 auto ProductName = CI.getFrontendOpts().ProductName;
397
398 if (CI.getFrontendOpts().SymbolGraphOutputDir.empty())
399 OS = CI.createDefaultOutputFile(/*Binary*/ false, InFile,
400 /*Extension*/ "symbols.json",
401 /*RemoveFileOnSignal*/ false,
402 /*CreateMissingDirectories*/ true);
403 else
404 OS = createAdditionalSymbolGraphFile(CI, ProductName);
405
406 if (!OS)
407 return nullptr;
408
409 // Now that we have enough information about the language options and the
410 // target triple, let's create the APISet before anyone uses it.
411 API = std::make_unique<APISet>(
412 CI.getTarget().getTriple(),
413 CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);
414
415 auto LCF = std::make_unique<LocationFileChecker>(CI, KnownInputFiles);
416
417 CI.getPreprocessor().addPPCallbacks(std::make_unique<APIMacroCallback>(
418 CI.getSourceManager(), *API, CI.getPreprocessor(), *LCF));
419
420 // Do not include location in anonymous decls.
421 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
422 Policy.AnonymousTagLocations = false;
423 CI.getASTContext().setPrintingPolicy(Policy);
424
425 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
426 llvm::handleAllErrors(
427 APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,
428 CI.getFileManager())
429 .moveInto(IgnoresList),
430 [&CI](const IgnoresFileNotFound &Err) {
431 CI.getDiagnostics().Report(
432 diag::err_extract_api_ignores_file_not_found)
433 << Err.Path;
434 });
435 }
436
437 return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(),
438 std::move(LCF), *API);
439 }
440
PrepareToExecuteAction(CompilerInstance & CI)441 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
442 auto &Inputs = CI.getFrontendOpts().Inputs;
443 if (Inputs.empty())
444 return true;
445
446 if (!CI.hasFileManager())
447 if (!CI.createFileManager())
448 return false;
449
450 auto Kind = Inputs[0].getKind();
451
452 // Convert the header file inputs into a single input buffer.
453 SmallString<256> HeaderContents;
454 bool IsQuoted = false;
455 for (const FrontendInputFile &FIF : Inputs) {
456 if (Kind.isObjectiveC())
457 HeaderContents += "#import";
458 else
459 HeaderContents += "#include";
460
461 StringRef FilePath = FIF.getFile();
462 if (auto RelativeName = getRelativeIncludeName(CI, FilePath, &IsQuoted)) {
463 if (IsQuoted)
464 HeaderContents += " \"";
465 else
466 HeaderContents += " <";
467
468 HeaderContents += *RelativeName;
469
470 if (IsQuoted)
471 HeaderContents += "\"\n";
472 else
473 HeaderContents += ">\n";
474 KnownInputFiles.emplace_back(static_cast<SmallString<32>>(*RelativeName),
475 IsQuoted);
476 } else {
477 HeaderContents += " \"";
478 HeaderContents += FilePath;
479 HeaderContents += "\"\n";
480 KnownInputFiles.emplace_back(FilePath, true);
481 }
482 }
483
484 if (CI.getHeaderSearchOpts().Verbose)
485 CI.getVerboseOutputStream() << getInputBufferName() << ":\n"
486 << HeaderContents << "\n";
487
488 Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents,
489 getInputBufferName());
490
491 // Set that buffer up as our "real" input in the CompilerInstance.
492 Inputs.clear();
493 Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false);
494
495 return true;
496 }
497
EndSourceFileAction()498 void ExtractAPIAction::EndSourceFileAction() {
499 ImplEndSourceFileAction(getCompilerInstance());
500 }
501
502 std::unique_ptr<ASTConsumer>
CreateASTConsumer(CompilerInstance & CI,StringRef InFile)503 WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI,
504 StringRef InFile) {
505 auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
506 if (!OtherConsumer)
507 return nullptr;
508
509 CreatedASTConsumer = true;
510
511 ProductName = CI.getFrontendOpts().ProductName;
512 auto InputFilename = llvm::sys::path::filename(InFile);
513 OS = createAdditionalSymbolGraphFile(CI, InputFilename);
514
515 // Now that we have enough information about the language options and the
516 // target triple, let's create the APISet before anyone uses it.
517 API = std::make_unique<APISet>(
518 CI.getTarget().getTriple(),
519 CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);
520
521 CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
522 CI.getSourceManager(), *API, CI.getPreprocessor()));
523
524 // Do not include location in anonymous decls.
525 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
526 Policy.AnonymousTagLocations = false;
527 CI.getASTContext().setPrintingPolicy(Policy);
528
529 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
530 llvm::handleAllErrors(
531 APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,
532 CI.getFileManager())
533 .moveInto(IgnoresList),
534 [&CI](const IgnoresFileNotFound &Err) {
535 CI.getDiagnostics().Report(
536 diag::err_extract_api_ignores_file_not_found)
537 << Err.Path;
538 });
539 }
540
541 auto WrappingConsumer =
542 std::make_unique<WrappingExtractAPIConsumer>(CI.getASTContext(), *API);
543 std::vector<std::unique_ptr<ASTConsumer>> Consumers;
544 Consumers.push_back(std::move(OtherConsumer));
545 Consumers.push_back(std::move(WrappingConsumer));
546
547 return std::make_unique<MultiplexConsumer>(std::move(Consumers));
548 }
549
EndSourceFileAction()550 void WrappingExtractAPIAction::EndSourceFileAction() {
551 // Invoke wrapped action's method.
552 WrapperFrontendAction::EndSourceFileAction();
553
554 if (CreatedASTConsumer) {
555 ImplEndSourceFileAction(getCompilerInstance());
556 }
557 }
558