1 //===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file implements the ExtractAPIAction, and ASTConsumer to collect API 11 /// information. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #include "clang/AST/ASTConcept.h" 16 #include "clang/AST/ASTConsumer.h" 17 #include "clang/AST/ASTContext.h" 18 #include "clang/AST/DeclObjC.h" 19 #include "clang/Basic/DiagnosticFrontend.h" 20 #include "clang/Basic/FileEntry.h" 21 #include "clang/Basic/SourceLocation.h" 22 #include "clang/Basic/SourceManager.h" 23 #include "clang/Basic/TargetInfo.h" 24 #include "clang/ExtractAPI/API.h" 25 #include "clang/ExtractAPI/APIIgnoresList.h" 26 #include "clang/ExtractAPI/ExtractAPIVisitor.h" 27 #include "clang/ExtractAPI/FrontendActions.h" 28 #include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h" 29 #include "clang/Frontend/ASTConsumers.h" 30 #include "clang/Frontend/CompilerInstance.h" 31 #include "clang/Frontend/FrontendOptions.h" 32 #include "clang/Frontend/MultiplexConsumer.h" 33 #include "clang/Index/USRGeneration.h" 34 #include "clang/InstallAPI/HeaderFile.h" 35 #include "clang/Lex/MacroInfo.h" 36 #include "clang/Lex/PPCallbacks.h" 37 #include "clang/Lex/Preprocessor.h" 38 #include "clang/Lex/PreprocessorOptions.h" 39 #include "llvm/ADT/DenseSet.h" 40 #include "llvm/ADT/STLExtras.h" 41 #include "llvm/ADT/SmallString.h" 42 #include "llvm/ADT/SmallVector.h" 43 #include "llvm/ADT/StringRef.h" 44 #include "llvm/Support/Casting.h" 45 #include "llvm/Support/Error.h" 46 #include "llvm/Support/MemoryBuffer.h" 47 #include "llvm/Support/Path.h" 48 #include "llvm/Support/Regex.h" 49 #include "llvm/Support/raw_ostream.h" 50 #include <memory> 51 #include <optional> 52 #include <utility> 53 54 using namespace clang; 55 using namespace extractapi; 56 57 namespace { 58 59 std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI, 60 StringRef File, 61 bool *IsQuoted = nullptr) { 62 assert(CI.hasFileManager() && 63 "CompilerInstance does not have a FileNamager!"); 64 65 using namespace llvm::sys; 66 const auto &FS = CI.getVirtualFileSystem(); 67 68 SmallString<128> FilePath(File.begin(), File.end()); 69 FS.makeAbsolute(FilePath); 70 path::remove_dots(FilePath, true); 71 FilePath = path::convert_to_slash(FilePath); 72 File = FilePath; 73 74 // Checks whether `Dir` is a strict path prefix of `File`. If so returns 75 // the prefix length. Otherwise return 0. 76 auto CheckDir = [&](llvm::StringRef Dir) -> unsigned { 77 llvm::SmallString<32> DirPath(Dir.begin(), Dir.end()); 78 FS.makeAbsolute(DirPath); 79 path::remove_dots(DirPath, true); 80 Dir = DirPath; 81 for (auto NI = path::begin(File), NE = path::end(File), 82 DI = path::begin(Dir), DE = path::end(Dir); 83 /*termination condition in loop*/; ++NI, ++DI) { 84 // '.' components in File are ignored. 85 while (NI != NE && *NI == ".") 86 ++NI; 87 if (NI == NE) 88 break; 89 90 // '.' components in Dir are ignored. 91 while (DI != DE && *DI == ".") 92 ++DI; 93 94 // Dir is a prefix of File, up to '.' components and choice of path 95 // separators. 96 if (DI == DE) 97 return NI - path::begin(File); 98 99 // Consider all path separators equal. 100 if (NI->size() == 1 && DI->size() == 1 && 101 path::is_separator(NI->front()) && path::is_separator(DI->front())) 102 continue; 103 104 // Special case Apple .sdk folders since the search path is typically a 105 // symlink like `iPhoneSimulator14.5.sdk` while the file is instead 106 // located in `iPhoneSimulator.sdk` (the real folder). 107 if (NI->ends_with(".sdk") && DI->ends_with(".sdk")) { 108 StringRef NBasename = path::stem(*NI); 109 StringRef DBasename = path::stem(*DI); 110 if (DBasename.starts_with(NBasename)) 111 continue; 112 } 113 114 if (*NI != *DI) 115 break; 116 } 117 return 0; 118 }; 119 120 unsigned PrefixLength = 0; 121 122 // Go through the search paths and find the first one that is a prefix of 123 // the header. 124 for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) { 125 // Note whether the match is found in a quoted entry. 126 if (IsQuoted) 127 *IsQuoted = Entry.Group == frontend::Quoted; 128 129 if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Entry.Path)) { 130 if (auto HMap = HeaderMap::Create(*EntryFile, CI.getFileManager())) { 131 // If this is a headermap entry, try to reverse lookup the full path 132 // for a spelled name before mapping. 133 StringRef SpelledFilename = HMap->reverseLookupFilename(File); 134 if (!SpelledFilename.empty()) 135 return SpelledFilename.str(); 136 137 // No matching mapping in this headermap, try next search entry. 138 continue; 139 } 140 } 141 142 // Entry is a directory search entry, try to check if it's a prefix of File. 143 PrefixLength = CheckDir(Entry.Path); 144 if (PrefixLength > 0) { 145 // The header is found in a framework path, construct the framework-style 146 // include name `<Framework/Header.h>` 147 if (Entry.IsFramework) { 148 SmallVector<StringRef, 4> Matches; 149 clang::installapi::HeaderFile::getFrameworkIncludeRule().match( 150 File, &Matches); 151 // Returned matches are always in stable order. 152 if (Matches.size() != 4) 153 return std::nullopt; 154 155 return path::convert_to_slash( 156 (Matches[1].drop_front(Matches[1].rfind('/') + 1) + "/" + 157 Matches[3]) 158 .str()); 159 } 160 161 // The header is found in a normal search path, strip the search path 162 // prefix to get an include name. 163 return path::convert_to_slash(File.drop_front(PrefixLength)); 164 } 165 } 166 167 // Couldn't determine a include name, use full path instead. 168 return std::nullopt; 169 } 170 171 std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI, 172 FileEntryRef FE, 173 bool *IsQuoted = nullptr) { 174 return getRelativeIncludeName(CI, FE.getNameAsRequested(), IsQuoted); 175 } 176 177 struct LocationFileChecker { 178 bool operator()(SourceLocation Loc) { 179 // If the loc refers to a macro expansion we need to first get the file 180 // location of the expansion. 181 auto &SM = CI.getSourceManager(); 182 auto FileLoc = SM.getFileLoc(Loc); 183 FileID FID = SM.getFileID(FileLoc); 184 if (FID.isInvalid()) 185 return false; 186 187 OptionalFileEntryRef File = SM.getFileEntryRefForID(FID); 188 if (!File) 189 return false; 190 191 if (KnownFileEntries.count(*File)) 192 return true; 193 194 if (ExternalFileEntries.count(*File)) 195 return false; 196 197 // Try to reduce the include name the same way we tried to include it. 198 bool IsQuoted = false; 199 if (auto IncludeName = getRelativeIncludeName(CI, *File, &IsQuoted)) 200 if (llvm::any_of(KnownFiles, 201 [&IsQuoted, &IncludeName](const auto &KnownFile) { 202 return KnownFile.first.equals(*IncludeName) && 203 KnownFile.second == IsQuoted; 204 })) { 205 KnownFileEntries.insert(*File); 206 return true; 207 } 208 209 // Record that the file was not found to avoid future reverse lookup for 210 // the same file. 211 ExternalFileEntries.insert(*File); 212 return false; 213 } 214 215 LocationFileChecker(const CompilerInstance &CI, 216 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles) 217 : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() { 218 for (const auto &KnownFile : KnownFiles) 219 if (auto FE = CI.getFileManager().getOptionalFileRef(KnownFile.first)) 220 KnownFileEntries.insert(*FE); 221 } 222 223 private: 224 const CompilerInstance &CI; 225 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles; 226 llvm::DenseSet<const FileEntry *> KnownFileEntries; 227 llvm::DenseSet<const FileEntry *> ExternalFileEntries; 228 }; 229 230 struct BatchExtractAPIVisitor : ExtractAPIVisitor<BatchExtractAPIVisitor> { 231 bool shouldDeclBeIncluded(const Decl *D) const { 232 bool ShouldBeIncluded = true; 233 // Check that we have the definition for redeclarable types. 234 if (auto *TD = llvm::dyn_cast<TagDecl>(D)) 235 ShouldBeIncluded = TD->isThisDeclarationADefinition(); 236 else if (auto *Interface = llvm::dyn_cast<ObjCInterfaceDecl>(D)) 237 ShouldBeIncluded = Interface->isThisDeclarationADefinition(); 238 else if (auto *Protocol = llvm::dyn_cast<ObjCProtocolDecl>(D)) 239 ShouldBeIncluded = Protocol->isThisDeclarationADefinition(); 240 241 ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation()); 242 return ShouldBeIncluded; 243 } 244 245 BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context, 246 APISet &API) 247 : ExtractAPIVisitor<BatchExtractAPIVisitor>(Context, API), LCF(LCF) {} 248 249 private: 250 LocationFileChecker &LCF; 251 }; 252 253 class WrappingExtractAPIConsumer : public ASTConsumer { 254 public: 255 WrappingExtractAPIConsumer(ASTContext &Context, APISet &API) 256 : Visitor(Context, API) {} 257 258 void HandleTranslationUnit(ASTContext &Context) override { 259 // Use ExtractAPIVisitor to traverse symbol declarations in the context. 260 Visitor.TraverseDecl(Context.getTranslationUnitDecl()); 261 } 262 263 private: 264 ExtractAPIVisitor<> Visitor; 265 }; 266 267 class ExtractAPIConsumer : public ASTConsumer { 268 public: 269 ExtractAPIConsumer(ASTContext &Context, 270 std::unique_ptr<LocationFileChecker> LCF, APISet &API) 271 : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {} 272 273 void HandleTranslationUnit(ASTContext &Context) override { 274 // Use ExtractAPIVisitor to traverse symbol declarations in the context. 275 Visitor.TraverseDecl(Context.getTranslationUnitDecl()); 276 } 277 278 private: 279 BatchExtractAPIVisitor Visitor; 280 std::unique_ptr<LocationFileChecker> LCF; 281 }; 282 283 class MacroCallback : public PPCallbacks { 284 public: 285 MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP) 286 : SM(SM), API(API), PP(PP) {} 287 288 void EndOfMainFile() override { 289 for (const auto &M : PP.macros()) { 290 auto *II = M.getFirst(); 291 auto MD = PP.getMacroDefinition(II); 292 auto *MI = MD.getMacroInfo(); 293 294 if (!MI) 295 continue; 296 297 // Ignore header guard macros 298 if (MI->isUsedForHeaderGuard()) 299 continue; 300 301 // Ignore builtin macros and ones defined via the command line. 302 if (MI->isBuiltinMacro()) 303 continue; 304 305 auto DefLoc = MI->getDefinitionLoc(); 306 307 if (SM.isInPredefinedFile(DefLoc)) 308 continue; 309 310 auto AssociatedModuleMacros = MD.getModuleMacros(); 311 StringRef OwningModuleName; 312 if (!AssociatedModuleMacros.empty()) 313 OwningModuleName = AssociatedModuleMacros.back() 314 ->getOwningModule() 315 ->getTopLevelModuleName(); 316 317 if (!shouldMacroBeIncluded(DefLoc, OwningModuleName)) 318 continue; 319 320 StringRef Name = II->getName(); 321 PresumedLoc Loc = SM.getPresumedLoc(DefLoc); 322 SmallString<128> USR; 323 index::generateUSRForMacro(Name, DefLoc, SM, USR); 324 API.createRecord<extractapi::MacroDefinitionRecord>( 325 USR, Name, SymbolReference(), Loc, 326 DeclarationFragmentsBuilder::getFragmentsForMacro(Name, MI), 327 DeclarationFragmentsBuilder::getSubHeadingForMacro(Name), 328 SM.isInSystemHeader(DefLoc)); 329 } 330 } 331 332 virtual bool shouldMacroBeIncluded(const SourceLocation &MacroLoc, 333 StringRef ModuleName) { 334 return true; 335 } 336 337 const SourceManager &SM; 338 APISet &API; 339 Preprocessor &PP; 340 }; 341 342 class APIMacroCallback : public MacroCallback { 343 public: 344 APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP, 345 LocationFileChecker &LCF) 346 : MacroCallback(SM, API, PP), LCF(LCF) {} 347 348 bool shouldMacroBeIncluded(const SourceLocation &MacroLoc, 349 StringRef ModuleName) override { 350 // Do not include macros from external files 351 return LCF(MacroLoc); 352 } 353 354 private: 355 LocationFileChecker &LCF; 356 }; 357 358 std::unique_ptr<llvm::raw_pwrite_stream> 359 createAdditionalSymbolGraphFile(CompilerInstance &CI, Twine BaseName) { 360 auto OutputDirectory = CI.getFrontendOpts().SymbolGraphOutputDir; 361 362 SmallString<256> FileName; 363 llvm::sys::path::append(FileName, OutputDirectory, 364 BaseName + ".symbols.json"); 365 return CI.createOutputFile( 366 FileName, /*Binary*/ false, /*RemoveFileOnSignal*/ false, 367 /*UseTemporary*/ true, /*CreateMissingDirectories*/ true); 368 } 369 370 } // namespace 371 372 void ExtractAPIActionBase::ImplEndSourceFileAction(CompilerInstance &CI) { 373 SymbolGraphSerializerOption SerializationOptions; 374 SerializationOptions.Compact = !CI.getFrontendOpts().EmitPrettySymbolGraphs; 375 SerializationOptions.EmitSymbolLabelsForTesting = 376 CI.getFrontendOpts().EmitSymbolGraphSymbolLabelsForTesting; 377 378 if (CI.getFrontendOpts().EmitExtensionSymbolGraphs) { 379 auto ConstructOutputFile = [&CI](Twine BaseName) { 380 return createAdditionalSymbolGraphFile(CI, BaseName); 381 }; 382 383 SymbolGraphSerializer::serializeWithExtensionGraphs( 384 *OS, *API, IgnoresList, ConstructOutputFile, SerializationOptions); 385 } else { 386 SymbolGraphSerializer::serializeMainSymbolGraph(*OS, *API, IgnoresList, 387 SerializationOptions); 388 } 389 390 // Flush the stream and close the main output stream. 391 OS.reset(); 392 } 393 394 std::unique_ptr<ASTConsumer> 395 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { 396 auto ProductName = CI.getFrontendOpts().ProductName; 397 398 if (CI.getFrontendOpts().SymbolGraphOutputDir.empty()) 399 OS = CI.createDefaultOutputFile(/*Binary*/ false, InFile, 400 /*Extension*/ "symbols.json", 401 /*RemoveFileOnSignal*/ false, 402 /*CreateMissingDirectories*/ true); 403 else 404 OS = createAdditionalSymbolGraphFile(CI, ProductName); 405 406 if (!OS) 407 return nullptr; 408 409 // Now that we have enough information about the language options and the 410 // target triple, let's create the APISet before anyone uses it. 411 API = std::make_unique<APISet>( 412 CI.getTarget().getTriple(), 413 CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName); 414 415 auto LCF = std::make_unique<LocationFileChecker>(CI, KnownInputFiles); 416 417 CI.getPreprocessor().addPPCallbacks(std::make_unique<APIMacroCallback>( 418 CI.getSourceManager(), *API, CI.getPreprocessor(), *LCF)); 419 420 // Do not include location in anonymous decls. 421 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy(); 422 Policy.AnonymousTagLocations = false; 423 CI.getASTContext().setPrintingPolicy(Policy); 424 425 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) { 426 llvm::handleAllErrors( 427 APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList, 428 CI.getFileManager()) 429 .moveInto(IgnoresList), 430 [&CI](const IgnoresFileNotFound &Err) { 431 CI.getDiagnostics().Report( 432 diag::err_extract_api_ignores_file_not_found) 433 << Err.Path; 434 }); 435 } 436 437 return std::make_unique<ExtractAPIConsumer>(CI.getASTContext(), 438 std::move(LCF), *API); 439 } 440 441 bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) { 442 auto &Inputs = CI.getFrontendOpts().Inputs; 443 if (Inputs.empty()) 444 return true; 445 446 if (!CI.hasFileManager()) 447 if (!CI.createFileManager()) 448 return false; 449 450 auto Kind = Inputs[0].getKind(); 451 452 // Convert the header file inputs into a single input buffer. 453 SmallString<256> HeaderContents; 454 bool IsQuoted = false; 455 for (const FrontendInputFile &FIF : Inputs) { 456 if (Kind.isObjectiveC()) 457 HeaderContents += "#import"; 458 else 459 HeaderContents += "#include"; 460 461 StringRef FilePath = FIF.getFile(); 462 if (auto RelativeName = getRelativeIncludeName(CI, FilePath, &IsQuoted)) { 463 if (IsQuoted) 464 HeaderContents += " \""; 465 else 466 HeaderContents += " <"; 467 468 HeaderContents += *RelativeName; 469 470 if (IsQuoted) 471 HeaderContents += "\"\n"; 472 else 473 HeaderContents += ">\n"; 474 KnownInputFiles.emplace_back(static_cast<SmallString<32>>(*RelativeName), 475 IsQuoted); 476 } else { 477 HeaderContents += " \""; 478 HeaderContents += FilePath; 479 HeaderContents += "\"\n"; 480 KnownInputFiles.emplace_back(FilePath, true); 481 } 482 } 483 484 if (CI.getHeaderSearchOpts().Verbose) 485 CI.getVerboseOutputStream() << getInputBufferName() << ":\n" 486 << HeaderContents << "\n"; 487 488 Buffer = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents, 489 getInputBufferName()); 490 491 // Set that buffer up as our "real" input in the CompilerInstance. 492 Inputs.clear(); 493 Inputs.emplace_back(Buffer->getMemBufferRef(), Kind, /*IsSystem*/ false); 494 495 return true; 496 } 497 498 void ExtractAPIAction::EndSourceFileAction() { 499 ImplEndSourceFileAction(getCompilerInstance()); 500 } 501 502 std::unique_ptr<ASTConsumer> 503 WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, 504 StringRef InFile) { 505 auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); 506 if (!OtherConsumer) 507 return nullptr; 508 509 CreatedASTConsumer = true; 510 511 ProductName = CI.getFrontendOpts().ProductName; 512 auto InputFilename = llvm::sys::path::filename(InFile); 513 OS = createAdditionalSymbolGraphFile(CI, InputFilename); 514 515 // Now that we have enough information about the language options and the 516 // target triple, let's create the APISet before anyone uses it. 517 API = std::make_unique<APISet>( 518 CI.getTarget().getTriple(), 519 CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName); 520 521 CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>( 522 CI.getSourceManager(), *API, CI.getPreprocessor())); 523 524 // Do not include location in anonymous decls. 525 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy(); 526 Policy.AnonymousTagLocations = false; 527 CI.getASTContext().setPrintingPolicy(Policy); 528 529 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) { 530 llvm::handleAllErrors( 531 APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList, 532 CI.getFileManager()) 533 .moveInto(IgnoresList), 534 [&CI](const IgnoresFileNotFound &Err) { 535 CI.getDiagnostics().Report( 536 diag::err_extract_api_ignores_file_not_found) 537 << Err.Path; 538 }); 539 } 540 541 auto WrappingConsumer = 542 std::make_unique<WrappingExtractAPIConsumer>(CI.getASTContext(), *API); 543 std::vector<std::unique_ptr<ASTConsumer>> Consumers; 544 Consumers.push_back(std::move(OtherConsumer)); 545 Consumers.push_back(std::move(WrappingConsumer)); 546 547 return std::make_unique<MultiplexConsumer>(std::move(Consumers)); 548 } 549 550 void WrappingExtractAPIAction::EndSourceFileAction() { 551 // Invoke wrapped action's method. 552 WrapperFrontendAction::EndSourceFileAction(); 553 554 if (CreatedASTConsumer) { 555 ImplEndSourceFileAction(getCompilerInstance()); 556 } 557 } 558