1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" 10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h" 11 #include "clang/Frontend/CompilerInstance.h" 12 #include "clang/Frontend/CompilerInvocation.h" 13 #include "clang/Frontend/FrontendActions.h" 14 #include "clang/Frontend/TextDiagnosticPrinter.h" 15 #include "clang/Frontend/Utils.h" 16 #include "clang/Lex/PreprocessorOptions.h" 17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" 18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" 19 #include "clang/Tooling/Tooling.h" 20 21 using namespace clang; 22 using namespace tooling; 23 using namespace dependencies; 24 25 namespace { 26 27 /// Forwards the gatherered dependencies to the consumer. 28 class DependencyConsumerForwarder : public DependencyFileGenerator { 29 public: 30 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts, 31 DependencyConsumer &C) 32 : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {} 33 34 void finishedMainFile(DiagnosticsEngine &Diags) override { 35 C.handleDependencyOutputOpts(*Opts); 36 llvm::SmallString<256> CanonPath; 37 for (const auto &File : getDependencies()) { 38 CanonPath = File; 39 llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true); 40 C.handleFileDependency(CanonPath); 41 } 42 } 43 44 private: 45 std::unique_ptr<DependencyOutputOptions> Opts; 46 DependencyConsumer &C; 47 }; 48 49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); 50 51 /// A listener that collects the imported modules and optionally the input 52 /// files. 53 class PrebuiltModuleListener : public ASTReaderListener { 54 public: 55 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, 56 llvm::StringSet<> &InputFiles, bool VisitInputFiles, 57 llvm::SmallVector<std::string> &NewModuleFiles) 58 : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles), 59 VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {} 60 61 bool needsImportVisitation() const override { return true; } 62 bool needsInputFileVisitation() override { return VisitInputFiles; } 63 bool needsSystemInputFileVisitation() override { return VisitInputFiles; } 64 65 void visitImport(StringRef ModuleName, StringRef Filename) override { 66 if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second) 67 NewModuleFiles.push_back(Filename.str()); 68 } 69 70 bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, 71 bool isExplicitModule) override { 72 InputFiles.insert(Filename); 73 return true; 74 } 75 76 private: 77 PrebuiltModuleFilesT &PrebuiltModuleFiles; 78 llvm::StringSet<> &InputFiles; 79 bool VisitInputFiles; 80 llvm::SmallVector<std::string> &NewModuleFiles; 81 }; 82 83 /// Visit the given prebuilt module and collect all of the modules it 84 /// transitively imports and contributing input files. 85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename, 86 CompilerInstance &CI, 87 PrebuiltModuleFilesT &ModuleFiles, 88 llvm::StringSet<> &InputFiles, 89 bool VisitInputFiles) { 90 // List of module files to be processed. 91 llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()}; 92 PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles, 93 Worklist); 94 95 while (!Worklist.empty()) 96 ASTReader::readASTFileControlBlock( 97 Worklist.pop_back_val(), CI.getFileManager(), 98 CI.getPCHContainerReader(), 99 /*FindModuleFileExtensions=*/false, Listener, 100 /*ValidateDiagnosticOptions=*/false); 101 } 102 103 /// Transform arbitrary file name into an object-like file name. 104 static std::string makeObjFileName(StringRef FileName) { 105 SmallString<128> ObjFileName(FileName); 106 llvm::sys::path::replace_extension(ObjFileName, "o"); 107 return std::string(ObjFileName.str()); 108 } 109 110 /// Deduce the dependency target based on the output file and input files. 111 static std::string 112 deduceDepTarget(const std::string &OutputFile, 113 const SmallVectorImpl<FrontendInputFile> &InputFiles) { 114 if (OutputFile != "-") 115 return OutputFile; 116 117 if (InputFiles.empty() || !InputFiles.front().isFile()) 118 return "clang-scan-deps\\ dependency"; 119 120 return makeObjFileName(InputFiles.front().getFile()); 121 } 122 123 /// Sanitize diagnostic options for dependency scan. 124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) { 125 // Don't print 'X warnings and Y errors generated'. 126 DiagOpts.ShowCarets = false; 127 // Don't write out diagnostic file. 128 DiagOpts.DiagnosticSerializationFile.clear(); 129 // Don't treat warnings as errors. 130 DiagOpts.Warnings.push_back("no-error"); 131 } 132 133 /// A clang tool that runs the preprocessor in a mode that's optimized for 134 /// dependency scanning for the given compiler invocation. 135 class DependencyScanningAction : public tooling::ToolAction { 136 public: 137 DependencyScanningAction( 138 StringRef WorkingDirectory, DependencyConsumer &Consumer, 139 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS, 140 ScanningOutputFormat Format, bool OptimizeArgs, bool DisableFree, 141 llvm::Optional<StringRef> ModuleName = None) 142 : WorkingDirectory(WorkingDirectory), Consumer(Consumer), 143 DepFS(std::move(DepFS)), Format(Format), OptimizeArgs(OptimizeArgs), 144 DisableFree(DisableFree), ModuleName(ModuleName) {} 145 146 bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, 147 FileManager *FileMgr, 148 std::shared_ptr<PCHContainerOperations> PCHContainerOps, 149 DiagnosticConsumer *DiagConsumer) override { 150 // Make a deep copy of the original Clang invocation. 151 CompilerInvocation OriginalInvocation(*Invocation); 152 // Restore the value of DisableFree, which may be modified by Tooling. 153 OriginalInvocation.getFrontendOpts().DisableFree = DisableFree; 154 155 // Create a compiler instance to handle the actual work. 156 CompilerInstance ScanInstance(std::move(PCHContainerOps)); 157 ScanInstance.setInvocation(std::move(Invocation)); 158 159 // Create the compiler's actual diagnostics engine. 160 sanitizeDiagOpts(ScanInstance.getDiagnosticOpts()); 161 ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); 162 if (!ScanInstance.hasDiagnostics()) 163 return false; 164 165 ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = 166 true; 167 168 ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false; 169 ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false; 170 171 FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory); 172 ScanInstance.setFileManager(FileMgr); 173 ScanInstance.createSourceManager(*FileMgr); 174 175 llvm::StringSet<> PrebuiltModulesInputFiles; 176 // Store the list of prebuilt module files into header search options. This 177 // will prevent the implicit build to create duplicate modules and will 178 // force reuse of the existing prebuilt module files instead. 179 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) 180 visitPrebuiltModule( 181 ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, 182 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, 183 PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr); 184 185 // Use the dependency scanning optimized file system if requested to do so. 186 if (DepFS) { 187 // Support for virtual file system overlays on top of the caching 188 // filesystem. 189 FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation( 190 ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS)); 191 192 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> LocalDepFS = 193 DepFS; 194 ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile = 195 [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File) 196 -> Optional<ArrayRef<dependency_directives_scan::Directive>> { 197 if (llvm::ErrorOr<EntryRef> Entry = 198 LocalDepFS->getOrCreateFileSystemEntry(File.getName())) 199 return Entry->getDirectiveTokens(); 200 return None; 201 }; 202 } 203 204 // Create the dependency collector that will collect the produced 205 // dependencies. 206 // 207 // This also moves the existing dependency output options from the 208 // invocation to the collector. The options in the invocation are reset, 209 // which ensures that the compiler won't create new dependency collectors, 210 // and thus won't write out the extra '.d' files to disk. 211 auto Opts = std::make_unique<DependencyOutputOptions>(); 212 std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts()); 213 // We need at least one -MT equivalent for the generator of make dependency 214 // files to work. 215 if (Opts->Targets.empty()) 216 Opts->Targets = { 217 deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile, 218 ScanInstance.getFrontendOpts().Inputs)}; 219 Opts->IncludeSystemHeaders = true; 220 221 switch (Format) { 222 case ScanningOutputFormat::Make: 223 ScanInstance.addDependencyCollector( 224 std::make_shared<DependencyConsumerForwarder>(std::move(Opts), 225 Consumer)); 226 break; 227 case ScanningOutputFormat::Full: 228 ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>( 229 std::move(Opts), ScanInstance, Consumer, 230 std::move(OriginalInvocation), OptimizeArgs)); 231 break; 232 } 233 234 // Consider different header search and diagnostic options to create 235 // different modules. This avoids the unsound aliasing of module PCMs. 236 // 237 // TODO: Implement diagnostic bucketing to reduce the impact of strict 238 // context hashing. 239 ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true; 240 241 std::unique_ptr<FrontendAction> Action; 242 243 if (ModuleName) 244 Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName); 245 else 246 Action = std::make_unique<ReadPCHAndPreprocessAction>(); 247 248 const bool Result = ScanInstance.ExecuteAction(*Action); 249 if (!DepFS) 250 FileMgr->clearStatCache(); 251 return Result; 252 } 253 254 private: 255 StringRef WorkingDirectory; 256 DependencyConsumer &Consumer; 257 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS; 258 ScanningOutputFormat Format; 259 bool OptimizeArgs; 260 bool DisableFree; 261 llvm::Optional<StringRef> ModuleName; 262 }; 263 264 } // end anonymous namespace 265 266 DependencyScanningWorker::DependencyScanningWorker( 267 DependencyScanningService &Service, 268 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) 269 : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) { 270 PCHContainerOps = std::make_shared<PCHContainerOperations>(); 271 PCHContainerOps->registerReader( 272 std::make_unique<ObjectFilePCHContainerReader>()); 273 // We don't need to write object files, but the current PCH implementation 274 // requires the writer to be registered as well. 275 PCHContainerOps->registerWriter( 276 std::make_unique<ObjectFilePCHContainerWriter>()); 277 278 auto OverlayFS = 279 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(std::move(FS)); 280 InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); 281 OverlayFS->pushOverlay(InMemoryFS); 282 RealFS = OverlayFS; 283 284 if (Service.getMode() == ScanningMode::DependencyDirectivesScan) 285 DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(), 286 RealFS); 287 if (Service.canReuseFileManager()) 288 Files = new FileManager(FileSystemOptions(), RealFS); 289 } 290 291 static llvm::Error 292 runWithDiags(DiagnosticOptions *DiagOpts, 293 llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)> 294 BodyShouldSucceed) { 295 sanitizeDiagOpts(*DiagOpts); 296 297 // Capture the emitted diagnostics and report them to the client 298 // in the case of a failure. 299 std::string DiagnosticOutput; 300 llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); 301 TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts); 302 303 if (BodyShouldSucceed(DiagPrinter, *DiagOpts)) 304 return llvm::Error::success(); 305 return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(), 306 llvm::inconvertibleErrorCode()); 307 } 308 309 llvm::Error DependencyScanningWorker::computeDependencies( 310 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine, 311 DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) { 312 // Reset what might have been modified in the previous worker invocation. 313 RealFS->setCurrentWorkingDirectory(WorkingDirectory); 314 if (Files) 315 Files->setVirtualFileSystem(RealFS); 316 317 llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles = 318 Files ? Files : new FileManager(FileSystemOptions(), RealFS); 319 320 Optional<std::vector<std::string>> ModifiedCommandLine; 321 if (ModuleName) { 322 ModifiedCommandLine = CommandLine; 323 InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer("")); 324 ModifiedCommandLine->emplace_back(*ModuleName); 325 } 326 327 const std::vector<std::string> &FinalCommandLine = 328 ModifiedCommandLine ? *ModifiedCommandLine : CommandLine; 329 330 std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr); 331 llvm::transform(CommandLine, FinalCCommandLine.begin(), 332 [](const std::string &Str) { return Str.c_str(); }); 333 334 return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(), 335 [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) { 336 // DisableFree is modified by Tooling for running 337 // in-process; preserve the original value, which is 338 // always true for a driver invocation. 339 bool DisableFree = true; 340 DependencyScanningAction Action( 341 WorkingDirectory, Consumer, DepFS, Format, 342 OptimizeArgs, DisableFree, ModuleName); 343 // Create an invocation that uses the underlying file 344 // system to ensure that any file system requests that 345 // are made by the driver do not go through the 346 // dependency scanning filesystem. 347 ToolInvocation Invocation(FinalCommandLine, &Action, 348 CurrentFiles.get(), 349 PCHContainerOps); 350 Invocation.setDiagnosticConsumer(&DC); 351 Invocation.setDiagnosticOptions(&DiagOpts); 352 return Invocation.run(); 353 }); 354 } 355