xref: /freebsd/contrib/llvm-project/clang/lib/Frontend/PrecompiledPreamble.cpp (revision 725a9f47324d42037db93c27ceb40d4956872f3e)
1 //===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Helper class to build precompiled preamble.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Frontend/PrecompiledPreamble.h"
14 #include "clang/Basic/FileManager.h"
15 #include "clang/Basic/LangStandard.h"
16 #include "clang/Frontend/CompilerInstance.h"
17 #include "clang/Frontend/CompilerInvocation.h"
18 #include "clang/Frontend/FrontendActions.h"
19 #include "clang/Frontend/FrontendOptions.h"
20 #include "clang/Lex/HeaderSearch.h"
21 #include "clang/Lex/Lexer.h"
22 #include "clang/Lex/Preprocessor.h"
23 #include "clang/Lex/PreprocessorOptions.h"
24 #include "clang/Serialization/ASTWriter.h"
25 #include "llvm/ADT/SmallString.h"
26 #include "llvm/ADT/StringSet.h"
27 #include "llvm/ADT/iterator_range.h"
28 #include "llvm/Config/llvm-config.h"
29 #include "llvm/Support/CrashRecoveryContext.h"
30 #include "llvm/Support/FileSystem.h"
31 #include "llvm/Support/Path.h"
32 #include "llvm/Support/Process.h"
33 #include "llvm/Support/VirtualFileSystem.h"
34 #include <limits>
35 #include <mutex>
36 #include <utility>
37 
38 using namespace clang;
39 
40 namespace {
41 
42 StringRef getInMemoryPreamblePath() {
43 #if defined(LLVM_ON_UNIX)
44   return "/__clang_tmp/___clang_inmemory_preamble___";
45 #elif defined(_WIN32)
46   return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
47 #else
48 #warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
49   return "/__clang_tmp/___clang_inmemory_preamble___";
50 #endif
51 }
52 
53 IntrusiveRefCntPtr<llvm::vfs::FileSystem>
54 createVFSOverlayForPreamblePCH(StringRef PCHFilename,
55                                std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
56                                IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
57   // We want only the PCH file from the real filesystem to be available,
58   // so we create an in-memory VFS with just that and overlay it on top.
59   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> PCHFS(
60       new llvm::vfs::InMemoryFileSystem());
61   PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
62   IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> Overlay(
63       new llvm::vfs::OverlayFileSystem(VFS));
64   Overlay->pushOverlay(PCHFS);
65   return Overlay;
66 }
67 
68 class PreambleDependencyCollector : public DependencyCollector {
69 public:
70   // We want to collect all dependencies for correctness. Avoiding the real
71   // system dependencies (e.g. stl from /usr/lib) would probably be a good idea,
72   // but there is no way to distinguish between those and the ones that can be
73   // spuriously added by '-isystem' (e.g. to suppress warnings from those
74   // headers).
75   bool needSystemDependencies() override { return true; }
76 };
77 
78 // Collects files whose existence would invalidate the preamble.
79 // Collecting *all* of these would make validating it too slow though, so we
80 // just find all the candidates for 'file not found' diagnostics.
81 //
82 // A caveat that may be significant for generated files: we'll omit files under
83 // search path entries whose roots don't exist when the preamble is built.
84 // These are pruned by InitHeaderSearch and so we don't see the search path.
85 // It would be nice to include them but we don't want to duplicate all the rest
86 // of the InitHeaderSearch logic to reconstruct them.
87 class MissingFileCollector : public PPCallbacks {
88   llvm::StringSet<> &Out;
89   const HeaderSearch &Search;
90   const SourceManager &SM;
91 
92 public:
93   MissingFileCollector(llvm::StringSet<> &Out, const HeaderSearch &Search,
94                        const SourceManager &SM)
95       : Out(Out), Search(Search), SM(SM) {}
96 
97   void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
98                           StringRef FileName, bool IsAngled,
99                           CharSourceRange FilenameRange,
100                           OptionalFileEntryRef File, StringRef SearchPath,
101                           StringRef RelativePath, const Module *Imported,
102                           SrcMgr::CharacteristicKind FileType) override {
103     // File is std::nullopt if it wasn't found.
104     // (We have some false negatives if PP recovered e.g. <foo> -> "foo")
105     if (File)
106       return;
107 
108     // If it's a rare absolute include, we know the full path already.
109     if (llvm::sys::path::is_absolute(FileName)) {
110       Out.insert(FileName);
111       return;
112     }
113 
114     // Reconstruct the filenames that would satisfy this directive...
115     llvm::SmallString<256> Buf;
116     auto NotFoundRelativeTo = [&](DirectoryEntryRef DE) {
117       Buf = DE.getName();
118       llvm::sys::path::append(Buf, FileName);
119       llvm::sys::path::remove_dots(Buf, /*remove_dot_dot=*/true);
120       Out.insert(Buf);
121     };
122     // ...relative to the including file.
123     if (!IsAngled) {
124       if (OptionalFileEntryRef IncludingFile =
125               SM.getFileEntryRefForID(SM.getFileID(IncludeTok.getLocation())))
126         if (IncludingFile->getDir())
127           NotFoundRelativeTo(IncludingFile->getDir());
128     }
129     // ...relative to the search paths.
130     for (const auto &Dir : llvm::make_range(
131              IsAngled ? Search.angled_dir_begin() : Search.search_dir_begin(),
132              Search.search_dir_end())) {
133       // No support for frameworks or header maps yet.
134       if (Dir.isNormalDir())
135         NotFoundRelativeTo(*Dir.getDirRef());
136     }
137   }
138 };
139 
140 /// Keeps a track of files to be deleted in destructor.
141 class TemporaryFiles {
142 public:
143   // A static instance to be used by all clients.
144   static TemporaryFiles &getInstance();
145 
146 private:
147   // Disallow constructing the class directly.
148   TemporaryFiles() = default;
149   // Disallow copy.
150   TemporaryFiles(const TemporaryFiles &) = delete;
151 
152 public:
153   ~TemporaryFiles();
154 
155   /// Adds \p File to a set of tracked files.
156   void addFile(StringRef File);
157 
158   /// Remove \p File from disk and from the set of tracked files.
159   void removeFile(StringRef File);
160 
161 private:
162   std::mutex Mutex;
163   llvm::StringSet<> Files;
164 };
165 
166 TemporaryFiles &TemporaryFiles::getInstance() {
167   static TemporaryFiles Instance;
168   return Instance;
169 }
170 
171 TemporaryFiles::~TemporaryFiles() {
172   std::lock_guard<std::mutex> Guard(Mutex);
173   for (const auto &File : Files)
174     llvm::sys::fs::remove(File.getKey());
175 }
176 
177 void TemporaryFiles::addFile(StringRef File) {
178   std::lock_guard<std::mutex> Guard(Mutex);
179   auto IsInserted = Files.insert(File).second;
180   (void)IsInserted;
181   assert(IsInserted && "File has already been added");
182 }
183 
184 void TemporaryFiles::removeFile(StringRef File) {
185   std::lock_guard<std::mutex> Guard(Mutex);
186   auto WasPresent = Files.erase(File);
187   (void)WasPresent;
188   assert(WasPresent && "File was not tracked");
189   llvm::sys::fs::remove(File);
190 }
191 
192 // A temp file that would be deleted on destructor call. If destructor is not
193 // called for any reason, the file will be deleted at static objects'
194 // destruction.
195 // An assertion will fire if two TempPCHFiles are created with the same name,
196 // so it's not intended to be used outside preamble-handling.
197 class TempPCHFile {
198 public:
199   // A main method used to construct TempPCHFile.
200   static std::unique_ptr<TempPCHFile> create(StringRef StoragePath) {
201     // FIXME: This is a hack so that we can override the preamble file during
202     // crash-recovery testing, which is the only case where the preamble files
203     // are not necessarily cleaned up.
204     if (const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE"))
205       return std::unique_ptr<TempPCHFile>(new TempPCHFile(TmpFile));
206 
207     llvm::SmallString<128> File;
208     // Using the versions of createTemporaryFile() and
209     // createUniqueFile() with a file descriptor guarantees
210     // that we would never get a race condition in a multi-threaded setting
211     // (i.e., multiple threads getting the same temporary path).
212     int FD;
213     std::error_code EC;
214     if (StoragePath.empty())
215       EC = llvm::sys::fs::createTemporaryFile("preamble", "pch", FD, File);
216     else {
217       llvm::SmallString<128> TempPath = StoragePath;
218       // Use the same filename model as fs::createTemporaryFile().
219       llvm::sys::path::append(TempPath, "preamble-%%%%%%.pch");
220       namespace fs = llvm::sys::fs;
221       // Use the same owner-only file permissions as fs::createTemporaryFile().
222       EC = fs::createUniqueFile(TempPath, FD, File, fs::OF_None,
223                                 fs::owner_read | fs::owner_write);
224     }
225     if (EC)
226       return nullptr;
227     // We only needed to make sure the file exists, close the file right away.
228     llvm::sys::Process::SafelyCloseFileDescriptor(FD);
229     return std::unique_ptr<TempPCHFile>(new TempPCHFile(File.str().str()));
230   }
231 
232   TempPCHFile &operator=(const TempPCHFile &) = delete;
233   TempPCHFile(const TempPCHFile &) = delete;
234   ~TempPCHFile() { TemporaryFiles::getInstance().removeFile(FilePath); };
235 
236   /// A path where temporary file is stored.
237   llvm::StringRef getFilePath() const { return FilePath; };
238 
239 private:
240   TempPCHFile(std::string FilePath) : FilePath(std::move(FilePath)) {
241     TemporaryFiles::getInstance().addFile(this->FilePath);
242   }
243 
244   std::string FilePath;
245 };
246 
247 class PrecompilePreambleAction : public ASTFrontendAction {
248 public:
249   PrecompilePreambleAction(std::shared_ptr<PCHBuffer> Buffer, bool WritePCHFile,
250                            PreambleCallbacks &Callbacks)
251       : Buffer(std::move(Buffer)), WritePCHFile(WritePCHFile),
252         Callbacks(Callbacks) {}
253 
254   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
255                                                  StringRef InFile) override;
256 
257   bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
258 
259   void setEmittedPreamblePCH(ASTWriter &Writer) {
260     if (FileOS) {
261       *FileOS << Buffer->Data;
262       // Make sure it hits disk now.
263       FileOS.reset();
264     }
265 
266     this->HasEmittedPreamblePCH = true;
267     Callbacks.AfterPCHEmitted(Writer);
268   }
269 
270   bool BeginSourceFileAction(CompilerInstance &CI) override {
271     assert(CI.getLangOpts().CompilingPCH);
272     return ASTFrontendAction::BeginSourceFileAction(CI);
273   }
274 
275   bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
276   bool hasCodeCompletionSupport() const override { return false; }
277   bool hasASTFileSupport() const override { return false; }
278   TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
279 
280 private:
281   friend class PrecompilePreambleConsumer;
282 
283   bool HasEmittedPreamblePCH = false;
284   std::shared_ptr<PCHBuffer> Buffer;
285   bool WritePCHFile; // otherwise the PCH is written into the PCHBuffer only.
286   std::unique_ptr<llvm::raw_pwrite_stream> FileOS; // null if in-memory
287   PreambleCallbacks &Callbacks;
288 };
289 
290 class PrecompilePreambleConsumer : public PCHGenerator {
291 public:
292   PrecompilePreambleConsumer(PrecompilePreambleAction &Action,
293                              const Preprocessor &PP,
294                              InMemoryModuleCache &ModuleCache,
295                              StringRef isysroot,
296                              std::shared_ptr<PCHBuffer> Buffer)
297       : PCHGenerator(PP, ModuleCache, "", isysroot, std::move(Buffer),
298                      ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
299                      /*AllowASTWithErrors=*/true),
300         Action(Action) {}
301 
302   bool HandleTopLevelDecl(DeclGroupRef DG) override {
303     Action.Callbacks.HandleTopLevelDecl(DG);
304     return true;
305   }
306 
307   void HandleTranslationUnit(ASTContext &Ctx) override {
308     PCHGenerator::HandleTranslationUnit(Ctx);
309     if (!hasEmittedPCH())
310       return;
311     Action.setEmittedPreamblePCH(getWriter());
312   }
313 
314   bool shouldSkipFunctionBody(Decl *D) override {
315     return Action.Callbacks.shouldSkipFunctionBody(D);
316   }
317 
318 private:
319   PrecompilePreambleAction &Action;
320 };
321 
322 std::unique_ptr<ASTConsumer>
323 PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
324                                             StringRef InFile) {
325   std::string Sysroot;
326   if (!GeneratePCHAction::ComputeASTConsumerArguments(CI, Sysroot))
327     return nullptr;
328 
329   if (WritePCHFile) {
330     std::string OutputFile; // unused
331     FileOS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
332     if (!FileOS)
333       return nullptr;
334   }
335 
336   if (!CI.getFrontendOpts().RelocatablePCH)
337     Sysroot.clear();
338 
339   return std::make_unique<PrecompilePreambleConsumer>(
340       *this, CI.getPreprocessor(), CI.getModuleCache(), Sysroot, Buffer);
341 }
342 
343 template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
344   if (!Val)
345     return false;
346   Output = std::move(*Val);
347   return true;
348 }
349 
350 } // namespace
351 
352 PreambleBounds clang::ComputePreambleBounds(const LangOptions &LangOpts,
353                                             const llvm::MemoryBufferRef &Buffer,
354                                             unsigned MaxLines) {
355   return Lexer::ComputePreamble(Buffer.getBuffer(), LangOpts, MaxLines);
356 }
357 
358 class PrecompiledPreamble::PCHStorage {
359 public:
360   static std::unique_ptr<PCHStorage> file(std::unique_ptr<TempPCHFile> File) {
361     assert(File);
362     std::unique_ptr<PCHStorage> S(new PCHStorage());
363     S->File = std::move(File);
364     return S;
365   }
366   static std::unique_ptr<PCHStorage> inMemory(std::shared_ptr<PCHBuffer> Buf) {
367     std::unique_ptr<PCHStorage> S(new PCHStorage());
368     S->Memory = std::move(Buf);
369     return S;
370   }
371 
372   enum class Kind { InMemory, TempFile };
373   Kind getKind() const {
374     if (Memory)
375       return Kind::InMemory;
376     if (File)
377       return Kind::TempFile;
378     llvm_unreachable("Neither Memory nor File?");
379   }
380   llvm::StringRef filePath() const {
381     assert(getKind() == Kind::TempFile);
382     return File->getFilePath();
383   }
384   llvm::StringRef memoryContents() const {
385     assert(getKind() == Kind::InMemory);
386     return StringRef(Memory->Data.data(), Memory->Data.size());
387   }
388 
389   // Shrink in-memory buffers to fit.
390   // This incurs a copy, but preambles tend to be long-lived.
391   // Only safe to call once nothing can alias the buffer.
392   void shrink() {
393     if (!Memory)
394       return;
395     Memory->Data = decltype(Memory->Data)(Memory->Data);
396   }
397 
398 private:
399   PCHStorage() = default;
400   PCHStorage(const PCHStorage &) = delete;
401   PCHStorage &operator=(const PCHStorage &) = delete;
402 
403   std::shared_ptr<PCHBuffer> Memory;
404   std::unique_ptr<TempPCHFile> File;
405 };
406 
407 PrecompiledPreamble::~PrecompiledPreamble() = default;
408 PrecompiledPreamble::PrecompiledPreamble(PrecompiledPreamble &&) = default;
409 PrecompiledPreamble &
410 PrecompiledPreamble::operator=(PrecompiledPreamble &&) = default;
411 
412 llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
413     const CompilerInvocation &Invocation,
414     const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
415     DiagnosticsEngine &Diagnostics,
416     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
417     std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
418     StringRef StoragePath, PreambleCallbacks &Callbacks) {
419   assert(VFS && "VFS is null");
420 
421   auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
422   FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
423   PreprocessorOptions &PreprocessorOpts =
424       PreambleInvocation->getPreprocessorOpts();
425 
426   std::shared_ptr<PCHBuffer> Buffer = std::make_shared<PCHBuffer>();
427   std::unique_ptr<PCHStorage> Storage;
428   if (StoreInMemory) {
429     Storage = PCHStorage::inMemory(Buffer);
430   } else {
431     // Create a temporary file for the precompiled preamble. In rare
432     // circumstances, this can fail.
433     std::unique_ptr<TempPCHFile> PreamblePCHFile =
434         TempPCHFile::create(StoragePath);
435     if (!PreamblePCHFile)
436       return BuildPreambleError::CouldntCreateTempFile;
437     Storage = PCHStorage::file(std::move(PreamblePCHFile));
438   }
439 
440   // Save the preamble text for later; we'll need to compare against it for
441   // subsequent reparses.
442   std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
443                                   MainFileBuffer->getBufferStart() +
444                                       Bounds.Size);
445   bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
446 
447   // Tell the compiler invocation to generate a temporary precompiled header.
448   FrontendOpts.ProgramAction = frontend::GeneratePCH;
449   FrontendOpts.OutputFile = std::string(
450       StoreInMemory ? getInMemoryPreamblePath() : Storage->filePath());
451   PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
452   PreprocessorOpts.PrecompiledPreambleBytes.second = false;
453   // Inform preprocessor to record conditional stack when building the preamble.
454   PreprocessorOpts.GeneratePreamble = true;
455 
456   // Create the compiler instance to use for building the precompiled preamble.
457   std::unique_ptr<CompilerInstance> Clang(
458       new CompilerInstance(std::move(PCHContainerOps)));
459 
460   // Recover resources if we crash before exiting this method.
461   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
462       Clang.get());
463 
464   Clang->setInvocation(std::move(PreambleInvocation));
465   Clang->setDiagnostics(&Diagnostics);
466 
467   // Create the target instance.
468   if (!Clang->createTarget())
469     return BuildPreambleError::CouldntCreateTargetInfo;
470 
471   if (Clang->getFrontendOpts().Inputs.size() != 1 ||
472       Clang->getFrontendOpts().Inputs[0].getKind().getFormat() !=
473           InputKind::Source ||
474       Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() ==
475           Language::LLVM_IR) {
476     return BuildPreambleError::BadInputs;
477   }
478 
479   // Clear out old caches and data.
480   Diagnostics.Reset();
481   ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
482 
483   VFS =
484       createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
485 
486   // Create a file manager object to provide access to and cache the filesystem.
487   Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
488 
489   // Create the source manager.
490   Clang->setSourceManager(
491       new SourceManager(Diagnostics, Clang->getFileManager()));
492 
493   auto PreambleDepCollector = std::make_shared<PreambleDependencyCollector>();
494   Clang->addDependencyCollector(PreambleDepCollector);
495 
496   Clang->getLangOpts().CompilingPCH = true;
497 
498   // Remap the main source file to the preamble buffer.
499   StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
500   auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
501       MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
502   if (PreprocessorOpts.RetainRemappedFileBuffers) {
503     // MainFileBuffer will be deleted by unique_ptr after leaving the method.
504     PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
505   } else {
506     // In that case, remapped buffer will be deleted by CompilerInstance on
507     // BeginSourceFile, so we call release() to avoid double deletion.
508     PreprocessorOpts.addRemappedFile(MainFilePath,
509                                      PreambleInputBuffer.release());
510   }
511 
512   auto Act = std::make_unique<PrecompilePreambleAction>(
513       std::move(Buffer),
514       /*WritePCHFile=*/Storage->getKind() == PCHStorage::Kind::TempFile,
515       Callbacks);
516   if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
517     return BuildPreambleError::BeginSourceFileFailed;
518 
519   // Performed after BeginSourceFile to ensure Clang->Preprocessor can be
520   // referenced in the callback.
521   Callbacks.BeforeExecute(*Clang);
522 
523   std::unique_ptr<PPCallbacks> DelegatedPPCallbacks =
524       Callbacks.createPPCallbacks();
525   if (DelegatedPPCallbacks)
526     Clang->getPreprocessor().addPPCallbacks(std::move(DelegatedPPCallbacks));
527   if (auto CommentHandler = Callbacks.getCommentHandler())
528     Clang->getPreprocessor().addCommentHandler(CommentHandler);
529   llvm::StringSet<> MissingFiles;
530   Clang->getPreprocessor().addPPCallbacks(
531       std::make_unique<MissingFileCollector>(
532           MissingFiles, Clang->getPreprocessor().getHeaderSearchInfo(),
533           Clang->getSourceManager()));
534 
535   if (llvm::Error Err = Act->Execute())
536     return errorToErrorCode(std::move(Err));
537 
538   // Run the callbacks.
539   Callbacks.AfterExecute(*Clang);
540 
541   Act->EndSourceFile();
542 
543   if (!Act->hasEmittedPreamblePCH())
544     return BuildPreambleError::CouldntEmitPCH;
545   Act.reset(); // Frees the PCH buffer, unless Storage keeps it in memory.
546 
547   // Keep track of all of the files that the source manager knows about,
548   // so we can verify whether they have changed or not.
549   llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
550 
551   SourceManager &SourceMgr = Clang->getSourceManager();
552   for (auto &Filename : PreambleDepCollector->getDependencies()) {
553     auto MaybeFile = Clang->getFileManager().getOptionalFileRef(Filename);
554     if (!MaybeFile ||
555         MaybeFile == SourceMgr.getFileEntryRefForID(SourceMgr.getMainFileID()))
556       continue;
557     auto File = *MaybeFile;
558     if (time_t ModTime = File.getModificationTime()) {
559       FilesInPreamble[File.getName()] =
560           PrecompiledPreamble::PreambleFileHash::createForFile(File.getSize(),
561                                                                ModTime);
562     } else {
563       llvm::MemoryBufferRef Buffer =
564           SourceMgr.getMemoryBufferForFileOrFake(File);
565       FilesInPreamble[File.getName()] =
566           PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
567     }
568   }
569 
570   // Shrinking the storage requires extra temporary memory.
571   // Destroying clang first reduces peak memory usage.
572   CICleanup.unregister();
573   Clang.reset();
574   Storage->shrink();
575   return PrecompiledPreamble(
576       std::move(Storage), std::move(PreambleBytes), PreambleEndsAtStartOfLine,
577       std::move(FilesInPreamble), std::move(MissingFiles));
578 }
579 
580 PreambleBounds PrecompiledPreamble::getBounds() const {
581   return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
582 }
583 
584 std::size_t PrecompiledPreamble::getSize() const {
585   switch (Storage->getKind()) {
586   case PCHStorage::Kind::InMemory:
587     return Storage->memoryContents().size();
588   case PCHStorage::Kind::TempFile: {
589     uint64_t Result;
590     if (llvm::sys::fs::file_size(Storage->filePath(), Result))
591       return 0;
592 
593     assert(Result <= std::numeric_limits<std::size_t>::max() &&
594            "file size did not fit into size_t");
595     return Result;
596   }
597   }
598   llvm_unreachable("Unhandled storage kind");
599 }
600 
601 bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
602                                    const llvm::MemoryBufferRef &MainFileBuffer,
603                                    PreambleBounds Bounds,
604                                    llvm::vfs::FileSystem &VFS) const {
605 
606   assert(
607       Bounds.Size <= MainFileBuffer.getBufferSize() &&
608       "Buffer is too large. Bounds were calculated from a different buffer?");
609 
610   auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
611   PreprocessorOptions &PreprocessorOpts =
612       PreambleInvocation->getPreprocessorOpts();
613 
614   // We've previously computed a preamble. Check whether we have the same
615   // preamble now that we did before, and that there's enough space in
616   // the main-file buffer within the precompiled preamble to fit the
617   // new main file.
618   if (PreambleBytes.size() != Bounds.Size ||
619       PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
620       !std::equal(PreambleBytes.begin(), PreambleBytes.end(),
621                   MainFileBuffer.getBuffer().begin()))
622     return false;
623   // The preamble has not changed. We may be able to re-use the precompiled
624   // preamble.
625 
626   // Check that none of the files used by the preamble have changed.
627   // First, make a record of those files that have been overridden via
628   // remapping or unsaved_files.
629   std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
630   llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
631   for (const auto &R : PreprocessorOpts.RemappedFiles) {
632     llvm::vfs::Status Status;
633     if (!moveOnNoError(VFS.status(R.second), Status)) {
634       // If we can't stat the file we're remapping to, assume that something
635       // horrible happened.
636       return false;
637     }
638     // If a mapped file was previously missing, then it has changed.
639     llvm::SmallString<128> MappedPath(R.first);
640     if (!VFS.makeAbsolute(MappedPath))
641       OverriddenAbsPaths.insert(MappedPath);
642 
643     OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
644         Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
645   }
646 
647   // OverridenFileBuffers tracks only the files not found in VFS.
648   llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
649   for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
650     const PrecompiledPreamble::PreambleFileHash PreambleHash =
651         PreambleFileHash::createForMemoryBuffer(RB.second->getMemBufferRef());
652     llvm::vfs::Status Status;
653     if (moveOnNoError(VFS.status(RB.first), Status))
654       OverriddenFiles[Status.getUniqueID()] = PreambleHash;
655     else
656       OverridenFileBuffers[RB.first] = PreambleHash;
657 
658     llvm::SmallString<128> MappedPath(RB.first);
659     if (!VFS.makeAbsolute(MappedPath))
660       OverriddenAbsPaths.insert(MappedPath);
661   }
662 
663   // Check whether anything has changed.
664   for (const auto &F : FilesInPreamble) {
665     auto OverridenFileBuffer = OverridenFileBuffers.find(F.first());
666     if (OverridenFileBuffer != OverridenFileBuffers.end()) {
667       // The file's buffer was remapped and the file was not found in VFS.
668       // Check whether it matches up with the previous mapping.
669       if (OverridenFileBuffer->second != F.second)
670         return false;
671       continue;
672     }
673 
674     llvm::vfs::Status Status;
675     if (!moveOnNoError(VFS.status(F.first()), Status)) {
676       // If the file's buffer is not remapped and we can't stat it,
677       // assume that something horrible happened.
678       return false;
679     }
680 
681     std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
682         OverriddenFiles.find(Status.getUniqueID());
683     if (Overridden != OverriddenFiles.end()) {
684       // This file was remapped; check whether the newly-mapped file
685       // matches up with the previous mapping.
686       if (Overridden->second != F.second)
687         return false;
688       continue;
689     }
690 
691     // Neither the file's buffer nor the file itself was remapped;
692     // check whether it has changed on disk.
693     if (Status.getSize() != uint64_t(F.second.Size) ||
694         llvm::sys::toTimeT(Status.getLastModificationTime()) !=
695             F.second.ModTime)
696       return false;
697   }
698   for (const auto &F : MissingFiles) {
699     // A missing file may be "provided" by an override buffer or file.
700     if (OverriddenAbsPaths.count(F.getKey()))
701       return false;
702     // If a file previously recorded as missing exists as a regular file, then
703     // consider the preamble out-of-date.
704     if (auto Status = VFS.status(F.getKey())) {
705       if (Status->isRegularFile())
706         return false;
707     }
708   }
709   return true;
710 }
711 
712 void PrecompiledPreamble::AddImplicitPreamble(
713     CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
714     llvm::MemoryBuffer *MainFileBuffer) const {
715   PreambleBounds Bounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
716   configurePreamble(Bounds, CI, VFS, MainFileBuffer);
717 }
718 
719 void PrecompiledPreamble::OverridePreamble(
720     CompilerInvocation &CI, IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
721     llvm::MemoryBuffer *MainFileBuffer) const {
722   auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *MainFileBuffer, 0);
723   configurePreamble(Bounds, CI, VFS, MainFileBuffer);
724 }
725 
726 PrecompiledPreamble::PrecompiledPreamble(
727     std::unique_ptr<PCHStorage> Storage, std::vector<char> PreambleBytes,
728     bool PreambleEndsAtStartOfLine,
729     llvm::StringMap<PreambleFileHash> FilesInPreamble,
730     llvm::StringSet<> MissingFiles)
731     : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
732       MissingFiles(std::move(MissingFiles)),
733       PreambleBytes(std::move(PreambleBytes)),
734       PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
735   assert(this->Storage != nullptr);
736 }
737 
738 PrecompiledPreamble::PreambleFileHash
739 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
740                                                      time_t ModTime) {
741   PreambleFileHash Result;
742   Result.Size = Size;
743   Result.ModTime = ModTime;
744   Result.MD5 = {};
745   return Result;
746 }
747 
748 PrecompiledPreamble::PreambleFileHash
749 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
750     const llvm::MemoryBufferRef &Buffer) {
751   PreambleFileHash Result;
752   Result.Size = Buffer.getBufferSize();
753   Result.ModTime = 0;
754 
755   llvm::MD5 MD5Ctx;
756   MD5Ctx.update(Buffer.getBuffer().data());
757   MD5Ctx.final(Result.MD5);
758 
759   return Result;
760 }
761 
762 void PrecompiledPreamble::configurePreamble(
763     PreambleBounds Bounds, CompilerInvocation &CI,
764     IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS,
765     llvm::MemoryBuffer *MainFileBuffer) const {
766   assert(VFS);
767 
768   auto &PreprocessorOpts = CI.getPreprocessorOpts();
769 
770   // Remap main file to point to MainFileBuffer.
771   auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
772   PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
773 
774   // Configure ImpicitPCHInclude.
775   PreprocessorOpts.PrecompiledPreambleBytes.first = Bounds.Size;
776   PreprocessorOpts.PrecompiledPreambleBytes.second =
777       Bounds.PreambleEndsAtStartOfLine;
778   PreprocessorOpts.DisablePCHOrModuleValidation =
779       DisableValidationForModuleKind::PCH;
780 
781   // Don't bother generating the long version of the predefines buffer.
782   // The preamble is going to overwrite it anyway.
783   PreprocessorOpts.UsePredefines = false;
784 
785   setupPreambleStorage(*Storage, PreprocessorOpts, VFS);
786 }
787 
788 void PrecompiledPreamble::setupPreambleStorage(
789     const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
790     IntrusiveRefCntPtr<llvm::vfs::FileSystem> &VFS) {
791   if (Storage.getKind() == PCHStorage::Kind::TempFile) {
792     llvm::StringRef PCHPath = Storage.filePath();
793     PreprocessorOpts.ImplicitPCHInclude = PCHPath.str();
794 
795     // Make sure we can access the PCH file even if we're using a VFS
796     IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS =
797         llvm::vfs::getRealFileSystem();
798     if (VFS == RealFS || VFS->exists(PCHPath))
799       return;
800     auto Buf = RealFS->getBufferForFile(PCHPath);
801     if (!Buf) {
802       // We can't read the file even from RealFS, this is clearly an error,
803       // but we'll just leave the current VFS as is and let clang's code
804       // figure out what to do with missing PCH.
805       return;
806     }
807 
808     // We have a slight inconsistency here -- we're using the VFS to
809     // read files, but the PCH was generated in the real file system.
810     VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
811   } else {
812     assert(Storage.getKind() == PCHStorage::Kind::InMemory);
813     // For in-memory preamble, we have to provide a VFS overlay that makes it
814     // accessible.
815     StringRef PCHPath = getInMemoryPreamblePath();
816     PreprocessorOpts.ImplicitPCHInclude = std::string(PCHPath);
817 
818     auto Buf = llvm::MemoryBuffer::getMemBuffer(
819         Storage.memoryContents(), PCHPath, /*RequiresNullTerminator=*/false);
820     VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
821   }
822 }
823 
824 void PreambleCallbacks::BeforeExecute(CompilerInstance &CI) {}
825 void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
826 void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
827 void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}
828 std::unique_ptr<PPCallbacks> PreambleCallbacks::createPPCallbacks() {
829   return nullptr;
830 }
831 CommentHandler *PreambleCallbacks::getCommentHandler() { return nullptr; }
832 
833 static llvm::ManagedStatic<BuildPreambleErrorCategory> BuildPreambleErrCategory;
834 
835 std::error_code clang::make_error_code(BuildPreambleError Error) {
836   return std::error_code(static_cast<int>(Error), *BuildPreambleErrCategory);
837 }
838 
839 const char *BuildPreambleErrorCategory::name() const noexcept {
840   return "build-preamble.error";
841 }
842 
843 std::string BuildPreambleErrorCategory::message(int condition) const {
844   switch (static_cast<BuildPreambleError>(condition)) {
845   case BuildPreambleError::CouldntCreateTempFile:
846     return "Could not create temporary file for PCH";
847   case BuildPreambleError::CouldntCreateTargetInfo:
848     return "CreateTargetInfo() return null";
849   case BuildPreambleError::BeginSourceFileFailed:
850     return "BeginSourceFile() return an error";
851   case BuildPreambleError::CouldntEmitPCH:
852     return "Could not emit PCH";
853   case BuildPreambleError::BadInputs:
854     return "Command line arguments must contain exactly one source file";
855   }
856   llvm_unreachable("unexpected BuildPreambleError");
857 }
858