xref: /freebsd/contrib/llvm-project/clang/lib/Frontend/ASTUnit.cpp (revision 19261079b74319502c6ffa1249920079f0f69a72)
1 //===- ASTUnit.cpp - ASTUnit utility --------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ASTUnit Implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "clang/Frontend/ASTUnit.h"
14 #include "clang/AST/ASTConsumer.h"
15 #include "clang/AST/ASTContext.h"
16 #include "clang/AST/CommentCommandTraits.h"
17 #include "clang/AST/Decl.h"
18 #include "clang/AST/DeclBase.h"
19 #include "clang/AST/DeclCXX.h"
20 #include "clang/AST/DeclGroup.h"
21 #include "clang/AST/DeclObjC.h"
22 #include "clang/AST/DeclTemplate.h"
23 #include "clang/AST/DeclarationName.h"
24 #include "clang/AST/ExternalASTSource.h"
25 #include "clang/AST/PrettyPrinter.h"
26 #include "clang/AST/Type.h"
27 #include "clang/AST/TypeOrdering.h"
28 #include "clang/Basic/Diagnostic.h"
29 #include "clang/Basic/FileManager.h"
30 #include "clang/Basic/IdentifierTable.h"
31 #include "clang/Basic/LLVM.h"
32 #include "clang/Basic/LangOptions.h"
33 #include "clang/Basic/LangStandard.h"
34 #include "clang/Basic/Module.h"
35 #include "clang/Basic/SourceLocation.h"
36 #include "clang/Basic/SourceManager.h"
37 #include "clang/Basic/TargetInfo.h"
38 #include "clang/Basic/TargetOptions.h"
39 #include "clang/Frontend/CompilerInstance.h"
40 #include "clang/Frontend/CompilerInvocation.h"
41 #include "clang/Frontend/FrontendAction.h"
42 #include "clang/Frontend/FrontendActions.h"
43 #include "clang/Frontend/FrontendDiagnostic.h"
44 #include "clang/Frontend/FrontendOptions.h"
45 #include "clang/Frontend/MultiplexConsumer.h"
46 #include "clang/Frontend/PrecompiledPreamble.h"
47 #include "clang/Frontend/Utils.h"
48 #include "clang/Lex/HeaderSearch.h"
49 #include "clang/Lex/HeaderSearchOptions.h"
50 #include "clang/Lex/Lexer.h"
51 #include "clang/Lex/PPCallbacks.h"
52 #include "clang/Lex/PreprocessingRecord.h"
53 #include "clang/Lex/Preprocessor.h"
54 #include "clang/Lex/PreprocessorOptions.h"
55 #include "clang/Lex/Token.h"
56 #include "clang/Sema/CodeCompleteConsumer.h"
57 #include "clang/Sema/CodeCompleteOptions.h"
58 #include "clang/Sema/Sema.h"
59 #include "clang/Serialization/ASTBitCodes.h"
60 #include "clang/Serialization/ASTReader.h"
61 #include "clang/Serialization/ASTWriter.h"
62 #include "clang/Serialization/ContinuousRangeMap.h"
63 #include "clang/Serialization/InMemoryModuleCache.h"
64 #include "clang/Serialization/ModuleFile.h"
65 #include "clang/Serialization/PCHContainerOperations.h"
66 #include "llvm/ADT/ArrayRef.h"
67 #include "llvm/ADT/DenseMap.h"
68 #include "llvm/ADT/IntrusiveRefCntPtr.h"
69 #include "llvm/ADT/None.h"
70 #include "llvm/ADT/Optional.h"
71 #include "llvm/ADT/STLExtras.h"
72 #include "llvm/ADT/ScopeExit.h"
73 #include "llvm/ADT/SmallString.h"
74 #include "llvm/ADT/SmallVector.h"
75 #include "llvm/ADT/StringMap.h"
76 #include "llvm/ADT/StringRef.h"
77 #include "llvm/ADT/StringSet.h"
78 #include "llvm/ADT/Twine.h"
79 #include "llvm/ADT/iterator_range.h"
80 #include "llvm/Bitstream/BitstreamWriter.h"
81 #include "llvm/Support/Allocator.h"
82 #include "llvm/Support/Casting.h"
83 #include "llvm/Support/CrashRecoveryContext.h"
84 #include "llvm/Support/DJB.h"
85 #include "llvm/Support/ErrorHandling.h"
86 #include "llvm/Support/ErrorOr.h"
87 #include "llvm/Support/FileSystem.h"
88 #include "llvm/Support/FileUtilities.h"
89 #include "llvm/Support/MemoryBuffer.h"
90 #include "llvm/Support/Timer.h"
91 #include "llvm/Support/VirtualFileSystem.h"
92 #include "llvm/Support/raw_ostream.h"
93 #include <algorithm>
94 #include <atomic>
95 #include <cassert>
96 #include <cstdint>
97 #include <cstdio>
98 #include <cstdlib>
99 #include <memory>
100 #include <mutex>
101 #include <string>
102 #include <tuple>
103 #include <utility>
104 #include <vector>
105 
106 using namespace clang;
107 
108 using llvm::TimeRecord;
109 
110 namespace {
111 
112   class SimpleTimer {
113     bool WantTiming;
114     TimeRecord Start;
115     std::string Output;
116 
117   public:
118     explicit SimpleTimer(bool WantTiming) : WantTiming(WantTiming) {
119       if (WantTiming)
120         Start = TimeRecord::getCurrentTime();
121     }
122 
123     ~SimpleTimer() {
124       if (WantTiming) {
125         TimeRecord Elapsed = TimeRecord::getCurrentTime();
126         Elapsed -= Start;
127         llvm::errs() << Output << ':';
128         Elapsed.print(Elapsed, llvm::errs());
129         llvm::errs() << '\n';
130       }
131     }
132 
133     void setOutput(const Twine &Output) {
134       if (WantTiming)
135         this->Output = Output.str();
136     }
137   };
138 
139 } // namespace
140 
141 template <class T>
142 static std::unique_ptr<T> valueOrNull(llvm::ErrorOr<std::unique_ptr<T>> Val) {
143   if (!Val)
144     return nullptr;
145   return std::move(*Val);
146 }
147 
148 template <class T>
149 static bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
150   if (!Val)
151     return false;
152   Output = std::move(*Val);
153   return true;
154 }
155 
156 /// Get a source buffer for \p MainFilePath, handling all file-to-file
157 /// and file-to-buffer remappings inside \p Invocation.
158 static std::unique_ptr<llvm::MemoryBuffer>
159 getBufferForFileHandlingRemapping(const CompilerInvocation &Invocation,
160                                   llvm::vfs::FileSystem *VFS,
161                                   StringRef FilePath, bool isVolatile) {
162   const auto &PreprocessorOpts = Invocation.getPreprocessorOpts();
163 
164   // Try to determine if the main file has been remapped, either from the
165   // command line (to another file) or directly through the compiler
166   // invocation (to a memory buffer).
167   llvm::MemoryBuffer *Buffer = nullptr;
168   std::unique_ptr<llvm::MemoryBuffer> BufferOwner;
169   auto FileStatus = VFS->status(FilePath);
170   if (FileStatus) {
171     llvm::sys::fs::UniqueID MainFileID = FileStatus->getUniqueID();
172 
173     // Check whether there is a file-file remapping of the main file
174     for (const auto &RF : PreprocessorOpts.RemappedFiles) {
175       std::string MPath(RF.first);
176       auto MPathStatus = VFS->status(MPath);
177       if (MPathStatus) {
178         llvm::sys::fs::UniqueID MID = MPathStatus->getUniqueID();
179         if (MainFileID == MID) {
180           // We found a remapping. Try to load the resulting, remapped source.
181           BufferOwner = valueOrNull(VFS->getBufferForFile(RF.second, -1, true, isVolatile));
182           if (!BufferOwner)
183             return nullptr;
184         }
185       }
186     }
187 
188     // Check whether there is a file-buffer remapping. It supercedes the
189     // file-file remapping.
190     for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
191       std::string MPath(RB.first);
192       auto MPathStatus = VFS->status(MPath);
193       if (MPathStatus) {
194         llvm::sys::fs::UniqueID MID = MPathStatus->getUniqueID();
195         if (MainFileID == MID) {
196           // We found a remapping.
197           BufferOwner.reset();
198           Buffer = const_cast<llvm::MemoryBuffer *>(RB.second);
199         }
200       }
201     }
202   }
203 
204   // If the main source file was not remapped, load it now.
205   if (!Buffer && !BufferOwner) {
206     BufferOwner = valueOrNull(VFS->getBufferForFile(FilePath, -1, true, isVolatile));
207     if (!BufferOwner)
208       return nullptr;
209   }
210 
211   if (BufferOwner)
212     return BufferOwner;
213   if (!Buffer)
214     return nullptr;
215   return llvm::MemoryBuffer::getMemBufferCopy(Buffer->getBuffer(), FilePath);
216 }
217 
218 struct ASTUnit::ASTWriterData {
219   SmallString<128> Buffer;
220   llvm::BitstreamWriter Stream;
221   ASTWriter Writer;
222 
223   ASTWriterData(InMemoryModuleCache &ModuleCache)
224       : Stream(Buffer), Writer(Stream, Buffer, ModuleCache, {}) {}
225 };
226 
227 void ASTUnit::clearFileLevelDecls() {
228   FileDecls.clear();
229 }
230 
231 /// After failing to build a precompiled preamble (due to
232 /// errors in the source that occurs in the preamble), the number of
233 /// reparses during which we'll skip even trying to precompile the
234 /// preamble.
235 const unsigned DefaultPreambleRebuildInterval = 5;
236 
237 /// Tracks the number of ASTUnit objects that are currently active.
238 ///
239 /// Used for debugging purposes only.
240 static std::atomic<unsigned> ActiveASTUnitObjects;
241 
242 ASTUnit::ASTUnit(bool _MainFileIsAST)
243     : MainFileIsAST(_MainFileIsAST), WantTiming(getenv("LIBCLANG_TIMING")),
244       ShouldCacheCodeCompletionResults(false),
245       IncludeBriefCommentsInCodeCompletion(false), UserFilesAreVolatile(false),
246       UnsafeToFree(false) {
247   if (getenv("LIBCLANG_OBJTRACKING"))
248     fprintf(stderr, "+++ %u translation units\n", ++ActiveASTUnitObjects);
249 }
250 
251 ASTUnit::~ASTUnit() {
252   // If we loaded from an AST file, balance out the BeginSourceFile call.
253   if (MainFileIsAST && getDiagnostics().getClient()) {
254     getDiagnostics().getClient()->EndSourceFile();
255   }
256 
257   clearFileLevelDecls();
258 
259   // Free the buffers associated with remapped files. We are required to
260   // perform this operation here because we explicitly request that the
261   // compiler instance *not* free these buffers for each invocation of the
262   // parser.
263   if (Invocation && OwnsRemappedFileBuffers) {
264     PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
265     for (const auto &RB : PPOpts.RemappedFileBuffers)
266       delete RB.second;
267   }
268 
269   ClearCachedCompletionResults();
270 
271   if (getenv("LIBCLANG_OBJTRACKING"))
272     fprintf(stderr, "--- %u translation units\n", --ActiveASTUnitObjects);
273 }
274 
275 void ASTUnit::setPreprocessor(std::shared_ptr<Preprocessor> PP) {
276   this->PP = std::move(PP);
277 }
278 
279 void ASTUnit::enableSourceFileDiagnostics() {
280   assert(getDiagnostics().getClient() && Ctx &&
281       "Bad context for source file");
282   getDiagnostics().getClient()->BeginSourceFile(Ctx->getLangOpts(), PP.get());
283 }
284 
285 /// Determine the set of code-completion contexts in which this
286 /// declaration should be shown.
287 static uint64_t getDeclShowContexts(const NamedDecl *ND,
288                                     const LangOptions &LangOpts,
289                                     bool &IsNestedNameSpecifier) {
290   IsNestedNameSpecifier = false;
291 
292   if (isa<UsingShadowDecl>(ND))
293     ND = ND->getUnderlyingDecl();
294   if (!ND)
295     return 0;
296 
297   uint64_t Contexts = 0;
298   if (isa<TypeDecl>(ND) || isa<ObjCInterfaceDecl>(ND) ||
299       isa<ClassTemplateDecl>(ND) || isa<TemplateTemplateParmDecl>(ND) ||
300       isa<TypeAliasTemplateDecl>(ND)) {
301     // Types can appear in these contexts.
302     if (LangOpts.CPlusPlus || !isa<TagDecl>(ND))
303       Contexts |= (1LL << CodeCompletionContext::CCC_TopLevel)
304                |  (1LL << CodeCompletionContext::CCC_ObjCIvarList)
305                |  (1LL << CodeCompletionContext::CCC_ClassStructUnion)
306                |  (1LL << CodeCompletionContext::CCC_Statement)
307                |  (1LL << CodeCompletionContext::CCC_Type)
308                |  (1LL << CodeCompletionContext::CCC_ParenthesizedExpression);
309 
310     // In C++, types can appear in expressions contexts (for functional casts).
311     if (LangOpts.CPlusPlus)
312       Contexts |= (1LL << CodeCompletionContext::CCC_Expression);
313 
314     // In Objective-C, message sends can send interfaces. In Objective-C++,
315     // all types are available due to functional casts.
316     if (LangOpts.CPlusPlus || isa<ObjCInterfaceDecl>(ND))
317       Contexts |= (1LL << CodeCompletionContext::CCC_ObjCMessageReceiver);
318 
319     // In Objective-C, you can only be a subclass of another Objective-C class
320     if (const auto *ID = dyn_cast<ObjCInterfaceDecl>(ND)) {
321       // Objective-C interfaces can be used in a class property expression.
322       if (ID->getDefinition())
323         Contexts |= (1LL << CodeCompletionContext::CCC_Expression);
324       Contexts |= (1LL << CodeCompletionContext::CCC_ObjCInterfaceName);
325     }
326 
327     // Deal with tag names.
328     if (isa<EnumDecl>(ND)) {
329       Contexts |= (1LL << CodeCompletionContext::CCC_EnumTag);
330 
331       // Part of the nested-name-specifier in C++0x.
332       if (LangOpts.CPlusPlus11)
333         IsNestedNameSpecifier = true;
334     } else if (const auto *Record = dyn_cast<RecordDecl>(ND)) {
335       if (Record->isUnion())
336         Contexts |= (1LL << CodeCompletionContext::CCC_UnionTag);
337       else
338         Contexts |= (1LL << CodeCompletionContext::CCC_ClassOrStructTag);
339 
340       if (LangOpts.CPlusPlus)
341         IsNestedNameSpecifier = true;
342     } else if (isa<ClassTemplateDecl>(ND))
343       IsNestedNameSpecifier = true;
344   } else if (isa<ValueDecl>(ND) || isa<FunctionTemplateDecl>(ND)) {
345     // Values can appear in these contexts.
346     Contexts = (1LL << CodeCompletionContext::CCC_Statement)
347              | (1LL << CodeCompletionContext::CCC_Expression)
348              | (1LL << CodeCompletionContext::CCC_ParenthesizedExpression)
349              | (1LL << CodeCompletionContext::CCC_ObjCMessageReceiver);
350   } else if (isa<ObjCProtocolDecl>(ND)) {
351     Contexts = (1LL << CodeCompletionContext::CCC_ObjCProtocolName);
352   } else if (isa<ObjCCategoryDecl>(ND)) {
353     Contexts = (1LL << CodeCompletionContext::CCC_ObjCCategoryName);
354   } else if (isa<NamespaceDecl>(ND) || isa<NamespaceAliasDecl>(ND)) {
355     Contexts = (1LL << CodeCompletionContext::CCC_Namespace);
356 
357     // Part of the nested-name-specifier.
358     IsNestedNameSpecifier = true;
359   }
360 
361   return Contexts;
362 }
363 
364 void ASTUnit::CacheCodeCompletionResults() {
365   if (!TheSema)
366     return;
367 
368   SimpleTimer Timer(WantTiming);
369   Timer.setOutput("Cache global code completions for " + getMainFileName());
370 
371   // Clear out the previous results.
372   ClearCachedCompletionResults();
373 
374   // Gather the set of global code completions.
375   using Result = CodeCompletionResult;
376   SmallVector<Result, 8> Results;
377   CachedCompletionAllocator = std::make_shared<GlobalCodeCompletionAllocator>();
378   CodeCompletionTUInfo CCTUInfo(CachedCompletionAllocator);
379   TheSema->GatherGlobalCodeCompletions(*CachedCompletionAllocator,
380                                        CCTUInfo, Results);
381 
382   // Translate global code completions into cached completions.
383   llvm::DenseMap<CanQualType, unsigned> CompletionTypes;
384   CodeCompletionContext CCContext(CodeCompletionContext::CCC_TopLevel);
385 
386   for (auto &R : Results) {
387     switch (R.Kind) {
388     case Result::RK_Declaration: {
389       bool IsNestedNameSpecifier = false;
390       CachedCodeCompletionResult CachedResult;
391       CachedResult.Completion = R.CreateCodeCompletionString(
392           *TheSema, CCContext, *CachedCompletionAllocator, CCTUInfo,
393           IncludeBriefCommentsInCodeCompletion);
394       CachedResult.ShowInContexts = getDeclShowContexts(
395           R.Declaration, Ctx->getLangOpts(), IsNestedNameSpecifier);
396       CachedResult.Priority = R.Priority;
397       CachedResult.Kind = R.CursorKind;
398       CachedResult.Availability = R.Availability;
399 
400       // Keep track of the type of this completion in an ASTContext-agnostic
401       // way.
402       QualType UsageType = getDeclUsageType(*Ctx, R.Declaration);
403       if (UsageType.isNull()) {
404         CachedResult.TypeClass = STC_Void;
405         CachedResult.Type = 0;
406       } else {
407         CanQualType CanUsageType
408           = Ctx->getCanonicalType(UsageType.getUnqualifiedType());
409         CachedResult.TypeClass = getSimplifiedTypeClass(CanUsageType);
410 
411         // Determine whether we have already seen this type. If so, we save
412         // ourselves the work of formatting the type string by using the
413         // temporary, CanQualType-based hash table to find the associated value.
414         unsigned &TypeValue = CompletionTypes[CanUsageType];
415         if (TypeValue == 0) {
416           TypeValue = CompletionTypes.size();
417           CachedCompletionTypes[QualType(CanUsageType).getAsString()]
418             = TypeValue;
419         }
420 
421         CachedResult.Type = TypeValue;
422       }
423 
424       CachedCompletionResults.push_back(CachedResult);
425 
426       /// Handle nested-name-specifiers in C++.
427       if (TheSema->Context.getLangOpts().CPlusPlus && IsNestedNameSpecifier &&
428           !R.StartsNestedNameSpecifier) {
429         // The contexts in which a nested-name-specifier can appear in C++.
430         uint64_t NNSContexts
431           = (1LL << CodeCompletionContext::CCC_TopLevel)
432           | (1LL << CodeCompletionContext::CCC_ObjCIvarList)
433           | (1LL << CodeCompletionContext::CCC_ClassStructUnion)
434           | (1LL << CodeCompletionContext::CCC_Statement)
435           | (1LL << CodeCompletionContext::CCC_Expression)
436           | (1LL << CodeCompletionContext::CCC_ObjCMessageReceiver)
437           | (1LL << CodeCompletionContext::CCC_EnumTag)
438           | (1LL << CodeCompletionContext::CCC_UnionTag)
439           | (1LL << CodeCompletionContext::CCC_ClassOrStructTag)
440           | (1LL << CodeCompletionContext::CCC_Type)
441           | (1LL << CodeCompletionContext::CCC_SymbolOrNewName)
442           | (1LL << CodeCompletionContext::CCC_ParenthesizedExpression);
443 
444         if (isa<NamespaceDecl>(R.Declaration) ||
445             isa<NamespaceAliasDecl>(R.Declaration))
446           NNSContexts |= (1LL << CodeCompletionContext::CCC_Namespace);
447 
448         if (uint64_t RemainingContexts
449                                 = NNSContexts & ~CachedResult.ShowInContexts) {
450           // If there any contexts where this completion can be a
451           // nested-name-specifier but isn't already an option, create a
452           // nested-name-specifier completion.
453           R.StartsNestedNameSpecifier = true;
454           CachedResult.Completion = R.CreateCodeCompletionString(
455               *TheSema, CCContext, *CachedCompletionAllocator, CCTUInfo,
456               IncludeBriefCommentsInCodeCompletion);
457           CachedResult.ShowInContexts = RemainingContexts;
458           CachedResult.Priority = CCP_NestedNameSpecifier;
459           CachedResult.TypeClass = STC_Void;
460           CachedResult.Type = 0;
461           CachedCompletionResults.push_back(CachedResult);
462         }
463       }
464       break;
465     }
466 
467     case Result::RK_Keyword:
468     case Result::RK_Pattern:
469       // Ignore keywords and patterns; we don't care, since they are so
470       // easily regenerated.
471       break;
472 
473     case Result::RK_Macro: {
474       CachedCodeCompletionResult CachedResult;
475       CachedResult.Completion = R.CreateCodeCompletionString(
476           *TheSema, CCContext, *CachedCompletionAllocator, CCTUInfo,
477           IncludeBriefCommentsInCodeCompletion);
478       CachedResult.ShowInContexts
479         = (1LL << CodeCompletionContext::CCC_TopLevel)
480         | (1LL << CodeCompletionContext::CCC_ObjCInterface)
481         | (1LL << CodeCompletionContext::CCC_ObjCImplementation)
482         | (1LL << CodeCompletionContext::CCC_ObjCIvarList)
483         | (1LL << CodeCompletionContext::CCC_ClassStructUnion)
484         | (1LL << CodeCompletionContext::CCC_Statement)
485         | (1LL << CodeCompletionContext::CCC_Expression)
486         | (1LL << CodeCompletionContext::CCC_ObjCMessageReceiver)
487         | (1LL << CodeCompletionContext::CCC_MacroNameUse)
488         | (1LL << CodeCompletionContext::CCC_PreprocessorExpression)
489         | (1LL << CodeCompletionContext::CCC_ParenthesizedExpression)
490         | (1LL << CodeCompletionContext::CCC_OtherWithMacros);
491 
492       CachedResult.Priority = R.Priority;
493       CachedResult.Kind = R.CursorKind;
494       CachedResult.Availability = R.Availability;
495       CachedResult.TypeClass = STC_Void;
496       CachedResult.Type = 0;
497       CachedCompletionResults.push_back(CachedResult);
498       break;
499     }
500     }
501   }
502 
503   // Save the current top-level hash value.
504   CompletionCacheTopLevelHashValue = CurrentTopLevelHashValue;
505 }
506 
507 void ASTUnit::ClearCachedCompletionResults() {
508   CachedCompletionResults.clear();
509   CachedCompletionTypes.clear();
510   CachedCompletionAllocator = nullptr;
511 }
512 
513 namespace {
514 
515 /// Gathers information from ASTReader that will be used to initialize
516 /// a Preprocessor.
517 class ASTInfoCollector : public ASTReaderListener {
518   Preprocessor &PP;
519   ASTContext *Context;
520   HeaderSearchOptions &HSOpts;
521   PreprocessorOptions &PPOpts;
522   LangOptions &LangOpt;
523   std::shared_ptr<TargetOptions> &TargetOpts;
524   IntrusiveRefCntPtr<TargetInfo> &Target;
525   unsigned &Counter;
526   bool InitializedLanguage = false;
527 
528 public:
529   ASTInfoCollector(Preprocessor &PP, ASTContext *Context,
530                    HeaderSearchOptions &HSOpts, PreprocessorOptions &PPOpts,
531                    LangOptions &LangOpt,
532                    std::shared_ptr<TargetOptions> &TargetOpts,
533                    IntrusiveRefCntPtr<TargetInfo> &Target, unsigned &Counter)
534       : PP(PP), Context(Context), HSOpts(HSOpts), PPOpts(PPOpts),
535         LangOpt(LangOpt), TargetOpts(TargetOpts), Target(Target),
536         Counter(Counter) {}
537 
538   bool ReadLanguageOptions(const LangOptions &LangOpts, bool Complain,
539                            bool AllowCompatibleDifferences) override {
540     if (InitializedLanguage)
541       return false;
542 
543     LangOpt = LangOpts;
544     InitializedLanguage = true;
545 
546     updated();
547     return false;
548   }
549 
550   bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
551                                StringRef SpecificModuleCachePath,
552                                bool Complain) override {
553     this->HSOpts = HSOpts;
554     return false;
555   }
556 
557   bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, bool Complain,
558                                std::string &SuggestedPredefines) override {
559     this->PPOpts = PPOpts;
560     return false;
561   }
562 
563   bool ReadTargetOptions(const TargetOptions &TargetOpts, bool Complain,
564                          bool AllowCompatibleDifferences) override {
565     // If we've already initialized the target, don't do it again.
566     if (Target)
567       return false;
568 
569     this->TargetOpts = std::make_shared<TargetOptions>(TargetOpts);
570     Target =
571         TargetInfo::CreateTargetInfo(PP.getDiagnostics(), this->TargetOpts);
572 
573     updated();
574     return false;
575   }
576 
577   void ReadCounter(const serialization::ModuleFile &M,
578                    unsigned Value) override {
579     Counter = Value;
580   }
581 
582 private:
583   void updated() {
584     if (!Target || !InitializedLanguage)
585       return;
586 
587     // Inform the target of the language options.
588     //
589     // FIXME: We shouldn't need to do this, the target should be immutable once
590     // created. This complexity should be lifted elsewhere.
591     Target->adjust(LangOpt);
592 
593     // Initialize the preprocessor.
594     PP.Initialize(*Target);
595 
596     if (!Context)
597       return;
598 
599     // Initialize the ASTContext
600     Context->InitBuiltinTypes(*Target);
601 
602     // Adjust printing policy based on language options.
603     Context->setPrintingPolicy(PrintingPolicy(LangOpt));
604 
605     // We didn't have access to the comment options when the ASTContext was
606     // constructed, so register them now.
607     Context->getCommentCommandTraits().registerCommentOptions(
608         LangOpt.CommentOpts);
609   }
610 };
611 
612 /// Diagnostic consumer that saves each diagnostic it is given.
613 class FilterAndStoreDiagnosticConsumer : public DiagnosticConsumer {
614   SmallVectorImpl<StoredDiagnostic> *StoredDiags;
615   SmallVectorImpl<ASTUnit::StandaloneDiagnostic> *StandaloneDiags;
616   bool CaptureNonErrorsFromIncludes = true;
617   const LangOptions *LangOpts = nullptr;
618   SourceManager *SourceMgr = nullptr;
619 
620 public:
621   FilterAndStoreDiagnosticConsumer(
622       SmallVectorImpl<StoredDiagnostic> *StoredDiags,
623       SmallVectorImpl<ASTUnit::StandaloneDiagnostic> *StandaloneDiags,
624       bool CaptureNonErrorsFromIncludes)
625       : StoredDiags(StoredDiags), StandaloneDiags(StandaloneDiags),
626         CaptureNonErrorsFromIncludes(CaptureNonErrorsFromIncludes) {
627     assert((StoredDiags || StandaloneDiags) &&
628            "No output collections were passed to StoredDiagnosticConsumer.");
629   }
630 
631   void BeginSourceFile(const LangOptions &LangOpts,
632                        const Preprocessor *PP = nullptr) override {
633     this->LangOpts = &LangOpts;
634     if (PP)
635       SourceMgr = &PP->getSourceManager();
636   }
637 
638   void HandleDiagnostic(DiagnosticsEngine::Level Level,
639                         const Diagnostic &Info) override;
640 };
641 
642 /// RAII object that optionally captures and filters diagnostics, if
643 /// there is no diagnostic client to capture them already.
644 class CaptureDroppedDiagnostics {
645   DiagnosticsEngine &Diags;
646   FilterAndStoreDiagnosticConsumer Client;
647   DiagnosticConsumer *PreviousClient = nullptr;
648   std::unique_ptr<DiagnosticConsumer> OwningPreviousClient;
649 
650 public:
651   CaptureDroppedDiagnostics(
652       CaptureDiagsKind CaptureDiagnostics, DiagnosticsEngine &Diags,
653       SmallVectorImpl<StoredDiagnostic> *StoredDiags,
654       SmallVectorImpl<ASTUnit::StandaloneDiagnostic> *StandaloneDiags)
655       : Diags(Diags),
656         Client(StoredDiags, StandaloneDiags,
657                CaptureDiagnostics !=
658                    CaptureDiagsKind::AllWithoutNonErrorsFromIncludes) {
659     if (CaptureDiagnostics != CaptureDiagsKind::None ||
660         Diags.getClient() == nullptr) {
661       OwningPreviousClient = Diags.takeClient();
662       PreviousClient = Diags.getClient();
663       Diags.setClient(&Client, false);
664     }
665   }
666 
667   ~CaptureDroppedDiagnostics() {
668     if (Diags.getClient() == &Client)
669       Diags.setClient(PreviousClient, !!OwningPreviousClient.release());
670   }
671 };
672 
673 } // namespace
674 
675 static ASTUnit::StandaloneDiagnostic
676 makeStandaloneDiagnostic(const LangOptions &LangOpts,
677                          const StoredDiagnostic &InDiag);
678 
679 static bool isInMainFile(const clang::Diagnostic &D) {
680   if (!D.hasSourceManager() || !D.getLocation().isValid())
681     return false;
682 
683   auto &M = D.getSourceManager();
684   return M.isWrittenInMainFile(M.getExpansionLoc(D.getLocation()));
685 }
686 
687 void FilterAndStoreDiagnosticConsumer::HandleDiagnostic(
688     DiagnosticsEngine::Level Level, const Diagnostic &Info) {
689   // Default implementation (Warnings/errors count).
690   DiagnosticConsumer::HandleDiagnostic(Level, Info);
691 
692   // Only record the diagnostic if it's part of the source manager we know
693   // about. This effectively drops diagnostics from modules we're building.
694   // FIXME: In the long run, ee don't want to drop source managers from modules.
695   if (!Info.hasSourceManager() || &Info.getSourceManager() == SourceMgr) {
696     if (!CaptureNonErrorsFromIncludes && Level <= DiagnosticsEngine::Warning &&
697         !isInMainFile(Info)) {
698       return;
699     }
700 
701     StoredDiagnostic *ResultDiag = nullptr;
702     if (StoredDiags) {
703       StoredDiags->emplace_back(Level, Info);
704       ResultDiag = &StoredDiags->back();
705     }
706 
707     if (StandaloneDiags) {
708       llvm::Optional<StoredDiagnostic> StoredDiag = None;
709       if (!ResultDiag) {
710         StoredDiag.emplace(Level, Info);
711         ResultDiag = StoredDiag.getPointer();
712       }
713       StandaloneDiags->push_back(
714           makeStandaloneDiagnostic(*LangOpts, *ResultDiag));
715     }
716   }
717 }
718 
719 IntrusiveRefCntPtr<ASTReader> ASTUnit::getASTReader() const {
720   return Reader;
721 }
722 
723 ASTMutationListener *ASTUnit::getASTMutationListener() {
724   if (WriterData)
725     return &WriterData->Writer;
726   return nullptr;
727 }
728 
729 ASTDeserializationListener *ASTUnit::getDeserializationListener() {
730   if (WriterData)
731     return &WriterData->Writer;
732   return nullptr;
733 }
734 
735 std::unique_ptr<llvm::MemoryBuffer>
736 ASTUnit::getBufferForFile(StringRef Filename, std::string *ErrorStr) {
737   assert(FileMgr);
738   auto Buffer = FileMgr->getBufferForFile(Filename, UserFilesAreVolatile);
739   if (Buffer)
740     return std::move(*Buffer);
741   if (ErrorStr)
742     *ErrorStr = Buffer.getError().message();
743   return nullptr;
744 }
745 
746 /// Configure the diagnostics object for use with ASTUnit.
747 void ASTUnit::ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
748                              ASTUnit &AST,
749                              CaptureDiagsKind CaptureDiagnostics) {
750   assert(Diags.get() && "no DiagnosticsEngine was provided");
751   if (CaptureDiagnostics != CaptureDiagsKind::None)
752     Diags->setClient(new FilterAndStoreDiagnosticConsumer(
753         &AST.StoredDiagnostics, nullptr,
754         CaptureDiagnostics != CaptureDiagsKind::AllWithoutNonErrorsFromIncludes));
755 }
756 
757 std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
758     const std::string &Filename, const PCHContainerReader &PCHContainerRdr,
759     WhatToLoad ToLoad, IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
760     const FileSystemOptions &FileSystemOpts, bool UseDebugInfo,
761     bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
762     bool AllowASTWithCompilerErrors, bool UserFilesAreVolatile) {
763   std::unique_ptr<ASTUnit> AST(new ASTUnit(true));
764 
765   // Recover resources if we crash before exiting this method.
766   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
767     ASTUnitCleanup(AST.get());
768   llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
769     llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine>>
770     DiagCleanup(Diags.get());
771 
772   ConfigureDiags(Diags, *AST, CaptureDiagnostics);
773 
774   AST->LangOpts = std::make_shared<LangOptions>();
775   AST->OnlyLocalDecls = OnlyLocalDecls;
776   AST->CaptureDiagnostics = CaptureDiagnostics;
777   AST->Diagnostics = Diags;
778   IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
779       llvm::vfs::getRealFileSystem();
780   AST->FileMgr = new FileManager(FileSystemOpts, VFS);
781   AST->UserFilesAreVolatile = UserFilesAreVolatile;
782   AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
783                                      AST->getFileManager(),
784                                      UserFilesAreVolatile);
785   AST->ModuleCache = new InMemoryModuleCache;
786   AST->HSOpts = std::make_shared<HeaderSearchOptions>();
787   AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormat());
788   AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
789                                          AST->getSourceManager(),
790                                          AST->getDiagnostics(),
791                                          AST->getLangOpts(),
792                                          /*Target=*/nullptr));
793   AST->PPOpts = std::make_shared<PreprocessorOptions>();
794 
795   // Gather Info for preprocessor construction later on.
796 
797   HeaderSearch &HeaderInfo = *AST->HeaderInfo;
798   unsigned Counter;
799 
800   AST->PP = std::make_shared<Preprocessor>(
801       AST->PPOpts, AST->getDiagnostics(), *AST->LangOpts,
802       AST->getSourceManager(), HeaderInfo, AST->ModuleLoader,
803       /*IILookup=*/nullptr,
804       /*OwnsHeaderSearch=*/false);
805   Preprocessor &PP = *AST->PP;
806 
807   if (ToLoad >= LoadASTOnly)
808     AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(),
809                               PP.getIdentifierTable(), PP.getSelectorTable(),
810                               PP.getBuiltinInfo());
811 
812   DisableValidationForModuleKind disableValid =
813       DisableValidationForModuleKind::None;
814   if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
815     disableValid = DisableValidationForModuleKind::All;
816   AST->Reader = new ASTReader(
817       PP, *AST->ModuleCache, AST->Ctx.get(), PCHContainerRdr, {},
818       /*isysroot=*/"",
819       /*DisableValidation=*/disableValid, AllowASTWithCompilerErrors);
820 
821   AST->Reader->setListener(std::make_unique<ASTInfoCollector>(
822       *AST->PP, AST->Ctx.get(), *AST->HSOpts, *AST->PPOpts, *AST->LangOpts,
823       AST->TargetOpts, AST->Target, Counter));
824 
825   // Attach the AST reader to the AST context as an external AST
826   // source, so that declarations will be deserialized from the
827   // AST file as needed.
828   // We need the external source to be set up before we read the AST, because
829   // eagerly-deserialized declarations may use it.
830   if (AST->Ctx)
831     AST->Ctx->setExternalSource(AST->Reader);
832 
833   switch (AST->Reader->ReadAST(Filename, serialization::MK_MainFile,
834                           SourceLocation(), ASTReader::ARR_None)) {
835   case ASTReader::Success:
836     break;
837 
838   case ASTReader::Failure:
839   case ASTReader::Missing:
840   case ASTReader::OutOfDate:
841   case ASTReader::VersionMismatch:
842   case ASTReader::ConfigurationMismatch:
843   case ASTReader::HadErrors:
844     AST->getDiagnostics().Report(diag::err_fe_unable_to_load_pch);
845     return nullptr;
846   }
847 
848   AST->OriginalSourceFile = std::string(AST->Reader->getOriginalSourceFile());
849 
850   PP.setCounterValue(Counter);
851 
852   // Create an AST consumer, even though it isn't used.
853   if (ToLoad >= LoadASTOnly)
854     AST->Consumer.reset(new ASTConsumer);
855 
856   // Create a semantic analysis object and tell the AST reader about it.
857   if (ToLoad >= LoadEverything) {
858     AST->TheSema.reset(new Sema(PP, *AST->Ctx, *AST->Consumer));
859     AST->TheSema->Initialize();
860     AST->Reader->InitializeSema(*AST->TheSema);
861   }
862 
863   // Tell the diagnostic client that we have started a source file.
864   AST->getDiagnostics().getClient()->BeginSourceFile(PP.getLangOpts(), &PP);
865 
866   return AST;
867 }
868 
869 /// Add the given macro to the hash of all top-level entities.
870 static void AddDefinedMacroToHash(const Token &MacroNameTok, unsigned &Hash) {
871   Hash = llvm::djbHash(MacroNameTok.getIdentifierInfo()->getName(), Hash);
872 }
873 
874 namespace {
875 
876 /// Preprocessor callback class that updates a hash value with the names
877 /// of all macros that have been defined by the translation unit.
878 class MacroDefinitionTrackerPPCallbacks : public PPCallbacks {
879   unsigned &Hash;
880 
881 public:
882   explicit MacroDefinitionTrackerPPCallbacks(unsigned &Hash) : Hash(Hash) {}
883 
884   void MacroDefined(const Token &MacroNameTok,
885                     const MacroDirective *MD) override {
886     AddDefinedMacroToHash(MacroNameTok, Hash);
887   }
888 };
889 
890 } // namespace
891 
892 /// Add the given declaration to the hash of all top-level entities.
893 static void AddTopLevelDeclarationToHash(Decl *D, unsigned &Hash) {
894   if (!D)
895     return;
896 
897   DeclContext *DC = D->getDeclContext();
898   if (!DC)
899     return;
900 
901   if (!(DC->isTranslationUnit() || DC->getLookupParent()->isTranslationUnit()))
902     return;
903 
904   if (const auto *ND = dyn_cast<NamedDecl>(D)) {
905     if (const auto *EnumD = dyn_cast<EnumDecl>(D)) {
906       // For an unscoped enum include the enumerators in the hash since they
907       // enter the top-level namespace.
908       if (!EnumD->isScoped()) {
909         for (const auto *EI : EnumD->enumerators()) {
910           if (EI->getIdentifier())
911             Hash = llvm::djbHash(EI->getIdentifier()->getName(), Hash);
912         }
913       }
914     }
915 
916     if (ND->getIdentifier())
917       Hash = llvm::djbHash(ND->getIdentifier()->getName(), Hash);
918     else if (DeclarationName Name = ND->getDeclName()) {
919       std::string NameStr = Name.getAsString();
920       Hash = llvm::djbHash(NameStr, Hash);
921     }
922     return;
923   }
924 
925   if (const auto *ImportD = dyn_cast<ImportDecl>(D)) {
926     if (const Module *Mod = ImportD->getImportedModule()) {
927       std::string ModName = Mod->getFullModuleName();
928       Hash = llvm::djbHash(ModName, Hash);
929     }
930     return;
931   }
932 }
933 
934 namespace {
935 
936 class TopLevelDeclTrackerConsumer : public ASTConsumer {
937   ASTUnit &Unit;
938   unsigned &Hash;
939 
940 public:
941   TopLevelDeclTrackerConsumer(ASTUnit &_Unit, unsigned &Hash)
942       : Unit(_Unit), Hash(Hash) {
943     Hash = 0;
944   }
945 
946   void handleTopLevelDecl(Decl *D) {
947     if (!D)
948       return;
949 
950     // FIXME: Currently ObjC method declarations are incorrectly being
951     // reported as top-level declarations, even though their DeclContext
952     // is the containing ObjC @interface/@implementation.  This is a
953     // fundamental problem in the parser right now.
954     if (isa<ObjCMethodDecl>(D))
955       return;
956 
957     AddTopLevelDeclarationToHash(D, Hash);
958     Unit.addTopLevelDecl(D);
959 
960     handleFileLevelDecl(D);
961   }
962 
963   void handleFileLevelDecl(Decl *D) {
964     Unit.addFileLevelDecl(D);
965     if (auto *NSD = dyn_cast<NamespaceDecl>(D)) {
966       for (auto *I : NSD->decls())
967         handleFileLevelDecl(I);
968     }
969   }
970 
971   bool HandleTopLevelDecl(DeclGroupRef D) override {
972     for (auto *TopLevelDecl : D)
973       handleTopLevelDecl(TopLevelDecl);
974     return true;
975   }
976 
977   // We're not interested in "interesting" decls.
978   void HandleInterestingDecl(DeclGroupRef) override {}
979 
980   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override {
981     for (auto *TopLevelDecl : D)
982       handleTopLevelDecl(TopLevelDecl);
983   }
984 
985   ASTMutationListener *GetASTMutationListener() override {
986     return Unit.getASTMutationListener();
987   }
988 
989   ASTDeserializationListener *GetASTDeserializationListener() override {
990     return Unit.getDeserializationListener();
991   }
992 };
993 
994 class TopLevelDeclTrackerAction : public ASTFrontendAction {
995 public:
996   ASTUnit &Unit;
997 
998   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
999                                                  StringRef InFile) override {
1000     CI.getPreprocessor().addPPCallbacks(
1001         std::make_unique<MacroDefinitionTrackerPPCallbacks>(
1002                                            Unit.getCurrentTopLevelHashValue()));
1003     return std::make_unique<TopLevelDeclTrackerConsumer>(
1004         Unit, Unit.getCurrentTopLevelHashValue());
1005   }
1006 
1007 public:
1008   TopLevelDeclTrackerAction(ASTUnit &_Unit) : Unit(_Unit) {}
1009 
1010   bool hasCodeCompletionSupport() const override { return false; }
1011 
1012   TranslationUnitKind getTranslationUnitKind() override {
1013     return Unit.getTranslationUnitKind();
1014   }
1015 };
1016 
1017 class ASTUnitPreambleCallbacks : public PreambleCallbacks {
1018 public:
1019   unsigned getHash() const { return Hash; }
1020 
1021   std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
1022 
1023   std::vector<serialization::DeclID> takeTopLevelDeclIDs() {
1024     return std::move(TopLevelDeclIDs);
1025   }
1026 
1027   void AfterPCHEmitted(ASTWriter &Writer) override {
1028     TopLevelDeclIDs.reserve(TopLevelDecls.size());
1029     for (const auto *D : TopLevelDecls) {
1030       // Invalid top-level decls may not have been serialized.
1031       if (D->isInvalidDecl())
1032         continue;
1033       TopLevelDeclIDs.push_back(Writer.getDeclID(D));
1034     }
1035   }
1036 
1037   void HandleTopLevelDecl(DeclGroupRef DG) override {
1038     for (auto *D : DG) {
1039       // FIXME: Currently ObjC method declarations are incorrectly being
1040       // reported as top-level declarations, even though their DeclContext
1041       // is the containing ObjC @interface/@implementation.  This is a
1042       // fundamental problem in the parser right now.
1043       if (isa<ObjCMethodDecl>(D))
1044         continue;
1045       AddTopLevelDeclarationToHash(D, Hash);
1046       TopLevelDecls.push_back(D);
1047     }
1048   }
1049 
1050   std::unique_ptr<PPCallbacks> createPPCallbacks() override {
1051     return std::make_unique<MacroDefinitionTrackerPPCallbacks>(Hash);
1052   }
1053 
1054 private:
1055   unsigned Hash = 0;
1056   std::vector<Decl *> TopLevelDecls;
1057   std::vector<serialization::DeclID> TopLevelDeclIDs;
1058   llvm::SmallVector<ASTUnit::StandaloneDiagnostic, 4> PreambleDiags;
1059 };
1060 
1061 } // namespace
1062 
1063 static bool isNonDriverDiag(const StoredDiagnostic &StoredDiag) {
1064   return StoredDiag.getLocation().isValid();
1065 }
1066 
1067 static void
1068 checkAndRemoveNonDriverDiags(SmallVectorImpl<StoredDiagnostic> &StoredDiags) {
1069   // Get rid of stored diagnostics except the ones from the driver which do not
1070   // have a source location.
1071   StoredDiags.erase(
1072       std::remove_if(StoredDiags.begin(), StoredDiags.end(), isNonDriverDiag),
1073       StoredDiags.end());
1074 }
1075 
1076 static void checkAndSanitizeDiags(SmallVectorImpl<StoredDiagnostic> &
1077                                                               StoredDiagnostics,
1078                                   SourceManager &SM) {
1079   // The stored diagnostic has the old source manager in it; update
1080   // the locations to refer into the new source manager. Since we've
1081   // been careful to make sure that the source manager's state
1082   // before and after are identical, so that we can reuse the source
1083   // location itself.
1084   for (auto &SD : StoredDiagnostics) {
1085     if (SD.getLocation().isValid()) {
1086       FullSourceLoc Loc(SD.getLocation(), SM);
1087       SD.setLocation(Loc);
1088     }
1089   }
1090 }
1091 
1092 /// Parse the source file into a translation unit using the given compiler
1093 /// invocation, replacing the current translation unit.
1094 ///
1095 /// \returns True if a failure occurred that causes the ASTUnit not to
1096 /// contain any translation-unit information, false otherwise.
1097 bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1098                     std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer,
1099                     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
1100   if (!Invocation)
1101     return true;
1102 
1103   if (VFS && FileMgr)
1104     assert(VFS == &FileMgr->getVirtualFileSystem() &&
1105            "VFS passed to Parse and VFS in FileMgr are different");
1106 
1107   auto CCInvocation = std::make_shared<CompilerInvocation>(*Invocation);
1108   if (OverrideMainBuffer) {
1109     assert(Preamble &&
1110            "No preamble was built, but OverrideMainBuffer is not null");
1111     Preamble->AddImplicitPreamble(*CCInvocation, VFS, OverrideMainBuffer.get());
1112     // VFS may have changed...
1113   }
1114 
1115   // Create the compiler instance to use for building the AST.
1116   std::unique_ptr<CompilerInstance> Clang(
1117       new CompilerInstance(std::move(PCHContainerOps)));
1118 
1119   // Clean up on error, disengage it if the function returns successfully.
1120   auto CleanOnError = llvm::make_scope_exit([&]() {
1121     // Remove the overridden buffer we used for the preamble.
1122     SavedMainFileBuffer = nullptr;
1123 
1124     // Keep the ownership of the data in the ASTUnit because the client may
1125     // want to see the diagnostics.
1126     transferASTDataFromCompilerInstance(*Clang);
1127     FailedParseDiagnostics.swap(StoredDiagnostics);
1128     StoredDiagnostics.clear();
1129     NumStoredDiagnosticsFromDriver = 0;
1130   });
1131 
1132   // Ensure that Clang has a FileManager with the right VFS, which may have
1133   // changed above in AddImplicitPreamble.  If VFS is nullptr, rely on
1134   // createFileManager to create one.
1135   if (VFS && FileMgr && &FileMgr->getVirtualFileSystem() == VFS)
1136     Clang->setFileManager(&*FileMgr);
1137   else
1138     FileMgr = Clang->createFileManager(std::move(VFS));
1139 
1140   // Recover resources if we crash before exiting this method.
1141   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
1142     CICleanup(Clang.get());
1143 
1144   Clang->setInvocation(CCInvocation);
1145   OriginalSourceFile =
1146       std::string(Clang->getFrontendOpts().Inputs[0].getFile());
1147 
1148   // Set up diagnostics, capturing any diagnostics that would
1149   // otherwise be dropped.
1150   Clang->setDiagnostics(&getDiagnostics());
1151 
1152   // Create the target instance.
1153   Clang->setTarget(TargetInfo::CreateTargetInfo(
1154       Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
1155   if (!Clang->hasTarget())
1156     return true;
1157 
1158   // Inform the target of the language options.
1159   //
1160   // FIXME: We shouldn't need to do this, the target should be immutable once
1161   // created. This complexity should be lifted elsewhere.
1162   Clang->getTarget().adjust(Clang->getLangOpts());
1163 
1164   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
1165          "Invocation must have exactly one source file!");
1166   assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() ==
1167              InputKind::Source &&
1168          "FIXME: AST inputs not yet supported here!");
1169   assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() !=
1170              Language::LLVM_IR &&
1171          "IR inputs not support here!");
1172 
1173   // Configure the various subsystems.
1174   LangOpts = Clang->getInvocation().LangOpts;
1175   FileSystemOpts = Clang->getFileSystemOpts();
1176 
1177   ResetForParse();
1178 
1179   SourceMgr = new SourceManager(getDiagnostics(), *FileMgr,
1180                                 UserFilesAreVolatile);
1181   if (!OverrideMainBuffer) {
1182     checkAndRemoveNonDriverDiags(StoredDiagnostics);
1183     TopLevelDeclsInPreamble.clear();
1184   }
1185 
1186   // Create the source manager.
1187   Clang->setSourceManager(&getSourceManager());
1188 
1189   // If the main file has been overridden due to the use of a preamble,
1190   // make that override happen and introduce the preamble.
1191   if (OverrideMainBuffer) {
1192     // The stored diagnostic has the old source manager in it; update
1193     // the locations to refer into the new source manager. Since we've
1194     // been careful to make sure that the source manager's state
1195     // before and after are identical, so that we can reuse the source
1196     // location itself.
1197     checkAndSanitizeDiags(StoredDiagnostics, getSourceManager());
1198 
1199     // Keep track of the override buffer;
1200     SavedMainFileBuffer = std::move(OverrideMainBuffer);
1201   }
1202 
1203   std::unique_ptr<TopLevelDeclTrackerAction> Act(
1204       new TopLevelDeclTrackerAction(*this));
1205 
1206   // Recover resources if we crash before exiting this method.
1207   llvm::CrashRecoveryContextCleanupRegistrar<TopLevelDeclTrackerAction>
1208     ActCleanup(Act.get());
1209 
1210   if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
1211     return true;
1212 
1213   if (SavedMainFileBuffer)
1214     TranslateStoredDiagnostics(getFileManager(), getSourceManager(),
1215                                PreambleDiagnostics, StoredDiagnostics);
1216   else
1217     PreambleSrcLocCache.clear();
1218 
1219   if (llvm::Error Err = Act->Execute()) {
1220     consumeError(std::move(Err)); // FIXME this drops errors on the floor.
1221     return true;
1222   }
1223 
1224   transferASTDataFromCompilerInstance(*Clang);
1225 
1226   Act->EndSourceFile();
1227 
1228   FailedParseDiagnostics.clear();
1229 
1230   CleanOnError.release();
1231 
1232   return false;
1233 }
1234 
1235 static std::pair<unsigned, unsigned>
1236 makeStandaloneRange(CharSourceRange Range, const SourceManager &SM,
1237                     const LangOptions &LangOpts) {
1238   CharSourceRange FileRange = Lexer::makeFileCharRange(Range, SM, LangOpts);
1239   unsigned Offset = SM.getFileOffset(FileRange.getBegin());
1240   unsigned EndOffset = SM.getFileOffset(FileRange.getEnd());
1241   return std::make_pair(Offset, EndOffset);
1242 }
1243 
1244 static ASTUnit::StandaloneFixIt makeStandaloneFixIt(const SourceManager &SM,
1245                                                     const LangOptions &LangOpts,
1246                                                     const FixItHint &InFix) {
1247   ASTUnit::StandaloneFixIt OutFix;
1248   OutFix.RemoveRange = makeStandaloneRange(InFix.RemoveRange, SM, LangOpts);
1249   OutFix.InsertFromRange = makeStandaloneRange(InFix.InsertFromRange, SM,
1250                                                LangOpts);
1251   OutFix.CodeToInsert = InFix.CodeToInsert;
1252   OutFix.BeforePreviousInsertions = InFix.BeforePreviousInsertions;
1253   return OutFix;
1254 }
1255 
1256 static ASTUnit::StandaloneDiagnostic
1257 makeStandaloneDiagnostic(const LangOptions &LangOpts,
1258                          const StoredDiagnostic &InDiag) {
1259   ASTUnit::StandaloneDiagnostic OutDiag;
1260   OutDiag.ID = InDiag.getID();
1261   OutDiag.Level = InDiag.getLevel();
1262   OutDiag.Message = std::string(InDiag.getMessage());
1263   OutDiag.LocOffset = 0;
1264   if (InDiag.getLocation().isInvalid())
1265     return OutDiag;
1266   const SourceManager &SM = InDiag.getLocation().getManager();
1267   SourceLocation FileLoc = SM.getFileLoc(InDiag.getLocation());
1268   OutDiag.Filename = std::string(SM.getFilename(FileLoc));
1269   if (OutDiag.Filename.empty())
1270     return OutDiag;
1271   OutDiag.LocOffset = SM.getFileOffset(FileLoc);
1272   for (const auto &Range : InDiag.getRanges())
1273     OutDiag.Ranges.push_back(makeStandaloneRange(Range, SM, LangOpts));
1274   for (const auto &FixIt : InDiag.getFixIts())
1275     OutDiag.FixIts.push_back(makeStandaloneFixIt(SM, LangOpts, FixIt));
1276 
1277   return OutDiag;
1278 }
1279 
1280 /// Attempt to build or re-use a precompiled preamble when (re-)parsing
1281 /// the source file.
1282 ///
1283 /// This routine will compute the preamble of the main source file. If a
1284 /// non-trivial preamble is found, it will precompile that preamble into a
1285 /// precompiled header so that the precompiled preamble can be used to reduce
1286 /// reparsing time. If a precompiled preamble has already been constructed,
1287 /// this routine will determine if it is still valid and, if so, avoid
1288 /// rebuilding the precompiled preamble.
1289 ///
1290 /// \param AllowRebuild When true (the default), this routine is
1291 /// allowed to rebuild the precompiled preamble if it is found to be
1292 /// out-of-date.
1293 ///
1294 /// \param MaxLines When non-zero, the maximum number of lines that
1295 /// can occur within the preamble.
1296 ///
1297 /// \returns If the precompiled preamble can be used, returns a newly-allocated
1298 /// buffer that should be used in place of the main file when doing so.
1299 /// Otherwise, returns a NULL pointer.
1300 std::unique_ptr<llvm::MemoryBuffer>
1301 ASTUnit::getMainBufferWithPrecompiledPreamble(
1302     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1303     CompilerInvocation &PreambleInvocationIn,
1304     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, bool AllowRebuild,
1305     unsigned MaxLines) {
1306   auto MainFilePath =
1307       PreambleInvocationIn.getFrontendOpts().Inputs[0].getFile();
1308   std::unique_ptr<llvm::MemoryBuffer> MainFileBuffer =
1309       getBufferForFileHandlingRemapping(PreambleInvocationIn, VFS.get(),
1310                                         MainFilePath, UserFilesAreVolatile);
1311   if (!MainFileBuffer)
1312     return nullptr;
1313 
1314   PreambleBounds Bounds = ComputePreambleBounds(
1315       *PreambleInvocationIn.getLangOpts(), *MainFileBuffer, MaxLines);
1316   if (!Bounds.Size)
1317     return nullptr;
1318 
1319   if (Preamble) {
1320     if (Preamble->CanReuse(PreambleInvocationIn, *MainFileBuffer, Bounds,
1321                            *VFS)) {
1322       // Okay! We can re-use the precompiled preamble.
1323 
1324       // Set the state of the diagnostic object to mimic its state
1325       // after parsing the preamble.
1326       getDiagnostics().Reset();
1327       ProcessWarningOptions(getDiagnostics(),
1328                             PreambleInvocationIn.getDiagnosticOpts());
1329       getDiagnostics().setNumWarnings(NumWarningsInPreamble);
1330 
1331       PreambleRebuildCountdown = 1;
1332       return MainFileBuffer;
1333     } else {
1334       Preamble.reset();
1335       PreambleDiagnostics.clear();
1336       TopLevelDeclsInPreamble.clear();
1337       PreambleSrcLocCache.clear();
1338       PreambleRebuildCountdown = 1;
1339     }
1340   }
1341 
1342   // If the preamble rebuild counter > 1, it's because we previously
1343   // failed to build a preamble and we're not yet ready to try
1344   // again. Decrement the counter and return a failure.
1345   if (PreambleRebuildCountdown > 1) {
1346     --PreambleRebuildCountdown;
1347     return nullptr;
1348   }
1349 
1350   assert(!Preamble && "No Preamble should be stored at that point");
1351   // If we aren't allowed to rebuild the precompiled preamble, just
1352   // return now.
1353   if (!AllowRebuild)
1354     return nullptr;
1355 
1356   ++PreambleCounter;
1357 
1358   SmallVector<StandaloneDiagnostic, 4> NewPreambleDiagsStandalone;
1359   SmallVector<StoredDiagnostic, 4> NewPreambleDiags;
1360   ASTUnitPreambleCallbacks Callbacks;
1361   {
1362     llvm::Optional<CaptureDroppedDiagnostics> Capture;
1363     if (CaptureDiagnostics != CaptureDiagsKind::None)
1364       Capture.emplace(CaptureDiagnostics, *Diagnostics, &NewPreambleDiags,
1365                       &NewPreambleDiagsStandalone);
1366 
1367     // We did not previously compute a preamble, or it can't be reused anyway.
1368     SimpleTimer PreambleTimer(WantTiming);
1369     PreambleTimer.setOutput("Precompiling preamble");
1370 
1371     const bool PreviousSkipFunctionBodies =
1372         PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies;
1373     if (SkipFunctionBodies == SkipFunctionBodiesScope::Preamble)
1374       PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies = true;
1375 
1376     llvm::ErrorOr<PrecompiledPreamble> NewPreamble = PrecompiledPreamble::Build(
1377         PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS,
1378         PCHContainerOps, /*StoreInMemory=*/false, Callbacks);
1379 
1380     PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies =
1381         PreviousSkipFunctionBodies;
1382 
1383     if (NewPreamble) {
1384       Preamble = std::move(*NewPreamble);
1385       PreambleRebuildCountdown = 1;
1386     } else {
1387       switch (static_cast<BuildPreambleError>(NewPreamble.getError().value())) {
1388       case BuildPreambleError::CouldntCreateTempFile:
1389         // Try again next time.
1390         PreambleRebuildCountdown = 1;
1391         return nullptr;
1392       case BuildPreambleError::CouldntCreateTargetInfo:
1393       case BuildPreambleError::BeginSourceFileFailed:
1394       case BuildPreambleError::CouldntEmitPCH:
1395       case BuildPreambleError::BadInputs:
1396         // These erros are more likely to repeat, retry after some period.
1397         PreambleRebuildCountdown = DefaultPreambleRebuildInterval;
1398         return nullptr;
1399       }
1400       llvm_unreachable("unexpected BuildPreambleError");
1401     }
1402   }
1403 
1404   assert(Preamble && "Preamble wasn't built");
1405 
1406   TopLevelDecls.clear();
1407   TopLevelDeclsInPreamble = Callbacks.takeTopLevelDeclIDs();
1408   PreambleTopLevelHashValue = Callbacks.getHash();
1409 
1410   NumWarningsInPreamble = getDiagnostics().getNumWarnings();
1411 
1412   checkAndRemoveNonDriverDiags(NewPreambleDiags);
1413   StoredDiagnostics = std::move(NewPreambleDiags);
1414   PreambleDiagnostics = std::move(NewPreambleDiagsStandalone);
1415 
1416   // If the hash of top-level entities differs from the hash of the top-level
1417   // entities the last time we rebuilt the preamble, clear out the completion
1418   // cache.
1419   if (CurrentTopLevelHashValue != PreambleTopLevelHashValue) {
1420     CompletionCacheTopLevelHashValue = 0;
1421     PreambleTopLevelHashValue = CurrentTopLevelHashValue;
1422   }
1423 
1424   return MainFileBuffer;
1425 }
1426 
1427 void ASTUnit::RealizeTopLevelDeclsFromPreamble() {
1428   assert(Preamble && "Should only be called when preamble was built");
1429 
1430   std::vector<Decl *> Resolved;
1431   Resolved.reserve(TopLevelDeclsInPreamble.size());
1432   ExternalASTSource &Source = *getASTContext().getExternalSource();
1433   for (const auto TopLevelDecl : TopLevelDeclsInPreamble) {
1434     // Resolve the declaration ID to an actual declaration, possibly
1435     // deserializing the declaration in the process.
1436     if (Decl *D = Source.GetExternalDecl(TopLevelDecl))
1437       Resolved.push_back(D);
1438   }
1439   TopLevelDeclsInPreamble.clear();
1440   TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end());
1441 }
1442 
1443 void ASTUnit::transferASTDataFromCompilerInstance(CompilerInstance &CI) {
1444   // Steal the created target, context, and preprocessor if they have been
1445   // created.
1446   assert(CI.hasInvocation() && "missing invocation");
1447   LangOpts = CI.getInvocation().LangOpts;
1448   TheSema = CI.takeSema();
1449   Consumer = CI.takeASTConsumer();
1450   if (CI.hasASTContext())
1451     Ctx = &CI.getASTContext();
1452   if (CI.hasPreprocessor())
1453     PP = CI.getPreprocessorPtr();
1454   CI.setSourceManager(nullptr);
1455   CI.setFileManager(nullptr);
1456   if (CI.hasTarget())
1457     Target = &CI.getTarget();
1458   Reader = CI.getASTReader();
1459   HadModuleLoaderFatalFailure = CI.hadModuleLoaderFatalFailure();
1460 }
1461 
1462 StringRef ASTUnit::getMainFileName() const {
1463   if (Invocation && !Invocation->getFrontendOpts().Inputs.empty()) {
1464     const FrontendInputFile &Input = Invocation->getFrontendOpts().Inputs[0];
1465     if (Input.isFile())
1466       return Input.getFile();
1467     else
1468       return Input.getBuffer().getBufferIdentifier();
1469   }
1470 
1471   if (SourceMgr) {
1472     if (const FileEntry *
1473           FE = SourceMgr->getFileEntryForID(SourceMgr->getMainFileID()))
1474       return FE->getName();
1475   }
1476 
1477   return {};
1478 }
1479 
1480 StringRef ASTUnit::getASTFileName() const {
1481   if (!isMainFileAST())
1482     return {};
1483 
1484   serialization::ModuleFile &
1485     Mod = Reader->getModuleManager().getPrimaryModule();
1486   return Mod.FileName;
1487 }
1488 
1489 std::unique_ptr<ASTUnit>
1490 ASTUnit::create(std::shared_ptr<CompilerInvocation> CI,
1491                 IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
1492                 CaptureDiagsKind CaptureDiagnostics,
1493                 bool UserFilesAreVolatile) {
1494   std::unique_ptr<ASTUnit> AST(new ASTUnit(false));
1495   ConfigureDiags(Diags, *AST, CaptureDiagnostics);
1496   IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
1497       createVFSFromCompilerInvocation(*CI, *Diags);
1498   AST->Diagnostics = Diags;
1499   AST->FileSystemOpts = CI->getFileSystemOpts();
1500   AST->Invocation = std::move(CI);
1501   AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
1502   AST->UserFilesAreVolatile = UserFilesAreVolatile;
1503   AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr,
1504                                      UserFilesAreVolatile);
1505   AST->ModuleCache = new InMemoryModuleCache;
1506 
1507   return AST;
1508 }
1509 
1510 ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(
1511     std::shared_ptr<CompilerInvocation> CI,
1512     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1513     IntrusiveRefCntPtr<DiagnosticsEngine> Diags, FrontendAction *Action,
1514     ASTUnit *Unit, bool Persistent, StringRef ResourceFilesPath,
1515     bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
1516     unsigned PrecompilePreambleAfterNParses, bool CacheCodeCompletionResults,
1517     bool UserFilesAreVolatile, std::unique_ptr<ASTUnit> *ErrAST) {
1518   assert(CI && "A CompilerInvocation is required");
1519 
1520   std::unique_ptr<ASTUnit> OwnAST;
1521   ASTUnit *AST = Unit;
1522   if (!AST) {
1523     // Create the AST unit.
1524     OwnAST = create(CI, Diags, CaptureDiagnostics, UserFilesAreVolatile);
1525     AST = OwnAST.get();
1526     if (!AST)
1527       return nullptr;
1528   }
1529 
1530   if (!ResourceFilesPath.empty()) {
1531     // Override the resources path.
1532     CI->getHeaderSearchOpts().ResourceDir = std::string(ResourceFilesPath);
1533   }
1534   AST->OnlyLocalDecls = OnlyLocalDecls;
1535   AST->CaptureDiagnostics = CaptureDiagnostics;
1536   if (PrecompilePreambleAfterNParses > 0)
1537     AST->PreambleRebuildCountdown = PrecompilePreambleAfterNParses;
1538   AST->TUKind = Action ? Action->getTranslationUnitKind() : TU_Complete;
1539   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
1540   AST->IncludeBriefCommentsInCodeCompletion = false;
1541 
1542   // Recover resources if we crash before exiting this method.
1543   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
1544     ASTUnitCleanup(OwnAST.get());
1545   llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
1546     llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine>>
1547     DiagCleanup(Diags.get());
1548 
1549   // We'll manage file buffers ourselves.
1550   CI->getPreprocessorOpts().RetainRemappedFileBuffers = true;
1551   CI->getFrontendOpts().DisableFree = false;
1552   ProcessWarningOptions(AST->getDiagnostics(), CI->getDiagnosticOpts());
1553 
1554   // Create the compiler instance to use for building the AST.
1555   std::unique_ptr<CompilerInstance> Clang(
1556       new CompilerInstance(std::move(PCHContainerOps)));
1557 
1558   // Recover resources if we crash before exiting this method.
1559   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
1560     CICleanup(Clang.get());
1561 
1562   Clang->setInvocation(std::move(CI));
1563   AST->OriginalSourceFile =
1564       std::string(Clang->getFrontendOpts().Inputs[0].getFile());
1565 
1566   // Set up diagnostics, capturing any diagnostics that would
1567   // otherwise be dropped.
1568   Clang->setDiagnostics(&AST->getDiagnostics());
1569 
1570   // Create the target instance.
1571   Clang->setTarget(TargetInfo::CreateTargetInfo(
1572       Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
1573   if (!Clang->hasTarget())
1574     return nullptr;
1575 
1576   // Inform the target of the language options.
1577   //
1578   // FIXME: We shouldn't need to do this, the target should be immutable once
1579   // created. This complexity should be lifted elsewhere.
1580   Clang->getTarget().adjust(Clang->getLangOpts());
1581 
1582   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
1583          "Invocation must have exactly one source file!");
1584   assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() ==
1585              InputKind::Source &&
1586          "FIXME: AST inputs not yet supported here!");
1587   assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() !=
1588              Language::LLVM_IR &&
1589          "IR inputs not support here!");
1590 
1591   // Configure the various subsystems.
1592   AST->TheSema.reset();
1593   AST->Ctx = nullptr;
1594   AST->PP = nullptr;
1595   AST->Reader = nullptr;
1596 
1597   // Create a file manager object to provide access to and cache the filesystem.
1598   Clang->setFileManager(&AST->getFileManager());
1599 
1600   // Create the source manager.
1601   Clang->setSourceManager(&AST->getSourceManager());
1602 
1603   FrontendAction *Act = Action;
1604 
1605   std::unique_ptr<TopLevelDeclTrackerAction> TrackerAct;
1606   if (!Act) {
1607     TrackerAct.reset(new TopLevelDeclTrackerAction(*AST));
1608     Act = TrackerAct.get();
1609   }
1610 
1611   // Recover resources if we crash before exiting this method.
1612   llvm::CrashRecoveryContextCleanupRegistrar<TopLevelDeclTrackerAction>
1613     ActCleanup(TrackerAct.get());
1614 
1615   if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0])) {
1616     AST->transferASTDataFromCompilerInstance(*Clang);
1617     if (OwnAST && ErrAST)
1618       ErrAST->swap(OwnAST);
1619 
1620     return nullptr;
1621   }
1622 
1623   if (Persistent && !TrackerAct) {
1624     Clang->getPreprocessor().addPPCallbacks(
1625         std::make_unique<MacroDefinitionTrackerPPCallbacks>(
1626                                            AST->getCurrentTopLevelHashValue()));
1627     std::vector<std::unique_ptr<ASTConsumer>> Consumers;
1628     if (Clang->hasASTConsumer())
1629       Consumers.push_back(Clang->takeASTConsumer());
1630     Consumers.push_back(std::make_unique<TopLevelDeclTrackerConsumer>(
1631         *AST, AST->getCurrentTopLevelHashValue()));
1632     Clang->setASTConsumer(
1633         std::make_unique<MultiplexConsumer>(std::move(Consumers)));
1634   }
1635   if (llvm::Error Err = Act->Execute()) {
1636     consumeError(std::move(Err)); // FIXME this drops errors on the floor.
1637     AST->transferASTDataFromCompilerInstance(*Clang);
1638     if (OwnAST && ErrAST)
1639       ErrAST->swap(OwnAST);
1640 
1641     return nullptr;
1642   }
1643 
1644   // Steal the created target, context, and preprocessor.
1645   AST->transferASTDataFromCompilerInstance(*Clang);
1646 
1647   Act->EndSourceFile();
1648 
1649   if (OwnAST)
1650     return OwnAST.release();
1651   else
1652     return AST;
1653 }
1654 
1655 bool ASTUnit::LoadFromCompilerInvocation(
1656     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1657     unsigned PrecompilePreambleAfterNParses,
1658     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
1659   if (!Invocation)
1660     return true;
1661 
1662   assert(VFS && "VFS is null");
1663 
1664   // We'll manage file buffers ourselves.
1665   Invocation->getPreprocessorOpts().RetainRemappedFileBuffers = true;
1666   Invocation->getFrontendOpts().DisableFree = false;
1667   getDiagnostics().Reset();
1668   ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
1669 
1670   std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer;
1671   if (PrecompilePreambleAfterNParses > 0) {
1672     PreambleRebuildCountdown = PrecompilePreambleAfterNParses;
1673     OverrideMainBuffer =
1674         getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS);
1675     getDiagnostics().Reset();
1676     ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
1677   }
1678 
1679   SimpleTimer ParsingTimer(WantTiming);
1680   ParsingTimer.setOutput("Parsing " + getMainFileName());
1681 
1682   // Recover resources if we crash before exiting this method.
1683   llvm::CrashRecoveryContextCleanupRegistrar<llvm::MemoryBuffer>
1684     MemBufferCleanup(OverrideMainBuffer.get());
1685 
1686   return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS);
1687 }
1688 
1689 std::unique_ptr<ASTUnit> ASTUnit::LoadFromCompilerInvocation(
1690     std::shared_ptr<CompilerInvocation> CI,
1691     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1692     IntrusiveRefCntPtr<DiagnosticsEngine> Diags, FileManager *FileMgr,
1693     bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
1694     unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind,
1695     bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion,
1696     bool UserFilesAreVolatile) {
1697   // Create the AST unit.
1698   std::unique_ptr<ASTUnit> AST(new ASTUnit(false));
1699   ConfigureDiags(Diags, *AST, CaptureDiagnostics);
1700   AST->Diagnostics = Diags;
1701   AST->OnlyLocalDecls = OnlyLocalDecls;
1702   AST->CaptureDiagnostics = CaptureDiagnostics;
1703   AST->TUKind = TUKind;
1704   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
1705   AST->IncludeBriefCommentsInCodeCompletion
1706     = IncludeBriefCommentsInCodeCompletion;
1707   AST->Invocation = std::move(CI);
1708   AST->FileSystemOpts = FileMgr->getFileSystemOpts();
1709   AST->FileMgr = FileMgr;
1710   AST->UserFilesAreVolatile = UserFilesAreVolatile;
1711 
1712   // Recover resources if we crash before exiting this method.
1713   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
1714     ASTUnitCleanup(AST.get());
1715   llvm::CrashRecoveryContextCleanupRegistrar<DiagnosticsEngine,
1716     llvm::CrashRecoveryContextReleaseRefCleanup<DiagnosticsEngine>>
1717     DiagCleanup(Diags.get());
1718 
1719   if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps),
1720                                       PrecompilePreambleAfterNParses,
1721                                       &AST->FileMgr->getVirtualFileSystem()))
1722     return nullptr;
1723   return AST;
1724 }
1725 
1726 ASTUnit *ASTUnit::LoadFromCommandLine(
1727     const char **ArgBegin, const char **ArgEnd,
1728     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1729     IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
1730     bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
1731     ArrayRef<RemappedFile> RemappedFiles, bool RemappedFilesKeepOriginalName,
1732     unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind,
1733     bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion,
1734     bool AllowPCHWithCompilerErrors, SkipFunctionBodiesScope SkipFunctionBodies,
1735     bool SingleFileParse, bool UserFilesAreVolatile, bool ForSerialization,
1736     bool RetainExcludedConditionalBlocks,
1737     llvm::Optional<StringRef> ModuleFormat, std::unique_ptr<ASTUnit> *ErrAST,
1738     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
1739   assert(Diags.get() && "no DiagnosticsEngine was provided");
1740 
1741   SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
1742 
1743   std::shared_ptr<CompilerInvocation> CI;
1744 
1745   {
1746     CaptureDroppedDiagnostics Capture(CaptureDiagnostics, *Diags,
1747                                       &StoredDiagnostics, nullptr);
1748 
1749     CI = createInvocationFromCommandLine(
1750         llvm::makeArrayRef(ArgBegin, ArgEnd), Diags, VFS);
1751     if (!CI)
1752       return nullptr;
1753   }
1754 
1755   // Override any files that need remapping
1756   for (const auto &RemappedFile : RemappedFiles) {
1757     CI->getPreprocessorOpts().addRemappedFile(RemappedFile.first,
1758                                               RemappedFile.second);
1759   }
1760   PreprocessorOptions &PPOpts = CI->getPreprocessorOpts();
1761   PPOpts.RemappedFilesKeepOriginalName = RemappedFilesKeepOriginalName;
1762   PPOpts.AllowPCHWithCompilerErrors = AllowPCHWithCompilerErrors;
1763   PPOpts.SingleFileParseMode = SingleFileParse;
1764   PPOpts.RetainExcludedConditionalBlocks = RetainExcludedConditionalBlocks;
1765 
1766   // Override the resources path.
1767   CI->getHeaderSearchOpts().ResourceDir = std::string(ResourceFilesPath);
1768 
1769   CI->getFrontendOpts().SkipFunctionBodies =
1770       SkipFunctionBodies == SkipFunctionBodiesScope::PreambleAndMainFile;
1771 
1772   if (ModuleFormat)
1773     CI->getHeaderSearchOpts().ModuleFormat =
1774         std::string(ModuleFormat.getValue());
1775 
1776   // Create the AST unit.
1777   std::unique_ptr<ASTUnit> AST;
1778   AST.reset(new ASTUnit(false));
1779   AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size();
1780   AST->StoredDiagnostics.swap(StoredDiagnostics);
1781   ConfigureDiags(Diags, *AST, CaptureDiagnostics);
1782   AST->Diagnostics = Diags;
1783   AST->FileSystemOpts = CI->getFileSystemOpts();
1784   if (!VFS)
1785     VFS = llvm::vfs::getRealFileSystem();
1786   VFS = createVFSFromCompilerInvocation(*CI, *Diags, VFS);
1787   AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS);
1788   AST->ModuleCache = new InMemoryModuleCache;
1789   AST->OnlyLocalDecls = OnlyLocalDecls;
1790   AST->CaptureDiagnostics = CaptureDiagnostics;
1791   AST->TUKind = TUKind;
1792   AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
1793   AST->IncludeBriefCommentsInCodeCompletion
1794     = IncludeBriefCommentsInCodeCompletion;
1795   AST->UserFilesAreVolatile = UserFilesAreVolatile;
1796   AST->Invocation = CI;
1797   AST->SkipFunctionBodies = SkipFunctionBodies;
1798   if (ForSerialization)
1799     AST->WriterData.reset(new ASTWriterData(*AST->ModuleCache));
1800   // Zero out now to ease cleanup during crash recovery.
1801   CI = nullptr;
1802   Diags = nullptr;
1803 
1804   // Recover resources if we crash before exiting this method.
1805   llvm::CrashRecoveryContextCleanupRegistrar<ASTUnit>
1806     ASTUnitCleanup(AST.get());
1807 
1808   if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps),
1809                                       PrecompilePreambleAfterNParses,
1810                                       VFS)) {
1811     // Some error occurred, if caller wants to examine diagnostics, pass it the
1812     // ASTUnit.
1813     if (ErrAST) {
1814       AST->StoredDiagnostics.swap(AST->FailedParseDiagnostics);
1815       ErrAST->swap(AST);
1816     }
1817     return nullptr;
1818   }
1819 
1820   return AST.release();
1821 }
1822 
1823 bool ASTUnit::Reparse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
1824                       ArrayRef<RemappedFile> RemappedFiles,
1825                       IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
1826   if (!Invocation)
1827     return true;
1828 
1829   if (!VFS) {
1830     assert(FileMgr && "FileMgr is null on Reparse call");
1831     VFS = &FileMgr->getVirtualFileSystem();
1832   }
1833 
1834   clearFileLevelDecls();
1835 
1836   SimpleTimer ParsingTimer(WantTiming);
1837   ParsingTimer.setOutput("Reparsing " + getMainFileName());
1838 
1839   // Remap files.
1840   PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
1841   for (const auto &RB : PPOpts.RemappedFileBuffers)
1842     delete RB.second;
1843 
1844   Invocation->getPreprocessorOpts().clearRemappedFiles();
1845   for (const auto &RemappedFile : RemappedFiles) {
1846     Invocation->getPreprocessorOpts().addRemappedFile(RemappedFile.first,
1847                                                       RemappedFile.second);
1848   }
1849 
1850   // If we have a preamble file lying around, or if we might try to
1851   // build a precompiled preamble, do so now.
1852   std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer;
1853   if (Preamble || PreambleRebuildCountdown > 0)
1854     OverrideMainBuffer =
1855         getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS);
1856 
1857   // Clear out the diagnostics state.
1858   FileMgr.reset();
1859   getDiagnostics().Reset();
1860   ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
1861   if (OverrideMainBuffer)
1862     getDiagnostics().setNumWarnings(NumWarningsInPreamble);
1863 
1864   // Parse the sources
1865   bool Result =
1866       Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS);
1867 
1868   // If we're caching global code-completion results, and the top-level
1869   // declarations have changed, clear out the code-completion cache.
1870   if (!Result && ShouldCacheCodeCompletionResults &&
1871       CurrentTopLevelHashValue != CompletionCacheTopLevelHashValue)
1872     CacheCodeCompletionResults();
1873 
1874   // We now need to clear out the completion info related to this translation
1875   // unit; it'll be recreated if necessary.
1876   CCTUInfo.reset();
1877 
1878   return Result;
1879 }
1880 
1881 void ASTUnit::ResetForParse() {
1882   SavedMainFileBuffer.reset();
1883 
1884   SourceMgr.reset();
1885   TheSema.reset();
1886   Ctx.reset();
1887   PP.reset();
1888   Reader.reset();
1889 
1890   TopLevelDecls.clear();
1891   clearFileLevelDecls();
1892 }
1893 
1894 //----------------------------------------------------------------------------//
1895 // Code completion
1896 //----------------------------------------------------------------------------//
1897 
1898 namespace {
1899 
1900   /// Code completion consumer that combines the cached code-completion
1901   /// results from an ASTUnit with the code-completion results provided to it,
1902   /// then passes the result on to
1903   class AugmentedCodeCompleteConsumer : public CodeCompleteConsumer {
1904     uint64_t NormalContexts;
1905     ASTUnit &AST;
1906     CodeCompleteConsumer &Next;
1907 
1908   public:
1909     AugmentedCodeCompleteConsumer(ASTUnit &AST, CodeCompleteConsumer &Next,
1910                                   const CodeCompleteOptions &CodeCompleteOpts)
1911         : CodeCompleteConsumer(CodeCompleteOpts), AST(AST), Next(Next) {
1912       // Compute the set of contexts in which we will look when we don't have
1913       // any information about the specific context.
1914       NormalContexts
1915         = (1LL << CodeCompletionContext::CCC_TopLevel)
1916         | (1LL << CodeCompletionContext::CCC_ObjCInterface)
1917         | (1LL << CodeCompletionContext::CCC_ObjCImplementation)
1918         | (1LL << CodeCompletionContext::CCC_ObjCIvarList)
1919         | (1LL << CodeCompletionContext::CCC_Statement)
1920         | (1LL << CodeCompletionContext::CCC_Expression)
1921         | (1LL << CodeCompletionContext::CCC_ObjCMessageReceiver)
1922         | (1LL << CodeCompletionContext::CCC_DotMemberAccess)
1923         | (1LL << CodeCompletionContext::CCC_ArrowMemberAccess)
1924         | (1LL << CodeCompletionContext::CCC_ObjCPropertyAccess)
1925         | (1LL << CodeCompletionContext::CCC_ObjCProtocolName)
1926         | (1LL << CodeCompletionContext::CCC_ParenthesizedExpression)
1927         | (1LL << CodeCompletionContext::CCC_Recovery);
1928 
1929       if (AST.getASTContext().getLangOpts().CPlusPlus)
1930         NormalContexts |= (1LL << CodeCompletionContext::CCC_EnumTag)
1931                        |  (1LL << CodeCompletionContext::CCC_UnionTag)
1932                        |  (1LL << CodeCompletionContext::CCC_ClassOrStructTag);
1933     }
1934 
1935     void ProcessCodeCompleteResults(Sema &S, CodeCompletionContext Context,
1936                                     CodeCompletionResult *Results,
1937                                     unsigned NumResults) override;
1938 
1939     void ProcessOverloadCandidates(Sema &S, unsigned CurrentArg,
1940                                    OverloadCandidate *Candidates,
1941                                    unsigned NumCandidates,
1942                                    SourceLocation OpenParLoc) override {
1943       Next.ProcessOverloadCandidates(S, CurrentArg, Candidates, NumCandidates,
1944                                      OpenParLoc);
1945     }
1946 
1947     CodeCompletionAllocator &getAllocator() override {
1948       return Next.getAllocator();
1949     }
1950 
1951     CodeCompletionTUInfo &getCodeCompletionTUInfo() override {
1952       return Next.getCodeCompletionTUInfo();
1953     }
1954   };
1955 
1956 } // namespace
1957 
1958 /// Helper function that computes which global names are hidden by the
1959 /// local code-completion results.
1960 static void CalculateHiddenNames(const CodeCompletionContext &Context,
1961                                  CodeCompletionResult *Results,
1962                                  unsigned NumResults,
1963                                  ASTContext &Ctx,
1964                           llvm::StringSet<llvm::BumpPtrAllocator> &HiddenNames){
1965   bool OnlyTagNames = false;
1966   switch (Context.getKind()) {
1967   case CodeCompletionContext::CCC_Recovery:
1968   case CodeCompletionContext::CCC_TopLevel:
1969   case CodeCompletionContext::CCC_ObjCInterface:
1970   case CodeCompletionContext::CCC_ObjCImplementation:
1971   case CodeCompletionContext::CCC_ObjCIvarList:
1972   case CodeCompletionContext::CCC_ClassStructUnion:
1973   case CodeCompletionContext::CCC_Statement:
1974   case CodeCompletionContext::CCC_Expression:
1975   case CodeCompletionContext::CCC_ObjCMessageReceiver:
1976   case CodeCompletionContext::CCC_DotMemberAccess:
1977   case CodeCompletionContext::CCC_ArrowMemberAccess:
1978   case CodeCompletionContext::CCC_ObjCPropertyAccess:
1979   case CodeCompletionContext::CCC_Namespace:
1980   case CodeCompletionContext::CCC_Type:
1981   case CodeCompletionContext::CCC_Symbol:
1982   case CodeCompletionContext::CCC_SymbolOrNewName:
1983   case CodeCompletionContext::CCC_ParenthesizedExpression:
1984   case CodeCompletionContext::CCC_ObjCInterfaceName:
1985     break;
1986 
1987   case CodeCompletionContext::CCC_EnumTag:
1988   case CodeCompletionContext::CCC_UnionTag:
1989   case CodeCompletionContext::CCC_ClassOrStructTag:
1990     OnlyTagNames = true;
1991     break;
1992 
1993   case CodeCompletionContext::CCC_ObjCProtocolName:
1994   case CodeCompletionContext::CCC_MacroName:
1995   case CodeCompletionContext::CCC_MacroNameUse:
1996   case CodeCompletionContext::CCC_PreprocessorExpression:
1997   case CodeCompletionContext::CCC_PreprocessorDirective:
1998   case CodeCompletionContext::CCC_NaturalLanguage:
1999   case CodeCompletionContext::CCC_SelectorName:
2000   case CodeCompletionContext::CCC_TypeQualifiers:
2001   case CodeCompletionContext::CCC_Other:
2002   case CodeCompletionContext::CCC_OtherWithMacros:
2003   case CodeCompletionContext::CCC_ObjCInstanceMessage:
2004   case CodeCompletionContext::CCC_ObjCClassMessage:
2005   case CodeCompletionContext::CCC_ObjCCategoryName:
2006   case CodeCompletionContext::CCC_IncludedFile:
2007   case CodeCompletionContext::CCC_NewName:
2008     // We're looking for nothing, or we're looking for names that cannot
2009     // be hidden.
2010     return;
2011   }
2012 
2013   using Result = CodeCompletionResult;
2014   for (unsigned I = 0; I != NumResults; ++I) {
2015     if (Results[I].Kind != Result::RK_Declaration)
2016       continue;
2017 
2018     unsigned IDNS
2019       = Results[I].Declaration->getUnderlyingDecl()->getIdentifierNamespace();
2020 
2021     bool Hiding = false;
2022     if (OnlyTagNames)
2023       Hiding = (IDNS & Decl::IDNS_Tag);
2024     else {
2025       unsigned HiddenIDNS = (Decl::IDNS_Type | Decl::IDNS_Member |
2026                              Decl::IDNS_Namespace | Decl::IDNS_Ordinary |
2027                              Decl::IDNS_NonMemberOperator);
2028       if (Ctx.getLangOpts().CPlusPlus)
2029         HiddenIDNS |= Decl::IDNS_Tag;
2030       Hiding = (IDNS & HiddenIDNS);
2031     }
2032 
2033     if (!Hiding)
2034       continue;
2035 
2036     DeclarationName Name = Results[I].Declaration->getDeclName();
2037     if (IdentifierInfo *Identifier = Name.getAsIdentifierInfo())
2038       HiddenNames.insert(Identifier->getName());
2039     else
2040       HiddenNames.insert(Name.getAsString());
2041   }
2042 }
2043 
2044 void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
2045                                             CodeCompletionContext Context,
2046                                             CodeCompletionResult *Results,
2047                                             unsigned NumResults) {
2048   // Merge the results we were given with the results we cached.
2049   bool AddedResult = false;
2050   uint64_t InContexts =
2051       Context.getKind() == CodeCompletionContext::CCC_Recovery
2052         ? NormalContexts : (1LL << Context.getKind());
2053   // Contains the set of names that are hidden by "local" completion results.
2054   llvm::StringSet<llvm::BumpPtrAllocator> HiddenNames;
2055   using Result = CodeCompletionResult;
2056   SmallVector<Result, 8> AllResults;
2057   for (ASTUnit::cached_completion_iterator
2058             C = AST.cached_completion_begin(),
2059          CEnd = AST.cached_completion_end();
2060        C != CEnd; ++C) {
2061     // If the context we are in matches any of the contexts we are
2062     // interested in, we'll add this result.
2063     if ((C->ShowInContexts & InContexts) == 0)
2064       continue;
2065 
2066     // If we haven't added any results previously, do so now.
2067     if (!AddedResult) {
2068       CalculateHiddenNames(Context, Results, NumResults, S.Context,
2069                            HiddenNames);
2070       AllResults.insert(AllResults.end(), Results, Results + NumResults);
2071       AddedResult = true;
2072     }
2073 
2074     // Determine whether this global completion result is hidden by a local
2075     // completion result. If so, skip it.
2076     if (C->Kind != CXCursor_MacroDefinition &&
2077         HiddenNames.count(C->Completion->getTypedText()))
2078       continue;
2079 
2080     // Adjust priority based on similar type classes.
2081     unsigned Priority = C->Priority;
2082     CodeCompletionString *Completion = C->Completion;
2083     if (!Context.getPreferredType().isNull()) {
2084       if (C->Kind == CXCursor_MacroDefinition) {
2085         Priority = getMacroUsagePriority(C->Completion->getTypedText(),
2086                                          S.getLangOpts(),
2087                                Context.getPreferredType()->isAnyPointerType());
2088       } else if (C->Type) {
2089         CanQualType Expected
2090           = S.Context.getCanonicalType(
2091                                Context.getPreferredType().getUnqualifiedType());
2092         SimplifiedTypeClass ExpectedSTC = getSimplifiedTypeClass(Expected);
2093         if (ExpectedSTC == C->TypeClass) {
2094           // We know this type is similar; check for an exact match.
2095           llvm::StringMap<unsigned> &CachedCompletionTypes
2096             = AST.getCachedCompletionTypes();
2097           llvm::StringMap<unsigned>::iterator Pos
2098             = CachedCompletionTypes.find(QualType(Expected).getAsString());
2099           if (Pos != CachedCompletionTypes.end() && Pos->second == C->Type)
2100             Priority /= CCF_ExactTypeMatch;
2101           else
2102             Priority /= CCF_SimilarTypeMatch;
2103         }
2104       }
2105     }
2106 
2107     // Adjust the completion string, if required.
2108     if (C->Kind == CXCursor_MacroDefinition &&
2109         Context.getKind() == CodeCompletionContext::CCC_MacroNameUse) {
2110       // Create a new code-completion string that just contains the
2111       // macro name, without its arguments.
2112       CodeCompletionBuilder Builder(getAllocator(), getCodeCompletionTUInfo(),
2113                                     CCP_CodePattern, C->Availability);
2114       Builder.AddTypedTextChunk(C->Completion->getTypedText());
2115       Priority = CCP_CodePattern;
2116       Completion = Builder.TakeString();
2117     }
2118 
2119     AllResults.push_back(Result(Completion, Priority, C->Kind,
2120                                 C->Availability));
2121   }
2122 
2123   // If we did not add any cached completion results, just forward the
2124   // results we were given to the next consumer.
2125   if (!AddedResult) {
2126     Next.ProcessCodeCompleteResults(S, Context, Results, NumResults);
2127     return;
2128   }
2129 
2130   Next.ProcessCodeCompleteResults(S, Context, AllResults.data(),
2131                                   AllResults.size());
2132 }
2133 
2134 void ASTUnit::CodeComplete(
2135     StringRef File, unsigned Line, unsigned Column,
2136     ArrayRef<RemappedFile> RemappedFiles, bool IncludeMacros,
2137     bool IncludeCodePatterns, bool IncludeBriefComments,
2138     CodeCompleteConsumer &Consumer,
2139     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
2140     DiagnosticsEngine &Diag, LangOptions &LangOpts, SourceManager &SourceMgr,
2141     FileManager &FileMgr, SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
2142     SmallVectorImpl<const llvm::MemoryBuffer *> &OwnedBuffers) {
2143   if (!Invocation)
2144     return;
2145 
2146   SimpleTimer CompletionTimer(WantTiming);
2147   CompletionTimer.setOutput("Code completion @ " + File + ":" +
2148                             Twine(Line) + ":" + Twine(Column));
2149 
2150   auto CCInvocation = std::make_shared<CompilerInvocation>(*Invocation);
2151 
2152   FrontendOptions &FrontendOpts = CCInvocation->getFrontendOpts();
2153   CodeCompleteOptions &CodeCompleteOpts = FrontendOpts.CodeCompleteOpts;
2154   PreprocessorOptions &PreprocessorOpts = CCInvocation->getPreprocessorOpts();
2155 
2156   CodeCompleteOpts.IncludeMacros = IncludeMacros &&
2157                                    CachedCompletionResults.empty();
2158   CodeCompleteOpts.IncludeCodePatterns = IncludeCodePatterns;
2159   CodeCompleteOpts.IncludeGlobals = CachedCompletionResults.empty();
2160   CodeCompleteOpts.IncludeBriefComments = IncludeBriefComments;
2161   CodeCompleteOpts.LoadExternal = Consumer.loadExternal();
2162   CodeCompleteOpts.IncludeFixIts = Consumer.includeFixIts();
2163 
2164   assert(IncludeBriefComments == this->IncludeBriefCommentsInCodeCompletion);
2165 
2166   FrontendOpts.CodeCompletionAt.FileName = std::string(File);
2167   FrontendOpts.CodeCompletionAt.Line = Line;
2168   FrontendOpts.CodeCompletionAt.Column = Column;
2169 
2170   // Set the language options appropriately.
2171   LangOpts = *CCInvocation->getLangOpts();
2172 
2173   // Spell-checking and warnings are wasteful during code-completion.
2174   LangOpts.SpellChecking = false;
2175   CCInvocation->getDiagnosticOpts().IgnoreWarnings = true;
2176 
2177   std::unique_ptr<CompilerInstance> Clang(
2178       new CompilerInstance(PCHContainerOps));
2179 
2180   // Recover resources if we crash before exiting this method.
2181   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
2182     CICleanup(Clang.get());
2183 
2184   auto &Inv = *CCInvocation;
2185   Clang->setInvocation(std::move(CCInvocation));
2186   OriginalSourceFile =
2187       std::string(Clang->getFrontendOpts().Inputs[0].getFile());
2188 
2189   // Set up diagnostics, capturing any diagnostics produced.
2190   Clang->setDiagnostics(&Diag);
2191   CaptureDroppedDiagnostics Capture(CaptureDiagsKind::All,
2192                                     Clang->getDiagnostics(),
2193                                     &StoredDiagnostics, nullptr);
2194   ProcessWarningOptions(Diag, Inv.getDiagnosticOpts());
2195 
2196   // Create the target instance.
2197   Clang->setTarget(TargetInfo::CreateTargetInfo(
2198       Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
2199   if (!Clang->hasTarget()) {
2200     Clang->setInvocation(nullptr);
2201     return;
2202   }
2203 
2204   // Inform the target of the language options.
2205   //
2206   // FIXME: We shouldn't need to do this, the target should be immutable once
2207   // created. This complexity should be lifted elsewhere.
2208   Clang->getTarget().adjust(Clang->getLangOpts());
2209 
2210   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
2211          "Invocation must have exactly one source file!");
2212   assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() ==
2213              InputKind::Source &&
2214          "FIXME: AST inputs not yet supported here!");
2215   assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() !=
2216              Language::LLVM_IR &&
2217          "IR inputs not support here!");
2218 
2219   // Use the source and file managers that we were given.
2220   Clang->setFileManager(&FileMgr);
2221   Clang->setSourceManager(&SourceMgr);
2222 
2223   // Remap files.
2224   PreprocessorOpts.clearRemappedFiles();
2225   PreprocessorOpts.RetainRemappedFileBuffers = true;
2226   for (const auto &RemappedFile : RemappedFiles) {
2227     PreprocessorOpts.addRemappedFile(RemappedFile.first, RemappedFile.second);
2228     OwnedBuffers.push_back(RemappedFile.second);
2229   }
2230 
2231   // Use the code completion consumer we were given, but adding any cached
2232   // code-completion results.
2233   AugmentedCodeCompleteConsumer *AugmentedConsumer
2234     = new AugmentedCodeCompleteConsumer(*this, Consumer, CodeCompleteOpts);
2235   Clang->setCodeCompletionConsumer(AugmentedConsumer);
2236 
2237   auto getUniqueID =
2238       [&FileMgr](StringRef Filename) -> Optional<llvm::sys::fs::UniqueID> {
2239     if (auto Status = FileMgr.getVirtualFileSystem().status(Filename))
2240       return Status->getUniqueID();
2241     return None;
2242   };
2243 
2244   auto hasSameUniqueID = [getUniqueID](StringRef LHS, StringRef RHS) {
2245     if (LHS == RHS)
2246       return true;
2247     if (auto LHSID = getUniqueID(LHS))
2248       if (auto RHSID = getUniqueID(RHS))
2249         return *LHSID == *RHSID;
2250     return false;
2251   };
2252 
2253   // If we have a precompiled preamble, try to use it. We only allow
2254   // the use of the precompiled preamble if we're if the completion
2255   // point is within the main file, after the end of the precompiled
2256   // preamble.
2257   std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer;
2258   if (Preamble && Line > 1 && hasSameUniqueID(File, OriginalSourceFile)) {
2259     OverrideMainBuffer = getMainBufferWithPrecompiledPreamble(
2260         PCHContainerOps, Inv, &FileMgr.getVirtualFileSystem(), false, Line - 1);
2261   }
2262 
2263   // If the main file has been overridden due to the use of a preamble,
2264   // make that override happen and introduce the preamble.
2265   if (OverrideMainBuffer) {
2266     assert(Preamble &&
2267            "No preamble was built, but OverrideMainBuffer is not null");
2268 
2269     IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
2270         &FileMgr.getVirtualFileSystem();
2271     Preamble->AddImplicitPreamble(Clang->getInvocation(), VFS,
2272                                   OverrideMainBuffer.get());
2273     // FIXME: there is no way to update VFS if it was changed by
2274     // AddImplicitPreamble as FileMgr is accepted as a parameter by this method.
2275     // We use on-disk preambles instead and rely on FileMgr's VFS to ensure the
2276     // PCH files are always readable.
2277     OwnedBuffers.push_back(OverrideMainBuffer.release());
2278   } else {
2279     PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
2280     PreprocessorOpts.PrecompiledPreambleBytes.second = false;
2281   }
2282 
2283   // Disable the preprocessing record if modules are not enabled.
2284   if (!Clang->getLangOpts().Modules)
2285     PreprocessorOpts.DetailedRecord = false;
2286 
2287   std::unique_ptr<SyntaxOnlyAction> Act;
2288   Act.reset(new SyntaxOnlyAction);
2289   if (Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0])) {
2290     if (llvm::Error Err = Act->Execute()) {
2291       consumeError(std::move(Err)); // FIXME this drops errors on the floor.
2292     }
2293     Act->EndSourceFile();
2294   }
2295 }
2296 
2297 bool ASTUnit::Save(StringRef File) {
2298   if (HadModuleLoaderFatalFailure)
2299     return true;
2300 
2301   // Write to a temporary file and later rename it to the actual file, to avoid
2302   // possible race conditions.
2303   SmallString<128> TempPath;
2304   TempPath = File;
2305   TempPath += "-%%%%%%%%";
2306   // FIXME: Can we somehow regenerate the stat cache here, or do we need to
2307   // unconditionally create a stat cache when we parse the file?
2308 
2309   if (llvm::Error Err = llvm::writeFileAtomically(
2310           TempPath, File, [this](llvm::raw_ostream &Out) {
2311             return serialize(Out) ? llvm::make_error<llvm::StringError>(
2312                                         "ASTUnit serialization failed",
2313                                         llvm::inconvertibleErrorCode())
2314                                   : llvm::Error::success();
2315           })) {
2316     consumeError(std::move(Err));
2317     return true;
2318   }
2319   return false;
2320 }
2321 
2322 static bool serializeUnit(ASTWriter &Writer,
2323                           SmallVectorImpl<char> &Buffer,
2324                           Sema &S,
2325                           bool hasErrors,
2326                           raw_ostream &OS) {
2327   Writer.WriteAST(S, std::string(), nullptr, "", hasErrors);
2328 
2329   // Write the generated bitstream to "Out".
2330   if (!Buffer.empty())
2331     OS.write(Buffer.data(), Buffer.size());
2332 
2333   return false;
2334 }
2335 
2336 bool ASTUnit::serialize(raw_ostream &OS) {
2337   // For serialization we are lenient if the errors were only warn-as-error kind.
2338   bool hasErrors = getDiagnostics().hasUncompilableErrorOccurred();
2339 
2340   if (WriterData)
2341     return serializeUnit(WriterData->Writer, WriterData->Buffer,
2342                          getSema(), hasErrors, OS);
2343 
2344   SmallString<128> Buffer;
2345   llvm::BitstreamWriter Stream(Buffer);
2346   InMemoryModuleCache ModuleCache;
2347   ASTWriter Writer(Stream, Buffer, ModuleCache, {});
2348   return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS);
2349 }
2350 
2351 using SLocRemap = ContinuousRangeMap<unsigned, int, 2>;
2352 
2353 void ASTUnit::TranslateStoredDiagnostics(
2354                           FileManager &FileMgr,
2355                           SourceManager &SrcMgr,
2356                           const SmallVectorImpl<StandaloneDiagnostic> &Diags,
2357                           SmallVectorImpl<StoredDiagnostic> &Out) {
2358   // Map the standalone diagnostic into the new source manager. We also need to
2359   // remap all the locations to the new view. This includes the diag location,
2360   // any associated source ranges, and the source ranges of associated fix-its.
2361   // FIXME: There should be a cleaner way to do this.
2362   SmallVector<StoredDiagnostic, 4> Result;
2363   Result.reserve(Diags.size());
2364 
2365   for (const auto &SD : Diags) {
2366     // Rebuild the StoredDiagnostic.
2367     if (SD.Filename.empty())
2368       continue;
2369     auto FE = FileMgr.getFile(SD.Filename);
2370     if (!FE)
2371       continue;
2372     SourceLocation FileLoc;
2373     auto ItFileID = PreambleSrcLocCache.find(SD.Filename);
2374     if (ItFileID == PreambleSrcLocCache.end()) {
2375       FileID FID = SrcMgr.translateFile(*FE);
2376       FileLoc = SrcMgr.getLocForStartOfFile(FID);
2377       PreambleSrcLocCache[SD.Filename] = FileLoc;
2378     } else {
2379       FileLoc = ItFileID->getValue();
2380     }
2381 
2382     if (FileLoc.isInvalid())
2383       continue;
2384     SourceLocation L = FileLoc.getLocWithOffset(SD.LocOffset);
2385     FullSourceLoc Loc(L, SrcMgr);
2386 
2387     SmallVector<CharSourceRange, 4> Ranges;
2388     Ranges.reserve(SD.Ranges.size());
2389     for (const auto &Range : SD.Ranges) {
2390       SourceLocation BL = FileLoc.getLocWithOffset(Range.first);
2391       SourceLocation EL = FileLoc.getLocWithOffset(Range.second);
2392       Ranges.push_back(CharSourceRange::getCharRange(BL, EL));
2393     }
2394 
2395     SmallVector<FixItHint, 2> FixIts;
2396     FixIts.reserve(SD.FixIts.size());
2397     for (const auto &FixIt : SD.FixIts) {
2398       FixIts.push_back(FixItHint());
2399       FixItHint &FH = FixIts.back();
2400       FH.CodeToInsert = FixIt.CodeToInsert;
2401       SourceLocation BL = FileLoc.getLocWithOffset(FixIt.RemoveRange.first);
2402       SourceLocation EL = FileLoc.getLocWithOffset(FixIt.RemoveRange.second);
2403       FH.RemoveRange = CharSourceRange::getCharRange(BL, EL);
2404     }
2405 
2406     Result.push_back(StoredDiagnostic(SD.Level, SD.ID,
2407                                       SD.Message, Loc, Ranges, FixIts));
2408   }
2409   Result.swap(Out);
2410 }
2411 
2412 void ASTUnit::addFileLevelDecl(Decl *D) {
2413   assert(D);
2414 
2415   // We only care about local declarations.
2416   if (D->isFromASTFile())
2417     return;
2418 
2419   SourceManager &SM = *SourceMgr;
2420   SourceLocation Loc = D->getLocation();
2421   if (Loc.isInvalid() || !SM.isLocalSourceLocation(Loc))
2422     return;
2423 
2424   // We only keep track of the file-level declarations of each file.
2425   if (!D->getLexicalDeclContext()->isFileContext())
2426     return;
2427 
2428   SourceLocation FileLoc = SM.getFileLoc(Loc);
2429   assert(SM.isLocalSourceLocation(FileLoc));
2430   FileID FID;
2431   unsigned Offset;
2432   std::tie(FID, Offset) = SM.getDecomposedLoc(FileLoc);
2433   if (FID.isInvalid())
2434     return;
2435 
2436   std::unique_ptr<LocDeclsTy> &Decls = FileDecls[FID];
2437   if (!Decls)
2438     Decls = std::make_unique<LocDeclsTy>();
2439 
2440   std::pair<unsigned, Decl *> LocDecl(Offset, D);
2441 
2442   if (Decls->empty() || Decls->back().first <= Offset) {
2443     Decls->push_back(LocDecl);
2444     return;
2445   }
2446 
2447   LocDeclsTy::iterator I =
2448       llvm::upper_bound(*Decls, LocDecl, llvm::less_first());
2449 
2450   Decls->insert(I, LocDecl);
2451 }
2452 
2453 void ASTUnit::findFileRegionDecls(FileID File, unsigned Offset, unsigned Length,
2454                                   SmallVectorImpl<Decl *> &Decls) {
2455   if (File.isInvalid())
2456     return;
2457 
2458   if (SourceMgr->isLoadedFileID(File)) {
2459     assert(Ctx->getExternalSource() && "No external source!");
2460     return Ctx->getExternalSource()->FindFileRegionDecls(File, Offset, Length,
2461                                                          Decls);
2462   }
2463 
2464   FileDeclsTy::iterator I = FileDecls.find(File);
2465   if (I == FileDecls.end())
2466     return;
2467 
2468   LocDeclsTy &LocDecls = *I->second;
2469   if (LocDecls.empty())
2470     return;
2471 
2472   LocDeclsTy::iterator BeginIt =
2473       llvm::partition_point(LocDecls, [=](std::pair<unsigned, Decl *> LD) {
2474         return LD.first < Offset;
2475       });
2476   if (BeginIt != LocDecls.begin())
2477     --BeginIt;
2478 
2479   // If we are pointing at a top-level decl inside an objc container, we need
2480   // to backtrack until we find it otherwise we will fail to report that the
2481   // region overlaps with an objc container.
2482   while (BeginIt != LocDecls.begin() &&
2483          BeginIt->second->isTopLevelDeclInObjCContainer())
2484     --BeginIt;
2485 
2486   LocDeclsTy::iterator EndIt = llvm::upper_bound(
2487       LocDecls, std::make_pair(Offset + Length, (Decl *)nullptr),
2488       llvm::less_first());
2489   if (EndIt != LocDecls.end())
2490     ++EndIt;
2491 
2492   for (LocDeclsTy::iterator DIt = BeginIt; DIt != EndIt; ++DIt)
2493     Decls.push_back(DIt->second);
2494 }
2495 
2496 SourceLocation ASTUnit::getLocation(const FileEntry *File,
2497                                     unsigned Line, unsigned Col) const {
2498   const SourceManager &SM = getSourceManager();
2499   SourceLocation Loc = SM.translateFileLineCol(File, Line, Col);
2500   return SM.getMacroArgExpandedLocation(Loc);
2501 }
2502 
2503 SourceLocation ASTUnit::getLocation(const FileEntry *File,
2504                                     unsigned Offset) const {
2505   const SourceManager &SM = getSourceManager();
2506   SourceLocation FileLoc = SM.translateFileLineCol(File, 1, 1);
2507   return SM.getMacroArgExpandedLocation(FileLoc.getLocWithOffset(Offset));
2508 }
2509 
2510 /// If \arg Loc is a loaded location from the preamble, returns
2511 /// the corresponding local location of the main file, otherwise it returns
2512 /// \arg Loc.
2513 SourceLocation ASTUnit::mapLocationFromPreamble(SourceLocation Loc) const {
2514   FileID PreambleID;
2515   if (SourceMgr)
2516     PreambleID = SourceMgr->getPreambleFileID();
2517 
2518   if (Loc.isInvalid() || !Preamble || PreambleID.isInvalid())
2519     return Loc;
2520 
2521   unsigned Offs;
2522   if (SourceMgr->isInFileID(Loc, PreambleID, &Offs) && Offs < Preamble->getBounds().Size) {
2523     SourceLocation FileLoc
2524         = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
2525     return FileLoc.getLocWithOffset(Offs);
2526   }
2527 
2528   return Loc;
2529 }
2530 
2531 /// If \arg Loc is a local location of the main file but inside the
2532 /// preamble chunk, returns the corresponding loaded location from the
2533 /// preamble, otherwise it returns \arg Loc.
2534 SourceLocation ASTUnit::mapLocationToPreamble(SourceLocation Loc) const {
2535   FileID PreambleID;
2536   if (SourceMgr)
2537     PreambleID = SourceMgr->getPreambleFileID();
2538 
2539   if (Loc.isInvalid() || !Preamble || PreambleID.isInvalid())
2540     return Loc;
2541 
2542   unsigned Offs;
2543   if (SourceMgr->isInFileID(Loc, SourceMgr->getMainFileID(), &Offs) &&
2544       Offs < Preamble->getBounds().Size) {
2545     SourceLocation FileLoc = SourceMgr->getLocForStartOfFile(PreambleID);
2546     return FileLoc.getLocWithOffset(Offs);
2547   }
2548 
2549   return Loc;
2550 }
2551 
2552 bool ASTUnit::isInPreambleFileID(SourceLocation Loc) const {
2553   FileID FID;
2554   if (SourceMgr)
2555     FID = SourceMgr->getPreambleFileID();
2556 
2557   if (Loc.isInvalid() || FID.isInvalid())
2558     return false;
2559 
2560   return SourceMgr->isInFileID(Loc, FID);
2561 }
2562 
2563 bool ASTUnit::isInMainFileID(SourceLocation Loc) const {
2564   FileID FID;
2565   if (SourceMgr)
2566     FID = SourceMgr->getMainFileID();
2567 
2568   if (Loc.isInvalid() || FID.isInvalid())
2569     return false;
2570 
2571   return SourceMgr->isInFileID(Loc, FID);
2572 }
2573 
2574 SourceLocation ASTUnit::getEndOfPreambleFileID() const {
2575   FileID FID;
2576   if (SourceMgr)
2577     FID = SourceMgr->getPreambleFileID();
2578 
2579   if (FID.isInvalid())
2580     return {};
2581 
2582   return SourceMgr->getLocForEndOfFile(FID);
2583 }
2584 
2585 SourceLocation ASTUnit::getStartOfMainFileID() const {
2586   FileID FID;
2587   if (SourceMgr)
2588     FID = SourceMgr->getMainFileID();
2589 
2590   if (FID.isInvalid())
2591     return {};
2592 
2593   return SourceMgr->getLocForStartOfFile(FID);
2594 }
2595 
2596 llvm::iterator_range<PreprocessingRecord::iterator>
2597 ASTUnit::getLocalPreprocessingEntities() const {
2598   if (isMainFileAST()) {
2599     serialization::ModuleFile &
2600       Mod = Reader->getModuleManager().getPrimaryModule();
2601     return Reader->getModulePreprocessedEntities(Mod);
2602   }
2603 
2604   if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
2605     return llvm::make_range(PPRec->local_begin(), PPRec->local_end());
2606 
2607   return llvm::make_range(PreprocessingRecord::iterator(),
2608                           PreprocessingRecord::iterator());
2609 }
2610 
2611 bool ASTUnit::visitLocalTopLevelDecls(void *context, DeclVisitorFn Fn) {
2612   if (isMainFileAST()) {
2613     serialization::ModuleFile &
2614       Mod = Reader->getModuleManager().getPrimaryModule();
2615     for (const auto *D : Reader->getModuleFileLevelDecls(Mod)) {
2616       if (!Fn(context, D))
2617         return false;
2618     }
2619 
2620     return true;
2621   }
2622 
2623   for (ASTUnit::top_level_iterator TL = top_level_begin(),
2624                                 TLEnd = top_level_end();
2625          TL != TLEnd; ++TL) {
2626     if (!Fn(context, *TL))
2627       return false;
2628   }
2629 
2630   return true;
2631 }
2632 
2633 const FileEntry *ASTUnit::getPCHFile() {
2634   if (!Reader)
2635     return nullptr;
2636 
2637   serialization::ModuleFile *Mod = nullptr;
2638   Reader->getModuleManager().visit([&Mod](serialization::ModuleFile &M) {
2639     switch (M.Kind) {
2640     case serialization::MK_ImplicitModule:
2641     case serialization::MK_ExplicitModule:
2642     case serialization::MK_PrebuiltModule:
2643       return true; // skip dependencies.
2644     case serialization::MK_PCH:
2645       Mod = &M;
2646       return true; // found it.
2647     case serialization::MK_Preamble:
2648       return false; // look in dependencies.
2649     case serialization::MK_MainFile:
2650       return false; // look in dependencies.
2651     }
2652 
2653     return true;
2654   });
2655   if (Mod)
2656     return Mod->File;
2657 
2658   return nullptr;
2659 }
2660 
2661 bool ASTUnit::isModuleFile() const {
2662   return isMainFileAST() && getLangOpts().isCompilingModule();
2663 }
2664 
2665 InputKind ASTUnit::getInputKind() const {
2666   auto &LangOpts = getLangOpts();
2667 
2668   Language Lang;
2669   if (LangOpts.OpenCL)
2670     Lang = Language::OpenCL;
2671   else if (LangOpts.CUDA)
2672     Lang = Language::CUDA;
2673   else if (LangOpts.RenderScript)
2674     Lang = Language::RenderScript;
2675   else if (LangOpts.CPlusPlus)
2676     Lang = LangOpts.ObjC ? Language::ObjCXX : Language::CXX;
2677   else
2678     Lang = LangOpts.ObjC ? Language::ObjC : Language::C;
2679 
2680   InputKind::Format Fmt = InputKind::Source;
2681   if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap)
2682     Fmt = InputKind::ModuleMap;
2683 
2684   // We don't know if input was preprocessed. Assume not.
2685   bool PP = false;
2686 
2687   return InputKind(Lang, Fmt, PP);
2688 }
2689 
2690 #ifndef NDEBUG
2691 ASTUnit::ConcurrencyState::ConcurrencyState() {
2692   Mutex = new std::recursive_mutex;
2693 }
2694 
2695 ASTUnit::ConcurrencyState::~ConcurrencyState() {
2696   delete static_cast<std::recursive_mutex *>(Mutex);
2697 }
2698 
2699 void ASTUnit::ConcurrencyState::start() {
2700   bool acquired = static_cast<std::recursive_mutex *>(Mutex)->try_lock();
2701   assert(acquired && "Concurrent access to ASTUnit!");
2702 }
2703 
2704 void ASTUnit::ConcurrencyState::finish() {
2705   static_cast<std::recursive_mutex *>(Mutex)->unlock();
2706 }
2707 
2708 #else // NDEBUG
2709 
2710 ASTUnit::ConcurrencyState::ConcurrencyState() { Mutex = nullptr; }
2711 ASTUnit::ConcurrencyState::~ConcurrencyState() {}
2712 void ASTUnit::ConcurrencyState::start() {}
2713 void ASTUnit::ConcurrencyState::finish() {}
2714 
2715 #endif // NDEBUG
2716