xref: /freebsd/contrib/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp (revision b64c5a0ace59af62eff52bfe110a521dc73c937b) 
1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 
15 #include "clang/AST/DeclContextInternals.h"
16 #include "clang/CodeGen/BackendUtil.h"
17 #include "clang/CodeGen/CodeGenAction.h"
18 #include "clang/CodeGen/ModuleBuilder.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/FrontendAction.h"
21 #include "clang/FrontendTool/Utils.h"
22 #include "clang/Interpreter/Interpreter.h"
23 #include "clang/Parse/Parser.h"
24 #include "clang/Sema/Sema.h"
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29 
30 #include <sstream>
31 
32 namespace clang {
33 
34 class IncrementalASTConsumer final : public ASTConsumer {
35   Interpreter &Interp;
36   std::unique_ptr<ASTConsumer> Consumer;
37 
38 public:
39   IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
40       : Interp(InterpRef), Consumer(std::move(C)) {}
41 
42   bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
43     if (DGR.isNull())
44       return true;
45     if (!Consumer)
46       return true;
47 
48     for (Decl *D : DGR)
49       if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
50           TSD && TSD->isSemiMissing())
51         TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
52 
53     return Consumer->HandleTopLevelDecl(DGR);
54   }
55   void HandleTranslationUnit(ASTContext &Ctx) override final {
56     Consumer->HandleTranslationUnit(Ctx);
57   }
58   void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
59     Consumer->HandleInlineFunctionDefinition(D);
60   }
61   void HandleInterestingDecl(DeclGroupRef D) override final {
62     Consumer->HandleInterestingDecl(D);
63   }
64   void HandleTagDeclDefinition(TagDecl *D) override final {
65     Consumer->HandleTagDeclDefinition(D);
66   }
67   void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
68     Consumer->HandleTagDeclRequiredDefinition(D);
69   }
70   void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
71     Consumer->HandleCXXImplicitFunctionInstantiation(D);
72   }
73   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
74     Consumer->HandleTopLevelDeclInObjCContainer(D);
75   }
76   void HandleImplicitImportDecl(ImportDecl *D) override final {
77     Consumer->HandleImplicitImportDecl(D);
78   }
79   void CompleteTentativeDefinition(VarDecl *D) override final {
80     Consumer->CompleteTentativeDefinition(D);
81   }
82   void CompleteExternalDeclaration(DeclaratorDecl *D) override final {
83     Consumer->CompleteExternalDeclaration(D);
84   }
85   void AssignInheritanceModel(CXXRecordDecl *RD) override final {
86     Consumer->AssignInheritanceModel(RD);
87   }
88   void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
89     Consumer->HandleCXXStaticMemberVarInstantiation(D);
90   }
91   void HandleVTable(CXXRecordDecl *RD) override final {
92     Consumer->HandleVTable(RD);
93   }
94   ASTMutationListener *GetASTMutationListener() override final {
95     return Consumer->GetASTMutationListener();
96   }
97   ASTDeserializationListener *GetASTDeserializationListener() override final {
98     return Consumer->GetASTDeserializationListener();
99   }
100   void PrintStats() override final { Consumer->PrintStats(); }
101   bool shouldSkipFunctionBody(Decl *D) override final {
102     return Consumer->shouldSkipFunctionBody(D);
103   }
104   static bool classof(const clang::ASTConsumer *) { return true; }
105 };
106 
107 /// A custom action enabling the incremental processing functionality.
108 ///
109 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
110 /// sees a call to \p EndSourceFile it deletes some of the important objects
111 /// such as \p Preprocessor and \p Sema assuming no further input will come.
112 ///
113 /// \p IncrementalAction ensures it keep its underlying action's objects alive
114 /// as long as the \p IncrementalParser needs them.
115 ///
116 class IncrementalAction : public WrapperFrontendAction {
117 private:
118   bool IsTerminating = false;
119 
120 public:
121   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
122                     llvm::Error &Err)
123       : WrapperFrontendAction([&]() {
124           llvm::ErrorAsOutParameter EAO(&Err);
125           std::unique_ptr<FrontendAction> Act;
126           switch (CI.getFrontendOpts().ProgramAction) {
127           default:
128             Err = llvm::createStringError(
129                 std::errc::state_not_recoverable,
130                 "Driver initialization failed. "
131                 "Incremental mode for action %d is not supported",
132                 CI.getFrontendOpts().ProgramAction);
133             return Act;
134           case frontend::ASTDump:
135             [[fallthrough]];
136           case frontend::ASTPrint:
137             [[fallthrough]];
138           case frontend::ParseSyntaxOnly:
139             Act = CreateFrontendAction(CI);
140             break;
141           case frontend::PluginAction:
142             [[fallthrough]];
143           case frontend::EmitAssembly:
144             [[fallthrough]];
145           case frontend::EmitBC:
146             [[fallthrough]];
147           case frontend::EmitObj:
148             [[fallthrough]];
149           case frontend::PrintPreprocessedInput:
150             [[fallthrough]];
151           case frontend::EmitLLVMOnly:
152             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
153             break;
154           }
155           return Act;
156         }()) {}
157   FrontendAction *getWrapped() const { return WrappedAction.get(); }
158   TranslationUnitKind getTranslationUnitKind() override {
159     return TU_Incremental;
160   }
161 
162   void ExecuteAction() override {
163     CompilerInstance &CI = getCompilerInstance();
164     assert(CI.hasPreprocessor() && "No PP!");
165 
166     // Use a code completion consumer?
167     CodeCompleteConsumer *CompletionConsumer = nullptr;
168     if (CI.hasCodeCompletionConsumer())
169       CompletionConsumer = &CI.getCodeCompletionConsumer();
170 
171     Preprocessor &PP = CI.getPreprocessor();
172     PP.EnterMainSourceFile();
173 
174     if (!CI.hasSema())
175       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
176   }
177 
178   // Do not terminate after processing the input. This allows us to keep various
179   // clang objects alive and to incrementally grow the current TU.
180   void EndSourceFile() override {
181     // The WrappedAction can be nullptr if we issued an error in the ctor.
182     if (IsTerminating && getWrapped())
183       WrapperFrontendAction::EndSourceFile();
184   }
185 
186   void FinalizeAction() {
187     assert(!IsTerminating && "Already finalized!");
188     IsTerminating = true;
189     EndSourceFile();
190   }
191 };
192 
193 CodeGenerator *IncrementalParser::getCodeGen() const {
194   FrontendAction *WrappedAct = Act->getWrapped();
195   if (!WrappedAct->hasIRSupport())
196     return nullptr;
197   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
198 }
199 
200 IncrementalParser::IncrementalParser() {}
201 
202 IncrementalParser::IncrementalParser(Interpreter &Interp,
203                                      std::unique_ptr<CompilerInstance> Instance,
204                                      llvm::LLVMContext &LLVMCtx,
205                                      llvm::Error &Err)
206     : CI(std::move(Instance)) {
207   llvm::ErrorAsOutParameter EAO(&Err);
208   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
209   if (Err)
210     return;
211   CI->ExecuteAction(*Act);
212 
213   if (getCodeGen())
214     CachedInCodeGenModule = GenModule();
215 
216   std::unique_ptr<ASTConsumer> IncrConsumer =
217       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
218   CI->setASTConsumer(std::move(IncrConsumer));
219   Consumer = &CI->getASTConsumer();
220   P.reset(
221       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
222   P->Initialize();
223 
224   // An initial PTU is needed as CUDA includes some headers automatically
225   auto PTU = ParseOrWrapTopLevelDecl();
226   if (auto E = PTU.takeError()) {
227     consumeError(std::move(E)); // FIXME
228     return;                     // PTU.takeError();
229   }
230 
231   if (getCodeGen()) {
232     PTU->TheModule = GenModule();
233     assert(PTU->TheModule && "Failed to create initial PTU");
234   }
235 }
236 
237 IncrementalParser::~IncrementalParser() {
238   P.reset();
239   Act->FinalizeAction();
240 }
241 
242 llvm::Expected<PartialTranslationUnit &>
243 IncrementalParser::ParseOrWrapTopLevelDecl() {
244   // Recover resources if we crash before exiting this method.
245   Sema &S = CI->getSema();
246   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
247   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
248   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
249 
250   PTUs.emplace_back(PartialTranslationUnit());
251   PartialTranslationUnit &LastPTU = PTUs.back();
252   // Add a new PTU.
253   ASTContext &C = S.getASTContext();
254   C.addTranslationUnitDecl();
255   LastPTU.TUPart = C.getTranslationUnitDecl();
256 
257   // Skip previous eof due to last incremental input.
258   if (P->getCurToken().is(tok::annot_repl_input_end)) {
259     P->ConsumeAnyToken();
260     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
261     // might want to do that around HandleEndOfTranslationUnit.
262     P->ExitScope();
263     S.CurContext = nullptr;
264     // Start a new PTU.
265     P->EnterScope(Scope::DeclScope);
266     S.ActOnTranslationUnitScope(P->getCurScope());
267   }
268 
269   Parser::DeclGroupPtrTy ADecl;
270   Sema::ModuleImportState ImportState;
271   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
272        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
273     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
274       return llvm::make_error<llvm::StringError>("Parsing failed. "
275                                                  "The consumer rejected a decl",
276                                                  std::error_code());
277   }
278 
279   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
280   if (Diags.hasErrorOccurred()) {
281     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
282                                             nullptr};
283     CleanUpPTU(MostRecentPTU);
284 
285     Diags.Reset(/*soft=*/true);
286     Diags.getClient()->clear();
287     return llvm::make_error<llvm::StringError>("Parsing failed.",
288                                                std::error_code());
289   }
290 
291   // Process any TopLevelDecls generated by #pragma weak.
292   for (Decl *D : S.WeakTopLevelDecls()) {
293     DeclGroupRef DGR(D);
294     Consumer->HandleTopLevelDecl(DGR);
295   }
296 
297   LocalInstantiations.perform();
298   GlobalInstantiations.perform();
299 
300   Consumer->HandleTranslationUnit(C);
301 
302   return LastPTU;
303 }
304 
305 llvm::Expected<PartialTranslationUnit &>
306 IncrementalParser::Parse(llvm::StringRef input) {
307   Preprocessor &PP = CI->getPreprocessor();
308   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
309 
310   std::ostringstream SourceName;
311   SourceName << "input_line_" << InputCount++;
312 
313   // Create an uninitialized memory buffer, copy code in and append "\n"
314   size_t InputSize = input.size(); // don't include trailing 0
315   // MemBuffer size should *not* include terminating zero
316   std::unique_ptr<llvm::MemoryBuffer> MB(
317       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
318                                                         SourceName.str()));
319   char *MBStart = const_cast<char *>(MB->getBufferStart());
320   memcpy(MBStart, input.data(), InputSize);
321   MBStart[InputSize] = '\n';
322 
323   SourceManager &SM = CI->getSourceManager();
324 
325   // FIXME: Create SourceLocation, which will allow clang to order the overload
326   // candidates for example
327   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
328 
329   // Create FileID for the current buffer.
330   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
331                                /*LoadedOffset=*/0, NewLoc);
332 
333   // NewLoc only used for diags.
334   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
335     return llvm::make_error<llvm::StringError>("Parsing failed. "
336                                                "Cannot enter source file.",
337                                                std::error_code());
338 
339   auto PTU = ParseOrWrapTopLevelDecl();
340   if (!PTU)
341     return PTU.takeError();
342 
343   if (PP.getLangOpts().DelayedTemplateParsing) {
344     // Microsoft-specific:
345     // Late parsed templates can leave unswallowed "macro"-like tokens.
346     // They will seriously confuse the Parser when entering the next
347     // source file. So lex until we are EOF.
348     Token Tok;
349     do {
350       PP.Lex(Tok);
351     } while (Tok.isNot(tok::annot_repl_input_end));
352   } else {
353     Token AssertTok;
354     PP.Lex(AssertTok);
355     assert(AssertTok.is(tok::annot_repl_input_end) &&
356            "Lexer must be EOF when starting incremental parse!");
357   }
358 
359   if (std::unique_ptr<llvm::Module> M = GenModule())
360     PTU->TheModule = std::move(M);
361 
362   return PTU;
363 }
364 
365 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
366   static unsigned ID = 0;
367   if (CodeGenerator *CG = getCodeGen()) {
368     // Clang's CodeGen is designed to work with a single llvm::Module. In many
369     // cases for convenience various CodeGen parts have a reference to the
370     // llvm::Module (TheModule or Module) which does not change when a new
371     // module is pushed. However, the execution engine wants to take ownership
372     // of the module which does not map well to CodeGen's design. To work this
373     // around we created an empty module to make CodeGen happy. We should make
374     // sure it always stays empty.
375     assert((!CachedInCodeGenModule ||
376             (CachedInCodeGenModule->empty() &&
377              CachedInCodeGenModule->global_empty() &&
378              CachedInCodeGenModule->alias_empty() &&
379              CachedInCodeGenModule->ifunc_empty())) &&
380            "CodeGen wrote to a readonly module");
381     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
382     CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
383     return M;
384   }
385   return nullptr;
386 }
387 
388 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
389   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
390   if (StoredDeclsMap *Map = MostRecentTU->getPrimaryContext()->getLookupPtr()) {
391     for (auto &&[Key, List] : *Map) {
392       DeclContextLookupResult R = List.getLookupResult();
393       std::vector<NamedDecl *> NamedDeclsToRemove;
394       bool RemoveAll = true;
395       for (NamedDecl *D : R) {
396         if (D->getTranslationUnitDecl() == MostRecentTU)
397           NamedDeclsToRemove.push_back(D);
398         else
399           RemoveAll = false;
400       }
401       if (LLVM_LIKELY(RemoveAll)) {
402         Map->erase(Key);
403       } else {
404         for (NamedDecl *D : NamedDeclsToRemove)
405           List.remove(D);
406       }
407     }
408   }
409 
410   // FIXME: We should de-allocate MostRecentTU
411   for (Decl *D : MostRecentTU->decls()) {
412     auto *ND = dyn_cast<NamedDecl>(D);
413     if (!ND)
414       continue;
415     // Check if we need to clean up the IdResolver chain.
416     if (ND->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC &&
417         !D->getLangOpts().CPlusPlus)
418       getCI()->getSema().IdResolver.RemoveDecl(ND);
419   }
420 }
421 
422 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
423   CodeGenerator *CG = getCodeGen();
424   assert(CG);
425   return CG->GetMangledName(GD);
426 }
427 } // end namespace clang
428