xref: /freebsd/contrib/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp (revision 9a7f7b2480a9f332f7f7f19ee00dc1606dfd2265) 
1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 
15 #include "clang/AST/DeclContextInternals.h"
16 #include "clang/CodeGen/BackendUtil.h"
17 #include "clang/CodeGen/CodeGenAction.h"
18 #include "clang/CodeGen/ModuleBuilder.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/FrontendAction.h"
21 #include "clang/FrontendTool/Utils.h"
22 #include "clang/Interpreter/Interpreter.h"
23 #include "clang/Parse/Parser.h"
24 #include "clang/Sema/Sema.h"
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29 
30 #include <sstream>
31 
32 namespace clang {
33 
34 class IncrementalASTConsumer final : public ASTConsumer {
35   Interpreter &Interp;
36   std::unique_ptr<ASTConsumer> Consumer;
37 
38 public:
39   IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
40       : Interp(InterpRef), Consumer(std::move(C)) {}
41 
42   bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
43     if (DGR.isNull())
44       return true;
45     if (!Consumer)
46       return true;
47 
48     for (Decl *D : DGR)
49       if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
50           TSD && TSD->isSemiMissing())
51         TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
52 
53     return Consumer->HandleTopLevelDecl(DGR);
54   }
55   void HandleTranslationUnit(ASTContext &Ctx) override final {
56     Consumer->HandleTranslationUnit(Ctx);
57   }
58   void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
59     Consumer->HandleInlineFunctionDefinition(D);
60   }
61   void HandleInterestingDecl(DeclGroupRef D) override final {
62     Consumer->HandleInterestingDecl(D);
63   }
64   void HandleTagDeclDefinition(TagDecl *D) override final {
65     Consumer->HandleTagDeclDefinition(D);
66   }
67   void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
68     Consumer->HandleTagDeclRequiredDefinition(D);
69   }
70   void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
71     Consumer->HandleCXXImplicitFunctionInstantiation(D);
72   }
73   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
74     Consumer->HandleTopLevelDeclInObjCContainer(D);
75   }
76   void HandleImplicitImportDecl(ImportDecl *D) override final {
77     Consumer->HandleImplicitImportDecl(D);
78   }
79   void CompleteTentativeDefinition(VarDecl *D) override final {
80     Consumer->CompleteTentativeDefinition(D);
81   }
82   void CompleteExternalDeclaration(VarDecl *D) override final {
83     Consumer->CompleteExternalDeclaration(D);
84   }
85   void AssignInheritanceModel(CXXRecordDecl *RD) override final {
86     Consumer->AssignInheritanceModel(RD);
87   }
88   void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
89     Consumer->HandleCXXStaticMemberVarInstantiation(D);
90   }
91   void HandleVTable(CXXRecordDecl *RD) override final {
92     Consumer->HandleVTable(RD);
93   }
94   ASTMutationListener *GetASTMutationListener() override final {
95     return Consumer->GetASTMutationListener();
96   }
97   ASTDeserializationListener *GetASTDeserializationListener() override final {
98     return Consumer->GetASTDeserializationListener();
99   }
100   void PrintStats() override final { Consumer->PrintStats(); }
101   bool shouldSkipFunctionBody(Decl *D) override final {
102     return Consumer->shouldSkipFunctionBody(D);
103   }
104   static bool classof(const clang::ASTConsumer *) { return true; }
105 };
106 
107 /// A custom action enabling the incremental processing functionality.
108 ///
109 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
110 /// sees a call to \p EndSourceFile it deletes some of the important objects
111 /// such as \p Preprocessor and \p Sema assuming no further input will come.
112 ///
113 /// \p IncrementalAction ensures it keep its underlying action's objects alive
114 /// as long as the \p IncrementalParser needs them.
115 ///
116 class IncrementalAction : public WrapperFrontendAction {
117 private:
118   bool IsTerminating = false;
119 
120 public:
121   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
122                     llvm::Error &Err)
123       : WrapperFrontendAction([&]() {
124           llvm::ErrorAsOutParameter EAO(&Err);
125           std::unique_ptr<FrontendAction> Act;
126           switch (CI.getFrontendOpts().ProgramAction) {
127           default:
128             Err = llvm::createStringError(
129                 std::errc::state_not_recoverable,
130                 "Driver initialization failed. "
131                 "Incremental mode for action %d is not supported",
132                 CI.getFrontendOpts().ProgramAction);
133             return Act;
134           case frontend::ASTDump:
135             [[fallthrough]];
136           case frontend::ASTPrint:
137             [[fallthrough]];
138           case frontend::ParseSyntaxOnly:
139             Act = CreateFrontendAction(CI);
140             break;
141           case frontend::PluginAction:
142             [[fallthrough]];
143           case frontend::EmitAssembly:
144             [[fallthrough]];
145           case frontend::EmitBC:
146             [[fallthrough]];
147           case frontend::EmitObj:
148             [[fallthrough]];
149           case frontend::PrintPreprocessedInput:
150             [[fallthrough]];
151           case frontend::EmitLLVMOnly:
152             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
153             break;
154           }
155           return Act;
156         }()) {}
157   FrontendAction *getWrapped() const { return WrappedAction.get(); }
158   TranslationUnitKind getTranslationUnitKind() override {
159     return TU_Incremental;
160   }
161 
162   void ExecuteAction() override {
163     CompilerInstance &CI = getCompilerInstance();
164     assert(CI.hasPreprocessor() && "No PP!");
165 
166     // Use a code completion consumer?
167     CodeCompleteConsumer *CompletionConsumer = nullptr;
168     if (CI.hasCodeCompletionConsumer())
169       CompletionConsumer = &CI.getCodeCompletionConsumer();
170 
171     Preprocessor &PP = CI.getPreprocessor();
172     PP.EnterMainSourceFile();
173 
174     if (!CI.hasSema())
175       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
176   }
177 
178   // Do not terminate after processing the input. This allows us to keep various
179   // clang objects alive and to incrementally grow the current TU.
180   void EndSourceFile() override {
181     // The WrappedAction can be nullptr if we issued an error in the ctor.
182     if (IsTerminating && getWrapped())
183       WrapperFrontendAction::EndSourceFile();
184   }
185 
186   void FinalizeAction() {
187     assert(!IsTerminating && "Already finalized!");
188     IsTerminating = true;
189     EndSourceFile();
190   }
191 };
192 
193 CodeGenerator *IncrementalParser::getCodeGen() const {
194   FrontendAction *WrappedAct = Act->getWrapped();
195   if (!WrappedAct->hasIRSupport())
196     return nullptr;
197   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
198 }
199 
200 IncrementalParser::IncrementalParser() {}
201 
202 IncrementalParser::IncrementalParser(Interpreter &Interp,
203                                      std::unique_ptr<CompilerInstance> Instance,
204                                      llvm::LLVMContext &LLVMCtx,
205                                      llvm::Error &Err)
206     : CI(std::move(Instance)) {
207   llvm::ErrorAsOutParameter EAO(&Err);
208   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
209   if (Err)
210     return;
211   CI->ExecuteAction(*Act);
212   std::unique_ptr<ASTConsumer> IncrConsumer =
213       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
214   CI->setASTConsumer(std::move(IncrConsumer));
215   Consumer = &CI->getASTConsumer();
216   P.reset(
217       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
218   P->Initialize();
219 
220   // An initial PTU is needed as CUDA includes some headers automatically
221   auto PTU = ParseOrWrapTopLevelDecl();
222   if (auto E = PTU.takeError()) {
223     consumeError(std::move(E)); // FIXME
224     return;                     // PTU.takeError();
225   }
226 
227   if (CodeGenerator *CG = getCodeGen()) {
228     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
229     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
230                     M->getContext());
231     PTU->TheModule = std::move(M);
232     assert(PTU->TheModule && "Failed to create initial PTU");
233   }
234 }
235 
236 IncrementalParser::~IncrementalParser() {
237   P.reset();
238   Act->FinalizeAction();
239 }
240 
241 llvm::Expected<PartialTranslationUnit &>
242 IncrementalParser::ParseOrWrapTopLevelDecl() {
243   // Recover resources if we crash before exiting this method.
244   Sema &S = CI->getSema();
245   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
246   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
247   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
248 
249   PTUs.emplace_back(PartialTranslationUnit());
250   PartialTranslationUnit &LastPTU = PTUs.back();
251   // Add a new PTU.
252   ASTContext &C = S.getASTContext();
253   C.addTranslationUnitDecl();
254   LastPTU.TUPart = C.getTranslationUnitDecl();
255 
256   // Skip previous eof due to last incremental input.
257   if (P->getCurToken().is(tok::annot_repl_input_end)) {
258     P->ConsumeAnyToken();
259     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
260     // might want to do that around HandleEndOfTranslationUnit.
261     P->ExitScope();
262     S.CurContext = nullptr;
263     // Start a new PTU.
264     P->EnterScope(Scope::DeclScope);
265     S.ActOnTranslationUnitScope(P->getCurScope());
266   }
267 
268   Parser::DeclGroupPtrTy ADecl;
269   Sema::ModuleImportState ImportState;
270   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
271        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
272     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
273       return llvm::make_error<llvm::StringError>("Parsing failed. "
274                                                  "The consumer rejected a decl",
275                                                  std::error_code());
276   }
277 
278   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
279   if (Diags.hasErrorOccurred()) {
280     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
281                                             nullptr};
282     CleanUpPTU(MostRecentPTU);
283 
284     Diags.Reset(/*soft=*/true);
285     Diags.getClient()->clear();
286     return llvm::make_error<llvm::StringError>("Parsing failed.",
287                                                std::error_code());
288   }
289 
290   // Process any TopLevelDecls generated by #pragma weak.
291   for (Decl *D : S.WeakTopLevelDecls()) {
292     DeclGroupRef DGR(D);
293     Consumer->HandleTopLevelDecl(DGR);
294   }
295 
296   LocalInstantiations.perform();
297   GlobalInstantiations.perform();
298 
299   Consumer->HandleTranslationUnit(C);
300 
301   return LastPTU;
302 }
303 
304 llvm::Expected<PartialTranslationUnit &>
305 IncrementalParser::Parse(llvm::StringRef input) {
306   Preprocessor &PP = CI->getPreprocessor();
307   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
308 
309   std::ostringstream SourceName;
310   SourceName << "input_line_" << InputCount++;
311 
312   // Create an uninitialized memory buffer, copy code in and append "\n"
313   size_t InputSize = input.size(); // don't include trailing 0
314   // MemBuffer size should *not* include terminating zero
315   std::unique_ptr<llvm::MemoryBuffer> MB(
316       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
317                                                         SourceName.str()));
318   char *MBStart = const_cast<char *>(MB->getBufferStart());
319   memcpy(MBStart, input.data(), InputSize);
320   MBStart[InputSize] = '\n';
321 
322   SourceManager &SM = CI->getSourceManager();
323 
324   // FIXME: Create SourceLocation, which will allow clang to order the overload
325   // candidates for example
326   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
327 
328   // Create FileID for the current buffer.
329   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
330                                /*LoadedOffset=*/0, NewLoc);
331 
332   // NewLoc only used for diags.
333   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
334     return llvm::make_error<llvm::StringError>("Parsing failed. "
335                                                "Cannot enter source file.",
336                                                std::error_code());
337 
338   auto PTU = ParseOrWrapTopLevelDecl();
339   if (!PTU)
340     return PTU.takeError();
341 
342   if (PP.getLangOpts().DelayedTemplateParsing) {
343     // Microsoft-specific:
344     // Late parsed templates can leave unswallowed "macro"-like tokens.
345     // They will seriously confuse the Parser when entering the next
346     // source file. So lex until we are EOF.
347     Token Tok;
348     do {
349       PP.Lex(Tok);
350     } while (Tok.isNot(tok::annot_repl_input_end));
351   } else {
352     Token AssertTok;
353     PP.Lex(AssertTok);
354     assert(AssertTok.is(tok::annot_repl_input_end) &&
355            "Lexer must be EOF when starting incremental parse!");
356   }
357 
358   if (std::unique_ptr<llvm::Module> M = GenModule())
359     PTU->TheModule = std::move(M);
360 
361   return PTU;
362 }
363 
364 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
365   static unsigned ID = 0;
366   if (CodeGenerator *CG = getCodeGen()) {
367     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
368     CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
369     return M;
370   }
371   return nullptr;
372 }
373 
374 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
375   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
376   TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
377   if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
378     for (auto I = Map->begin(); I != Map->end(); ++I) {
379       StoredDeclsList &List = I->second;
380       DeclContextLookupResult R = List.getLookupResult();
381       for (NamedDecl *D : R) {
382         if (D->getTranslationUnitDecl() == MostRecentTU) {
383           List.remove(D);
384         }
385       }
386       if (List.isNull())
387         Map->erase(I);
388     }
389   }
390 }
391 
392 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
393   CodeGenerator *CG = getCodeGen();
394   assert(CG);
395   return CG->GetMangledName(GD);
396 }
397 } // end namespace clang
398