xref: /freebsd/contrib/llvm-project/clang/lib/Interpreter/IncrementalParser.cpp (revision 78cd75393ec79565c63927bf200f06f839a1dc05) 
1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 #include "clang/AST/DeclContextInternals.h"
15 #include "clang/CodeGen/BackendUtil.h"
16 #include "clang/CodeGen/CodeGenAction.h"
17 #include "clang/CodeGen/ModuleBuilder.h"
18 #include "clang/Frontend/CompilerInstance.h"
19 #include "clang/Frontend/FrontendAction.h"
20 #include "clang/FrontendTool/Utils.h"
21 #include "clang/Interpreter/Interpreter.h"
22 #include "clang/Parse/Parser.h"
23 #include "clang/Sema/Sema.h"
24 #include "llvm/Option/ArgList.h"
25 #include "llvm/Support/CrashRecoveryContext.h"
26 #include "llvm/Support/Error.h"
27 #include "llvm/Support/Timer.h"
28 
29 #include <sstream>
30 
31 namespace clang {
32 
33 class IncrementalASTConsumer final : public ASTConsumer {
34   Interpreter &Interp;
35   std::unique_ptr<ASTConsumer> Consumer;
36 
37 public:
38   IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
39       : Interp(InterpRef), Consumer(std::move(C)) {}
40 
41   bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
42     if (DGR.isNull())
43       return true;
44     if (!Consumer)
45       return true;
46 
47     for (Decl *D : DGR)
48       if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
49           TSD && TSD->isSemiMissing())
50         TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
51 
52     return Consumer->HandleTopLevelDecl(DGR);
53   }
54   void HandleTranslationUnit(ASTContext &Ctx) override final {
55     Consumer->HandleTranslationUnit(Ctx);
56   }
57   void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
58     Consumer->HandleInlineFunctionDefinition(D);
59   }
60   void HandleInterestingDecl(DeclGroupRef D) override final {
61     Consumer->HandleInterestingDecl(D);
62   }
63   void HandleTagDeclDefinition(TagDecl *D) override final {
64     Consumer->HandleTagDeclDefinition(D);
65   }
66   void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
67     Consumer->HandleTagDeclRequiredDefinition(D);
68   }
69   void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
70     Consumer->HandleCXXImplicitFunctionInstantiation(D);
71   }
72   void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
73     Consumer->HandleTopLevelDeclInObjCContainer(D);
74   }
75   void HandleImplicitImportDecl(ImportDecl *D) override final {
76     Consumer->HandleImplicitImportDecl(D);
77   }
78   void CompleteTentativeDefinition(VarDecl *D) override final {
79     Consumer->CompleteTentativeDefinition(D);
80   }
81   void CompleteExternalDeclaration(VarDecl *D) override final {
82     Consumer->CompleteExternalDeclaration(D);
83   }
84   void AssignInheritanceModel(CXXRecordDecl *RD) override final {
85     Consumer->AssignInheritanceModel(RD);
86   }
87   void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
88     Consumer->HandleCXXStaticMemberVarInstantiation(D);
89   }
90   void HandleVTable(CXXRecordDecl *RD) override final {
91     Consumer->HandleVTable(RD);
92   }
93   ASTMutationListener *GetASTMutationListener() override final {
94     return Consumer->GetASTMutationListener();
95   }
96   ASTDeserializationListener *GetASTDeserializationListener() override final {
97     return Consumer->GetASTDeserializationListener();
98   }
99   void PrintStats() override final { Consumer->PrintStats(); }
100   bool shouldSkipFunctionBody(Decl *D) override final {
101     return Consumer->shouldSkipFunctionBody(D);
102   }
103   static bool classof(const clang::ASTConsumer *) { return true; }
104 };
105 
106 /// A custom action enabling the incremental processing functionality.
107 ///
108 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
109 /// sees a call to \p EndSourceFile it deletes some of the important objects
110 /// such as \p Preprocessor and \p Sema assuming no further input will come.
111 ///
112 /// \p IncrementalAction ensures it keep its underlying action's objects alive
113 /// as long as the \p IncrementalParser needs them.
114 ///
115 class IncrementalAction : public WrapperFrontendAction {
116 private:
117   bool IsTerminating = false;
118 
119 public:
120   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
121                     llvm::Error &Err)
122       : WrapperFrontendAction([&]() {
123           llvm::ErrorAsOutParameter EAO(&Err);
124           std::unique_ptr<FrontendAction> Act;
125           switch (CI.getFrontendOpts().ProgramAction) {
126           default:
127             Err = llvm::createStringError(
128                 std::errc::state_not_recoverable,
129                 "Driver initialization failed. "
130                 "Incremental mode for action %d is not supported",
131                 CI.getFrontendOpts().ProgramAction);
132             return Act;
133           case frontend::ASTDump:
134             [[fallthrough]];
135           case frontend::ASTPrint:
136             [[fallthrough]];
137           case frontend::ParseSyntaxOnly:
138             Act = CreateFrontendAction(CI);
139             break;
140           case frontend::PluginAction:
141             [[fallthrough]];
142           case frontend::EmitAssembly:
143             [[fallthrough]];
144           case frontend::EmitBC:
145             [[fallthrough]];
146           case frontend::EmitObj:
147             [[fallthrough]];
148           case frontend::PrintPreprocessedInput:
149             [[fallthrough]];
150           case frontend::EmitLLVMOnly:
151             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
152             break;
153           }
154           return Act;
155         }()) {}
156   FrontendAction *getWrapped() const { return WrappedAction.get(); }
157   TranslationUnitKind getTranslationUnitKind() override {
158     return TU_Incremental;
159   }
160   void ExecuteAction() override {
161     CompilerInstance &CI = getCompilerInstance();
162     assert(CI.hasPreprocessor() && "No PP!");
163 
164     // FIXME: Move the truncation aspect of this into Sema, we delayed this till
165     // here so the source manager would be initialized.
166     if (hasCodeCompletionSupport() &&
167         !CI.getFrontendOpts().CodeCompletionAt.FileName.empty())
168       CI.createCodeCompletionConsumer();
169 
170     // Use a code completion consumer?
171     CodeCompleteConsumer *CompletionConsumer = nullptr;
172     if (CI.hasCodeCompletionConsumer())
173       CompletionConsumer = &CI.getCodeCompletionConsumer();
174 
175     Preprocessor &PP = CI.getPreprocessor();
176     PP.EnterMainSourceFile();
177 
178     if (!CI.hasSema())
179       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
180   }
181 
182   // Do not terminate after processing the input. This allows us to keep various
183   // clang objects alive and to incrementally grow the current TU.
184   void EndSourceFile() override {
185     // The WrappedAction can be nullptr if we issued an error in the ctor.
186     if (IsTerminating && getWrapped())
187       WrapperFrontendAction::EndSourceFile();
188   }
189 
190   void FinalizeAction() {
191     assert(!IsTerminating && "Already finalized!");
192     IsTerminating = true;
193     EndSourceFile();
194   }
195 };
196 
197 CodeGenerator *IncrementalParser::getCodeGen() const {
198   FrontendAction *WrappedAct = Act->getWrapped();
199   if (!WrappedAct->hasIRSupport())
200     return nullptr;
201   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
202 }
203 
204 IncrementalParser::IncrementalParser() {}
205 
206 IncrementalParser::IncrementalParser(Interpreter &Interp,
207                                      std::unique_ptr<CompilerInstance> Instance,
208                                      llvm::LLVMContext &LLVMCtx,
209                                      llvm::Error &Err)
210     : CI(std::move(Instance)) {
211   llvm::ErrorAsOutParameter EAO(&Err);
212   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
213   if (Err)
214     return;
215   CI->ExecuteAction(*Act);
216   std::unique_ptr<ASTConsumer> IncrConsumer =
217       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
218   CI->setASTConsumer(std::move(IncrConsumer));
219   Consumer = &CI->getASTConsumer();
220   P.reset(
221       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
222   P->Initialize();
223 
224   // An initial PTU is needed as CUDA includes some headers automatically
225   auto PTU = ParseOrWrapTopLevelDecl();
226   if (auto E = PTU.takeError()) {
227     consumeError(std::move(E)); // FIXME
228     return;                     // PTU.takeError();
229   }
230 
231   if (CodeGenerator *CG = getCodeGen()) {
232     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
233     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
234                     M->getContext());
235     PTU->TheModule = std::move(M);
236     assert(PTU->TheModule && "Failed to create initial PTU");
237   }
238 }
239 
240 IncrementalParser::~IncrementalParser() {
241   P.reset();
242   Act->FinalizeAction();
243 }
244 
245 llvm::Expected<PartialTranslationUnit &>
246 IncrementalParser::ParseOrWrapTopLevelDecl() {
247   // Recover resources if we crash before exiting this method.
248   Sema &S = CI->getSema();
249   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
250   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
251   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
252 
253   PTUs.emplace_back(PartialTranslationUnit());
254   PartialTranslationUnit &LastPTU = PTUs.back();
255   // Add a new PTU.
256   ASTContext &C = S.getASTContext();
257   C.addTranslationUnitDecl();
258   LastPTU.TUPart = C.getTranslationUnitDecl();
259 
260   // Skip previous eof due to last incremental input.
261   if (P->getCurToken().is(tok::annot_repl_input_end)) {
262     P->ConsumeAnyToken();
263     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
264     // might want to do that around HandleEndOfTranslationUnit.
265     P->ExitScope();
266     S.CurContext = nullptr;
267     // Start a new PTU.
268     P->EnterScope(Scope::DeclScope);
269     S.ActOnTranslationUnitScope(P->getCurScope());
270   }
271 
272   Parser::DeclGroupPtrTy ADecl;
273   Sema::ModuleImportState ImportState;
274   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
275        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
276     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
277       return llvm::make_error<llvm::StringError>("Parsing failed. "
278                                                  "The consumer rejected a decl",
279                                                  std::error_code());
280   }
281 
282   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
283   if (Diags.hasErrorOccurred()) {
284     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
285                                             nullptr};
286     CleanUpPTU(MostRecentPTU);
287 
288     Diags.Reset(/*soft=*/true);
289     Diags.getClient()->clear();
290     return llvm::make_error<llvm::StringError>("Parsing failed.",
291                                                std::error_code());
292   }
293 
294   // Process any TopLevelDecls generated by #pragma weak.
295   for (Decl *D : S.WeakTopLevelDecls()) {
296     DeclGroupRef DGR(D);
297     Consumer->HandleTopLevelDecl(DGR);
298   }
299 
300   LocalInstantiations.perform();
301   GlobalInstantiations.perform();
302 
303   Consumer->HandleTranslationUnit(C);
304 
305   return LastPTU;
306 }
307 
308 llvm::Expected<PartialTranslationUnit &>
309 IncrementalParser::Parse(llvm::StringRef input) {
310   Preprocessor &PP = CI->getPreprocessor();
311   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
312 
313   std::ostringstream SourceName;
314   SourceName << "input_line_" << InputCount++;
315 
316   // Create an uninitialized memory buffer, copy code in and append "\n"
317   size_t InputSize = input.size(); // don't include trailing 0
318   // MemBuffer size should *not* include terminating zero
319   std::unique_ptr<llvm::MemoryBuffer> MB(
320       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
321                                                         SourceName.str()));
322   char *MBStart = const_cast<char *>(MB->getBufferStart());
323   memcpy(MBStart, input.data(), InputSize);
324   MBStart[InputSize] = '\n';
325 
326   SourceManager &SM = CI->getSourceManager();
327 
328   // FIXME: Create SourceLocation, which will allow clang to order the overload
329   // candidates for example
330   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
331 
332   // Create FileID for the current buffer.
333   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
334                                /*LoadedOffset=*/0, NewLoc);
335 
336   // NewLoc only used for diags.
337   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
338     return llvm::make_error<llvm::StringError>("Parsing failed. "
339                                                "Cannot enter source file.",
340                                                std::error_code());
341 
342   auto PTU = ParseOrWrapTopLevelDecl();
343   if (!PTU)
344     return PTU.takeError();
345 
346   if (PP.getLangOpts().DelayedTemplateParsing) {
347     // Microsoft-specific:
348     // Late parsed templates can leave unswallowed "macro"-like tokens.
349     // They will seriously confuse the Parser when entering the next
350     // source file. So lex until we are EOF.
351     Token Tok;
352     do {
353       PP.Lex(Tok);
354     } while (Tok.isNot(tok::annot_repl_input_end));
355   } else {
356     Token AssertTok;
357     PP.Lex(AssertTok);
358     assert(AssertTok.is(tok::annot_repl_input_end) &&
359            "Lexer must be EOF when starting incremental parse!");
360   }
361 
362   if (std::unique_ptr<llvm::Module> M = GenModule())
363     PTU->TheModule = std::move(M);
364 
365   return PTU;
366 }
367 
368 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
369   static unsigned ID = 0;
370   if (CodeGenerator *CG = getCodeGen()) {
371     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
372     CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
373     return M;
374   }
375   return nullptr;
376 }
377 
378 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
379   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
380   TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
381   if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
382     for (auto I = Map->begin(); I != Map->end(); ++I) {
383       StoredDeclsList &List = I->second;
384       DeclContextLookupResult R = List.getLookupResult();
385       for (NamedDecl *D : R) {
386         if (D->getTranslationUnitDecl() == MostRecentTU) {
387           List.remove(D);
388         }
389       }
390       if (List.isNull())
391         Map->erase(I);
392     }
393   }
394 }
395 
396 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
397   CodeGenerator *CG = getCodeGen();
398   assert(CG);
399   return CG->GetMangledName(GD);
400 }
401 
402 } // end namespace clang
403