1 //===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "IncrementalParser.h"
14
15 #include "clang/AST/DeclContextInternals.h"
16 #include "clang/CodeGen/BackendUtil.h"
17 #include "clang/CodeGen/CodeGenAction.h"
18 #include "clang/CodeGen/ModuleBuilder.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/FrontendAction.h"
21 #include "clang/FrontendTool/Utils.h"
22 #include "clang/Interpreter/Interpreter.h"
23 #include "clang/Parse/Parser.h"
24 #include "clang/Sema/Sema.h"
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29
30 #include <sstream>
31
32 namespace clang {
33
34 class IncrementalASTConsumer final : public ASTConsumer {
35 Interpreter &Interp;
36 std::unique_ptr<ASTConsumer> Consumer;
37
38 public:
IncrementalASTConsumer(Interpreter & InterpRef,std::unique_ptr<ASTConsumer> C)39 IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
40 : Interp(InterpRef), Consumer(std::move(C)) {}
41
HandleTopLevelDecl(DeclGroupRef DGR)42 bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
43 if (DGR.isNull())
44 return true;
45 if (!Consumer)
46 return true;
47
48 for (Decl *D : DGR)
49 if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(D);
50 TSD && TSD->isSemiMissing())
51 TSD->setStmt(Interp.SynthesizeExpr(cast<Expr>(TSD->getStmt())));
52
53 return Consumer->HandleTopLevelDecl(DGR);
54 }
HandleTranslationUnit(ASTContext & Ctx)55 void HandleTranslationUnit(ASTContext &Ctx) override final {
56 Consumer->HandleTranslationUnit(Ctx);
57 }
HandleInlineFunctionDefinition(FunctionDecl * D)58 void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
59 Consumer->HandleInlineFunctionDefinition(D);
60 }
HandleInterestingDecl(DeclGroupRef D)61 void HandleInterestingDecl(DeclGroupRef D) override final {
62 Consumer->HandleInterestingDecl(D);
63 }
HandleTagDeclDefinition(TagDecl * D)64 void HandleTagDeclDefinition(TagDecl *D) override final {
65 Consumer->HandleTagDeclDefinition(D);
66 }
HandleTagDeclRequiredDefinition(const TagDecl * D)67 void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
68 Consumer->HandleTagDeclRequiredDefinition(D);
69 }
HandleCXXImplicitFunctionInstantiation(FunctionDecl * D)70 void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
71 Consumer->HandleCXXImplicitFunctionInstantiation(D);
72 }
HandleTopLevelDeclInObjCContainer(DeclGroupRef D)73 void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
74 Consumer->HandleTopLevelDeclInObjCContainer(D);
75 }
HandleImplicitImportDecl(ImportDecl * D)76 void HandleImplicitImportDecl(ImportDecl *D) override final {
77 Consumer->HandleImplicitImportDecl(D);
78 }
CompleteTentativeDefinition(VarDecl * D)79 void CompleteTentativeDefinition(VarDecl *D) override final {
80 Consumer->CompleteTentativeDefinition(D);
81 }
CompleteExternalDeclaration(DeclaratorDecl * D)82 void CompleteExternalDeclaration(DeclaratorDecl *D) override final {
83 Consumer->CompleteExternalDeclaration(D);
84 }
AssignInheritanceModel(CXXRecordDecl * RD)85 void AssignInheritanceModel(CXXRecordDecl *RD) override final {
86 Consumer->AssignInheritanceModel(RD);
87 }
HandleCXXStaticMemberVarInstantiation(VarDecl * D)88 void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
89 Consumer->HandleCXXStaticMemberVarInstantiation(D);
90 }
HandleVTable(CXXRecordDecl * RD)91 void HandleVTable(CXXRecordDecl *RD) override final {
92 Consumer->HandleVTable(RD);
93 }
GetASTMutationListener()94 ASTMutationListener *GetASTMutationListener() override final {
95 return Consumer->GetASTMutationListener();
96 }
GetASTDeserializationListener()97 ASTDeserializationListener *GetASTDeserializationListener() override final {
98 return Consumer->GetASTDeserializationListener();
99 }
PrintStats()100 void PrintStats() override final { Consumer->PrintStats(); }
shouldSkipFunctionBody(Decl * D)101 bool shouldSkipFunctionBody(Decl *D) override final {
102 return Consumer->shouldSkipFunctionBody(D);
103 }
classof(const clang::ASTConsumer *)104 static bool classof(const clang::ASTConsumer *) { return true; }
105 };
106
107 /// A custom action enabling the incremental processing functionality.
108 ///
109 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
110 /// sees a call to \p EndSourceFile it deletes some of the important objects
111 /// such as \p Preprocessor and \p Sema assuming no further input will come.
112 ///
113 /// \p IncrementalAction ensures it keep its underlying action's objects alive
114 /// as long as the \p IncrementalParser needs them.
115 ///
116 class IncrementalAction : public WrapperFrontendAction {
117 private:
118 bool IsTerminating = false;
119
120 public:
IncrementalAction(CompilerInstance & CI,llvm::LLVMContext & LLVMCtx,llvm::Error & Err)121 IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
122 llvm::Error &Err)
123 : WrapperFrontendAction([&]() {
124 llvm::ErrorAsOutParameter EAO(&Err);
125 std::unique_ptr<FrontendAction> Act;
126 switch (CI.getFrontendOpts().ProgramAction) {
127 default:
128 Err = llvm::createStringError(
129 std::errc::state_not_recoverable,
130 "Driver initialization failed. "
131 "Incremental mode for action %d is not supported",
132 CI.getFrontendOpts().ProgramAction);
133 return Act;
134 case frontend::ASTDump:
135 [[fallthrough]];
136 case frontend::ASTPrint:
137 [[fallthrough]];
138 case frontend::ParseSyntaxOnly:
139 Act = CreateFrontendAction(CI);
140 break;
141 case frontend::PluginAction:
142 [[fallthrough]];
143 case frontend::EmitAssembly:
144 [[fallthrough]];
145 case frontend::EmitBC:
146 [[fallthrough]];
147 case frontend::EmitObj:
148 [[fallthrough]];
149 case frontend::PrintPreprocessedInput:
150 [[fallthrough]];
151 case frontend::EmitLLVMOnly:
152 Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
153 break;
154 }
155 return Act;
156 }()) {}
getWrapped() const157 FrontendAction *getWrapped() const { return WrappedAction.get(); }
getTranslationUnitKind()158 TranslationUnitKind getTranslationUnitKind() override {
159 return TU_Incremental;
160 }
161
ExecuteAction()162 void ExecuteAction() override {
163 CompilerInstance &CI = getCompilerInstance();
164 assert(CI.hasPreprocessor() && "No PP!");
165
166 // Use a code completion consumer?
167 CodeCompleteConsumer *CompletionConsumer = nullptr;
168 if (CI.hasCodeCompletionConsumer())
169 CompletionConsumer = &CI.getCodeCompletionConsumer();
170
171 Preprocessor &PP = CI.getPreprocessor();
172 PP.EnterMainSourceFile();
173
174 if (!CI.hasSema())
175 CI.createSema(getTranslationUnitKind(), CompletionConsumer);
176 }
177
178 // Do not terminate after processing the input. This allows us to keep various
179 // clang objects alive and to incrementally grow the current TU.
EndSourceFile()180 void EndSourceFile() override {
181 // The WrappedAction can be nullptr if we issued an error in the ctor.
182 if (IsTerminating && getWrapped())
183 WrapperFrontendAction::EndSourceFile();
184 }
185
FinalizeAction()186 void FinalizeAction() {
187 assert(!IsTerminating && "Already finalized!");
188 IsTerminating = true;
189 EndSourceFile();
190 }
191 };
192
getCodeGen() const193 CodeGenerator *IncrementalParser::getCodeGen() const {
194 FrontendAction *WrappedAct = Act->getWrapped();
195 if (!WrappedAct->hasIRSupport())
196 return nullptr;
197 return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
198 }
199
IncrementalParser()200 IncrementalParser::IncrementalParser() {}
201
IncrementalParser(Interpreter & Interp,std::unique_ptr<CompilerInstance> Instance,llvm::LLVMContext & LLVMCtx,llvm::Error & Err)202 IncrementalParser::IncrementalParser(Interpreter &Interp,
203 std::unique_ptr<CompilerInstance> Instance,
204 llvm::LLVMContext &LLVMCtx,
205 llvm::Error &Err)
206 : CI(std::move(Instance)) {
207 llvm::ErrorAsOutParameter EAO(&Err);
208 Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
209 if (Err)
210 return;
211 CI->ExecuteAction(*Act);
212
213 if (getCodeGen())
214 CachedInCodeGenModule = GenModule();
215
216 std::unique_ptr<ASTConsumer> IncrConsumer =
217 std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
218 CI->setASTConsumer(std::move(IncrConsumer));
219 Consumer = &CI->getASTConsumer();
220 P.reset(
221 new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
222 P->Initialize();
223
224 // An initial PTU is needed as CUDA includes some headers automatically
225 auto PTU = ParseOrWrapTopLevelDecl();
226 if (auto E = PTU.takeError()) {
227 consumeError(std::move(E)); // FIXME
228 return; // PTU.takeError();
229 }
230
231 if (getCodeGen()) {
232 PTU->TheModule = GenModule();
233 assert(PTU->TheModule && "Failed to create initial PTU");
234 }
235 }
236
~IncrementalParser()237 IncrementalParser::~IncrementalParser() {
238 P.reset();
239 Act->FinalizeAction();
240 }
241
242 llvm::Expected<PartialTranslationUnit &>
ParseOrWrapTopLevelDecl()243 IncrementalParser::ParseOrWrapTopLevelDecl() {
244 // Recover resources if we crash before exiting this method.
245 Sema &S = CI->getSema();
246 llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
247 Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
248 Sema::LocalEagerInstantiationScope LocalInstantiations(S);
249
250 PTUs.emplace_back(PartialTranslationUnit());
251 PartialTranslationUnit &LastPTU = PTUs.back();
252 // Add a new PTU.
253 ASTContext &C = S.getASTContext();
254 C.addTranslationUnitDecl();
255 LastPTU.TUPart = C.getTranslationUnitDecl();
256
257 // Skip previous eof due to last incremental input.
258 if (P->getCurToken().is(tok::annot_repl_input_end)) {
259 P->ConsumeAnyToken();
260 // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
261 // might want to do that around HandleEndOfTranslationUnit.
262 P->ExitScope();
263 S.CurContext = nullptr;
264 // Start a new PTU.
265 P->EnterScope(Scope::DeclScope);
266 S.ActOnTranslationUnitScope(P->getCurScope());
267 }
268
269 Parser::DeclGroupPtrTy ADecl;
270 Sema::ModuleImportState ImportState;
271 for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
272 AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
273 if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
274 return llvm::make_error<llvm::StringError>("Parsing failed. "
275 "The consumer rejected a decl",
276 std::error_code());
277 }
278
279 DiagnosticsEngine &Diags = getCI()->getDiagnostics();
280 if (Diags.hasErrorOccurred()) {
281 PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
282 nullptr};
283 CleanUpPTU(MostRecentPTU);
284
285 Diags.Reset(/*soft=*/true);
286 Diags.getClient()->clear();
287 return llvm::make_error<llvm::StringError>("Parsing failed.",
288 std::error_code());
289 }
290
291 // Process any TopLevelDecls generated by #pragma weak.
292 for (Decl *D : S.WeakTopLevelDecls()) {
293 DeclGroupRef DGR(D);
294 Consumer->HandleTopLevelDecl(DGR);
295 }
296
297 LocalInstantiations.perform();
298 GlobalInstantiations.perform();
299
300 Consumer->HandleTranslationUnit(C);
301
302 return LastPTU;
303 }
304
305 llvm::Expected<PartialTranslationUnit &>
Parse(llvm::StringRef input)306 IncrementalParser::Parse(llvm::StringRef input) {
307 Preprocessor &PP = CI->getPreprocessor();
308 assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
309
310 std::ostringstream SourceName;
311 SourceName << "input_line_" << InputCount++;
312
313 // Create an uninitialized memory buffer, copy code in and append "\n"
314 size_t InputSize = input.size(); // don't include trailing 0
315 // MemBuffer size should *not* include terminating zero
316 std::unique_ptr<llvm::MemoryBuffer> MB(
317 llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
318 SourceName.str()));
319 char *MBStart = const_cast<char *>(MB->getBufferStart());
320 memcpy(MBStart, input.data(), InputSize);
321 MBStart[InputSize] = '\n';
322
323 SourceManager &SM = CI->getSourceManager();
324
325 // FIXME: Create SourceLocation, which will allow clang to order the overload
326 // candidates for example
327 SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
328
329 // Create FileID for the current buffer.
330 FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
331 /*LoadedOffset=*/0, NewLoc);
332
333 // NewLoc only used for diags.
334 if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
335 return llvm::make_error<llvm::StringError>("Parsing failed. "
336 "Cannot enter source file.",
337 std::error_code());
338
339 auto PTU = ParseOrWrapTopLevelDecl();
340 if (!PTU)
341 return PTU.takeError();
342
343 if (PP.getLangOpts().DelayedTemplateParsing) {
344 // Microsoft-specific:
345 // Late parsed templates can leave unswallowed "macro"-like tokens.
346 // They will seriously confuse the Parser when entering the next
347 // source file. So lex until we are EOF.
348 Token Tok;
349 do {
350 PP.Lex(Tok);
351 } while (Tok.isNot(tok::annot_repl_input_end));
352 } else {
353 Token AssertTok;
354 PP.Lex(AssertTok);
355 assert(AssertTok.is(tok::annot_repl_input_end) &&
356 "Lexer must be EOF when starting incremental parse!");
357 }
358
359 if (std::unique_ptr<llvm::Module> M = GenModule())
360 PTU->TheModule = std::move(M);
361
362 return PTU;
363 }
364
GenModule()365 std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
366 static unsigned ID = 0;
367 if (CodeGenerator *CG = getCodeGen()) {
368 // Clang's CodeGen is designed to work with a single llvm::Module. In many
369 // cases for convenience various CodeGen parts have a reference to the
370 // llvm::Module (TheModule or Module) which does not change when a new
371 // module is pushed. However, the execution engine wants to take ownership
372 // of the module which does not map well to CodeGen's design. To work this
373 // around we created an empty module to make CodeGen happy. We should make
374 // sure it always stays empty.
375 assert((!CachedInCodeGenModule ||
376 (CachedInCodeGenModule->empty() &&
377 CachedInCodeGenModule->global_empty() &&
378 CachedInCodeGenModule->alias_empty() &&
379 CachedInCodeGenModule->ifunc_empty())) &&
380 "CodeGen wrote to a readonly module");
381 std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
382 CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
383 return M;
384 }
385 return nullptr;
386 }
387
CleanUpPTU(PartialTranslationUnit & PTU)388 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
389 TranslationUnitDecl *MostRecentTU = PTU.TUPart;
390 if (StoredDeclsMap *Map = MostRecentTU->getPrimaryContext()->getLookupPtr()) {
391 for (auto &&[Key, List] : *Map) {
392 DeclContextLookupResult R = List.getLookupResult();
393 std::vector<NamedDecl *> NamedDeclsToRemove;
394 bool RemoveAll = true;
395 for (NamedDecl *D : R) {
396 if (D->getTranslationUnitDecl() == MostRecentTU)
397 NamedDeclsToRemove.push_back(D);
398 else
399 RemoveAll = false;
400 }
401 if (LLVM_LIKELY(RemoveAll)) {
402 Map->erase(Key);
403 } else {
404 for (NamedDecl *D : NamedDeclsToRemove)
405 List.remove(D);
406 }
407 }
408 }
409
410 // FIXME: We should de-allocate MostRecentTU
411 for (Decl *D : MostRecentTU->decls()) {
412 auto *ND = dyn_cast<NamedDecl>(D);
413 if (!ND)
414 continue;
415 // Check if we need to clean up the IdResolver chain.
416 if (ND->getDeclName().getFETokenInfo() && !D->getLangOpts().ObjC &&
417 !D->getLangOpts().CPlusPlus)
418 getCI()->getSema().IdResolver.RemoveDecl(ND);
419 }
420 }
421
GetMangledName(GlobalDecl GD) const422 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
423 CodeGenerator *CG = getCodeGen();
424 assert(CG);
425 return CG->GetMangledName(GD);
426 }
427 } // end namespace clang
428