xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp (revision 093cf790569775b80662926efea6d9d3464bde94)
1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
12 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
13 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
14 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
15 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
16 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
17 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
18 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
19 #include "llvm/IR/GlobalVariable.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Mangler.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/Support/DynamicLibrary.h"
24 
25 #include <map>
26 
27 #define DEBUG_TYPE "orc"
28 
29 using namespace llvm;
30 using namespace llvm::orc;
31 
32 namespace {
33 
34 /// Adds helper function decls and wrapper functions that call the helper with
35 /// some additional prefix arguments.
36 ///
37 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
38 /// args i32 4 and i16 12345, this function will add:
39 ///
40 /// declare i8 @bar(i32, i16, i8, i64)
41 ///
42 /// define i8 @foo(i8, i64) {
43 /// entry:
44 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
45 ///   ret i8 %2
46 /// }
47 ///
48 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
49                               FunctionType *WrapperFnType,
50                               GlobalValue::VisibilityTypes WrapperVisibility,
51                               StringRef HelperName,
52                               ArrayRef<Value *> HelperPrefixArgs) {
53   std::vector<Type *> HelperArgTypes;
54   for (auto *Arg : HelperPrefixArgs)
55     HelperArgTypes.push_back(Arg->getType());
56   for (auto *T : WrapperFnType->params())
57     HelperArgTypes.push_back(T);
58   auto *HelperFnType =
59       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
60   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
61                                     HelperName, M);
62 
63   auto *WrapperFn = Function::Create(
64       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
65   WrapperFn->setVisibility(WrapperVisibility);
66 
67   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
68   IRBuilder<> IB(EntryBlock);
69 
70   std::vector<Value *> HelperArgs;
71   for (auto *Arg : HelperPrefixArgs)
72     HelperArgs.push_back(Arg);
73   for (auto &Arg : WrapperFn->args())
74     HelperArgs.push_back(&Arg);
75   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
76   if (HelperFn->getReturnType()->isVoidTy())
77     IB.CreateRetVoid();
78   else
79     IB.CreateRet(HelperResult);
80 
81   return WrapperFn;
82 }
83 
84 class GenericLLVMIRPlatformSupport;
85 
86 /// orc::Platform component of Generic LLVM IR Platform support.
87 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
88 class GenericLLVMIRPlatform : public Platform {
89 public:
90   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
91   Error setupJITDylib(JITDylib &JD) override;
92   Error notifyAdding(ResourceTracker &RT,
93                      const MaterializationUnit &MU) override;
94   Error notifyRemoving(ResourceTracker &RT) override {
95     // Noop -- Nothing to do (yet).
96     return Error::success();
97   }
98 
99 private:
100   GenericLLVMIRPlatformSupport &S;
101 };
102 
103 /// This transform parses llvm.global_ctors to produce a single initialization
104 /// function for the module, records the function, then deletes
105 /// llvm.global_ctors.
106 class GlobalCtorDtorScraper {
107 public:
108 
109   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
110                         StringRef InitFunctionPrefix)
111     : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {}
112   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
113                                         MaterializationResponsibility &R);
114 
115 private:
116   GenericLLVMIRPlatformSupport &PS;
117   StringRef InitFunctionPrefix;
118 };
119 
120 /// Generic IR Platform Support
121 ///
122 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
123 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
124 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
125 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
126 public:
127   GenericLLVMIRPlatformSupport(LLJIT &J)
128       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) {
129 
130     getExecutionSession().setPlatform(
131         std::make_unique<GenericLLVMIRPlatform>(*this));
132 
133     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix));
134 
135     SymbolMap StdInterposes;
136 
137     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
138         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
139                            JITSymbolFlags::Exported);
140     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
141         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
142                            JITSymbolFlags());
143 
144     cantFail(
145         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
146     cantFail(setupJITDylib(J.getMainJITDylib()));
147     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
148   }
149 
150   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
151 
152   /// Adds a module that defines the __dso_handle global.
153   Error setupJITDylib(JITDylib &JD) {
154 
155     // Add per-jitdylib standard interposes.
156     SymbolMap PerJDInterposes;
157     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
158         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
159                            JITSymbolFlags());
160     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
161 
162     auto Ctx = std::make_unique<LLVMContext>();
163     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
164     M->setDataLayout(J.getDataLayout());
165 
166     auto *Int64Ty = Type::getInt64Ty(*Ctx);
167     auto *DSOHandle = new GlobalVariable(
168         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
169         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
170         "__dso_handle");
171     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
172     DSOHandle->setInitializer(
173         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
174 
175     auto *GenericIRPlatformSupportTy =
176         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
177 
178     auto *PlatformInstanceDecl = new GlobalVariable(
179         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
180         nullptr, "__lljit.platform_support_instance");
181 
182     auto *VoidTy = Type::getVoidTy(*Ctx);
183     addHelperAndWrapper(
184         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
185         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
186         {PlatformInstanceDecl, DSOHandle});
187 
188     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
189   }
190 
191   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
192     auto &JD = RT.getJITDylib();
193     if (auto &InitSym = MU.getInitializerSymbol())
194       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
195     else {
196       // If there's no identified init symbol attached, but there is a symbol
197       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
198       // an init function. Add the symbol to both the InitSymbols map (which
199       // will trigger a lookup to materialize the module) and the InitFunctions
200       // map (which holds the names of the symbols to execute).
201       for (auto &KV : MU.getSymbols())
202         if ((*KV.first).startswith(InitFunctionPrefix)) {
203           InitSymbols[&JD].add(KV.first,
204                                SymbolLookupFlags::WeaklyReferencedSymbol);
205           InitFunctions[&JD].add(KV.first);
206         }
207     }
208     return Error::success();
209   }
210 
211   Error initialize(JITDylib &JD) override {
212     LLVM_DEBUG({
213       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
214     });
215     if (auto Initializers = getInitializers(JD)) {
216       LLVM_DEBUG(
217           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
218       for (auto InitFnAddr : *Initializers) {
219         LLVM_DEBUG({
220           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
221                  << "...\n";
222         });
223         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
224         InitFn();
225       }
226     } else
227       return Initializers.takeError();
228     return Error::success();
229   }
230 
231   Error deinitialize(JITDylib &JD) override {
232     LLVM_DEBUG({
233       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
234     });
235     if (auto Deinitializers = getDeinitializers(JD)) {
236       LLVM_DEBUG({
237         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
238       });
239       for (auto DeinitFnAddr : *Deinitializers) {
240         LLVM_DEBUG({
241           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
242                  << "...\n";
243         });
244         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
245         DeinitFn();
246       }
247     } else
248       return Deinitializers.takeError();
249 
250     return Error::success();
251   }
252 
253   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
254     getExecutionSession().runSessionLocked([&]() {
255         InitFunctions[&JD].add(InitName);
256       });
257   }
258 
259 private:
260 
261   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
262     if (auto Err = issueInitLookups(JD))
263       return std::move(Err);
264 
265     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
266     std::vector<JITDylibSP> DFSLinkOrder;
267 
268     getExecutionSession().runSessionLocked([&]() {
269       DFSLinkOrder = JD.getDFSLinkOrder();
270 
271       for (auto &NextJD : DFSLinkOrder) {
272         auto IFItr = InitFunctions.find(NextJD.get());
273         if (IFItr != InitFunctions.end()) {
274           LookupSymbols[NextJD.get()] = std::move(IFItr->second);
275           InitFunctions.erase(IFItr);
276         }
277       }
278     });
279 
280     LLVM_DEBUG({
281       dbgs() << "JITDylib init order is [ ";
282       for (auto &JD : llvm::reverse(DFSLinkOrder))
283         dbgs() << "\"" << JD->getName() << "\" ";
284       dbgs() << "]\n";
285       dbgs() << "Looking up init functions:\n";
286       for (auto &KV : LookupSymbols)
287         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
288     });
289 
290     auto &ES = getExecutionSession();
291     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
292 
293     if (!LookupResult)
294       return LookupResult.takeError();
295 
296     std::vector<JITTargetAddress> Initializers;
297     while (!DFSLinkOrder.empty()) {
298       auto &NextJD = *DFSLinkOrder.back();
299       DFSLinkOrder.pop_back();
300       auto InitsItr = LookupResult->find(&NextJD);
301       if (InitsItr == LookupResult->end())
302         continue;
303       for (auto &KV : InitsItr->second)
304         Initializers.push_back(KV.second.getAddress());
305     }
306 
307     return Initializers;
308   }
309 
310   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
311     auto &ES = getExecutionSession();
312 
313     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
314 
315     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
316     std::vector<JITDylibSP> DFSLinkOrder;
317 
318     ES.runSessionLocked([&]() {
319       DFSLinkOrder = JD.getDFSLinkOrder();
320 
321       for (auto &NextJD : DFSLinkOrder) {
322         auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
323         auto DIFItr = DeInitFunctions.find(NextJD.get());
324         if (DIFItr != DeInitFunctions.end()) {
325           LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
326           DeInitFunctions.erase(DIFItr);
327         }
328         JDLookupSymbols.add(LLJITRunAtExits,
329                             SymbolLookupFlags::WeaklyReferencedSymbol);
330       }
331     });
332 
333     LLVM_DEBUG({
334       dbgs() << "JITDylib deinit order is [ ";
335       for (auto &JD : DFSLinkOrder)
336         dbgs() << "\"" << JD->getName() << "\" ";
337       dbgs() << "]\n";
338       dbgs() << "Looking up deinit functions:\n";
339       for (auto &KV : LookupSymbols)
340         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
341     });
342 
343     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
344 
345     if (!LookupResult)
346       return LookupResult.takeError();
347 
348     std::vector<JITTargetAddress> DeInitializers;
349     for (auto &NextJD : DFSLinkOrder) {
350       auto DeInitsItr = LookupResult->find(NextJD.get());
351       assert(DeInitsItr != LookupResult->end() &&
352              "Every JD should have at least __lljit_run_atexits");
353 
354       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
355       if (RunAtExitsItr != DeInitsItr->second.end())
356         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
357 
358       for (auto &KV : DeInitsItr->second)
359         if (KV.first != LLJITRunAtExits)
360           DeInitializers.push_back(KV.second.getAddress());
361     }
362 
363     return DeInitializers;
364   }
365 
366   /// Issue lookups for all init symbols required to initialize JD (and any
367   /// JITDylibs that it depends on).
368   Error issueInitLookups(JITDylib &JD) {
369     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
370     std::vector<JITDylibSP> DFSLinkOrder;
371 
372     getExecutionSession().runSessionLocked([&]() {
373       DFSLinkOrder = JD.getDFSLinkOrder();
374 
375       for (auto &NextJD : DFSLinkOrder) {
376         auto ISItr = InitSymbols.find(NextJD.get());
377         if (ISItr != InitSymbols.end()) {
378           RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
379           InitSymbols.erase(ISItr);
380         }
381       }
382     });
383 
384     return Platform::lookupInitSymbols(getExecutionSession(),
385                                        RequiredInitSymbols)
386         .takeError();
387   }
388 
389   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
390                                    void *DSOHandle) {
391     LLVM_DEBUG({
392       dbgs() << "Registering atexit function " << (void *)F << " for JD "
393              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
394     });
395     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
396         F, Ctx, DSOHandle);
397   }
398 
399   static void runAtExitsHelper(void *Self, void *DSOHandle) {
400     LLVM_DEBUG({
401       dbgs() << "Running atexit functions for JD "
402              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
403     });
404     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
405         DSOHandle);
406   }
407 
408   // Constructs an LLVM IR module containing platform runtime globals,
409   // functions, and interposes.
410   ThreadSafeModule createPlatformRuntimeModule() {
411     auto Ctx = std::make_unique<LLVMContext>();
412     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
413     M->setDataLayout(J.getDataLayout());
414 
415     auto *GenericIRPlatformSupportTy =
416         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
417 
418     auto *PlatformInstanceDecl = new GlobalVariable(
419         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
420         nullptr, "__lljit.platform_support_instance");
421 
422     auto *Int8Ty = Type::getInt8Ty(*Ctx);
423     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
424     auto *VoidTy = Type::getVoidTy(*Ctx);
425     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
426     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
427     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
428 
429     addHelperAndWrapper(
430         *M, "__cxa_atexit",
431         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
432                           false),
433         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
434         {PlatformInstanceDecl});
435 
436     return ThreadSafeModule(std::move(M), std::move(Ctx));
437   }
438 
439   LLJIT &J;
440   std::string InitFunctionPrefix;
441   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
442   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
443   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
444   ItaniumCXAAtExitSupport AtExitMgr;
445 };
446 
447 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
448   return S.setupJITDylib(JD);
449 }
450 
451 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
452                                           const MaterializationUnit &MU) {
453   return S.notifyAdding(RT, MU);
454 }
455 
456 Expected<ThreadSafeModule>
457 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
458                                   MaterializationResponsibility &R) {
459   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
460     auto &Ctx = M.getContext();
461     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
462 
463     // If there's no llvm.global_ctors or it's just a decl then skip.
464     if (!GlobalCtors || GlobalCtors->isDeclaration())
465       return Error::success();
466 
467     std::string InitFunctionName;
468     raw_string_ostream(InitFunctionName)
469         << InitFunctionPrefix << M.getModuleIdentifier();
470 
471     MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
472     auto InternedName = Mangle(InitFunctionName);
473     if (auto Err =
474             R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}}))
475       return Err;
476 
477     auto *InitFunc =
478         Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false),
479                          GlobalValue::ExternalLinkage, InitFunctionName, &M);
480     InitFunc->setVisibility(GlobalValue::HiddenVisibility);
481     std::vector<std::pair<Function *, unsigned>> Inits;
482     for (auto E : getConstructors(M))
483       Inits.push_back(std::make_pair(E.Func, E.Priority));
484     llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS,
485                          const std::pair<Function *, unsigned> &RHS) {
486       return LHS.first < RHS.first;
487     });
488     auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc);
489     IRBuilder<> IB(EntryBlock);
490     for (auto &KV : Inits)
491       IB.CreateCall(KV.first);
492     IB.CreateRetVoid();
493 
494     PS.registerInitFunc(R.getTargetJITDylib(), InternedName);
495     GlobalCtors->eraseFromParent();
496     return Error::success();
497   });
498 
499   if (Err)
500     return std::move(Err);
501 
502   return std::move(TSM);
503 }
504 
505 /// Inactive Platform Support
506 ///
507 /// Explicitly disables platform support. JITDylibs are not scanned for special
508 /// init/deinit symbols. No runtime API interposes are injected.
509 class InactivePlatformSupport : public LLJIT::PlatformSupport {
510 public:
511   InactivePlatformSupport() = default;
512 
513   Error initialize(JITDylib &JD) override {
514     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
515                       << JD.getName() << "\n");
516     return Error::success();
517   }
518 
519   Error deinitialize(JITDylib &JD) override {
520     LLVM_DEBUG(
521         dbgs() << "InactivePlatformSupport: no deinitializers running for "
522                << JD.getName() << "\n");
523     return Error::success();
524   }
525 };
526 
527 } // end anonymous namespace
528 
529 namespace llvm {
530 namespace orc {
531 
532 void LLJIT::PlatformSupport::setInitTransform(
533     LLJIT &J, IRTransformLayer::TransformFunction T) {
534   J.InitHelperTransformLayer->setTransform(std::move(T));
535 }
536 
537 LLJIT::PlatformSupport::~PlatformSupport() {}
538 
539 Error LLJITBuilderState::prepareForConstruction() {
540 
541   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
542 
543   if (!JTMB) {
544     LLVM_DEBUG({
545       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
546                 "Detecting host...\n";
547     });
548     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
549       JTMB = std::move(*JTMBOrErr);
550     else
551       return JTMBOrErr.takeError();
552   }
553 
554   LLVM_DEBUG({
555     dbgs() << "  JITTargetMachineBuilder is "
556            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
557            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
558            << "\n"
559            << "  DataLayout: ";
560     if (DL)
561       dbgs() << DL->getStringRepresentation() << "\n";
562     else
563       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
564 
565     dbgs() << "  Custom object-linking-layer creator: "
566            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
567            << "  Custom compile-function creator: "
568            << (CreateCompileFunction ? "Yes" : "No") << "\n"
569            << "  Custom platform-setup function: "
570            << (SetUpPlatform ? "Yes" : "No") << "\n"
571            << "  Number of compile threads: " << NumCompileThreads;
572     if (!NumCompileThreads)
573       dbgs() << " (code will be compiled on the execution thread)\n";
574     else
575       dbgs() << "\n";
576   });
577 
578   // If neither ES nor EPC has been set then create an EPC instance.
579   if (!ES && !EPC) {
580     LLVM_DEBUG({
581       dbgs() << "ExecutorProcessControl not specified, "
582                 "Creating SelfExecutorProcessControl instance\n";
583     });
584     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
585       EPC = std::move(*EPCOrErr);
586     else
587       return EPCOrErr.takeError();
588   } else
589     LLVM_DEBUG({
590       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
591              << EPC.get() << "\n";
592     });
593 
594   // If the client didn't configure any linker options then auto-configure the
595   // JIT linker.
596   if (!CreateObjectLinkingLayer) {
597     auto &TT = JTMB->getTargetTriple();
598     if (TT.isOSBinFormatMachO() &&
599         (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) {
600 
601       JTMB->setRelocationModel(Reloc::PIC_);
602       JTMB->setCodeModel(CodeModel::Small);
603       CreateObjectLinkingLayer =
604           [](ExecutionSession &ES,
605              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
606         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
607         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
608             ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>()));
609         return std::move(ObjLinkingLayer);
610       };
611     }
612   }
613 
614   return Error::success();
615 }
616 
617 LLJIT::~LLJIT() {
618   if (CompileThreads)
619     CompileThreads->wait();
620   if (auto Err = ES->endSession())
621     ES->reportError(std::move(Err));
622 }
623 
624 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
625   assert(TSM && "Can not add null module");
626 
627   if (auto Err =
628           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
629     return Err;
630 
631   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
632 }
633 
634 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
635   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
636 }
637 
638 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
639                            std::unique_ptr<MemoryBuffer> Obj) {
640   assert(Obj && "Can not add null object");
641 
642   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
643 }
644 
645 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
646   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
647 }
648 
649 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
650                                                         SymbolStringPtr Name) {
651   return ES->lookup(
652       makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name);
653 }
654 
655 Expected<std::unique_ptr<ObjectLayer>>
656 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
657 
658   // If the config state provided an ObjectLinkingLayer factory then use it.
659   if (S.CreateObjectLinkingLayer)
660     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
661 
662   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
663   // a new SectionMemoryManager for each object.
664   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
665   auto Layer =
666       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
667 
668   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
669     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
670     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
671   }
672 
673   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
674   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
675   //        just return ObjLinkingLayer) once those bots are upgraded.
676   return std::unique_ptr<ObjectLayer>(std::move(Layer));
677 }
678 
679 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
680 LLJIT::createCompileFunction(LLJITBuilderState &S,
681                              JITTargetMachineBuilder JTMB) {
682 
683   /// If there is a custom compile function creator set then use it.
684   if (S.CreateCompileFunction)
685     return S.CreateCompileFunction(std::move(JTMB));
686 
687   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
688   // depending on the number of threads requested.
689   if (S.NumCompileThreads > 0)
690     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
691 
692   auto TM = JTMB.createTargetMachine();
693   if (!TM)
694     return TM.takeError();
695 
696   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
697 }
698 
699 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
700     : DL(""), TT(S.JTMB->getTargetTriple()) {
701 
702   ErrorAsOutParameter _(&Err);
703 
704   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
705 
706   if (S.EPC) {
707     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
708   } else if (S.ES)
709     ES = std::move(S.ES);
710   else {
711     if (auto EPC = SelfExecutorProcessControl::Create()) {
712       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
713     } else {
714       Err = EPC.takeError();
715       return;
716     }
717   }
718 
719   if (auto MainOrErr = this->ES->createJITDylib("main"))
720     Main = &*MainOrErr;
721   else {
722     Err = MainOrErr.takeError();
723     return;
724   }
725 
726   if (S.DL)
727     DL = std::move(*S.DL);
728   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
729     DL = std::move(*DLOrErr);
730   else {
731     Err = DLOrErr.takeError();
732     return;
733   }
734 
735   auto ObjLayer = createObjectLinkingLayer(S, *ES);
736   if (!ObjLayer) {
737     Err = ObjLayer.takeError();
738     return;
739   }
740   ObjLinkingLayer = std::move(*ObjLayer);
741   ObjTransformLayer =
742       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
743 
744   {
745     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
746     if (!CompileFunction) {
747       Err = CompileFunction.takeError();
748       return;
749     }
750     CompileLayer = std::make_unique<IRCompileLayer>(
751         *ES, *ObjTransformLayer, std::move(*CompileFunction));
752     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
753     InitHelperTransformLayer =
754         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
755   }
756 
757   if (S.NumCompileThreads > 0) {
758     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
759     CompileThreads =
760         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
761     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
762       // FIXME: We should be able to use move-capture here, but ThreadPool's
763       // AsyncTaskTys are std::functions rather than unique_functions
764       // (because MSVC's std::packaged_tasks don't support move-only types).
765       // Fix this when all the above gets sorted out.
766       CompileThreads->async([UnownedT = T.release()]() mutable {
767         std::unique_ptr<Task> T(UnownedT);
768         T->run();
769       });
770     });
771   }
772 
773   if (S.SetUpPlatform)
774     Err = S.SetUpPlatform(*this);
775   else
776     setUpGenericLLVMIRPlatform(*this);
777 }
778 
779 std::string LLJIT::mangle(StringRef UnmangledName) const {
780   std::string MangledName;
781   {
782     raw_string_ostream MangledNameStream(MangledName);
783     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
784   }
785   return MangledName;
786 }
787 
788 Error LLJIT::applyDataLayout(Module &M) {
789   if (M.getDataLayout().isDefault())
790     M.setDataLayout(DL);
791 
792   if (M.getDataLayout() != DL)
793     return make_error<StringError>(
794         "Added modules have incompatible data layouts: " +
795             M.getDataLayout().getStringRepresentation() + " (module) vs " +
796             DL.getStringRepresentation() + " (jit)",
797         inconvertibleErrorCode());
798 
799   return Error::success();
800 }
801 
802 void setUpGenericLLVMIRPlatform(LLJIT &J) {
803   LLVM_DEBUG(
804       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
805   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
806 }
807 
808 Error setUpInactivePlatform(LLJIT &J) {
809   LLVM_DEBUG(
810       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
811   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
812   return Error::success();
813 }
814 
815 Error LLLazyJITBuilderState::prepareForConstruction() {
816   if (auto Err = LLJITBuilderState::prepareForConstruction())
817     return Err;
818   TT = JTMB->getTargetTriple();
819   return Error::success();
820 }
821 
822 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
823   assert(TSM && "Can not add null module");
824 
825   if (auto Err = TSM.withModuleDo(
826           [&](Module &M) -> Error { return applyDataLayout(M); }))
827     return Err;
828 
829   return CODLayer->add(JD, std::move(TSM));
830 }
831 
832 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
833 
834   // If LLJIT construction failed then bail out.
835   if (Err)
836     return;
837 
838   ErrorAsOutParameter _(&Err);
839 
840   /// Take/Create the lazy-compile callthrough manager.
841   if (S.LCTMgr)
842     LCTMgr = std::move(S.LCTMgr);
843   else {
844     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
845             S.TT, *ES, S.LazyCompileFailureAddr))
846       LCTMgr = std::move(*LCTMgrOrErr);
847     else {
848       Err = LCTMgrOrErr.takeError();
849       return;
850     }
851   }
852 
853   // Take/Create the indirect stubs manager builder.
854   auto ISMBuilder = std::move(S.ISMBuilder);
855 
856   // If none was provided, try to build one.
857   if (!ISMBuilder)
858     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
859 
860   // No luck. Bail out.
861   if (!ISMBuilder) {
862     Err = make_error<StringError>("Could not construct "
863                                   "IndirectStubsManagerBuilder for target " +
864                                       S.TT.str(),
865                                   inconvertibleErrorCode());
866     return;
867   }
868 
869   // Create the COD layer.
870   CODLayer = std::make_unique<CompileOnDemandLayer>(
871       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
872 
873   if (S.NumCompileThreads > 0)
874     CODLayer->setCloneToNewContextOnEmit(true);
875 }
876 
877 } // End namespace orc.
878 } // End namespace llvm.
879