1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/LLJIT.h" 10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" 11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" 12 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" 13 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 14 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" 15 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" 16 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" 17 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" 18 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" 19 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" 20 #include "llvm/ExecutionEngine/SectionMemoryManager.h" 21 #include "llvm/IR/GlobalVariable.h" 22 #include "llvm/IR/IRBuilder.h" 23 #include "llvm/IR/Mangler.h" 24 #include "llvm/IR/Module.h" 25 #include "llvm/Support/DynamicLibrary.h" 26 27 #include <map> 28 29 #define DEBUG_TYPE "orc" 30 31 using namespace llvm; 32 using namespace llvm::orc; 33 34 namespace { 35 36 /// Adds helper function decls and wrapper functions that call the helper with 37 /// some additional prefix arguments. 38 /// 39 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix 40 /// args i32 4 and i16 12345, this function will add: 41 /// 42 /// declare i8 @bar(i32, i16, i8, i64) 43 /// 44 /// define i8 @foo(i8, i64) { 45 /// entry: 46 /// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1) 47 /// ret i8 %2 48 /// } 49 /// 50 Function *addHelperAndWrapper(Module &M, StringRef WrapperName, 51 FunctionType *WrapperFnType, 52 GlobalValue::VisibilityTypes WrapperVisibility, 53 StringRef HelperName, 54 ArrayRef<Value *> HelperPrefixArgs) { 55 std::vector<Type *> HelperArgTypes; 56 for (auto *Arg : HelperPrefixArgs) 57 HelperArgTypes.push_back(Arg->getType()); 58 for (auto *T : WrapperFnType->params()) 59 HelperArgTypes.push_back(T); 60 auto *HelperFnType = 61 FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false); 62 auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage, 63 HelperName, M); 64 65 auto *WrapperFn = Function::Create( 66 WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M); 67 WrapperFn->setVisibility(WrapperVisibility); 68 69 auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn); 70 IRBuilder<> IB(EntryBlock); 71 72 std::vector<Value *> HelperArgs; 73 for (auto *Arg : HelperPrefixArgs) 74 HelperArgs.push_back(Arg); 75 for (auto &Arg : WrapperFn->args()) 76 HelperArgs.push_back(&Arg); 77 auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs); 78 if (HelperFn->getReturnType()->isVoidTy()) 79 IB.CreateRetVoid(); 80 else 81 IB.CreateRet(HelperResult); 82 83 return WrapperFn; 84 } 85 86 class ORCPlatformSupport : public LLJIT::PlatformSupport { 87 public: 88 ORCPlatformSupport(orc::LLJIT &J) : J(J) {} 89 90 Error initialize(orc::JITDylib &JD) override { 91 using llvm::orc::shared::SPSExecutorAddr; 92 using llvm::orc::shared::SPSString; 93 using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t); 94 enum dlopen_mode : int32_t { 95 ORC_RT_RTLD_LAZY = 0x1, 96 ORC_RT_RTLD_NOW = 0x2, 97 ORC_RT_RTLD_LOCAL = 0x4, 98 ORC_RT_RTLD_GLOBAL = 0x8 99 }; 100 101 if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlopen_wrapper")) { 102 return J.getExecutionSession().callSPSWrapper<SPSDLOpenSig>( 103 *WrapperAddr, DSOHandles[&JD], JD.getName(), 104 int32_t(ORC_RT_RTLD_LAZY)); 105 } else 106 return WrapperAddr.takeError(); 107 } 108 109 Error deinitialize(orc::JITDylib &JD) override { 110 using llvm::orc::shared::SPSExecutorAddr; 111 using SPSDLCloseSig = int32_t(SPSExecutorAddr); 112 113 if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlclose_wrapper")) { 114 int32_t result; 115 auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>( 116 *WrapperAddr, result, DSOHandles[&JD]); 117 if (E) 118 return E; 119 else if (result) 120 return make_error<StringError>("dlclose failed", 121 inconvertibleErrorCode()); 122 DSOHandles.erase(&JD); 123 } else 124 return WrapperAddr.takeError(); 125 return Error::success(); 126 } 127 128 private: 129 orc::LLJIT &J; 130 DenseMap<orc::JITDylib *, orc::ExecutorAddr> DSOHandles; 131 }; 132 133 class GenericLLVMIRPlatformSupport; 134 135 /// orc::Platform component of Generic LLVM IR Platform support. 136 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below. 137 class GenericLLVMIRPlatform : public Platform { 138 public: 139 GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {} 140 Error setupJITDylib(JITDylib &JD) override; 141 Error teardownJITDylib(JITDylib &JD) override; 142 Error notifyAdding(ResourceTracker &RT, 143 const MaterializationUnit &MU) override; 144 Error notifyRemoving(ResourceTracker &RT) override { 145 // Noop -- Nothing to do (yet). 146 return Error::success(); 147 } 148 149 private: 150 GenericLLVMIRPlatformSupport &S; 151 }; 152 153 /// This transform parses llvm.global_ctors to produce a single initialization 154 /// function for the module, records the function, then deletes 155 /// llvm.global_ctors. 156 class GlobalCtorDtorScraper { 157 public: 158 GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS, 159 StringRef InitFunctionPrefix, 160 StringRef DeInitFunctionPrefix) 161 : PS(PS), InitFunctionPrefix(InitFunctionPrefix), 162 DeInitFunctionPrefix(DeInitFunctionPrefix) {} 163 Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM, 164 MaterializationResponsibility &R); 165 166 private: 167 GenericLLVMIRPlatformSupport &PS; 168 StringRef InitFunctionPrefix; 169 StringRef DeInitFunctionPrefix; 170 }; 171 172 /// Generic IR Platform Support 173 /// 174 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with 175 /// specially named 'init' and 'deinit'. Injects definitions / interposes for 176 /// some runtime API, including __cxa_atexit, dlopen, and dlclose. 177 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { 178 public: 179 GenericLLVMIRPlatformSupport(LLJIT &J) 180 : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")), 181 DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) { 182 183 getExecutionSession().setPlatform( 184 std::make_unique<GenericLLVMIRPlatform>(*this)); 185 186 setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix, 187 DeInitFunctionPrefix)); 188 189 SymbolMap StdInterposes; 190 191 StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = 192 JITEvaluatedSymbol(pointerToJITTargetAddress(this), 193 JITSymbolFlags::Exported); 194 StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = 195 JITEvaluatedSymbol(pointerToJITTargetAddress(registerCxaAtExitHelper), 196 JITSymbolFlags()); 197 198 cantFail( 199 J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); 200 cantFail(setupJITDylib(J.getMainJITDylib())); 201 cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); 202 } 203 204 ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } 205 206 /// Adds a module that defines the __dso_handle global. 207 Error setupJITDylib(JITDylib &JD) { 208 209 // Add per-jitdylib standard interposes. 210 SymbolMap PerJDInterposes; 211 PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = 212 JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), 213 JITSymbolFlags()); 214 PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = 215 JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), 216 JITSymbolFlags()); 217 cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes)))); 218 219 auto Ctx = std::make_unique<LLVMContext>(); 220 auto M = std::make_unique<Module>("__standard_lib", *Ctx); 221 M->setDataLayout(J.getDataLayout()); 222 223 auto *Int64Ty = Type::getInt64Ty(*Ctx); 224 auto *DSOHandle = new GlobalVariable( 225 *M, Int64Ty, true, GlobalValue::ExternalLinkage, 226 ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)), 227 "__dso_handle"); 228 DSOHandle->setVisibility(GlobalValue::DefaultVisibility); 229 DSOHandle->setInitializer( 230 ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); 231 232 auto *GenericIRPlatformSupportTy = 233 StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); 234 235 auto *PlatformInstanceDecl = new GlobalVariable( 236 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, 237 nullptr, "__lljit.platform_support_instance"); 238 239 auto *VoidTy = Type::getVoidTy(*Ctx); 240 addHelperAndWrapper( 241 *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false), 242 GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper", 243 {PlatformInstanceDecl, DSOHandle}); 244 245 auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); 246 auto *AtExitCallbackTy = FunctionType::get(VoidTy, {}, false); 247 auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); 248 addHelperAndWrapper(*M, "atexit", 249 FunctionType::get(IntTy, {AtExitCallbackPtrTy}, false), 250 GlobalValue::HiddenVisibility, "__lljit.atexit_helper", 251 {PlatformInstanceDecl, DSOHandle}); 252 253 return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx))); 254 } 255 256 Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) { 257 auto &JD = RT.getJITDylib(); 258 if (auto &InitSym = MU.getInitializerSymbol()) 259 InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol); 260 else { 261 // If there's no identified init symbol attached, but there is a symbol 262 // with the GenericIRPlatform::InitFunctionPrefix, then treat that as 263 // an init function. Add the symbol to both the InitSymbols map (which 264 // will trigger a lookup to materialize the module) and the InitFunctions 265 // map (which holds the names of the symbols to execute). 266 for (auto &KV : MU.getSymbols()) 267 if ((*KV.first).startswith(InitFunctionPrefix)) { 268 InitSymbols[&JD].add(KV.first, 269 SymbolLookupFlags::WeaklyReferencedSymbol); 270 InitFunctions[&JD].add(KV.first); 271 } else if ((*KV.first).startswith(DeInitFunctionPrefix)) { 272 DeInitFunctions[&JD].add(KV.first); 273 } 274 } 275 return Error::success(); 276 } 277 278 Error initialize(JITDylib &JD) override { 279 LLVM_DEBUG({ 280 dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n"; 281 }); 282 if (auto Initializers = getInitializers(JD)) { 283 LLVM_DEBUG( 284 { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; }); 285 for (auto InitFnAddr : *Initializers) { 286 LLVM_DEBUG({ 287 dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr) 288 << "...\n"; 289 }); 290 auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr); 291 InitFn(); 292 } 293 } else 294 return Initializers.takeError(); 295 return Error::success(); 296 } 297 298 Error deinitialize(JITDylib &JD) override { 299 LLVM_DEBUG({ 300 dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n"; 301 }); 302 if (auto Deinitializers = getDeinitializers(JD)) { 303 LLVM_DEBUG({ 304 dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n"; 305 }); 306 for (auto DeinitFnAddr : *Deinitializers) { 307 LLVM_DEBUG({ 308 dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr) 309 << "...\n"; 310 }); 311 auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr); 312 DeinitFn(); 313 } 314 } else 315 return Deinitializers.takeError(); 316 317 return Error::success(); 318 } 319 320 void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) { 321 getExecutionSession().runSessionLocked([&]() { 322 InitFunctions[&JD].add(InitName); 323 }); 324 } 325 326 void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) { 327 getExecutionSession().runSessionLocked( 328 [&]() { DeInitFunctions[&JD].add(DeInitName); }); 329 } 330 331 private: 332 333 Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) { 334 if (auto Err = issueInitLookups(JD)) 335 return std::move(Err); 336 337 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; 338 std::vector<JITDylibSP> DFSLinkOrder; 339 340 if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error { 341 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder()) 342 DFSLinkOrder = std::move(*DFSLinkOrderOrErr); 343 else 344 return DFSLinkOrderOrErr.takeError(); 345 346 for (auto &NextJD : DFSLinkOrder) { 347 auto IFItr = InitFunctions.find(NextJD.get()); 348 if (IFItr != InitFunctions.end()) { 349 LookupSymbols[NextJD.get()] = std::move(IFItr->second); 350 InitFunctions.erase(IFItr); 351 } 352 } 353 return Error::success(); 354 })) 355 return std::move(Err); 356 357 LLVM_DEBUG({ 358 dbgs() << "JITDylib init order is [ "; 359 for (auto &JD : llvm::reverse(DFSLinkOrder)) 360 dbgs() << "\"" << JD->getName() << "\" "; 361 dbgs() << "]\n"; 362 dbgs() << "Looking up init functions:\n"; 363 for (auto &KV : LookupSymbols) 364 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; 365 }); 366 367 auto &ES = getExecutionSession(); 368 auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); 369 370 if (!LookupResult) 371 return LookupResult.takeError(); 372 373 std::vector<JITTargetAddress> Initializers; 374 while (!DFSLinkOrder.empty()) { 375 auto &NextJD = *DFSLinkOrder.back(); 376 DFSLinkOrder.pop_back(); 377 auto InitsItr = LookupResult->find(&NextJD); 378 if (InitsItr == LookupResult->end()) 379 continue; 380 for (auto &KV : InitsItr->second) 381 Initializers.push_back(KV.second.getAddress()); 382 } 383 384 return Initializers; 385 } 386 387 Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) { 388 auto &ES = getExecutionSession(); 389 390 auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits"); 391 392 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; 393 std::vector<JITDylibSP> DFSLinkOrder; 394 395 if (auto Err = ES.runSessionLocked([&]() -> Error { 396 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder()) 397 DFSLinkOrder = std::move(*DFSLinkOrderOrErr); 398 else 399 return DFSLinkOrderOrErr.takeError(); 400 401 for (auto &NextJD : DFSLinkOrder) { 402 auto &JDLookupSymbols = LookupSymbols[NextJD.get()]; 403 auto DIFItr = DeInitFunctions.find(NextJD.get()); 404 if (DIFItr != DeInitFunctions.end()) { 405 LookupSymbols[NextJD.get()] = std::move(DIFItr->second); 406 DeInitFunctions.erase(DIFItr); 407 } 408 JDLookupSymbols.add(LLJITRunAtExits, 409 SymbolLookupFlags::WeaklyReferencedSymbol); 410 } 411 return Error::success(); 412 })) 413 return std::move(Err); 414 415 LLVM_DEBUG({ 416 dbgs() << "JITDylib deinit order is [ "; 417 for (auto &JD : DFSLinkOrder) 418 dbgs() << "\"" << JD->getName() << "\" "; 419 dbgs() << "]\n"; 420 dbgs() << "Looking up deinit functions:\n"; 421 for (auto &KV : LookupSymbols) 422 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; 423 }); 424 425 auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); 426 427 if (!LookupResult) 428 return LookupResult.takeError(); 429 430 std::vector<JITTargetAddress> DeInitializers; 431 for (auto &NextJD : DFSLinkOrder) { 432 auto DeInitsItr = LookupResult->find(NextJD.get()); 433 assert(DeInitsItr != LookupResult->end() && 434 "Every JD should have at least __lljit_run_atexits"); 435 436 auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits); 437 if (RunAtExitsItr != DeInitsItr->second.end()) 438 DeInitializers.push_back(RunAtExitsItr->second.getAddress()); 439 440 for (auto &KV : DeInitsItr->second) 441 if (KV.first != LLJITRunAtExits) 442 DeInitializers.push_back(KV.second.getAddress()); 443 } 444 445 return DeInitializers; 446 } 447 448 /// Issue lookups for all init symbols required to initialize JD (and any 449 /// JITDylibs that it depends on). 450 Error issueInitLookups(JITDylib &JD) { 451 DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols; 452 std::vector<JITDylibSP> DFSLinkOrder; 453 454 if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error { 455 if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder()) 456 DFSLinkOrder = std::move(*DFSLinkOrderOrErr); 457 else 458 return DFSLinkOrderOrErr.takeError(); 459 460 for (auto &NextJD : DFSLinkOrder) { 461 auto ISItr = InitSymbols.find(NextJD.get()); 462 if (ISItr != InitSymbols.end()) { 463 RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second); 464 InitSymbols.erase(ISItr); 465 } 466 } 467 return Error::success(); 468 })) 469 return Err; 470 471 return Platform::lookupInitSymbols(getExecutionSession(), 472 RequiredInitSymbols) 473 .takeError(); 474 } 475 476 static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx, 477 void *DSOHandle) { 478 LLVM_DEBUG({ 479 dbgs() << "Registering cxa atexit function " << (void *)F << " for JD " 480 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; 481 }); 482 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit( 483 F, Ctx, DSOHandle); 484 } 485 486 static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) { 487 LLVM_DEBUG({ 488 dbgs() << "Registering atexit function " << (void *)F << " for JD " 489 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; 490 }); 491 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit( 492 reinterpret_cast<void (*)(void *)>(F), nullptr, DSOHandle); 493 } 494 495 static void runAtExitsHelper(void *Self, void *DSOHandle) { 496 LLVM_DEBUG({ 497 dbgs() << "Running atexit functions for JD " 498 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; 499 }); 500 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits( 501 DSOHandle); 502 } 503 504 // Constructs an LLVM IR module containing platform runtime globals, 505 // functions, and interposes. 506 ThreadSafeModule createPlatformRuntimeModule() { 507 auto Ctx = std::make_unique<LLVMContext>(); 508 auto M = std::make_unique<Module>("__standard_lib", *Ctx); 509 M->setDataLayout(J.getDataLayout()); 510 511 auto *GenericIRPlatformSupportTy = 512 StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); 513 514 auto *PlatformInstanceDecl = new GlobalVariable( 515 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, 516 nullptr, "__lljit.platform_support_instance"); 517 518 auto *Int8Ty = Type::getInt8Ty(*Ctx); 519 auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); 520 auto *VoidTy = Type::getVoidTy(*Ctx); 521 auto *BytePtrTy = PointerType::getUnqual(Int8Ty); 522 auto *CxaAtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); 523 auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(CxaAtExitCallbackTy); 524 525 addHelperAndWrapper( 526 *M, "__cxa_atexit", 527 FunctionType::get(IntTy, {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, 528 false), 529 GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", 530 {PlatformInstanceDecl}); 531 532 return ThreadSafeModule(std::move(M), std::move(Ctx)); 533 } 534 535 LLJIT &J; 536 std::string InitFunctionPrefix; 537 std::string DeInitFunctionPrefix; 538 DenseMap<JITDylib *, SymbolLookupSet> InitSymbols; 539 DenseMap<JITDylib *, SymbolLookupSet> InitFunctions; 540 DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions; 541 ItaniumCXAAtExitSupport AtExitMgr; 542 }; 543 544 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) { 545 return S.setupJITDylib(JD); 546 } 547 548 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) { 549 return Error::success(); 550 } 551 552 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT, 553 const MaterializationUnit &MU) { 554 return S.notifyAdding(RT, MU); 555 } 556 557 Expected<ThreadSafeModule> 558 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, 559 MaterializationResponsibility &R) { 560 auto Err = TSM.withModuleDo([&](Module &M) -> Error { 561 auto &Ctx = M.getContext(); 562 auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors"); 563 auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors"); 564 565 auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors, 566 bool isCtor) -> Error { 567 // If there's no llvm.global_c/dtor or it's just a decl then skip. 568 if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration()) 569 return Error::success(); 570 std::string InitOrDeInitFunctionName; 571 if (isCtor) 572 raw_string_ostream(InitOrDeInitFunctionName) 573 << InitFunctionPrefix << M.getModuleIdentifier(); 574 else 575 raw_string_ostream(InitOrDeInitFunctionName) 576 << DeInitFunctionPrefix << M.getModuleIdentifier(); 577 578 MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); 579 auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName); 580 if (auto Err = R.defineMaterializing( 581 {{InternedInitOrDeInitName, JITSymbolFlags::Callable}})) 582 return Err; 583 584 auto *InitOrDeInitFunc = Function::Create( 585 FunctionType::get(Type::getVoidTy(Ctx), {}, false), 586 GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M); 587 InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility); 588 std::vector<std::pair<Function *, unsigned>> InitsOrDeInits; 589 auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M); 590 591 for (auto E : COrDtors) 592 InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority)); 593 llvm::sort(InitsOrDeInits, llvm::less_second()); 594 595 auto *InitOrDeInitFuncEntryBlock = 596 BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc); 597 IRBuilder<> IB(InitOrDeInitFuncEntryBlock); 598 for (auto &KV : InitsOrDeInits) 599 IB.CreateCall(KV.first); 600 IB.CreateRetVoid(); 601 602 if (isCtor) 603 PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName); 604 else 605 PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName); 606 607 GlobalCOrDtors->eraseFromParent(); 608 return Error::success(); 609 }; 610 611 if (auto Err = RegisterCOrDtors(GlobalCtors, true)) 612 return Err; 613 if (auto Err = RegisterCOrDtors(GlobalDtors, false)) 614 return Err; 615 616 return Error::success(); 617 }); 618 619 if (Err) 620 return std::move(Err); 621 622 return std::move(TSM); 623 } 624 625 /// Inactive Platform Support 626 /// 627 /// Explicitly disables platform support. JITDylibs are not scanned for special 628 /// init/deinit symbols. No runtime API interposes are injected. 629 class InactivePlatformSupport : public LLJIT::PlatformSupport { 630 public: 631 InactivePlatformSupport() = default; 632 633 Error initialize(JITDylib &JD) override { 634 LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for " 635 << JD.getName() << "\n"); 636 return Error::success(); 637 } 638 639 Error deinitialize(JITDylib &JD) override { 640 LLVM_DEBUG( 641 dbgs() << "InactivePlatformSupport: no deinitializers running for " 642 << JD.getName() << "\n"); 643 return Error::success(); 644 } 645 }; 646 647 } // end anonymous namespace 648 649 namespace llvm { 650 namespace orc { 651 652 void LLJIT::PlatformSupport::setInitTransform( 653 LLJIT &J, IRTransformLayer::TransformFunction T) { 654 J.InitHelperTransformLayer->setTransform(std::move(T)); 655 } 656 657 LLJIT::PlatformSupport::~PlatformSupport() = default; 658 659 Error LLJITBuilderState::prepareForConstruction() { 660 661 LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n"); 662 663 if (!JTMB) { 664 LLVM_DEBUG({ 665 dbgs() << " No explicitly set JITTargetMachineBuilder. " 666 "Detecting host...\n"; 667 }); 668 if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) 669 JTMB = std::move(*JTMBOrErr); 670 else 671 return JTMBOrErr.takeError(); 672 } 673 674 LLVM_DEBUG({ 675 dbgs() << " JITTargetMachineBuilder is " 676 << JITTargetMachineBuilderPrinter(*JTMB, " ") 677 << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No") 678 << "\n" 679 << " DataLayout: "; 680 if (DL) 681 dbgs() << DL->getStringRepresentation() << "\n"; 682 else 683 dbgs() << "None (will be created by JITTargetMachineBuilder)\n"; 684 685 dbgs() << " Custom object-linking-layer creator: " 686 << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n" 687 << " Custom compile-function creator: " 688 << (CreateCompileFunction ? "Yes" : "No") << "\n" 689 << " Custom platform-setup function: " 690 << (SetUpPlatform ? "Yes" : "No") << "\n" 691 << " Number of compile threads: " << NumCompileThreads; 692 if (!NumCompileThreads) 693 dbgs() << " (code will be compiled on the execution thread)\n"; 694 else 695 dbgs() << "\n"; 696 }); 697 698 // If neither ES nor EPC has been set then create an EPC instance. 699 if (!ES && !EPC) { 700 LLVM_DEBUG({ 701 dbgs() << "ExecutorProcessControl not specified, " 702 "Creating SelfExecutorProcessControl instance\n"; 703 }); 704 if (auto EPCOrErr = SelfExecutorProcessControl::Create()) 705 EPC = std::move(*EPCOrErr); 706 else 707 return EPCOrErr.takeError(); 708 } else 709 LLVM_DEBUG({ 710 dbgs() << "Using explicitly specified ExecutorProcessControl instance " 711 << EPC.get() << "\n"; 712 }); 713 714 // If the client didn't configure any linker options then auto-configure the 715 // JIT linker. 716 if (!CreateObjectLinkingLayer) { 717 auto &TT = JTMB->getTargetTriple(); 718 if (TT.getArch() == Triple::riscv64 || 719 TT.getArch() == Triple::loongarch64 || 720 (TT.isOSBinFormatMachO() && 721 (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64))) { 722 723 JTMB->setRelocationModel(Reloc::PIC_); 724 JTMB->setCodeModel(CodeModel::Small); 725 CreateObjectLinkingLayer = 726 [](ExecutionSession &ES, 727 const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { 728 auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES); 729 if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES)) 730 ObjLinkingLayer->addPlugin( 731 std::make_unique<EHFrameRegistrationPlugin>( 732 ES, std::move(*EHFrameRegistrar))); 733 else 734 return EHFrameRegistrar.takeError(); 735 return std::move(ObjLinkingLayer); 736 }; 737 } 738 } 739 740 return Error::success(); 741 } 742 743 LLJIT::~LLJIT() { 744 if (CompileThreads) 745 CompileThreads->wait(); 746 if (auto Err = ES->endSession()) 747 ES->reportError(std::move(Err)); 748 } 749 750 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) { 751 assert(TSM && "Can not add null module"); 752 753 if (auto Err = 754 TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) 755 return Err; 756 757 return InitHelperTransformLayer->add(std::move(RT), std::move(TSM)); 758 } 759 760 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { 761 return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM)); 762 } 763 764 Error LLJIT::addObjectFile(ResourceTrackerSP RT, 765 std::unique_ptr<MemoryBuffer> Obj) { 766 assert(Obj && "Can not add null object"); 767 768 return ObjTransformLayer->add(std::move(RT), std::move(Obj)); 769 } 770 771 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { 772 return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj)); 773 } 774 775 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD, 776 SymbolStringPtr Name) { 777 if (auto Sym = ES->lookup( 778 makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), 779 Name)) 780 return ExecutorAddr(Sym->getAddress()); 781 else 782 return Sym.takeError(); 783 } 784 785 Expected<std::unique_ptr<ObjectLayer>> 786 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { 787 788 // If the config state provided an ObjectLinkingLayer factory then use it. 789 if (S.CreateObjectLinkingLayer) 790 return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple()); 791 792 // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs 793 // a new SectionMemoryManager for each object. 794 auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); }; 795 auto Layer = 796 std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); 797 798 if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) { 799 Layer->setOverrideObjectFlagsWithResponsibilityFlags(true); 800 Layer->setAutoClaimResponsibilityForObjectSymbols(true); 801 } 802 803 if (S.JTMB->getTargetTriple().isOSBinFormatELF() && 804 (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 || 805 S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le)) 806 Layer->setAutoClaimResponsibilityForObjectSymbols(true); 807 808 // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence 809 // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. 810 // just return ObjLinkingLayer) once those bots are upgraded. 811 return std::unique_ptr<ObjectLayer>(std::move(Layer)); 812 } 813 814 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> 815 LLJIT::createCompileFunction(LLJITBuilderState &S, 816 JITTargetMachineBuilder JTMB) { 817 818 /// If there is a custom compile function creator set then use it. 819 if (S.CreateCompileFunction) 820 return S.CreateCompileFunction(std::move(JTMB)); 821 822 // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, 823 // depending on the number of threads requested. 824 if (S.NumCompileThreads > 0) 825 return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB)); 826 827 auto TM = JTMB.createTargetMachine(); 828 if (!TM) 829 return TM.takeError(); 830 831 return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM)); 832 } 833 834 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) 835 : DL(""), TT(S.JTMB->getTargetTriple()) { 836 837 ErrorAsOutParameter _(&Err); 838 839 assert(!(S.EPC && S.ES) && "EPC and ES should not both be set"); 840 841 if (S.EPC) { 842 ES = std::make_unique<ExecutionSession>(std::move(S.EPC)); 843 } else if (S.ES) 844 ES = std::move(S.ES); 845 else { 846 if (auto EPC = SelfExecutorProcessControl::Create()) { 847 ES = std::make_unique<ExecutionSession>(std::move(*EPC)); 848 } else { 849 Err = EPC.takeError(); 850 return; 851 } 852 } 853 854 if (auto MainOrErr = this->ES->createJITDylib("main")) 855 Main = &*MainOrErr; 856 else { 857 Err = MainOrErr.takeError(); 858 return; 859 } 860 861 if (S.DL) 862 DL = std::move(*S.DL); 863 else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) 864 DL = std::move(*DLOrErr); 865 else { 866 Err = DLOrErr.takeError(); 867 return; 868 } 869 870 auto ObjLayer = createObjectLinkingLayer(S, *ES); 871 if (!ObjLayer) { 872 Err = ObjLayer.takeError(); 873 return; 874 } 875 ObjLinkingLayer = std::move(*ObjLayer); 876 ObjTransformLayer = 877 std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer); 878 879 { 880 auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); 881 if (!CompileFunction) { 882 Err = CompileFunction.takeError(); 883 return; 884 } 885 CompileLayer = std::make_unique<IRCompileLayer>( 886 *ES, *ObjTransformLayer, std::move(*CompileFunction)); 887 TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer); 888 InitHelperTransformLayer = 889 std::make_unique<IRTransformLayer>(*ES, *TransformLayer); 890 } 891 892 if (S.NumCompileThreads > 0) { 893 InitHelperTransformLayer->setCloneToNewContextOnEmit(true); 894 CompileThreads = 895 std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads)); 896 ES->setDispatchTask([this](std::unique_ptr<Task> T) { 897 // FIXME: We should be able to use move-capture here, but ThreadPool's 898 // AsyncTaskTys are std::functions rather than unique_functions 899 // (because MSVC's std::packaged_tasks don't support move-only types). 900 // Fix this when all the above gets sorted out. 901 CompileThreads->async([UnownedT = T.release()]() mutable { 902 std::unique_ptr<Task> T(UnownedT); 903 T->run(); 904 }); 905 }); 906 } 907 908 if (S.SetUpPlatform) 909 Err = S.SetUpPlatform(*this); 910 else 911 setUpGenericLLVMIRPlatform(*this); 912 } 913 914 std::string LLJIT::mangle(StringRef UnmangledName) const { 915 std::string MangledName; 916 { 917 raw_string_ostream MangledNameStream(MangledName); 918 Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL); 919 } 920 return MangledName; 921 } 922 923 Error LLJIT::applyDataLayout(Module &M) { 924 if (M.getDataLayout().isDefault()) 925 M.setDataLayout(DL); 926 927 if (M.getDataLayout() != DL) 928 return make_error<StringError>( 929 "Added modules have incompatible data layouts: " + 930 M.getDataLayout().getStringRepresentation() + " (module) vs " + 931 DL.getStringRepresentation() + " (jit)", 932 inconvertibleErrorCode()); 933 934 return Error::success(); 935 } 936 937 Error setUpOrcPlatform(LLJIT& J) { 938 LLVM_DEBUG( 939 { dbgs() << "Setting up orc platform support for LLJIT\n"; }); 940 J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); 941 return Error::success(); 942 } 943 944 void setUpGenericLLVMIRPlatform(LLJIT &J) { 945 LLVM_DEBUG( 946 { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; }); 947 J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); 948 } 949 950 Error setUpInactivePlatform(LLJIT &J) { 951 LLVM_DEBUG( 952 { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; }); 953 J.setPlatformSupport(std::make_unique<InactivePlatformSupport>()); 954 return Error::success(); 955 } 956 957 Error LLLazyJITBuilderState::prepareForConstruction() { 958 if (auto Err = LLJITBuilderState::prepareForConstruction()) 959 return Err; 960 TT = JTMB->getTargetTriple(); 961 return Error::success(); 962 } 963 964 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { 965 assert(TSM && "Can not add null module"); 966 967 if (auto Err = TSM.withModuleDo( 968 [&](Module &M) -> Error { return applyDataLayout(M); })) 969 return Err; 970 971 return CODLayer->add(JD, std::move(TSM)); 972 } 973 974 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { 975 976 // If LLJIT construction failed then bail out. 977 if (Err) 978 return; 979 980 ErrorAsOutParameter _(&Err); 981 982 /// Take/Create the lazy-compile callthrough manager. 983 if (S.LCTMgr) 984 LCTMgr = std::move(S.LCTMgr); 985 else { 986 if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( 987 S.TT, *ES, S.LazyCompileFailureAddr.getValue())) 988 LCTMgr = std::move(*LCTMgrOrErr); 989 else { 990 Err = LCTMgrOrErr.takeError(); 991 return; 992 } 993 } 994 995 // Take/Create the indirect stubs manager builder. 996 auto ISMBuilder = std::move(S.ISMBuilder); 997 998 // If none was provided, try to build one. 999 if (!ISMBuilder) 1000 ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT); 1001 1002 // No luck. Bail out. 1003 if (!ISMBuilder) { 1004 Err = make_error<StringError>("Could not construct " 1005 "IndirectStubsManagerBuilder for target " + 1006 S.TT.str(), 1007 inconvertibleErrorCode()); 1008 return; 1009 } 1010 1011 // Create the COD layer. 1012 CODLayer = std::make_unique<CompileOnDemandLayer>( 1013 *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder)); 1014 1015 if (S.NumCompileThreads > 0) 1016 CODLayer->setCloneToNewContextOnEmit(true); 1017 } 1018 1019 // In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force 1020 // them to be linked in. 1021 LLVM_ATTRIBUTE_USED void linkComponents() { 1022 errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper 1023 << (void *)&llvm_orc_deregisterEHFrameSectionWrapper; 1024 } 1025 1026 } // End namespace orc. 1027 } // End namespace llvm. 1028