1 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h" 10 #include "llvm/ExecutionEngine/Orc/DebugUtils.h" 11 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" 12 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" 13 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" 14 15 #include "llvm/Object/COFF.h" 16 17 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" 18 19 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 20 21 #define DEBUG_TYPE "orc" 22 23 using namespace llvm; 24 using namespace llvm::orc; 25 using namespace llvm::orc::shared; 26 27 namespace llvm { 28 namespace orc { 29 namespace shared { 30 31 using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>; 32 using SPSCOFFJITDylibDepInfoMap = 33 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>; 34 using SPSCOFFObjectSectionsMap = 35 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; 36 using SPSCOFFRegisterObjectSectionsArgs = 37 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>; 38 using SPSCOFFDeregisterObjectSectionsArgs = 39 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>; 40 41 } // namespace shared 42 } // namespace orc 43 } // namespace llvm 44 namespace { 45 46 class COFFHeaderMaterializationUnit : public MaterializationUnit { 47 public: 48 COFFHeaderMaterializationUnit(COFFPlatform &CP, 49 const SymbolStringPtr &HeaderStartSymbol) 50 : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)), 51 CP(CP) {} 52 53 StringRef getName() const override { return "COFFHeaderMU"; } 54 55 void materialize(std::unique_ptr<MaterializationResponsibility> R) override { 56 unsigned PointerSize; 57 llvm::endianness Endianness; 58 const auto &TT = CP.getExecutionSession().getTargetTriple(); 59 60 switch (TT.getArch()) { 61 case Triple::x86_64: 62 PointerSize = 8; 63 Endianness = llvm::endianness::little; 64 break; 65 default: 66 llvm_unreachable("Unrecognized architecture"); 67 } 68 69 auto G = std::make_unique<jitlink::LinkGraph>( 70 "<COFFHeaderMU>", TT, PointerSize, Endianness, 71 jitlink::getGenericEdgeKindName); 72 auto &HeaderSection = G->createSection("__header", MemProt::Read); 73 auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); 74 75 // Init symbol is __ImageBase symbol. 76 auto &ImageBaseSymbol = G->addDefinedSymbol( 77 HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(), 78 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true); 79 80 addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol); 81 82 CP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); 83 } 84 85 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} 86 87 private: 88 struct HeaderSymbol { 89 const char *Name; 90 uint64_t Offset; 91 }; 92 93 struct NTHeader { 94 support::ulittle32_t PEMagic; 95 object::coff_file_header FileHeader; 96 struct PEHeader { 97 object::pe32plus_header Header; 98 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1]; 99 } OptionalHeader; 100 }; 101 102 struct HeaderBlockContent { 103 object::dos_header DOSHeader; 104 COFFHeaderMaterializationUnit::NTHeader NTHeader; 105 }; 106 107 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, 108 jitlink::Section &HeaderSection) { 109 HeaderBlockContent Hdr = {}; 110 111 // Set up magic 112 Hdr.DOSHeader.Magic[0] = 'M'; 113 Hdr.DOSHeader.Magic[1] = 'Z'; 114 Hdr.DOSHeader.AddressOfNewExeHeader = 115 offsetof(HeaderBlockContent, NTHeader); 116 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic); 117 Hdr.NTHeader.PEMagic = PEMagic; 118 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS; 119 120 switch (G.getTargetTriple().getArch()) { 121 case Triple::x86_64: 122 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64; 123 break; 124 default: 125 llvm_unreachable("Unrecognized architecture"); 126 } 127 128 auto HeaderContent = G.allocateContent( 129 ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); 130 131 return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, 132 0); 133 } 134 135 static void addImageBaseRelocationEdge(jitlink::Block &B, 136 jitlink::Symbol &ImageBase) { 137 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) + 138 offsetof(NTHeader, OptionalHeader) + 139 offsetof(object::pe32plus_header, ImageBase); 140 B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0); 141 } 142 143 static MaterializationUnit::Interface 144 createHeaderInterface(COFFPlatform &MOP, 145 const SymbolStringPtr &HeaderStartSymbol) { 146 SymbolFlagsMap HeaderSymbolFlags; 147 148 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; 149 150 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), 151 HeaderStartSymbol); 152 } 153 154 COFFPlatform &CP; 155 }; 156 157 } // end anonymous namespace 158 159 namespace llvm { 160 namespace orc { 161 162 Expected<std::unique_ptr<COFFPlatform>> COFFPlatform::Create( 163 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, 164 JITDylib &PlatformJD, std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer, 165 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, 166 const char *VCRuntimePath, std::optional<SymbolAliasMap> RuntimeAliases) { 167 168 // If the target is not supported then bail out immediately. 169 if (!supportedTarget(ES.getTargetTriple())) 170 return make_error<StringError>("Unsupported COFFPlatform triple: " + 171 ES.getTargetTriple().str(), 172 inconvertibleErrorCode()); 173 174 auto &EPC = ES.getExecutorProcessControl(); 175 176 auto GeneratorArchive = 177 object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()); 178 if (!GeneratorArchive) 179 return GeneratorArchive.takeError(); 180 181 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create( 182 ObjLinkingLayer, nullptr, std::move(*GeneratorArchive)); 183 if (!OrcRuntimeArchiveGenerator) 184 return OrcRuntimeArchiveGenerator.takeError(); 185 186 // We need a second instance of the archive (for now) for the Platform. We 187 // can `cantFail` this call, since if it were going to fail it would have 188 // failed above. 189 auto RuntimeArchive = cantFail( 190 object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef())); 191 192 // Create default aliases if the caller didn't supply any. 193 if (!RuntimeAliases) 194 RuntimeAliases = standardPlatformAliases(ES); 195 196 // Define the aliases. 197 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) 198 return std::move(Err); 199 200 auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>"); 201 202 // Add JIT-dispatch function support symbols. 203 if (auto Err = HostFuncJD.define( 204 absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"), 205 {EPC.getJITDispatchInfo().JITDispatchFunction, 206 JITSymbolFlags::Exported}}, 207 {ES.intern("__orc_rt_jit_dispatch_ctx"), 208 {EPC.getJITDispatchInfo().JITDispatchContext, 209 JITSymbolFlags::Exported}}}))) 210 return std::move(Err); 211 212 PlatformJD.addToLinkOrder(HostFuncJD); 213 214 // Create the instance. 215 Error Err = Error::success(); 216 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform( 217 ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), 218 std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive), 219 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err)); 220 if (Err) 221 return std::move(Err); 222 return std::move(P); 223 } 224 225 Expected<std::unique_ptr<COFFPlatform>> 226 COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, 227 JITDylib &PlatformJD, const char *OrcRuntimePath, 228 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, 229 const char *VCRuntimePath, 230 std::optional<SymbolAliasMap> RuntimeAliases) { 231 232 auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath); 233 if (!ArchiveBuffer) 234 return createFileError(OrcRuntimePath, ArchiveBuffer.getError()); 235 236 return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer), 237 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, 238 std::move(RuntimeAliases)); 239 } 240 241 Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() { 242 auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker"); 243 if (!PerJDObj) 244 return PerJDObj.takeError(); 245 246 if (!*PerJDObj) 247 return make_error<StringError>("Could not find per jd object file", 248 inconvertibleErrorCode()); 249 250 auto Buffer = (*PerJDObj)->getAsBinary(); 251 if (!Buffer) 252 return Buffer.takeError(); 253 254 return (*Buffer)->getMemoryBufferRef(); 255 } 256 257 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, 258 ArrayRef<std::pair<const char *, const char *>> AL) { 259 for (auto &KV : AL) { 260 auto AliasName = ES.intern(KV.first); 261 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); 262 Aliases[std::move(AliasName)] = {ES.intern(KV.second), 263 JITSymbolFlags::Exported}; 264 } 265 } 266 267 Error COFFPlatform::setupJITDylib(JITDylib &JD) { 268 if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>( 269 *this, COFFHeaderStartSymbol))) 270 return Err; 271 272 if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError()) 273 return Err; 274 275 // Define the CXX aliases. 276 SymbolAliasMap CXXAliases; 277 addAliases(ES, CXXAliases, requiredCXXAliases()); 278 if (auto Err = JD.define(symbolAliases(std::move(CXXAliases)))) 279 return Err; 280 281 auto PerJDObj = getPerJDObjectFile(); 282 if (!PerJDObj) 283 return PerJDObj.takeError(); 284 285 auto I = getObjectFileInterface(ES, *PerJDObj); 286 if (!I) 287 return I.takeError(); 288 289 if (auto Err = ObjLinkingLayer.add( 290 JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I))) 291 return Err; 292 293 if (!Bootstrapping) { 294 auto ImportedLibs = StaticVCRuntime 295 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD) 296 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD); 297 if (!ImportedLibs) 298 return ImportedLibs.takeError(); 299 for (auto &Lib : *ImportedLibs) 300 if (auto Err = LoadDynLibrary(JD, Lib)) 301 return Err; 302 if (StaticVCRuntime) 303 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD)) 304 return Err; 305 } 306 307 JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer)); 308 return Error::success(); 309 } 310 311 Error COFFPlatform::teardownJITDylib(JITDylib &JD) { 312 std::lock_guard<std::mutex> Lock(PlatformMutex); 313 auto I = JITDylibToHeaderAddr.find(&JD); 314 if (I != JITDylibToHeaderAddr.end()) { 315 assert(HeaderAddrToJITDylib.count(I->second) && 316 "HeaderAddrToJITDylib missing entry"); 317 HeaderAddrToJITDylib.erase(I->second); 318 JITDylibToHeaderAddr.erase(I); 319 } 320 return Error::success(); 321 } 322 323 Error COFFPlatform::notifyAdding(ResourceTracker &RT, 324 const MaterializationUnit &MU) { 325 auto &JD = RT.getJITDylib(); 326 const auto &InitSym = MU.getInitializerSymbol(); 327 if (!InitSym) 328 return Error::success(); 329 330 RegisteredInitSymbols[&JD].add(InitSym, 331 SymbolLookupFlags::WeaklyReferencedSymbol); 332 333 LLVM_DEBUG({ 334 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU " 335 << MU.getName() << "\n"; 336 }); 337 return Error::success(); 338 } 339 340 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) { 341 llvm_unreachable("Not supported yet"); 342 } 343 344 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) { 345 SymbolAliasMap Aliases; 346 addAliases(ES, Aliases, standardRuntimeUtilityAliases()); 347 return Aliases; 348 } 349 350 ArrayRef<std::pair<const char *, const char *>> 351 COFFPlatform::requiredCXXAliases() { 352 static const std::pair<const char *, const char *> RequiredCXXAliases[] = { 353 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"}, 354 {"_onexit", "__orc_rt_coff_onexit_per_jd"}, 355 {"atexit", "__orc_rt_coff_atexit_per_jd"}}; 356 357 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases); 358 } 359 360 ArrayRef<std::pair<const char *, const char *>> 361 COFFPlatform::standardRuntimeUtilityAliases() { 362 static const std::pair<const char *, const char *> 363 StandardRuntimeUtilityAliases[] = { 364 {"__orc_rt_run_program", "__orc_rt_coff_run_program"}, 365 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"}, 366 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"}, 367 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"}, 368 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"}, 369 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}}; 370 371 return ArrayRef<std::pair<const char *, const char *>>( 372 StandardRuntimeUtilityAliases); 373 } 374 375 bool COFFPlatform::supportedTarget(const Triple &TT) { 376 switch (TT.getArch()) { 377 case Triple::x86_64: 378 return true; 379 default: 380 return false; 381 } 382 } 383 384 COFFPlatform::COFFPlatform( 385 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, 386 JITDylib &PlatformJD, 387 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator, 388 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer, 389 std::unique_ptr<object::Archive> OrcRuntimeArchive, 390 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, 391 const char *VCRuntimePath, Error &Err) 392 : ES(ES), ObjLinkingLayer(ObjLinkingLayer), 393 LoadDynLibrary(std::move(LoadDynLibrary)), 394 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)), 395 OrcRuntimeArchive(std::move(OrcRuntimeArchive)), 396 StaticVCRuntime(StaticVCRuntime), 397 COFFHeaderStartSymbol(ES.intern("__ImageBase")) { 398 ErrorAsOutParameter _(&Err); 399 400 Bootstrapping.store(true); 401 ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this)); 402 403 // Load vc runtime 404 auto VCRT = 405 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath); 406 if (!VCRT) { 407 Err = VCRT.takeError(); 408 return; 409 } 410 VCRuntimeBootstrap = std::move(*VCRT); 411 412 for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries()) 413 DylibsToPreload.insert(Lib); 414 415 auto ImportedLibs = 416 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD) 417 : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD); 418 if (!ImportedLibs) { 419 Err = ImportedLibs.takeError(); 420 return; 421 } 422 423 for (auto &Lib : *ImportedLibs) 424 DylibsToPreload.insert(Lib); 425 426 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); 427 428 // PlatformJD hasn't been set up by the platform yet (since we're creating 429 // the platform now), so set it up. 430 if (auto E2 = setupJITDylib(PlatformJD)) { 431 Err = std::move(E2); 432 return; 433 } 434 435 for (auto& Lib : DylibsToPreload) 436 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) { 437 Err = std::move(E2); 438 return; 439 } 440 441 if (StaticVCRuntime) 442 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) { 443 Err = std::move(E2); 444 return; 445 } 446 447 // Associate wrapper function tags with JIT-side function implementations. 448 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { 449 Err = std::move(E2); 450 return; 451 } 452 453 // Lookup addresses of runtime functions callable by the platform, 454 // call the platform bootstrap function to initialize the platform-state 455 // object in the executor. 456 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) { 457 Err = std::move(E2); 458 return; 459 } 460 461 Bootstrapping.store(false); 462 JDBootstrapStates.clear(); 463 } 464 465 Expected<COFFPlatform::JITDylibDepMap> 466 COFFPlatform::buildJDDepMap(JITDylib &JD) { 467 return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> { 468 JITDylibDepMap JDDepMap; 469 470 SmallVector<JITDylib *, 16> Worklist({&JD}); 471 while (!Worklist.empty()) { 472 auto CurJD = Worklist.back(); 473 Worklist.pop_back(); 474 475 auto &DM = JDDepMap[CurJD]; 476 CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) { 477 DM.reserve(O.size()); 478 for (auto &KV : O) { 479 if (KV.first == CurJD) 480 continue; 481 { 482 // Bare jitdylibs not known to the platform 483 std::lock_guard<std::mutex> Lock(PlatformMutex); 484 if (!JITDylibToHeaderAddr.count(KV.first)) { 485 LLVM_DEBUG({ 486 dbgs() << "JITDylib unregistered to COFFPlatform detected in " 487 "LinkOrder: " 488 << CurJD->getName() << "\n"; 489 }); 490 continue; 491 } 492 } 493 DM.push_back(KV.first); 494 // Push unvisited entry. 495 if (!JDDepMap.count(KV.first)) { 496 Worklist.push_back(KV.first); 497 JDDepMap[KV.first] = {}; 498 } 499 } 500 }); 501 } 502 return std::move(JDDepMap); 503 }); 504 } 505 506 void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult, 507 JITDylibSP JD, 508 JITDylibDepMap &JDDepMap) { 509 SmallVector<JITDylib *, 16> Worklist({JD.get()}); 510 DenseSet<JITDylib *> Visited({JD.get()}); 511 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; 512 ES.runSessionLocked([&]() { 513 while (!Worklist.empty()) { 514 auto CurJD = Worklist.back(); 515 Worklist.pop_back(); 516 517 auto RISItr = RegisteredInitSymbols.find(CurJD); 518 if (RISItr != RegisteredInitSymbols.end()) { 519 NewInitSymbols[CurJD] = std::move(RISItr->second); 520 RegisteredInitSymbols.erase(RISItr); 521 } 522 523 for (auto *DepJD : JDDepMap[CurJD]) 524 if (!Visited.count(DepJD)) { 525 Worklist.push_back(DepJD); 526 Visited.insert(DepJD); 527 } 528 } 529 }); 530 531 // If there are no further init symbols to look up then send the link order 532 // (as a list of header addresses) to the caller. 533 if (NewInitSymbols.empty()) { 534 // Build the dep info map to return. 535 COFFJITDylibDepInfoMap DIM; 536 DIM.reserve(JDDepMap.size()); 537 for (auto &KV : JDDepMap) { 538 std::lock_guard<std::mutex> Lock(PlatformMutex); 539 COFFJITDylibDepInfo DepInfo; 540 DepInfo.reserve(KV.second.size()); 541 for (auto &Dep : KV.second) { 542 DepInfo.push_back(JITDylibToHeaderAddr[Dep]); 543 } 544 auto H = JITDylibToHeaderAddr[KV.first]; 545 DIM.push_back(std::make_pair(H, std::move(DepInfo))); 546 } 547 SendResult(DIM); 548 return; 549 } 550 551 // Otherwise issue a lookup and re-run this phase when it completes. 552 lookupInitSymbolsAsync( 553 [this, SendResult = std::move(SendResult), &JD, 554 JDDepMap = std::move(JDDepMap)](Error Err) mutable { 555 if (Err) 556 SendResult(std::move(Err)); 557 else 558 pushInitializersLoop(std::move(SendResult), JD, JDDepMap); 559 }, 560 ES, std::move(NewInitSymbols)); 561 } 562 563 void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, 564 ExecutorAddr JDHeaderAddr) { 565 JITDylibSP JD; 566 { 567 std::lock_guard<std::mutex> Lock(PlatformMutex); 568 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr); 569 if (I != HeaderAddrToJITDylib.end()) 570 JD = I->second; 571 } 572 573 LLVM_DEBUG({ 574 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") "; 575 if (JD) 576 dbgs() << "pushing initializers for " << JD->getName() << "\n"; 577 else 578 dbgs() << "No JITDylib for header address.\n"; 579 }); 580 581 if (!JD) { 582 SendResult(make_error<StringError>("No JITDylib with header addr " + 583 formatv("{0:x}", JDHeaderAddr), 584 inconvertibleErrorCode())); 585 return; 586 } 587 588 auto JDDepMap = buildJDDepMap(*JD); 589 if (!JDDepMap) { 590 SendResult(JDDepMap.takeError()); 591 return; 592 } 593 594 pushInitializersLoop(std::move(SendResult), JD, *JDDepMap); 595 } 596 597 void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, 598 ExecutorAddr Handle, StringRef SymbolName) { 599 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"); 600 601 JITDylib *JD = nullptr; 602 603 { 604 std::lock_guard<std::mutex> Lock(PlatformMutex); 605 auto I = HeaderAddrToJITDylib.find(Handle); 606 if (I != HeaderAddrToJITDylib.end()) 607 JD = I->second; 608 } 609 610 if (!JD) { 611 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); 612 SendResult(make_error<StringError>("No JITDylib associated with handle " + 613 formatv("{0:x}", Handle), 614 inconvertibleErrorCode())); 615 return; 616 } 617 618 // Use functor class to work around XL build compiler issue on AIX. 619 class RtLookupNotifyComplete { 620 public: 621 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) 622 : SendResult(std::move(SendResult)) {} 623 void operator()(Expected<SymbolMap> Result) { 624 if (Result) { 625 assert(Result->size() == 1 && "Unexpected result map count"); 626 SendResult(Result->begin()->second.getAddress()); 627 } else { 628 SendResult(Result.takeError()); 629 } 630 } 631 632 private: 633 SendSymbolAddressFn SendResult; 634 }; 635 636 ES.lookup( 637 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, 638 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready, 639 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); 640 } 641 642 Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { 643 ExecutionSession::JITDispatchHandlerAssociationMap WFs; 644 645 using LookupSymbolSPSSig = 646 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); 647 WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] = 648 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, 649 &COFFPlatform::rt_lookupSymbol); 650 using PushInitializersSPSSig = 651 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr); 652 WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] = 653 ES.wrapAsyncWithSPS<PushInitializersSPSSig>( 654 this, &COFFPlatform::rt_pushInitializers); 655 656 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); 657 } 658 659 Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) { 660 llvm::sort(BState.Initializers); 661 if (auto Err = 662 runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ")) 663 return Err; 664 665 if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init")) 666 return Err; 667 668 if (auto Err = 669 runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ")) 670 return Err; 671 return Error::success(); 672 } 673 674 Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState, 675 StringRef Start, 676 StringRef End) { 677 for (auto &Initializer : BState.Initializers) 678 if (Initializer.first >= Start && Initializer.first <= End && 679 Initializer.second) { 680 auto Res = 681 ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second); 682 if (!Res) 683 return Res.takeError(); 684 } 685 return Error::success(); 686 } 687 688 Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) { 689 // Lookup of runtime symbols causes the collection of initializers if 690 // it's static linking setting. 691 if (auto Err = lookupAndRecordAddrs( 692 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), 693 { 694 {ES.intern("__orc_rt_coff_platform_bootstrap"), 695 &orc_rt_coff_platform_bootstrap}, 696 {ES.intern("__orc_rt_coff_platform_shutdown"), 697 &orc_rt_coff_platform_shutdown}, 698 {ES.intern("__orc_rt_coff_register_jitdylib"), 699 &orc_rt_coff_register_jitdylib}, 700 {ES.intern("__orc_rt_coff_deregister_jitdylib"), 701 &orc_rt_coff_deregister_jitdylib}, 702 {ES.intern("__orc_rt_coff_register_object_sections"), 703 &orc_rt_coff_register_object_sections}, 704 {ES.intern("__orc_rt_coff_deregister_object_sections"), 705 &orc_rt_coff_deregister_object_sections}, 706 })) 707 return Err; 708 709 // Call bootstrap functions 710 if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap)) 711 return Err; 712 713 // Do the pending jitdylib registration actions that we couldn't do 714 // because orc runtime was not linked fully. 715 for (auto KV : JDBootstrapStates) { 716 auto &JDBState = KV.second; 717 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>( 718 orc_rt_coff_register_jitdylib, JDBState.JDName, 719 JDBState.HeaderAddr)) 720 return Err; 721 722 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps) 723 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr, 724 SPSCOFFObjectSectionsMap, bool)>( 725 orc_rt_coff_register_object_sections, JDBState.HeaderAddr, 726 ObjSectionMap, false)) 727 return Err; 728 } 729 730 // Run static initializers collected in bootstrap stage. 731 for (auto KV : JDBootstrapStates) { 732 auto &JDBState = KV.second; 733 if (auto Err = runBootstrapInitializers(JDBState)) 734 return Err; 735 } 736 737 return Error::success(); 738 } 739 740 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD, 741 StringRef SymbolName) { 742 ExecutorAddr jit_function; 743 auto AfterCLookupErr = lookupAndRecordAddrs( 744 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), 745 {{ES.intern(SymbolName), &jit_function}}); 746 if (!AfterCLookupErr) { 747 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function); 748 if (!Res) 749 return Res.takeError(); 750 return Error::success(); 751 } 752 if (!AfterCLookupErr.isA<SymbolsNotFound>()) 753 return AfterCLookupErr; 754 consumeError(std::move(AfterCLookupErr)); 755 return Error::success(); 756 } 757 758 void COFFPlatform::COFFPlatformPlugin::modifyPassConfig( 759 MaterializationResponsibility &MR, jitlink::LinkGraph &LG, 760 jitlink::PassConfiguration &Config) { 761 762 bool IsBootstrapping = CP.Bootstrapping.load(); 763 764 if (auto InitSymbol = MR.getInitializerSymbol()) { 765 if (InitSymbol == CP.COFFHeaderStartSymbol) { 766 Config.PostAllocationPasses.push_back( 767 [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) { 768 return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping); 769 }); 770 return; 771 } 772 Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { 773 return preserveInitializerSections(G, MR); 774 }); 775 } 776 777 if (!IsBootstrapping) 778 Config.PostFixupPasses.push_back( 779 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { 780 return registerObjectPlatformSections(G, JD); 781 }); 782 else 783 Config.PostFixupPasses.push_back( 784 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { 785 return registerObjectPlatformSectionsInBootstrap(G, JD); 786 }); 787 } 788 789 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap 790 COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies( 791 MaterializationResponsibility &MR) { 792 std::lock_guard<std::mutex> Lock(PluginMutex); 793 auto I = InitSymbolDeps.find(&MR); 794 if (I != InitSymbolDeps.end()) { 795 SyntheticSymbolDependenciesMap Result; 796 Result[MR.getInitializerSymbol()] = std::move(I->second); 797 InitSymbolDeps.erase(&MR); 798 return Result; 799 } 800 return SyntheticSymbolDependenciesMap(); 801 } 802 803 Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol( 804 jitlink::LinkGraph &G, MaterializationResponsibility &MR, 805 bool IsBootstraping) { 806 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { 807 return Sym->getName() == *CP.COFFHeaderStartSymbol; 808 }); 809 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol"); 810 811 auto &JD = MR.getTargetJITDylib(); 812 std::lock_guard<std::mutex> Lock(CP.PlatformMutex); 813 auto HeaderAddr = (*I)->getAddress(); 814 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr; 815 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD; 816 if (!IsBootstraping) { 817 G.allocActions().push_back( 818 {cantFail(WrapperFunctionCall::Create< 819 SPSArgList<SPSString, SPSExecutorAddr>>( 820 CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)), 821 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( 822 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))}); 823 } else { 824 G.allocActions().push_back( 825 {{}, 826 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( 827 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))}); 828 JDBootstrapState BState; 829 BState.JD = &JD; 830 BState.JDName = JD.getName(); 831 BState.HeaderAddr = HeaderAddr; 832 CP.JDBootstrapStates.emplace(&JD, BState); 833 } 834 835 return Error::success(); 836 } 837 838 Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections( 839 jitlink::LinkGraph &G, JITDylib &JD) { 840 COFFObjectSectionsMap ObjSecs; 841 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD]; 842 assert(HeaderAddr && "Must be registered jitdylib"); 843 for (auto &S : G.sections()) { 844 jitlink::SectionRange Range(S); 845 if (Range.getSize()) 846 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange())); 847 } 848 849 G.allocActions().push_back( 850 {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>( 851 CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)), 852 cantFail( 853 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>( 854 CP.orc_rt_coff_deregister_object_sections, HeaderAddr, 855 ObjSecs))}); 856 857 return Error::success(); 858 } 859 860 Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections( 861 jitlink::LinkGraph &G, MaterializationResponsibility &MR) { 862 JITLinkSymbolSet InitSectionSymbols; 863 for (auto &Sec : G.sections()) 864 if (isCOFFInitializerSection(Sec.getName())) 865 for (auto *B : Sec.blocks()) 866 if (!B->edges_empty()) 867 InitSectionSymbols.insert( 868 &G.addAnonymousSymbol(*B, 0, 0, false, true)); 869 870 std::lock_guard<std::mutex> Lock(PluginMutex); 871 InitSymbolDeps[&MR] = InitSectionSymbols; 872 return Error::success(); 873 } 874 875 Error COFFPlatform::COFFPlatformPlugin:: 876 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G, 877 JITDylib &JD) { 878 std::lock_guard<std::mutex> Lock(CP.PlatformMutex); 879 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD]; 880 COFFObjectSectionsMap ObjSecs; 881 for (auto &S : G.sections()) { 882 jitlink::SectionRange Range(S); 883 if (Range.getSize()) 884 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange())); 885 } 886 887 G.allocActions().push_back( 888 {{}, 889 cantFail( 890 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>( 891 CP.orc_rt_coff_deregister_object_sections, HeaderAddr, 892 ObjSecs))}); 893 894 auto &BState = CP.JDBootstrapStates[&JD]; 895 BState.ObjectSectionsMaps.push_back(std::move(ObjSecs)); 896 897 // Collect static initializers 898 for (auto &S : G.sections()) 899 if (isCOFFInitializerSection(S.getName())) 900 for (auto *B : S.blocks()) { 901 if (B->edges_empty()) 902 continue; 903 for (auto &E : B->edges()) 904 BState.Initializers.push_back(std::make_pair( 905 S.getName().str(), E.getTarget().getAddress() + E.getAddend())); 906 } 907 908 return Error::success(); 909 } 910 911 } // End namespace orc. 912 } // End namespace llvm. 913