1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" 10 11 #include "llvm/BinaryFormat/ELF.h" 12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" 13 #include "llvm/ExecutionEngine/JITLink/aarch64.h" 14 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 15 #include "llvm/ExecutionEngine/Orc/DebugUtils.h" 16 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" 17 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" 18 #include "llvm/Support/BinaryByteStream.h" 19 #include "llvm/Support/Debug.h" 20 #include <optional> 21 22 #define DEBUG_TYPE "orc" 23 24 using namespace llvm; 25 using namespace llvm::orc; 26 using namespace llvm::orc::shared; 27 28 namespace { 29 30 class DSOHandleMaterializationUnit : public MaterializationUnit { 31 public: 32 DSOHandleMaterializationUnit(ELFNixPlatform &ENP, 33 const SymbolStringPtr &DSOHandleSymbol) 34 : MaterializationUnit( 35 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)), 36 ENP(ENP) {} 37 38 StringRef getName() const override { return "DSOHandleMU"; } 39 40 void materialize(std::unique_ptr<MaterializationResponsibility> R) override { 41 unsigned PointerSize; 42 support::endianness Endianness; 43 jitlink::Edge::Kind EdgeKind; 44 const auto &TT = ENP.getExecutionSession().getTargetTriple(); 45 46 switch (TT.getArch()) { 47 case Triple::x86_64: 48 PointerSize = 8; 49 Endianness = support::endianness::little; 50 EdgeKind = jitlink::x86_64::Pointer64; 51 break; 52 case Triple::aarch64: 53 PointerSize = 8; 54 Endianness = support::endianness::little; 55 EdgeKind = jitlink::aarch64::Pointer64; 56 break; 57 default: 58 llvm_unreachable("Unrecognized architecture"); 59 } 60 61 // void *__dso_handle = &__dso_handle; 62 auto G = std::make_unique<jitlink::LinkGraph>( 63 "<DSOHandleMU>", TT, PointerSize, Endianness, 64 jitlink::getGenericEdgeKindName); 65 auto &DSOHandleSection = 66 G->createSection(".data.__dso_handle", MemProt::Read); 67 auto &DSOHandleBlock = G->createContentBlock( 68 DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(), 69 8, 0); 70 auto &DSOHandleSymbol = G->addDefinedSymbol( 71 DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(), 72 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true); 73 DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0); 74 75 ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); 76 } 77 78 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} 79 80 private: 81 static MaterializationUnit::Interface 82 createDSOHandleSectionInterface(ELFNixPlatform &ENP, 83 const SymbolStringPtr &DSOHandleSymbol) { 84 SymbolFlagsMap SymbolFlags; 85 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported; 86 return MaterializationUnit::Interface(std::move(SymbolFlags), 87 DSOHandleSymbol); 88 } 89 90 ArrayRef<char> getDSOHandleContent(size_t PointerSize) { 91 static const char Content[8] = {0}; 92 assert(PointerSize <= sizeof Content); 93 return {Content, PointerSize}; 94 } 95 96 ELFNixPlatform &ENP; 97 }; 98 99 } // end anonymous namespace 100 101 namespace llvm { 102 namespace orc { 103 104 Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create( 105 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, 106 JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime, 107 std::optional<SymbolAliasMap> RuntimeAliases) { 108 109 // If the target is not supported then bail out immediately. 110 if (!supportedTarget(ES.getTargetTriple())) 111 return make_error<StringError>("Unsupported ELFNixPlatform triple: " + 112 ES.getTargetTriple().str(), 113 inconvertibleErrorCode()); 114 115 auto &EPC = ES.getExecutorProcessControl(); 116 117 // Create default aliases if the caller didn't supply any. 118 if (!RuntimeAliases) { 119 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD); 120 if (!StandardRuntimeAliases) 121 return StandardRuntimeAliases.takeError(); 122 RuntimeAliases = std::move(*StandardRuntimeAliases); 123 } 124 125 // Define the aliases. 126 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) 127 return std::move(Err); 128 129 // Add JIT-dispatch function support symbols. 130 if (auto Err = PlatformJD.define( 131 absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"), 132 {EPC.getJITDispatchInfo().JITDispatchFunction, 133 JITSymbolFlags::Exported}}, 134 {ES.intern("__orc_rt_jit_dispatch_ctx"), 135 {EPC.getJITDispatchInfo().JITDispatchContext, 136 JITSymbolFlags::Exported}}}))) 137 return std::move(Err); 138 139 // Create the instance. 140 Error Err = Error::success(); 141 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform( 142 ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); 143 if (Err) 144 return std::move(Err); 145 return std::move(P); 146 } 147 148 Expected<std::unique_ptr<ELFNixPlatform>> 149 ELFNixPlatform::Create(ExecutionSession &ES, 150 ObjectLinkingLayer &ObjLinkingLayer, 151 JITDylib &PlatformJD, const char *OrcRuntimePath, 152 std::optional<SymbolAliasMap> RuntimeAliases) { 153 154 // Create a generator for the ORC runtime archive. 155 auto OrcRuntimeArchiveGenerator = 156 StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); 157 if (!OrcRuntimeArchiveGenerator) 158 return OrcRuntimeArchiveGenerator.takeError(); 159 160 return Create(ES, ObjLinkingLayer, PlatformJD, 161 std::move(*OrcRuntimeArchiveGenerator), 162 std::move(RuntimeAliases)); 163 } 164 165 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) { 166 return JD.define( 167 std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol)); 168 } 169 170 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) { 171 return Error::success(); 172 } 173 174 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT, 175 const MaterializationUnit &MU) { 176 auto &JD = RT.getJITDylib(); 177 const auto &InitSym = MU.getInitializerSymbol(); 178 if (!InitSym) 179 return Error::success(); 180 181 RegisteredInitSymbols[&JD].add(InitSym, 182 SymbolLookupFlags::WeaklyReferencedSymbol); 183 LLVM_DEBUG({ 184 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym 185 << " for MU " << MU.getName() << "\n"; 186 }); 187 return Error::success(); 188 } 189 190 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) { 191 llvm_unreachable("Not supported yet"); 192 } 193 194 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, 195 ArrayRef<std::pair<const char *, const char *>> AL) { 196 for (auto &KV : AL) { 197 auto AliasName = ES.intern(KV.first); 198 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); 199 Aliases[std::move(AliasName)] = {ES.intern(KV.second), 200 JITSymbolFlags::Exported}; 201 } 202 } 203 204 Expected<SymbolAliasMap> 205 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES, 206 JITDylib &PlatformJD) { 207 SymbolAliasMap Aliases; 208 addAliases(ES, Aliases, requiredCXXAliases()); 209 addAliases(ES, Aliases, standardRuntimeUtilityAliases()); 210 return Aliases; 211 } 212 213 ArrayRef<std::pair<const char *, const char *>> 214 ELFNixPlatform::requiredCXXAliases() { 215 static const std::pair<const char *, const char *> RequiredCXXAliases[] = { 216 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"}, 217 {"atexit", "__orc_rt_elfnix_atexit"}}; 218 219 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases); 220 } 221 222 ArrayRef<std::pair<const char *, const char *>> 223 ELFNixPlatform::standardRuntimeUtilityAliases() { 224 static const std::pair<const char *, const char *> 225 StandardRuntimeUtilityAliases[] = { 226 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"}, 227 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"}, 228 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"}, 229 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"}, 230 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"}, 231 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}}; 232 233 return ArrayRef<std::pair<const char *, const char *>>( 234 StandardRuntimeUtilityAliases); 235 } 236 237 bool ELFNixPlatform::supportedTarget(const Triple &TT) { 238 switch (TT.getArch()) { 239 case Triple::x86_64: 240 case Triple::aarch64: 241 return true; 242 default: 243 return false; 244 } 245 } 246 247 ELFNixPlatform::ELFNixPlatform( 248 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, 249 JITDylib &PlatformJD, 250 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err) 251 : ES(ES), ObjLinkingLayer(ObjLinkingLayer), 252 DSOHandleSymbol(ES.intern("__dso_handle")) { 253 ErrorAsOutParameter _(&Err); 254 255 ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this)); 256 257 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); 258 259 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating 260 // the platform now), so set it up. 261 if (auto E2 = setupJITDylib(PlatformJD)) { 262 Err = std::move(E2); 263 return; 264 } 265 266 RegisteredInitSymbols[&PlatformJD].add( 267 DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); 268 269 // Associate wrapper function tags with JIT-side function implementations. 270 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { 271 Err = std::move(E2); 272 return; 273 } 274 275 // Lookup addresses of runtime functions callable by the platform, 276 // call the platform bootstrap function to initialize the platform-state 277 // object in the executor. 278 if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) { 279 Err = std::move(E2); 280 return; 281 } 282 } 283 284 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { 285 ExecutionSession::JITDispatchHandlerAssociationMap WFs; 286 287 using GetInitializersSPSSig = 288 SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString); 289 WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] = 290 ES.wrapAsyncWithSPS<GetInitializersSPSSig>( 291 this, &ELFNixPlatform::rt_getInitializers); 292 293 using GetDeinitializersSPSSig = 294 SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr); 295 WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] = 296 ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>( 297 this, &ELFNixPlatform::rt_getDeinitializers); 298 299 using LookupSymbolSPSSig = 300 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); 301 WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] = 302 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, 303 &ELFNixPlatform::rt_lookupSymbol); 304 305 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); 306 } 307 308 void ELFNixPlatform::getInitializersBuildSequencePhase( 309 SendInitializerSequenceFn SendResult, JITDylib &JD, 310 std::vector<JITDylibSP> DFSLinkOrder) { 311 ELFNixJITDylibInitializerSequence FullInitSeq; 312 { 313 std::lock_guard<std::mutex> Lock(PlatformMutex); 314 for (auto &InitJD : reverse(DFSLinkOrder)) { 315 LLVM_DEBUG({ 316 dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName() 317 << "\" to sequence\n"; 318 }); 319 auto ISItr = InitSeqs.find(InitJD.get()); 320 if (ISItr != InitSeqs.end()) { 321 FullInitSeq.emplace_back(std::move(ISItr->second)); 322 InitSeqs.erase(ISItr); 323 } 324 } 325 } 326 327 SendResult(std::move(FullInitSeq)); 328 } 329 330 void ELFNixPlatform::getInitializersLookupPhase( 331 SendInitializerSequenceFn SendResult, JITDylib &JD) { 332 333 auto DFSLinkOrder = JD.getDFSLinkOrder(); 334 if (!DFSLinkOrder) { 335 SendResult(DFSLinkOrder.takeError()); 336 return; 337 } 338 339 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; 340 ES.runSessionLocked([&]() { 341 for (auto &InitJD : *DFSLinkOrder) { 342 auto RISItr = RegisteredInitSymbols.find(InitJD.get()); 343 if (RISItr != RegisteredInitSymbols.end()) { 344 NewInitSymbols[InitJD.get()] = std::move(RISItr->second); 345 RegisteredInitSymbols.erase(RISItr); 346 } 347 } 348 }); 349 350 // If there are no further init symbols to look up then move on to the next 351 // phase. 352 if (NewInitSymbols.empty()) { 353 getInitializersBuildSequencePhase(std::move(SendResult), JD, 354 std::move(*DFSLinkOrder)); 355 return; 356 } 357 358 // Otherwise issue a lookup and re-run this phase when it completes. 359 lookupInitSymbolsAsync( 360 [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { 361 if (Err) 362 SendResult(std::move(Err)); 363 else 364 getInitializersLookupPhase(std::move(SendResult), JD); 365 }, 366 ES, std::move(NewInitSymbols)); 367 } 368 369 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, 370 StringRef JDName) { 371 LLVM_DEBUG({ 372 dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n"; 373 }); 374 375 JITDylib *JD = ES.getJITDylibByName(JDName); 376 if (!JD) { 377 LLVM_DEBUG({ 378 dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; 379 }); 380 SendResult(make_error<StringError>("No JITDylib named " + JDName, 381 inconvertibleErrorCode())); 382 return; 383 } 384 385 getInitializersLookupPhase(std::move(SendResult), *JD); 386 } 387 388 void ELFNixPlatform::rt_getDeinitializers( 389 SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) { 390 LLVM_DEBUG({ 391 dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n"; 392 }); 393 394 JITDylib *JD = nullptr; 395 396 { 397 std::lock_guard<std::mutex> Lock(PlatformMutex); 398 auto I = HandleAddrToJITDylib.find(Handle); 399 if (I != HandleAddrToJITDylib.end()) 400 JD = I->second; 401 } 402 403 if (!JD) { 404 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); 405 SendResult(make_error<StringError>("No JITDylib associated with handle " + 406 formatv("{0:x}", Handle), 407 inconvertibleErrorCode())); 408 return; 409 } 410 411 SendResult(ELFNixJITDylibDeinitializerSequence()); 412 } 413 414 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, 415 ExecutorAddr Handle, 416 StringRef SymbolName) { 417 LLVM_DEBUG({ 418 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; 419 }); 420 421 JITDylib *JD = nullptr; 422 423 { 424 std::lock_guard<std::mutex> Lock(PlatformMutex); 425 auto I = HandleAddrToJITDylib.find(Handle); 426 if (I != HandleAddrToJITDylib.end()) 427 JD = I->second; 428 } 429 430 if (!JD) { 431 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); 432 SendResult(make_error<StringError>("No JITDylib associated with handle " + 433 formatv("{0:x}", Handle), 434 inconvertibleErrorCode())); 435 return; 436 } 437 438 // Use functor class to work around XL build compiler issue on AIX. 439 class RtLookupNotifyComplete { 440 public: 441 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) 442 : SendResult(std::move(SendResult)) {} 443 void operator()(Expected<SymbolMap> Result) { 444 if (Result) { 445 assert(Result->size() == 1 && "Unexpected result map count"); 446 SendResult(Result->begin()->second.getAddress()); 447 } else { 448 SendResult(Result.takeError()); 449 } 450 } 451 452 private: 453 SendSymbolAddressFn SendResult; 454 }; 455 456 ES.lookup( 457 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, 458 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready, 459 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); 460 } 461 462 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) { 463 464 std::pair<const char *, ExecutorAddr *> Symbols[] = { 465 {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap}, 466 {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown}, 467 {"__orc_rt_elfnix_register_object_sections", 468 &orc_rt_elfnix_register_object_sections}, 469 {"__orc_rt_elfnix_create_pthread_key", 470 &orc_rt_elfnix_create_pthread_key}}; 471 472 SymbolLookupSet RuntimeSymbols; 473 std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord; 474 for (const auto &KV : Symbols) { 475 auto Name = ES.intern(KV.first); 476 RuntimeSymbols.add(Name); 477 AddrsToRecord.push_back({std::move(Name), KV.second}); 478 } 479 480 auto RuntimeSymbolAddrs = ES.lookup( 481 {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); 482 if (!RuntimeSymbolAddrs) 483 return RuntimeSymbolAddrs.takeError(); 484 485 for (const auto &KV : AddrsToRecord) { 486 auto &Name = KV.first; 487 assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); 488 *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress(); 489 } 490 491 auto PJDDSOHandle = ES.lookup( 492 {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol); 493 if (!PJDDSOHandle) 494 return PJDDSOHandle.takeError(); 495 496 if (auto Err = ES.callSPSWrapper<void(uint64_t)>( 497 orc_rt_elfnix_platform_bootstrap, 498 PJDDSOHandle->getAddress().getValue())) 499 return Err; 500 501 // FIXME: Ordering is fuzzy here. We're probably best off saying 502 // "behavior is undefined if code that uses the runtime is added before 503 // the platform constructor returns", then move all this to the constructor. 504 RuntimeBootstrapped = true; 505 std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs; 506 { 507 std::lock_guard<std::mutex> Lock(PlatformMutex); 508 DeferredPOSRs = std::move(BootstrapPOSRs); 509 } 510 511 for (auto &D : DeferredPOSRs) 512 if (auto Err = registerPerObjectSections(D)) 513 return Err; 514 515 return Error::success(); 516 } 517 518 Error ELFNixPlatform::registerInitInfo( 519 JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) { 520 521 std::unique_lock<std::mutex> Lock(PlatformMutex); 522 523 ELFNixJITDylibInitializers *InitSeq = nullptr; 524 { 525 auto I = InitSeqs.find(&JD); 526 if (I == InitSeqs.end()) { 527 // If there's no init sequence entry yet then we need to look up the 528 // header symbol to force creation of one. 529 Lock.unlock(); 530 531 auto SearchOrder = 532 JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); 533 if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError()) 534 return Err; 535 536 Lock.lock(); 537 I = InitSeqs.find(&JD); 538 assert(I != InitSeqs.end() && 539 "Entry missing after header symbol lookup?"); 540 } 541 InitSeq = &I->second; 542 } 543 544 for (auto *Sec : InitSections) { 545 // FIXME: Avoid copy here. 546 jitlink::SectionRange R(*Sec); 547 InitSeq->InitSections[Sec->getName()].push_back(R.getRange()); 548 } 549 550 return Error::success(); 551 } 552 553 Error ELFNixPlatform::registerPerObjectSections( 554 const ELFPerObjectSectionsToRegister &POSR) { 555 556 if (!orc_rt_elfnix_register_object_sections) 557 return make_error<StringError>("Attempting to register per-object " 558 "sections, but runtime support has not " 559 "been loaded yet", 560 inconvertibleErrorCode()); 561 562 Error ErrResult = Error::success(); 563 if (auto Err = ES.callSPSWrapper<shared::SPSError( 564 SPSELFPerObjectSectionsToRegister)>( 565 orc_rt_elfnix_register_object_sections, ErrResult, POSR)) 566 return Err; 567 return ErrResult; 568 } 569 570 Expected<uint64_t> ELFNixPlatform::createPThreadKey() { 571 if (!orc_rt_elfnix_create_pthread_key) 572 return make_error<StringError>( 573 "Attempting to create pthread key in target, but runtime support has " 574 "not been loaded yet", 575 inconvertibleErrorCode()); 576 577 Expected<uint64_t> Result(0); 578 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>( 579 orc_rt_elfnix_create_pthread_key, Result)) 580 return std::move(Err); 581 return Result; 582 } 583 584 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig( 585 MaterializationResponsibility &MR, jitlink::LinkGraph &LG, 586 jitlink::PassConfiguration &Config) { 587 588 // If the initializer symbol is the __dso_handle symbol then just add 589 // the DSO handle support passes. 590 if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) { 591 addDSOHandleSupportPasses(MR, Config); 592 // The DSOHandle materialization unit doesn't require any other 593 // support, so we can bail out early. 594 return; 595 } 596 597 // If the object contains initializers then add passes to record them. 598 if (MR.getInitializerSymbol()) 599 addInitializerSupportPasses(MR, Config); 600 601 // Add passes for eh-frame and TLV support. 602 addEHAndTLVSupportPasses(MR, Config); 603 } 604 605 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap 606 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies( 607 MaterializationResponsibility &MR) { 608 std::lock_guard<std::mutex> Lock(PluginMutex); 609 auto I = InitSymbolDeps.find(&MR); 610 if (I != InitSymbolDeps.end()) { 611 SyntheticSymbolDependenciesMap Result; 612 Result[MR.getInitializerSymbol()] = std::move(I->second); 613 InitSymbolDeps.erase(&MR); 614 return Result; 615 } 616 return SyntheticSymbolDependenciesMap(); 617 } 618 619 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses( 620 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { 621 622 /// Preserve init sections. 623 Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { 624 if (auto Err = preserveInitSections(G, MR)) 625 return Err; 626 return Error::success(); 627 }); 628 629 Config.PostFixupPasses.push_back( 630 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { 631 return registerInitSections(G, JD); 632 }); 633 } 634 635 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses( 636 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { 637 638 Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( 639 jitlink::LinkGraph &G) -> Error { 640 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { 641 return Sym->getName() == *MP.DSOHandleSymbol; 642 }); 643 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol"); 644 { 645 std::lock_guard<std::mutex> Lock(MP.PlatformMutex); 646 auto HandleAddr = (*I)->getAddress(); 647 MP.HandleAddrToJITDylib[HandleAddr] = &JD; 648 assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); 649 MP.InitSeqs.insert(std::make_pair( 650 &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr))); 651 } 652 return Error::success(); 653 }); 654 } 655 656 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses( 657 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { 658 659 // Insert TLV lowering at the start of the PostPrunePasses, since we want 660 // it to run before GOT/PLT lowering. 661 662 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build 663 // pass has done. Because the TLS descriptor need to be allocate in GOT. 664 Config.PostPrunePasses.push_back( 665 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { 666 return fixTLVSectionsAndEdges(G, JD); 667 }); 668 669 // Add a pass to register the final addresses of the eh-frame and TLV sections 670 // with the runtime. 671 Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { 672 ELFPerObjectSectionsToRegister POSR; 673 674 if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) { 675 jitlink::SectionRange R(*EHFrameSection); 676 if (!R.empty()) 677 POSR.EHFrameSection = R.getRange(); 678 } 679 680 // Get a pointer to the thread data section if there is one. It will be used 681 // below. 682 jitlink::Section *ThreadDataSection = 683 G.findSectionByName(ELFThreadDataSectionName); 684 685 // Handle thread BSS section if there is one. 686 if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) { 687 // If there's already a thread data section in this graph then merge the 688 // thread BSS section content into it, otherwise just treat the thread 689 // BSS section as the thread data section. 690 if (ThreadDataSection) 691 G.mergeSections(*ThreadDataSection, *ThreadBSSSection); 692 else 693 ThreadDataSection = ThreadBSSSection; 694 } 695 696 // Having merged thread BSS (if present) and thread data (if present), 697 // record the resulting section range. 698 if (ThreadDataSection) { 699 jitlink::SectionRange R(*ThreadDataSection); 700 if (!R.empty()) 701 POSR.ThreadDataSection = R.getRange(); 702 } 703 704 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) { 705 706 // If we're still bootstrapping the runtime then just record this 707 // frame for now. 708 if (!MP.RuntimeBootstrapped) { 709 std::lock_guard<std::mutex> Lock(MP.PlatformMutex); 710 MP.BootstrapPOSRs.push_back(POSR); 711 return Error::success(); 712 } 713 714 // Otherwise register it immediately. 715 if (auto Err = MP.registerPerObjectSections(POSR)) 716 return Err; 717 } 718 719 return Error::success(); 720 }); 721 } 722 723 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections( 724 jitlink::LinkGraph &G, MaterializationResponsibility &MR) { 725 726 JITLinkSymbolSet InitSectionSymbols; 727 for (auto &InitSection : G.sections()) { 728 // Skip non-init sections. 729 if (!isELFInitializerSection(InitSection.getName())) 730 continue; 731 732 // Make a pass over live symbols in the section: those blocks are already 733 // preserved. 734 DenseSet<jitlink::Block *> AlreadyLiveBlocks; 735 for (auto &Sym : InitSection.symbols()) { 736 auto &B = Sym->getBlock(); 737 if (Sym->isLive() && Sym->getOffset() == 0 && 738 Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) { 739 InitSectionSymbols.insert(Sym); 740 AlreadyLiveBlocks.insert(&B); 741 } 742 } 743 744 // Add anonymous symbols to preserve any not-already-preserved blocks. 745 for (auto *B : InitSection.blocks()) 746 if (!AlreadyLiveBlocks.count(B)) 747 InitSectionSymbols.insert( 748 &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true)); 749 } 750 751 if (!InitSectionSymbols.empty()) { 752 std::lock_guard<std::mutex> Lock(PluginMutex); 753 InitSymbolDeps[&MR] = std::move(InitSectionSymbols); 754 } 755 756 return Error::success(); 757 } 758 759 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections( 760 jitlink::LinkGraph &G, JITDylib &JD) { 761 762 SmallVector<jitlink::Section *> InitSections; 763 764 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n"); 765 766 for (auto &Sec : G.sections()) { 767 if (isELFInitializerSection(Sec.getName())) { 768 InitSections.push_back(&Sec); 769 } 770 } 771 772 // Dump the scraped inits. 773 LLVM_DEBUG({ 774 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n"; 775 for (auto *Sec : InitSections) { 776 jitlink::SectionRange R(*Sec); 777 dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n"; 778 } 779 }); 780 781 return MP.registerInitInfo(JD, InitSections); 782 } 783 784 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges( 785 jitlink::LinkGraph &G, JITDylib &JD) { 786 787 for (auto *Sym : G.external_symbols()) { 788 if (Sym->getName() == "__tls_get_addr") { 789 Sym->setName("___orc_rt_elfnix_tls_get_addr"); 790 } else if (Sym->getName() == "__tlsdesc_resolver") { 791 Sym->setName("___orc_rt_elfnix_tlsdesc_resolver"); 792 } 793 } 794 795 auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO"); 796 797 if (TLSInfoEntrySection) { 798 std::optional<uint64_t> Key; 799 { 800 std::lock_guard<std::mutex> Lock(MP.PlatformMutex); 801 auto I = MP.JITDylibToPThreadKey.find(&JD); 802 if (I != MP.JITDylibToPThreadKey.end()) 803 Key = I->second; 804 } 805 if (!Key) { 806 if (auto KeyOrErr = MP.createPThreadKey()) 807 Key = *KeyOrErr; 808 else 809 return KeyOrErr.takeError(); 810 } 811 812 uint64_t PlatformKeyBits = 813 support::endian::byte_swap(*Key, G.getEndianness()); 814 815 for (auto *B : TLSInfoEntrySection->blocks()) { 816 // FIXME: The TLS descriptor byte length may different with different 817 // ISA 818 assert(B->getSize() == (G.getPointerSize() * 2) && 819 "TLS descriptor must be 2 words length"); 820 auto TLSInfoEntryContent = B->getMutableContent(G); 821 memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize()); 822 } 823 } 824 825 return Error::success(); 826 } 827 828 } // End namespace orc. 829 } // End namespace llvm. 830