1 //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Contains utilities for adding indirections and breaking up modules. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 14 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 15 16 #include "llvm/ADT/StringMap.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/ExecutionEngine/JITSymbol.h" 19 #include "llvm/ExecutionEngine/Orc/Core.h" 20 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 21 #include "llvm/Support/Error.h" 22 #include "llvm/Support/Memory.h" 23 #include "llvm/Support/Process.h" 24 #include "llvm/Transforms/Utils/ValueMapper.h" 25 #include <algorithm> 26 #include <cassert> 27 #include <cstdint> 28 #include <functional> 29 #include <future> 30 #include <map> 31 #include <memory> 32 #include <system_error> 33 #include <utility> 34 #include <vector> 35 36 namespace llvm { 37 38 class Constant; 39 class Function; 40 class FunctionType; 41 class GlobalAlias; 42 class GlobalVariable; 43 class Module; 44 class PointerType; 45 class Triple; 46 class Twine; 47 class Value; 48 class MCDisassembler; 49 class MCInstrAnalysis; 50 51 namespace jitlink { 52 class LinkGraph; 53 class Symbol; 54 } // namespace jitlink 55 56 namespace orc { 57 58 /// Base class for pools of compiler re-entry trampolines. 59 /// These trampolines are callable addresses that save all register state 60 /// before calling a supplied function to return the trampoline landing 61 /// address, then restore all state before jumping to that address. They 62 /// are used by various ORC APIs to support lazy compilation 63 class TrampolinePool { 64 public: 65 using NotifyLandingResolvedFunction = 66 unique_function<void(ExecutorAddr) const>; 67 68 using ResolveLandingFunction = unique_function<void( 69 ExecutorAddr TrampolineAddr, 70 NotifyLandingResolvedFunction OnLandingResolved) const>; 71 72 virtual ~TrampolinePool(); 73 74 /// Get an available trampoline address. 75 /// Returns an error if no trampoline can be created. 76 Expected<ExecutorAddr> getTrampoline() { 77 std::lock_guard<std::mutex> Lock(TPMutex); 78 if (AvailableTrampolines.empty()) { 79 if (auto Err = grow()) 80 return std::move(Err); 81 } 82 assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool"); 83 auto TrampolineAddr = AvailableTrampolines.back(); 84 AvailableTrampolines.pop_back(); 85 return TrampolineAddr; 86 } 87 88 /// Returns the given trampoline to the pool for re-use. 89 void releaseTrampoline(ExecutorAddr TrampolineAddr) { 90 std::lock_guard<std::mutex> Lock(TPMutex); 91 AvailableTrampolines.push_back(TrampolineAddr); 92 } 93 94 protected: 95 virtual Error grow() = 0; 96 97 std::mutex TPMutex; 98 std::vector<ExecutorAddr> AvailableTrampolines; 99 }; 100 101 /// A trampoline pool for trampolines within the current process. 102 template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool { 103 public: 104 /// Creates a LocalTrampolinePool with the given RunCallback function. 105 /// Returns an error if this function is unable to correctly allocate, write 106 /// and protect the resolver code block. 107 static Expected<std::unique_ptr<LocalTrampolinePool>> 108 Create(ResolveLandingFunction ResolveLanding) { 109 Error Err = Error::success(); 110 111 auto LTP = std::unique_ptr<LocalTrampolinePool>( 112 new LocalTrampolinePool(std::move(ResolveLanding), Err)); 113 114 if (Err) 115 return std::move(Err); 116 return std::move(LTP); 117 } 118 119 private: 120 static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) { 121 LocalTrampolinePool<ORCABI> *TrampolinePool = 122 static_cast<LocalTrampolinePool *>(TrampolinePoolPtr); 123 124 std::promise<ExecutorAddr> LandingAddressP; 125 auto LandingAddressF = LandingAddressP.get_future(); 126 127 TrampolinePool->ResolveLanding(ExecutorAddr::fromPtr(TrampolineId), 128 [&](ExecutorAddr LandingAddress) { 129 LandingAddressP.set_value(LandingAddress); 130 }); 131 return LandingAddressF.get().getValue(); 132 } 133 134 LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err) 135 : ResolveLanding(std::move(ResolveLanding)) { 136 137 ErrorAsOutParameter _(&Err); 138 139 /// Try to set up the resolver block. 140 std::error_code EC; 141 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 142 ORCABI::ResolverCodeSize, nullptr, 143 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 144 if (EC) { 145 Err = errorCodeToError(EC); 146 return; 147 } 148 149 ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()), 150 ExecutorAddr::fromPtr(ResolverBlock.base()), 151 ExecutorAddr::fromPtr(&reenter), 152 ExecutorAddr::fromPtr(this)); 153 154 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), 155 sys::Memory::MF_READ | 156 sys::Memory::MF_EXEC); 157 if (EC) { 158 Err = errorCodeToError(EC); 159 return; 160 } 161 } 162 163 Error grow() override { 164 assert(AvailableTrampolines.empty() && "Growing prematurely?"); 165 166 std::error_code EC; 167 auto TrampolineBlock = 168 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 169 sys::Process::getPageSizeEstimate(), nullptr, 170 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 171 if (EC) 172 return errorCodeToError(EC); 173 174 unsigned NumTrampolines = 175 (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) / 176 ORCABI::TrampolineSize; 177 178 char *TrampolineMem = static_cast<char *>(TrampolineBlock.base()); 179 ORCABI::writeTrampolines( 180 TrampolineMem, ExecutorAddr::fromPtr(TrampolineMem), 181 ExecutorAddr::fromPtr(ResolverBlock.base()), NumTrampolines); 182 183 for (unsigned I = 0; I < NumTrampolines; ++I) 184 AvailableTrampolines.push_back( 185 ExecutorAddr::fromPtr(TrampolineMem + (I * ORCABI::TrampolineSize))); 186 187 if (auto EC = sys::Memory::protectMappedMemory( 188 TrampolineBlock.getMemoryBlock(), 189 sys::Memory::MF_READ | sys::Memory::MF_EXEC)) 190 return errorCodeToError(EC); 191 192 TrampolineBlocks.push_back(std::move(TrampolineBlock)); 193 return Error::success(); 194 } 195 196 ResolveLandingFunction ResolveLanding; 197 198 sys::OwningMemoryBlock ResolverBlock; 199 std::vector<sys::OwningMemoryBlock> TrampolineBlocks; 200 }; 201 202 /// Target-independent base class for compile callback management. 203 class JITCompileCallbackManager { 204 public: 205 using CompileFunction = std::function<ExecutorAddr()>; 206 207 virtual ~JITCompileCallbackManager() = default; 208 209 /// Reserve a compile callback. 210 Expected<ExecutorAddr> getCompileCallback(CompileFunction Compile); 211 212 /// Execute the callback for the given trampoline id. Called by the JIT 213 /// to compile functions on demand. 214 ExecutorAddr executeCompileCallback(ExecutorAddr TrampolineAddr); 215 216 protected: 217 /// Construct a JITCompileCallbackManager. 218 JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP, 219 ExecutionSession &ES, 220 ExecutorAddr ErrorHandlerAddress) 221 : TP(std::move(TP)), ES(ES), 222 CallbacksJD(ES.createBareJITDylib("<Callbacks>")), 223 ErrorHandlerAddress(ErrorHandlerAddress) {} 224 225 void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) { 226 this->TP = std::move(TP); 227 } 228 229 private: 230 std::mutex CCMgrMutex; 231 std::unique_ptr<TrampolinePool> TP; 232 ExecutionSession &ES; 233 JITDylib &CallbacksJD; 234 ExecutorAddr ErrorHandlerAddress; 235 std::map<ExecutorAddr, SymbolStringPtr> AddrToSymbol; 236 size_t NextCallbackId = 0; 237 }; 238 239 /// Manage compile callbacks for in-process JITs. 240 template <typename ORCABI> 241 class LocalJITCompileCallbackManager : public JITCompileCallbackManager { 242 public: 243 /// Create a new LocalJITCompileCallbackManager. 244 static Expected<std::unique_ptr<LocalJITCompileCallbackManager>> 245 Create(ExecutionSession &ES, ExecutorAddr ErrorHandlerAddress) { 246 Error Err = Error::success(); 247 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>( 248 new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err)); 249 if (Err) 250 return std::move(Err); 251 return std::move(CCMgr); 252 } 253 254 private: 255 /// Construct a InProcessJITCompileCallbackManager. 256 /// @param ErrorHandlerAddress The address of an error handler in the target 257 /// process to be used if a compile callback fails. 258 LocalJITCompileCallbackManager(ExecutionSession &ES, 259 ExecutorAddr ErrorHandlerAddress, Error &Err) 260 : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) { 261 using NotifyLandingResolvedFunction = 262 TrampolinePool::NotifyLandingResolvedFunction; 263 264 ErrorAsOutParameter _(&Err); 265 auto TP = LocalTrampolinePool<ORCABI>::Create( 266 [this](ExecutorAddr TrampolineAddr, 267 NotifyLandingResolvedFunction NotifyLandingResolved) { 268 NotifyLandingResolved(executeCompileCallback(TrampolineAddr)); 269 }); 270 271 if (!TP) { 272 Err = TP.takeError(); 273 return; 274 } 275 276 setTrampolinePool(std::move(*TP)); 277 } 278 }; 279 280 /// Base class for managing collections of named indirect stubs. 281 class IndirectStubsManager { 282 public: 283 /// Map type for initializing the manager. See init. 284 using StubInitsMap = StringMap<std::pair<ExecutorAddr, JITSymbolFlags>>; 285 286 virtual ~IndirectStubsManager() = default; 287 288 /// Create a single stub with the given name, target address and flags. 289 virtual Error createStub(StringRef StubName, ExecutorAddr StubAddr, 290 JITSymbolFlags StubFlags) = 0; 291 292 /// Create StubInits.size() stubs with the given names, target 293 /// addresses, and flags. 294 virtual Error createStubs(const StubInitsMap &StubInits) = 0; 295 296 /// Find the stub with the given name. If ExportedStubsOnly is true, 297 /// this will only return a result if the stub's flags indicate that it 298 /// is exported. 299 virtual ExecutorSymbolDef findStub(StringRef Name, 300 bool ExportedStubsOnly) = 0; 301 302 /// Find the implementation-pointer for the stub. 303 virtual ExecutorSymbolDef findPointer(StringRef Name) = 0; 304 305 /// Change the value of the implementation pointer for the stub. 306 virtual Error updatePointer(StringRef Name, ExecutorAddr NewAddr) = 0; 307 308 private: 309 virtual void anchor(); 310 }; 311 312 template <typename ORCABI> class LocalIndirectStubsInfo { 313 public: 314 LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem) 315 : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {} 316 317 static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs, 318 unsigned PageSize) { 319 auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize); 320 321 assert((ISAS.StubBytes % PageSize == 0) && 322 "StubBytes is not a page size multiple"); 323 uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize); 324 325 // Allocate memory for stubs and pointers in one call. 326 std::error_code EC; 327 auto StubsAndPtrsMem = 328 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 329 ISAS.StubBytes + PointerAlloc, nullptr, 330 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 331 if (EC) 332 return errorCodeToError(EC); 333 334 sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes); 335 auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base()); 336 auto PtrBlockAddress = 337 ExecutorAddr::fromPtr(StubsBlockMem) + ISAS.StubBytes; 338 339 ORCABI::writeIndirectStubsBlock(StubsBlockMem, 340 ExecutorAddr::fromPtr(StubsBlockMem), 341 PtrBlockAddress, ISAS.NumStubs); 342 343 if (auto EC = sys::Memory::protectMappedMemory( 344 StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) 345 return errorCodeToError(EC); 346 347 return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem)); 348 } 349 350 unsigned getNumStubs() const { return NumStubs; } 351 352 void *getStub(unsigned Idx) const { 353 return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize; 354 } 355 356 void **getPtr(unsigned Idx) const { 357 char *PtrsBase = 358 static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize; 359 return reinterpret_cast<void **>(PtrsBase) + Idx; 360 } 361 362 private: 363 unsigned NumStubs = 0; 364 sys::OwningMemoryBlock StubsMem; 365 }; 366 367 /// IndirectStubsManager implementation for the host architecture, e.g. 368 /// OrcX86_64. (See OrcArchitectureSupport.h). 369 template <typename TargetT> 370 class LocalIndirectStubsManager : public IndirectStubsManager { 371 public: 372 Error createStub(StringRef StubName, ExecutorAddr StubAddr, 373 JITSymbolFlags StubFlags) override { 374 std::lock_guard<std::mutex> Lock(StubsMutex); 375 if (auto Err = reserveStubs(1)) 376 return Err; 377 378 createStubInternal(StubName, StubAddr, StubFlags); 379 380 return Error::success(); 381 } 382 383 Error createStubs(const StubInitsMap &StubInits) override { 384 std::lock_guard<std::mutex> Lock(StubsMutex); 385 if (auto Err = reserveStubs(StubInits.size())) 386 return Err; 387 388 for (const auto &Entry : StubInits) 389 createStubInternal(Entry.first(), Entry.second.first, 390 Entry.second.second); 391 392 return Error::success(); 393 } 394 395 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override { 396 std::lock_guard<std::mutex> Lock(StubsMutex); 397 auto I = StubIndexes.find(Name); 398 if (I == StubIndexes.end()) 399 return ExecutorSymbolDef(); 400 auto Key = I->second.first; 401 void *StubPtr = IndirectStubsInfos[Key.first].getStub(Key.second); 402 assert(StubPtr && "Missing stub address"); 403 auto StubAddr = ExecutorAddr::fromPtr(StubPtr); 404 auto StubSymbol = ExecutorSymbolDef(StubAddr, I->second.second); 405 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) 406 return ExecutorSymbolDef(); 407 return StubSymbol; 408 } 409 410 ExecutorSymbolDef findPointer(StringRef Name) override { 411 std::lock_guard<std::mutex> Lock(StubsMutex); 412 auto I = StubIndexes.find(Name); 413 if (I == StubIndexes.end()) 414 return ExecutorSymbolDef(); 415 auto Key = I->second.first; 416 void *PtrPtr = IndirectStubsInfos[Key.first].getPtr(Key.second); 417 assert(PtrPtr && "Missing pointer address"); 418 auto PtrAddr = ExecutorAddr::fromPtr(PtrPtr); 419 return ExecutorSymbolDef(PtrAddr, I->second.second); 420 } 421 422 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override { 423 using AtomicIntPtr = std::atomic<uintptr_t>; 424 425 std::lock_guard<std::mutex> Lock(StubsMutex); 426 auto I = StubIndexes.find(Name); 427 assert(I != StubIndexes.end() && "No stub pointer for symbol"); 428 auto Key = I->second.first; 429 AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>( 430 IndirectStubsInfos[Key.first].getPtr(Key.second)); 431 *AtomicStubPtr = static_cast<uintptr_t>(NewAddr.getValue()); 432 return Error::success(); 433 } 434 435 private: 436 Error reserveStubs(unsigned NumStubs) { 437 if (NumStubs <= FreeStubs.size()) 438 return Error::success(); 439 440 unsigned NewStubsRequired = NumStubs - FreeStubs.size(); 441 unsigned NewBlockId = IndirectStubsInfos.size(); 442 auto ISI = 443 LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize); 444 if (!ISI) 445 return ISI.takeError(); 446 for (unsigned I = 0; I < ISI->getNumStubs(); ++I) 447 FreeStubs.push_back(std::make_pair(NewBlockId, I)); 448 IndirectStubsInfos.push_back(std::move(*ISI)); 449 return Error::success(); 450 } 451 452 void createStubInternal(StringRef StubName, ExecutorAddr InitAddr, 453 JITSymbolFlags StubFlags) { 454 auto Key = FreeStubs.back(); 455 FreeStubs.pop_back(); 456 *IndirectStubsInfos[Key.first].getPtr(Key.second) = 457 InitAddr.toPtr<void *>(); 458 StubIndexes[StubName] = std::make_pair(Key, StubFlags); 459 } 460 461 unsigned PageSize = sys::Process::getPageSizeEstimate(); 462 std::mutex StubsMutex; 463 std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos; 464 using StubKey = std::pair<uint16_t, uint16_t>; 465 std::vector<StubKey> FreeStubs; 466 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; 467 }; 468 469 /// Create a local compile callback manager. 470 /// 471 /// The given target triple will determine the ABI, and the given 472 /// ErrorHandlerAddress will be used by the resulting compile callback 473 /// manager if a compile callback fails. 474 Expected<std::unique_ptr<JITCompileCallbackManager>> 475 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 476 ExecutorAddr ErrorHandlerAddress); 477 478 /// Create a local indirect stubs manager builder. 479 /// 480 /// The given target triple will determine the ABI. 481 std::function<std::unique_ptr<IndirectStubsManager>()> 482 createLocalIndirectStubsManagerBuilder(const Triple &T); 483 484 /// Build a function pointer of FunctionType with the given constant 485 /// address. 486 /// 487 /// Usage example: Turn a trampoline address into a function pointer constant 488 /// for use in a stub. 489 Constant *createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr); 490 491 /// Create a function pointer with the given type, name, and initializer 492 /// in the given Module. 493 GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name, 494 Constant *Initializer); 495 496 /// Turn a function declaration into a stub function that makes an 497 /// indirect call using the given function pointer. 498 void makeStub(Function &F, Value &ImplPointer); 499 500 /// Promotes private symbols to global hidden, and renames to prevent clashes 501 /// with other promoted symbols. The same SymbolPromoter instance should be 502 /// used for all symbols to be added to a single JITDylib. 503 class SymbolLinkagePromoter { 504 public: 505 /// Promote symbols in the given module. Returns the set of global values 506 /// that have been renamed/promoted. 507 std::vector<GlobalValue *> operator()(Module &M); 508 509 private: 510 unsigned NextId = 0; 511 }; 512 513 /// Clone a function declaration into a new module. 514 /// 515 /// This function can be used as the first step towards creating a callback 516 /// stub (see makeStub). 517 /// 518 /// If the VMap argument is non-null, a mapping will be added between F and 519 /// the new declaration, and between each of F's arguments and the new 520 /// declaration's arguments. This map can then be passed in to moveFunction to 521 /// move the function body if required. Note: When moving functions between 522 /// modules with these utilities, all decls should be cloned (and added to a 523 /// single VMap) before any bodies are moved. This will ensure that references 524 /// between functions all refer to the versions in the new module. 525 Function *cloneFunctionDecl(Module &Dst, const Function &F, 526 ValueToValueMapTy *VMap = nullptr); 527 528 /// Clone a global variable declaration into a new module. 529 GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 530 ValueToValueMapTy *VMap = nullptr); 531 532 /// Clone a global alias declaration into a new module. 533 GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 534 ValueToValueMapTy &VMap); 535 536 /// Introduce relocations to \p Sym in its own definition if there are any 537 /// pointers formed via PC-relative address that do not already have a 538 /// relocation. 539 /// 540 /// This is useful when introducing indirection via a stub function at link time 541 /// without compiler support. If a function pointer is formed without a 542 /// relocation, e.g. in the definition of \c foo 543 /// 544 /// \code 545 /// _foo: 546 /// leaq -7(%rip), rax # form pointer to _foo without relocation 547 /// _bar: 548 /// leaq (%rip), %rax # uses X86_64_RELOC_SIGNED to '_foo' 549 /// \endcode 550 /// 551 /// the pointer to \c _foo computed by \c _foo and \c _bar may differ if we 552 /// introduce a stub for _foo. If the pointer is used as a key, this may be 553 /// observable to the program. This pass will attempt to introduce the missing 554 /// "self-relocation" on the leaq instruction. 555 /// 556 /// This is based on disassembly and should be considered "best effort". It may 557 /// silently fail to add relocations. 558 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, 559 jitlink::LinkGraph &G, 560 MCDisassembler &Disassembler, 561 MCInstrAnalysis &MIA); 562 563 } // end namespace orc 564 565 } // end namespace llvm 566 567 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 568