1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 std::promise<MSVCPError> DeallocResultP; 92 auto DeallocResultF = DeallocResultP.get_future(); 93 94 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 95 std::move(TrampolineBlocks), 96 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 97 98 return DeallocResultF.get(); 99 } 100 101 Error EPCTrampolinePool::grow() { 102 using namespace jitlink; 103 104 assert(AvailableTrampolines.empty() && 105 "Grow called with trampolines still available"); 106 107 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 108 assert(ResolverAddress && "Resolver address can not be null"); 109 110 auto &EPC = EPCIU.getExecutorProcessControl(); 111 auto PageSize = EPC.getPageSize(); 112 auto Alloc = SimpleSegmentAlloc::Create( 113 EPC.getMemMgr(), nullptr, 114 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 115 if (!Alloc) 116 return Alloc.takeError(); 117 118 unsigned NumTrampolines = TrampolinesPerPage; 119 120 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 121 EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(), 122 SegInfo.Addr.getValue(), 123 ResolverAddress, NumTrampolines); 124 for (unsigned I = 0; I < NumTrampolines; ++I) 125 AvailableTrampolines.push_back(SegInfo.Addr.getValue() + 126 (I * TrampolineSize)); 127 128 auto FA = Alloc->finalize(); 129 if (!FA) 130 return FA.takeError(); 131 132 TrampolineBlocks.push_back(std::move(*FA)); 133 134 return Error::success(); 135 } 136 137 Error EPCIndirectStubsManager::createStub(StringRef StubName, 138 JITTargetAddress StubAddr, 139 JITSymbolFlags StubFlags) { 140 StubInitsMap SIM; 141 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 142 return createStubs(SIM); 143 } 144 145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 146 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 147 if (!AvailableStubInfos) 148 return AvailableStubInfos.takeError(); 149 150 { 151 std::lock_guard<std::mutex> Lock(ISMMutex); 152 unsigned ASIdx = 0; 153 for (auto &SI : StubInits) { 154 auto &A = (*AvailableStubInfos)[ASIdx++]; 155 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 156 } 157 } 158 159 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 160 switch (EPCIU.getABISupport().getPointerSize()) { 161 case 4: { 162 unsigned ASIdx = 0; 163 std::vector<tpctypes::UInt32Write> PtrUpdates; 164 for (auto &SI : StubInits) 165 PtrUpdates.push_back( 166 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 167 static_cast<uint32_t>(SI.second.first)}); 168 return MemAccess.writeUInt32s(PtrUpdates); 169 } 170 case 8: { 171 unsigned ASIdx = 0; 172 std::vector<tpctypes::UInt64Write> PtrUpdates; 173 for (auto &SI : StubInits) 174 PtrUpdates.push_back( 175 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 176 static_cast<uint64_t>(SI.second.first)}); 177 return MemAccess.writeUInt64s(PtrUpdates); 178 } 179 default: 180 return make_error<StringError>("Unsupported pointer size", 181 inconvertibleErrorCode()); 182 } 183 } 184 185 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 186 bool ExportedStubsOnly) { 187 std::lock_guard<std::mutex> Lock(ISMMutex); 188 auto I = StubInfos.find(Name); 189 if (I == StubInfos.end()) 190 return nullptr; 191 return {I->second.first.StubAddress, I->second.second}; 192 } 193 194 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 195 std::lock_guard<std::mutex> Lock(ISMMutex); 196 auto I = StubInfos.find(Name); 197 if (I == StubInfos.end()) 198 return nullptr; 199 return {I->second.first.PointerAddress, I->second.second}; 200 } 201 202 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 203 JITTargetAddress NewAddr) { 204 205 JITTargetAddress PtrAddr = 0; 206 { 207 std::lock_guard<std::mutex> Lock(ISMMutex); 208 auto I = StubInfos.find(Name); 209 if (I == StubInfos.end()) 210 return make_error<StringError>("Unknown stub name", 211 inconvertibleErrorCode()); 212 PtrAddr = I->second.first.PointerAddress; 213 } 214 215 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 216 switch (EPCIU.getABISupport().getPointerSize()) { 217 case 4: { 218 tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 219 return MemAccess.writeUInt32s(PUpdate); 220 } 221 case 8: { 222 tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 223 return MemAccess.writeUInt64s(PUpdate); 224 } 225 default: 226 return make_error<StringError>("Unsupported pointer size", 227 inconvertibleErrorCode()); 228 } 229 } 230 231 } // end anonymous namespace. 232 233 namespace llvm { 234 namespace orc { 235 236 EPCIndirectionUtils::ABISupport::~ABISupport() = default; 237 238 Expected<std::unique_ptr<EPCIndirectionUtils>> 239 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 240 const auto &TT = EPC.getTargetTriple(); 241 switch (TT.getArch()) { 242 default: 243 return make_error<StringError>( 244 std::string("No EPCIndirectionUtils available for ") + TT.str(), 245 inconvertibleErrorCode()); 246 case Triple::aarch64: 247 case Triple::aarch64_32: 248 return CreateWithABI<OrcAArch64>(EPC); 249 250 case Triple::x86: 251 return CreateWithABI<OrcI386>(EPC); 252 253 case Triple::loongarch64: 254 return CreateWithABI<OrcLoongArch64>(EPC); 255 256 case Triple::mips: 257 return CreateWithABI<OrcMips32Be>(EPC); 258 259 case Triple::mipsel: 260 return CreateWithABI<OrcMips32Le>(EPC); 261 262 case Triple::mips64: 263 case Triple::mips64el: 264 return CreateWithABI<OrcMips64>(EPC); 265 266 case Triple::riscv64: 267 return CreateWithABI<OrcRiscv64>(EPC); 268 269 case Triple::x86_64: 270 if (TT.getOS() == Triple::OSType::Win32) 271 return CreateWithABI<OrcX86_64_Win32>(EPC); 272 else 273 return CreateWithABI<OrcX86_64_SysV>(EPC); 274 } 275 } 276 277 Error EPCIndirectionUtils::cleanup() { 278 279 auto &MemMgr = EPC.getMemMgr(); 280 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 281 282 if (TP) 283 Err = joinErrors(std::move(Err), 284 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 285 286 if (ResolverBlock) 287 Err = 288 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 289 290 return Err; 291 } 292 293 Expected<JITTargetAddress> 294 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 295 JITTargetAddress ReentryCtxAddr) { 296 using namespace jitlink; 297 298 assert(ABI && "ABI can not be null"); 299 auto ResolverSize = ABI->getResolverCodeSize(); 300 301 auto Alloc = 302 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 303 {{MemProt::Read | MemProt::Exec, 304 {ResolverSize, Align(EPC.getPageSize())}}}); 305 306 if (!Alloc) 307 return Alloc.takeError(); 308 309 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 310 ResolverBlockAddr = SegInfo.Addr.getValue(); 311 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr, 312 ReentryFnAddr, ReentryCtxAddr); 313 314 auto FA = Alloc->finalize(); 315 if (!FA) 316 return FA.takeError(); 317 318 ResolverBlock = std::move(*FA); 319 return ResolverBlockAddr; 320 } 321 322 std::unique_ptr<IndirectStubsManager> 323 EPCIndirectionUtils::createIndirectStubsManager() { 324 return std::make_unique<EPCIndirectStubsManager>(*this); 325 } 326 327 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 328 if (!TP) 329 TP = std::make_unique<EPCTrampolinePool>(*this); 330 return *TP; 331 } 332 333 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 334 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 335 assert(!LCTM && 336 "createLazyCallThroughManager can not have been called before"); 337 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 338 &getTrampolinePool()); 339 return *LCTM; 340 } 341 342 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 343 std::unique_ptr<ABISupport> ABI) 344 : EPC(EPC), ABI(std::move(ABI)) { 345 assert(this->ABI && "ABI can not be null"); 346 347 assert(EPC.getPageSize() > getABISupport().getStubSize() && 348 "Stubs larger than one page are not supported"); 349 } 350 351 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 352 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 353 using namespace jitlink; 354 355 std::lock_guard<std::mutex> Lock(EPCUIMutex); 356 357 // If there aren't enough stubs available then allocate some more. 358 if (NumStubs > AvailableIndirectStubs.size()) { 359 auto NumStubsToAllocate = NumStubs; 360 auto PageSize = EPC.getPageSize(); 361 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 362 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 363 auto PtrBytes = 364 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 365 366 auto StubProt = MemProt::Read | MemProt::Exec; 367 auto PtrProt = MemProt::Read | MemProt::Write; 368 369 auto Alloc = SimpleSegmentAlloc::Create( 370 EPC.getMemMgr(), nullptr, 371 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 372 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); 373 374 if (!Alloc) 375 return Alloc.takeError(); 376 377 auto StubSeg = Alloc->getSegInfo(StubProt); 378 auto PtrSeg = Alloc->getSegInfo(PtrProt); 379 380 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), 381 StubSeg.Addr.getValue(), 382 PtrSeg.Addr.getValue(), NumStubsToAllocate); 383 384 auto FA = Alloc->finalize(); 385 if (!FA) 386 return FA.takeError(); 387 388 IndirectStubAllocs.push_back(std::move(*FA)); 389 390 auto StubExecutorAddr = StubSeg.Addr; 391 auto PtrExecutorAddr = PtrSeg.Addr; 392 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 393 AvailableIndirectStubs.push_back(IndirectStubInfo( 394 StubExecutorAddr.getValue(), PtrExecutorAddr.getValue())); 395 StubExecutorAddr += ABI->getStubSize(); 396 PtrExecutorAddr += ABI->getPointerSize(); 397 } 398 } 399 400 assert(NumStubs <= AvailableIndirectStubs.size() && 401 "Sufficient stubs should have been allocated above"); 402 403 IndirectStubInfoVector Result; 404 while (NumStubs--) { 405 Result.push_back(AvailableIndirectStubs.back()); 406 AvailableIndirectStubs.pop_back(); 407 } 408 409 return std::move(Result); 410 } 411 412 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 413 JITTargetAddress TrampolineAddr) { 414 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 415 std::promise<JITTargetAddress> LandingAddrP; 416 auto LandingAddrF = LandingAddrP.get_future(); 417 LCTM.resolveTrampolineLandingAddress( 418 TrampolineAddr, 419 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 420 return LandingAddrF.get(); 421 } 422 423 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 424 auto &LCTM = EPCIU.getLazyCallThroughManager(); 425 return EPCIU 426 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 427 pointerToJITTargetAddress(&LCTM)) 428 .takeError(); 429 } 430 431 } // end namespace orc 432 } // end namespace llvm 433