1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, ExecutorAddr StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 ExecutorSymbolDef findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 std::promise<MSVCPError> DeallocResultP; 92 auto DeallocResultF = DeallocResultP.get_future(); 93 94 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 95 std::move(TrampolineBlocks), 96 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 97 98 return DeallocResultF.get(); 99 } 100 101 Error EPCTrampolinePool::grow() { 102 using namespace jitlink; 103 104 assert(AvailableTrampolines.empty() && 105 "Grow called with trampolines still available"); 106 107 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 108 assert(ResolverAddress && "Resolver address can not be null"); 109 110 auto &EPC = EPCIU.getExecutorProcessControl(); 111 auto PageSize = EPC.getPageSize(); 112 auto Alloc = SimpleSegmentAlloc::Create( 113 EPC.getMemMgr(), nullptr, 114 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 115 if (!Alloc) 116 return Alloc.takeError(); 117 118 unsigned NumTrampolines = TrampolinesPerPage; 119 120 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 121 EPCIU.getABISupport().writeTrampolines( 122 SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); 123 for (unsigned I = 0; I < NumTrampolines; ++I) 124 AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); 125 126 auto FA = Alloc->finalize(); 127 if (!FA) 128 return FA.takeError(); 129 130 TrampolineBlocks.push_back(std::move(*FA)); 131 132 return Error::success(); 133 } 134 135 Error EPCIndirectStubsManager::createStub(StringRef StubName, 136 ExecutorAddr StubAddr, 137 JITSymbolFlags StubFlags) { 138 StubInitsMap SIM; 139 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 140 return createStubs(SIM); 141 } 142 143 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 144 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 145 if (!AvailableStubInfos) 146 return AvailableStubInfos.takeError(); 147 148 { 149 std::lock_guard<std::mutex> Lock(ISMMutex); 150 unsigned ASIdx = 0; 151 for (auto &SI : StubInits) { 152 auto &A = (*AvailableStubInfos)[ASIdx++]; 153 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 154 } 155 } 156 157 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 158 switch (EPCIU.getABISupport().getPointerSize()) { 159 case 4: { 160 unsigned ASIdx = 0; 161 std::vector<tpctypes::UInt32Write> PtrUpdates; 162 for (auto &SI : StubInits) 163 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, 164 static_cast<uint32_t>(SI.second.first.getValue())}); 165 return MemAccess.writeUInt32s(PtrUpdates); 166 } 167 case 8: { 168 unsigned ASIdx = 0; 169 std::vector<tpctypes::UInt64Write> PtrUpdates; 170 for (auto &SI : StubInits) 171 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, 172 static_cast<uint64_t>(SI.second.first.getValue())}); 173 return MemAccess.writeUInt64s(PtrUpdates); 174 } 175 default: 176 return make_error<StringError>("Unsupported pointer size", 177 inconvertibleErrorCode()); 178 } 179 } 180 181 ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name, 182 bool ExportedStubsOnly) { 183 std::lock_guard<std::mutex> Lock(ISMMutex); 184 auto I = StubInfos.find(Name); 185 if (I == StubInfos.end()) 186 return ExecutorSymbolDef(); 187 return {I->second.first.StubAddress, I->second.second}; 188 } 189 190 ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) { 191 std::lock_guard<std::mutex> Lock(ISMMutex); 192 auto I = StubInfos.find(Name); 193 if (I == StubInfos.end()) 194 return ExecutorSymbolDef(); 195 return {I->second.first.PointerAddress, I->second.second}; 196 } 197 198 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 199 ExecutorAddr NewAddr) { 200 201 ExecutorAddr PtrAddr; 202 { 203 std::lock_guard<std::mutex> Lock(ISMMutex); 204 auto I = StubInfos.find(Name); 205 if (I == StubInfos.end()) 206 return make_error<StringError>("Unknown stub name", 207 inconvertibleErrorCode()); 208 PtrAddr = I->second.first.PointerAddress; 209 } 210 211 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 212 switch (EPCIU.getABISupport().getPointerSize()) { 213 case 4: { 214 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue()); 215 return MemAccess.writeUInt32s(PUpdate); 216 } 217 case 8: { 218 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue()); 219 return MemAccess.writeUInt64s(PUpdate); 220 } 221 default: 222 return make_error<StringError>("Unsupported pointer size", 223 inconvertibleErrorCode()); 224 } 225 } 226 227 } // end anonymous namespace. 228 229 namespace llvm { 230 namespace orc { 231 232 EPCIndirectionUtils::ABISupport::~ABISupport() = default; 233 234 Expected<std::unique_ptr<EPCIndirectionUtils>> 235 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 236 const auto &TT = EPC.getTargetTriple(); 237 switch (TT.getArch()) { 238 default: 239 return make_error<StringError>( 240 std::string("No EPCIndirectionUtils available for ") + TT.str(), 241 inconvertibleErrorCode()); 242 case Triple::aarch64: 243 case Triple::aarch64_32: 244 return CreateWithABI<OrcAArch64>(EPC); 245 246 case Triple::x86: 247 return CreateWithABI<OrcI386>(EPC); 248 249 case Triple::loongarch64: 250 return CreateWithABI<OrcLoongArch64>(EPC); 251 252 case Triple::mips: 253 return CreateWithABI<OrcMips32Be>(EPC); 254 255 case Triple::mipsel: 256 return CreateWithABI<OrcMips32Le>(EPC); 257 258 case Triple::mips64: 259 case Triple::mips64el: 260 return CreateWithABI<OrcMips64>(EPC); 261 262 case Triple::riscv64: 263 return CreateWithABI<OrcRiscv64>(EPC); 264 265 case Triple::x86_64: 266 if (TT.getOS() == Triple::OSType::Win32) 267 return CreateWithABI<OrcX86_64_Win32>(EPC); 268 else 269 return CreateWithABI<OrcX86_64_SysV>(EPC); 270 } 271 } 272 273 Error EPCIndirectionUtils::cleanup() { 274 275 auto &MemMgr = EPC.getMemMgr(); 276 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 277 278 if (TP) 279 Err = joinErrors(std::move(Err), 280 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 281 282 if (ResolverBlock) 283 Err = 284 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 285 286 return Err; 287 } 288 289 Expected<ExecutorAddr> 290 EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr, 291 ExecutorAddr ReentryCtxAddr) { 292 using namespace jitlink; 293 294 assert(ABI && "ABI can not be null"); 295 auto ResolverSize = ABI->getResolverCodeSize(); 296 297 auto Alloc = 298 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 299 {{MemProt::Read | MemProt::Exec, 300 {ResolverSize, Align(EPC.getPageSize())}}}); 301 302 if (!Alloc) 303 return Alloc.takeError(); 304 305 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 306 ResolverBlockAddr = SegInfo.Addr; 307 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr, 308 ReentryFnAddr, ReentryCtxAddr); 309 310 auto FA = Alloc->finalize(); 311 if (!FA) 312 return FA.takeError(); 313 314 ResolverBlock = std::move(*FA); 315 return ResolverBlockAddr; 316 } 317 318 std::unique_ptr<IndirectStubsManager> 319 EPCIndirectionUtils::createIndirectStubsManager() { 320 return std::make_unique<EPCIndirectStubsManager>(*this); 321 } 322 323 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 324 if (!TP) 325 TP = std::make_unique<EPCTrampolinePool>(*this); 326 return *TP; 327 } 328 329 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 330 ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) { 331 assert(!LCTM && 332 "createLazyCallThroughManager can not have been called before"); 333 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 334 &getTrampolinePool()); 335 return *LCTM; 336 } 337 338 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 339 std::unique_ptr<ABISupport> ABI) 340 : EPC(EPC), ABI(std::move(ABI)) { 341 assert(this->ABI && "ABI can not be null"); 342 343 assert(EPC.getPageSize() > getABISupport().getStubSize() && 344 "Stubs larger than one page are not supported"); 345 } 346 347 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 348 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 349 using namespace jitlink; 350 351 std::lock_guard<std::mutex> Lock(EPCUIMutex); 352 353 // If there aren't enough stubs available then allocate some more. 354 if (NumStubs > AvailableIndirectStubs.size()) { 355 auto NumStubsToAllocate = NumStubs; 356 auto PageSize = EPC.getPageSize(); 357 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 358 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 359 auto PtrBytes = 360 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 361 362 auto StubProt = MemProt::Read | MemProt::Exec; 363 auto PtrProt = MemProt::Read | MemProt::Write; 364 365 auto Alloc = SimpleSegmentAlloc::Create( 366 EPC.getMemMgr(), nullptr, 367 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 368 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); 369 370 if (!Alloc) 371 return Alloc.takeError(); 372 373 auto StubSeg = Alloc->getSegInfo(StubProt); 374 auto PtrSeg = Alloc->getSegInfo(PtrProt); 375 376 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, 377 PtrSeg.Addr, NumStubsToAllocate); 378 379 auto FA = Alloc->finalize(); 380 if (!FA) 381 return FA.takeError(); 382 383 IndirectStubAllocs.push_back(std::move(*FA)); 384 385 auto StubExecutorAddr = StubSeg.Addr; 386 auto PtrExecutorAddr = PtrSeg.Addr; 387 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 388 AvailableIndirectStubs.push_back( 389 IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); 390 StubExecutorAddr += ABI->getStubSize(); 391 PtrExecutorAddr += ABI->getPointerSize(); 392 } 393 } 394 395 assert(NumStubs <= AvailableIndirectStubs.size() && 396 "Sufficient stubs should have been allocated above"); 397 398 IndirectStubInfoVector Result; 399 while (NumStubs--) { 400 Result.push_back(AvailableIndirectStubs.back()); 401 AvailableIndirectStubs.pop_back(); 402 } 403 404 return std::move(Result); 405 } 406 407 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 408 JITTargetAddress TrampolineAddr) { 409 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 410 std::promise<ExecutorAddr> LandingAddrP; 411 auto LandingAddrF = LandingAddrP.get_future(); 412 LCTM.resolveTrampolineLandingAddress( 413 ExecutorAddr(TrampolineAddr), 414 [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); }); 415 return LandingAddrF.get().getValue(); 416 } 417 418 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 419 auto &LCTM = EPCIU.getLazyCallThroughManager(); 420 return EPCIU 421 .writeResolverBlock(ExecutorAddr::fromPtr(&reentry), 422 ExecutorAddr::fromPtr(&LCTM)) 423 .takeError(); 424 } 425 426 } // end namespace orc 427 } // end namespace llvm 428