1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 Error Err = Error::success(); 92 std::promise<MSVCPError> DeallocResultP; 93 auto DeallocResultF = DeallocResultP.get_future(); 94 95 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 96 std::move(TrampolineBlocks), 97 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 98 99 return DeallocResultF.get(); 100 } 101 102 Error EPCTrampolinePool::grow() { 103 using namespace jitlink; 104 105 assert(AvailableTrampolines.empty() && 106 "Grow called with trampolines still available"); 107 108 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 109 assert(ResolverAddress && "Resolver address can not be null"); 110 111 auto &EPC = EPCIU.getExecutorProcessControl(); 112 auto PageSize = EPC.getPageSize(); 113 auto Alloc = SimpleSegmentAlloc::Create( 114 EPC.getMemMgr(), nullptr, 115 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 116 if (!Alloc) 117 return Alloc.takeError(); 118 119 unsigned NumTrampolines = TrampolinesPerPage; 120 121 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 122 EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(), 123 SegInfo.Addr.getValue(), 124 ResolverAddress, NumTrampolines); 125 for (unsigned I = 0; I < NumTrampolines; ++I) 126 AvailableTrampolines.push_back(SegInfo.Addr.getValue() + 127 (I * TrampolineSize)); 128 129 auto FA = Alloc->finalize(); 130 if (!FA) 131 return FA.takeError(); 132 133 TrampolineBlocks.push_back(std::move(*FA)); 134 135 return Error::success(); 136 } 137 138 Error EPCIndirectStubsManager::createStub(StringRef StubName, 139 JITTargetAddress StubAddr, 140 JITSymbolFlags StubFlags) { 141 StubInitsMap SIM; 142 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 143 return createStubs(SIM); 144 } 145 146 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 147 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 148 if (!AvailableStubInfos) 149 return AvailableStubInfos.takeError(); 150 151 { 152 std::lock_guard<std::mutex> Lock(ISMMutex); 153 unsigned ASIdx = 0; 154 for (auto &SI : StubInits) { 155 auto &A = (*AvailableStubInfos)[ASIdx++]; 156 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 157 } 158 } 159 160 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 161 switch (EPCIU.getABISupport().getPointerSize()) { 162 case 4: { 163 unsigned ASIdx = 0; 164 std::vector<tpctypes::UInt32Write> PtrUpdates; 165 for (auto &SI : StubInits) 166 PtrUpdates.push_back( 167 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 168 static_cast<uint32_t>(SI.second.first)}); 169 return MemAccess.writeUInt32s(PtrUpdates); 170 } 171 case 8: { 172 unsigned ASIdx = 0; 173 std::vector<tpctypes::UInt64Write> PtrUpdates; 174 for (auto &SI : StubInits) 175 PtrUpdates.push_back( 176 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 177 static_cast<uint64_t>(SI.second.first)}); 178 return MemAccess.writeUInt64s(PtrUpdates); 179 } 180 default: 181 return make_error<StringError>("Unsupported pointer size", 182 inconvertibleErrorCode()); 183 } 184 } 185 186 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 187 bool ExportedStubsOnly) { 188 std::lock_guard<std::mutex> Lock(ISMMutex); 189 auto I = StubInfos.find(Name); 190 if (I == StubInfos.end()) 191 return nullptr; 192 return {I->second.first.StubAddress, I->second.second}; 193 } 194 195 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 196 std::lock_guard<std::mutex> Lock(ISMMutex); 197 auto I = StubInfos.find(Name); 198 if (I == StubInfos.end()) 199 return nullptr; 200 return {I->second.first.PointerAddress, I->second.second}; 201 } 202 203 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 204 JITTargetAddress NewAddr) { 205 206 JITTargetAddress PtrAddr = 0; 207 { 208 std::lock_guard<std::mutex> Lock(ISMMutex); 209 auto I = StubInfos.find(Name); 210 if (I == StubInfos.end()) 211 return make_error<StringError>("Unknown stub name", 212 inconvertibleErrorCode()); 213 PtrAddr = I->second.first.PointerAddress; 214 } 215 216 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 217 switch (EPCIU.getABISupport().getPointerSize()) { 218 case 4: { 219 tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 220 return MemAccess.writeUInt32s(PUpdate); 221 } 222 case 8: { 223 tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 224 return MemAccess.writeUInt64s(PUpdate); 225 } 226 default: 227 return make_error<StringError>("Unsupported pointer size", 228 inconvertibleErrorCode()); 229 } 230 } 231 232 } // end anonymous namespace. 233 234 namespace llvm { 235 namespace orc { 236 237 EPCIndirectionUtils::ABISupport::~ABISupport() {} 238 239 Expected<std::unique_ptr<EPCIndirectionUtils>> 240 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 241 const auto &TT = EPC.getTargetTriple(); 242 switch (TT.getArch()) { 243 default: 244 return make_error<StringError>( 245 std::string("No EPCIndirectionUtils available for ") + TT.str(), 246 inconvertibleErrorCode()); 247 case Triple::aarch64: 248 case Triple::aarch64_32: 249 return CreateWithABI<OrcAArch64>(EPC); 250 251 case Triple::x86: 252 return CreateWithABI<OrcI386>(EPC); 253 254 case Triple::mips: 255 return CreateWithABI<OrcMips32Be>(EPC); 256 257 case Triple::mipsel: 258 return CreateWithABI<OrcMips32Le>(EPC); 259 260 case Triple::mips64: 261 case Triple::mips64el: 262 return CreateWithABI<OrcMips64>(EPC); 263 264 case Triple::x86_64: 265 if (TT.getOS() == Triple::OSType::Win32) 266 return CreateWithABI<OrcX86_64_Win32>(EPC); 267 else 268 return CreateWithABI<OrcX86_64_SysV>(EPC); 269 } 270 } 271 272 Error EPCIndirectionUtils::cleanup() { 273 274 auto &MemMgr = EPC.getMemMgr(); 275 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 276 277 if (TP) 278 Err = joinErrors(std::move(Err), 279 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 280 281 if (ResolverBlock) 282 Err = 283 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 284 285 return Err; 286 } 287 288 Expected<JITTargetAddress> 289 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 290 JITTargetAddress ReentryCtxAddr) { 291 using namespace jitlink; 292 293 assert(ABI && "ABI can not be null"); 294 auto ResolverSize = ABI->getResolverCodeSize(); 295 296 auto Alloc = 297 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 298 {{MemProt::Read | MemProt::Exec, 299 {ResolverSize, Align(EPC.getPageSize())}}}); 300 301 if (!Alloc) 302 return Alloc.takeError(); 303 304 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 305 ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr.getValue(), 306 ReentryFnAddr, ReentryCtxAddr); 307 308 auto FA = Alloc->finalize(); 309 if (!FA) 310 return FA.takeError(); 311 312 ResolverBlock = std::move(*FA); 313 return SegInfo.Addr.getValue(); 314 } 315 316 std::unique_ptr<IndirectStubsManager> 317 EPCIndirectionUtils::createIndirectStubsManager() { 318 return std::make_unique<EPCIndirectStubsManager>(*this); 319 } 320 321 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 322 if (!TP) 323 TP = std::make_unique<EPCTrampolinePool>(*this); 324 return *TP; 325 } 326 327 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 328 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 329 assert(!LCTM && 330 "createLazyCallThroughManager can not have been called before"); 331 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 332 &getTrampolinePool()); 333 return *LCTM; 334 } 335 336 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 337 std::unique_ptr<ABISupport> ABI) 338 : EPC(EPC), ABI(std::move(ABI)) { 339 assert(this->ABI && "ABI can not be null"); 340 341 assert(EPC.getPageSize() > getABISupport().getStubSize() && 342 "Stubs larger than one page are not supported"); 343 } 344 345 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 346 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 347 using namespace jitlink; 348 349 std::lock_guard<std::mutex> Lock(EPCUIMutex); 350 351 // If there aren't enough stubs available then allocate some more. 352 if (NumStubs > AvailableIndirectStubs.size()) { 353 auto NumStubsToAllocate = NumStubs; 354 auto PageSize = EPC.getPageSize(); 355 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 356 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 357 auto PtrBytes = 358 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 359 360 auto StubProt = MemProt::Read | MemProt::Exec; 361 auto PtrProt = MemProt::Read | MemProt::Write; 362 363 auto Alloc = SimpleSegmentAlloc::Create( 364 EPC.getMemMgr(), nullptr, 365 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 366 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); 367 368 if (!Alloc) 369 return Alloc.takeError(); 370 371 auto StubSeg = Alloc->getSegInfo(StubProt); 372 auto PtrSeg = Alloc->getSegInfo(PtrProt); 373 374 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), 375 StubSeg.Addr.getValue(), 376 PtrSeg.Addr.getValue(), NumStubsToAllocate); 377 378 auto FA = Alloc->finalize(); 379 if (!FA) 380 return FA.takeError(); 381 382 IndirectStubAllocs.push_back(std::move(*FA)); 383 384 auto StubExecutorAddr = StubSeg.Addr; 385 auto PtrExecutorAddr = PtrSeg.Addr; 386 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 387 AvailableIndirectStubs.push_back(IndirectStubInfo( 388 StubExecutorAddr.getValue(), PtrExecutorAddr.getValue())); 389 StubExecutorAddr += ABI->getStubSize(); 390 PtrExecutorAddr += ABI->getPointerSize(); 391 } 392 } 393 394 assert(NumStubs <= AvailableIndirectStubs.size() && 395 "Sufficient stubs should have been allocated above"); 396 397 IndirectStubInfoVector Result; 398 while (NumStubs--) { 399 Result.push_back(AvailableIndirectStubs.back()); 400 AvailableIndirectStubs.pop_back(); 401 } 402 403 return std::move(Result); 404 } 405 406 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 407 JITTargetAddress TrampolineAddr) { 408 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 409 std::promise<JITTargetAddress> LandingAddrP; 410 auto LandingAddrF = LandingAddrP.get_future(); 411 LCTM.resolveTrampolineLandingAddress( 412 TrampolineAddr, 413 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 414 return LandingAddrF.get(); 415 } 416 417 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 418 auto &LCTM = EPCIU.getLazyCallThroughManager(); 419 return EPCIU 420 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 421 pointerToJITTargetAddress(&LCTM)) 422 .takeError(); 423 } 424 425 } // end namespace orc 426 } // end namespace llvm 427