1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 Error Err = Error::success(); 92 std::promise<MSVCPError> DeallocResultP; 93 auto DeallocResultF = DeallocResultP.get_future(); 94 95 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 96 std::move(TrampolineBlocks), 97 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 98 99 return DeallocResultF.get(); 100 } 101 102 Error EPCTrampolinePool::grow() { 103 using namespace jitlink; 104 105 assert(AvailableTrampolines.empty() && 106 "Grow called with trampolines still available"); 107 108 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 109 assert(ResolverAddress && "Resolver address can not be null"); 110 111 auto &EPC = EPCIU.getExecutorProcessControl(); 112 auto PageSize = EPC.getPageSize(); 113 auto Alloc = SimpleSegmentAlloc::Create( 114 EPC.getMemMgr(), nullptr, 115 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 116 if (!Alloc) 117 return Alloc.takeError(); 118 119 unsigned NumTrampolines = TrampolinesPerPage; 120 121 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 122 EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(), 123 SegInfo.Addr.getValue(), 124 ResolverAddress, NumTrampolines); 125 for (unsigned I = 0; I < NumTrampolines; ++I) 126 AvailableTrampolines.push_back(SegInfo.Addr.getValue() + 127 (I * TrampolineSize)); 128 129 auto FA = Alloc->finalize(); 130 if (!FA) 131 return FA.takeError(); 132 133 TrampolineBlocks.push_back(std::move(*FA)); 134 135 return Error::success(); 136 } 137 138 Error EPCIndirectStubsManager::createStub(StringRef StubName, 139 JITTargetAddress StubAddr, 140 JITSymbolFlags StubFlags) { 141 StubInitsMap SIM; 142 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 143 return createStubs(SIM); 144 } 145 146 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 147 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 148 if (!AvailableStubInfos) 149 return AvailableStubInfos.takeError(); 150 151 { 152 std::lock_guard<std::mutex> Lock(ISMMutex); 153 unsigned ASIdx = 0; 154 for (auto &SI : StubInits) { 155 auto &A = (*AvailableStubInfos)[ASIdx++]; 156 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 157 } 158 } 159 160 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 161 switch (EPCIU.getABISupport().getPointerSize()) { 162 case 4: { 163 unsigned ASIdx = 0; 164 std::vector<tpctypes::UInt32Write> PtrUpdates; 165 for (auto &SI : StubInits) 166 PtrUpdates.push_back( 167 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 168 static_cast<uint32_t>(SI.second.first)}); 169 return MemAccess.writeUInt32s(PtrUpdates); 170 } 171 case 8: { 172 unsigned ASIdx = 0; 173 std::vector<tpctypes::UInt64Write> PtrUpdates; 174 for (auto &SI : StubInits) 175 PtrUpdates.push_back( 176 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 177 static_cast<uint64_t>(SI.second.first)}); 178 return MemAccess.writeUInt64s(PtrUpdates); 179 } 180 default: 181 return make_error<StringError>("Unsupported pointer size", 182 inconvertibleErrorCode()); 183 } 184 } 185 186 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 187 bool ExportedStubsOnly) { 188 std::lock_guard<std::mutex> Lock(ISMMutex); 189 auto I = StubInfos.find(Name); 190 if (I == StubInfos.end()) 191 return nullptr; 192 return {I->second.first.StubAddress, I->second.second}; 193 } 194 195 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 196 std::lock_guard<std::mutex> Lock(ISMMutex); 197 auto I = StubInfos.find(Name); 198 if (I == StubInfos.end()) 199 return nullptr; 200 return {I->second.first.PointerAddress, I->second.second}; 201 } 202 203 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 204 JITTargetAddress NewAddr) { 205 206 JITTargetAddress PtrAddr = 0; 207 { 208 std::lock_guard<std::mutex> Lock(ISMMutex); 209 auto I = StubInfos.find(Name); 210 if (I == StubInfos.end()) 211 return make_error<StringError>("Unknown stub name", 212 inconvertibleErrorCode()); 213 PtrAddr = I->second.first.PointerAddress; 214 } 215 216 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 217 switch (EPCIU.getABISupport().getPointerSize()) { 218 case 4: { 219 tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 220 return MemAccess.writeUInt32s(PUpdate); 221 } 222 case 8: { 223 tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 224 return MemAccess.writeUInt64s(PUpdate); 225 } 226 default: 227 return make_error<StringError>("Unsupported pointer size", 228 inconvertibleErrorCode()); 229 } 230 } 231 232 } // end anonymous namespace. 233 234 namespace llvm { 235 namespace orc { 236 237 EPCIndirectionUtils::ABISupport::~ABISupport() {} 238 239 Expected<std::unique_ptr<EPCIndirectionUtils>> 240 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 241 const auto &TT = EPC.getTargetTriple(); 242 switch (TT.getArch()) { 243 default: 244 return make_error<StringError>( 245 std::string("No EPCIndirectionUtils available for ") + TT.str(), 246 inconvertibleErrorCode()); 247 case Triple::aarch64: 248 case Triple::aarch64_32: 249 return CreateWithABI<OrcAArch64>(EPC); 250 251 case Triple::x86: 252 return CreateWithABI<OrcI386>(EPC); 253 254 case Triple::mips: 255 return CreateWithABI<OrcMips32Be>(EPC); 256 257 case Triple::mipsel: 258 return CreateWithABI<OrcMips32Le>(EPC); 259 260 case Triple::mips64: 261 case Triple::mips64el: 262 return CreateWithABI<OrcMips64>(EPC); 263 264 case Triple::x86_64: 265 if (TT.getOS() == Triple::OSType::Win32) 266 return CreateWithABI<OrcX86_64_Win32>(EPC); 267 else 268 return CreateWithABI<OrcX86_64_SysV>(EPC); 269 } 270 } 271 272 Error EPCIndirectionUtils::cleanup() { 273 274 auto &MemMgr = EPC.getMemMgr(); 275 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 276 277 if (TP) 278 Err = joinErrors(std::move(Err), 279 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 280 281 if (ResolverBlock) 282 Err = 283 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 284 285 return Err; 286 } 287 288 Expected<JITTargetAddress> 289 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 290 JITTargetAddress ReentryCtxAddr) { 291 using namespace jitlink; 292 293 assert(ABI && "ABI can not be null"); 294 auto ResolverSize = ABI->getResolverCodeSize(); 295 296 auto Alloc = 297 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 298 {{MemProt::Read | MemProt::Exec, 299 {ResolverSize, Align(EPC.getPageSize())}}}); 300 301 if (!Alloc) 302 return Alloc.takeError(); 303 304 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 305 ResolverBlockAddr = SegInfo.Addr.getValue(); 306 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr, 307 ReentryFnAddr, ReentryCtxAddr); 308 309 auto FA = Alloc->finalize(); 310 if (!FA) 311 return FA.takeError(); 312 313 ResolverBlock = std::move(*FA); 314 return ResolverBlockAddr; 315 } 316 317 std::unique_ptr<IndirectStubsManager> 318 EPCIndirectionUtils::createIndirectStubsManager() { 319 return std::make_unique<EPCIndirectStubsManager>(*this); 320 } 321 322 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 323 if (!TP) 324 TP = std::make_unique<EPCTrampolinePool>(*this); 325 return *TP; 326 } 327 328 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 329 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 330 assert(!LCTM && 331 "createLazyCallThroughManager can not have been called before"); 332 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 333 &getTrampolinePool()); 334 return *LCTM; 335 } 336 337 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 338 std::unique_ptr<ABISupport> ABI) 339 : EPC(EPC), ABI(std::move(ABI)) { 340 assert(this->ABI && "ABI can not be null"); 341 342 assert(EPC.getPageSize() > getABISupport().getStubSize() && 343 "Stubs larger than one page are not supported"); 344 } 345 346 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 347 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 348 using namespace jitlink; 349 350 std::lock_guard<std::mutex> Lock(EPCUIMutex); 351 352 // If there aren't enough stubs available then allocate some more. 353 if (NumStubs > AvailableIndirectStubs.size()) { 354 auto NumStubsToAllocate = NumStubs; 355 auto PageSize = EPC.getPageSize(); 356 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 357 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 358 auto PtrBytes = 359 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 360 361 auto StubProt = MemProt::Read | MemProt::Exec; 362 auto PtrProt = MemProt::Read | MemProt::Write; 363 364 auto Alloc = SimpleSegmentAlloc::Create( 365 EPC.getMemMgr(), nullptr, 366 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 367 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); 368 369 if (!Alloc) 370 return Alloc.takeError(); 371 372 auto StubSeg = Alloc->getSegInfo(StubProt); 373 auto PtrSeg = Alloc->getSegInfo(PtrProt); 374 375 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), 376 StubSeg.Addr.getValue(), 377 PtrSeg.Addr.getValue(), NumStubsToAllocate); 378 379 auto FA = Alloc->finalize(); 380 if (!FA) 381 return FA.takeError(); 382 383 IndirectStubAllocs.push_back(std::move(*FA)); 384 385 auto StubExecutorAddr = StubSeg.Addr; 386 auto PtrExecutorAddr = PtrSeg.Addr; 387 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 388 AvailableIndirectStubs.push_back(IndirectStubInfo( 389 StubExecutorAddr.getValue(), PtrExecutorAddr.getValue())); 390 StubExecutorAddr += ABI->getStubSize(); 391 PtrExecutorAddr += ABI->getPointerSize(); 392 } 393 } 394 395 assert(NumStubs <= AvailableIndirectStubs.size() && 396 "Sufficient stubs should have been allocated above"); 397 398 IndirectStubInfoVector Result; 399 while (NumStubs--) { 400 Result.push_back(AvailableIndirectStubs.back()); 401 AvailableIndirectStubs.pop_back(); 402 } 403 404 return std::move(Result); 405 } 406 407 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 408 JITTargetAddress TrampolineAddr) { 409 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 410 std::promise<JITTargetAddress> LandingAddrP; 411 auto LandingAddrF = LandingAddrP.get_future(); 412 LCTM.resolveTrampolineLandingAddress( 413 TrampolineAddr, 414 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 415 return LandingAddrF.get(); 416 } 417 418 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 419 auto &LCTM = EPCIU.getLazyCallThroughManager(); 420 return EPCIU 421 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 422 pointerToJITTargetAddress(&LCTM)) 423 .takeError(); 424 } 425 426 } // end namespace orc 427 } // end namespace llvm 428