1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<FinalizedAlloc> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 Error Err = Error::success(); 92 std::promise<MSVCPError> DeallocResultP; 93 auto DeallocResultF = DeallocResultP.get_future(); 94 95 EPCIU.getExecutorProcessControl().getMemMgr().deallocate( 96 std::move(TrampolineBlocks), 97 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); 98 99 return DeallocResultF.get(); 100 } 101 102 Error EPCTrampolinePool::grow() { 103 using namespace jitlink; 104 105 assert(AvailableTrampolines.empty() && 106 "Grow called with trampolines still available"); 107 108 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 109 assert(ResolverAddress && "Resolver address can not be null"); 110 111 auto &EPC = EPCIU.getExecutorProcessControl(); 112 auto PageSize = EPC.getPageSize(); 113 auto Alloc = SimpleSegmentAlloc::Create( 114 EPC.getMemMgr(), nullptr, 115 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); 116 if (!Alloc) 117 return Alloc.takeError(); 118 119 unsigned NumTrampolines = TrampolinesPerPage; 120 121 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 122 EPCIU.getABISupport().writeTrampolines( 123 SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); 124 for (unsigned I = 0; I < NumTrampolines; ++I) 125 AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); 126 127 auto FA = Alloc->finalize(); 128 if (!FA) 129 return FA.takeError(); 130 131 TrampolineBlocks.push_back(std::move(*FA)); 132 133 return Error::success(); 134 } 135 136 Error EPCIndirectStubsManager::createStub(StringRef StubName, 137 JITTargetAddress StubAddr, 138 JITSymbolFlags StubFlags) { 139 StubInitsMap SIM; 140 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 141 return createStubs(SIM); 142 } 143 144 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 145 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 146 if (!AvailableStubInfos) 147 return AvailableStubInfos.takeError(); 148 149 { 150 std::lock_guard<std::mutex> Lock(ISMMutex); 151 unsigned ASIdx = 0; 152 for (auto &SI : StubInits) { 153 auto &A = (*AvailableStubInfos)[ASIdx++]; 154 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 155 } 156 } 157 158 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 159 switch (EPCIU.getABISupport().getPointerSize()) { 160 case 4: { 161 unsigned ASIdx = 0; 162 std::vector<tpctypes::UInt32Write> PtrUpdates; 163 for (auto &SI : StubInits) 164 PtrUpdates.push_back( 165 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 166 static_cast<uint32_t>(SI.second.first)}); 167 return MemAccess.writeUInt32s(PtrUpdates); 168 } 169 case 8: { 170 unsigned ASIdx = 0; 171 std::vector<tpctypes::UInt64Write> PtrUpdates; 172 for (auto &SI : StubInits) 173 PtrUpdates.push_back( 174 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), 175 static_cast<uint64_t>(SI.second.first)}); 176 return MemAccess.writeUInt64s(PtrUpdates); 177 } 178 default: 179 return make_error<StringError>("Unsupported pointer size", 180 inconvertibleErrorCode()); 181 } 182 } 183 184 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 185 bool ExportedStubsOnly) { 186 std::lock_guard<std::mutex> Lock(ISMMutex); 187 auto I = StubInfos.find(Name); 188 if (I == StubInfos.end()) 189 return nullptr; 190 return {I->second.first.StubAddress, I->second.second}; 191 } 192 193 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 194 std::lock_guard<std::mutex> Lock(ISMMutex); 195 auto I = StubInfos.find(Name); 196 if (I == StubInfos.end()) 197 return nullptr; 198 return {I->second.first.PointerAddress, I->second.second}; 199 } 200 201 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 202 JITTargetAddress NewAddr) { 203 204 JITTargetAddress PtrAddr = 0; 205 { 206 std::lock_guard<std::mutex> Lock(ISMMutex); 207 auto I = StubInfos.find(Name); 208 if (I == StubInfos.end()) 209 return make_error<StringError>("Unknown stub name", 210 inconvertibleErrorCode()); 211 PtrAddr = I->second.first.PointerAddress; 212 } 213 214 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 215 switch (EPCIU.getABISupport().getPointerSize()) { 216 case 4: { 217 tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 218 return MemAccess.writeUInt32s(PUpdate); 219 } 220 case 8: { 221 tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); 222 return MemAccess.writeUInt64s(PUpdate); 223 } 224 default: 225 return make_error<StringError>("Unsupported pointer size", 226 inconvertibleErrorCode()); 227 } 228 } 229 230 } // end anonymous namespace. 231 232 namespace llvm { 233 namespace orc { 234 235 EPCIndirectionUtils::ABISupport::~ABISupport() {} 236 237 Expected<std::unique_ptr<EPCIndirectionUtils>> 238 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 239 const auto &TT = EPC.getTargetTriple(); 240 switch (TT.getArch()) { 241 default: 242 return make_error<StringError>( 243 std::string("No EPCIndirectionUtils available for ") + TT.str(), 244 inconvertibleErrorCode()); 245 case Triple::aarch64: 246 case Triple::aarch64_32: 247 return CreateWithABI<OrcAArch64>(EPC); 248 249 case Triple::x86: 250 return CreateWithABI<OrcI386>(EPC); 251 252 case Triple::mips: 253 return CreateWithABI<OrcMips32Be>(EPC); 254 255 case Triple::mipsel: 256 return CreateWithABI<OrcMips32Le>(EPC); 257 258 case Triple::mips64: 259 case Triple::mips64el: 260 return CreateWithABI<OrcMips64>(EPC); 261 262 case Triple::x86_64: 263 if (TT.getOS() == Triple::OSType::Win32) 264 return CreateWithABI<OrcX86_64_Win32>(EPC); 265 else 266 return CreateWithABI<OrcX86_64_SysV>(EPC); 267 } 268 } 269 270 Error EPCIndirectionUtils::cleanup() { 271 272 auto &MemMgr = EPC.getMemMgr(); 273 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); 274 275 if (TP) 276 Err = joinErrors(std::move(Err), 277 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 278 279 if (ResolverBlock) 280 Err = 281 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); 282 283 return Err; 284 } 285 286 Expected<JITTargetAddress> 287 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 288 JITTargetAddress ReentryCtxAddr) { 289 using namespace jitlink; 290 291 assert(ABI && "ABI can not be null"); 292 auto ResolverSize = ABI->getResolverCodeSize(); 293 294 auto Alloc = 295 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, 296 {{MemProt::Read | MemProt::Exec, 297 {ResolverSize, Align(EPC.getPageSize())}}}); 298 299 if (!Alloc) 300 return Alloc.takeError(); 301 302 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); 303 ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr, ReentryFnAddr, 304 ReentryCtxAddr); 305 306 auto FA = Alloc->finalize(); 307 if (!FA) 308 return FA.takeError(); 309 310 ResolverBlock = std::move(*FA); 311 return SegInfo.Addr; 312 } 313 314 std::unique_ptr<IndirectStubsManager> 315 EPCIndirectionUtils::createIndirectStubsManager() { 316 return std::make_unique<EPCIndirectStubsManager>(*this); 317 } 318 319 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 320 if (!TP) 321 TP = std::make_unique<EPCTrampolinePool>(*this); 322 return *TP; 323 } 324 325 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 326 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 327 assert(!LCTM && 328 "createLazyCallThroughManager can not have been called before"); 329 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 330 &getTrampolinePool()); 331 return *LCTM; 332 } 333 334 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 335 std::unique_ptr<ABISupport> ABI) 336 : EPC(EPC), ABI(std::move(ABI)) { 337 assert(this->ABI && "ABI can not be null"); 338 339 assert(EPC.getPageSize() > getABISupport().getStubSize() && 340 "Stubs larger than one page are not supported"); 341 } 342 343 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 344 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 345 using namespace jitlink; 346 347 std::lock_guard<std::mutex> Lock(EPCUIMutex); 348 349 // If there aren't enough stubs available then allocate some more. 350 if (NumStubs > AvailableIndirectStubs.size()) { 351 auto NumStubsToAllocate = NumStubs; 352 auto PageSize = EPC.getPageSize(); 353 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 354 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 355 auto PtrBytes = 356 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 357 358 auto StubProt = MemProt::Read | MemProt::Exec; 359 auto PtrProt = MemProt::Read | MemProt::Write; 360 361 auto Alloc = SimpleSegmentAlloc::Create( 362 EPC.getMemMgr(), nullptr, 363 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, 364 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); 365 366 if (!Alloc) 367 return Alloc.takeError(); 368 369 auto StubSeg = Alloc->getSegInfo(StubProt); 370 auto PtrSeg = Alloc->getSegInfo(PtrProt); 371 372 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, 373 PtrSeg.Addr, NumStubsToAllocate); 374 375 auto FA = Alloc->finalize(); 376 if (!FA) 377 return FA.takeError(); 378 379 IndirectStubAllocs.push_back(std::move(*FA)); 380 381 auto StubExecutorAddr = StubSeg.Addr; 382 auto PtrExecutorAddr = PtrSeg.Addr; 383 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 384 AvailableIndirectStubs.push_back( 385 IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); 386 StubExecutorAddr += ABI->getStubSize(); 387 PtrExecutorAddr += ABI->getPointerSize(); 388 } 389 } 390 391 assert(NumStubs <= AvailableIndirectStubs.size() && 392 "Sufficient stubs should have been allocated above"); 393 394 IndirectStubInfoVector Result; 395 while (NumStubs--) { 396 Result.push_back(AvailableIndirectStubs.back()); 397 AvailableIndirectStubs.pop_back(); 398 } 399 400 return std::move(Result); 401 } 402 403 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 404 JITTargetAddress TrampolineAddr) { 405 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 406 std::promise<JITTargetAddress> LandingAddrP; 407 auto LandingAddrF = LandingAddrP.get_future(); 408 LCTM.resolveTrampolineLandingAddress( 409 TrampolineAddr, 410 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 411 return LandingAddrF.get(); 412 } 413 414 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 415 auto &LCTM = EPCIU.getLazyCallThroughManager(); 416 return EPCIU 417 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 418 pointerToJITTargetAddress(&LCTM)) 419 .takeError(); 420 } 421 422 } // end namespace orc 423 } // end namespace llvm 424