1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" 10 11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" 12 #include "llvm/Support/MathExtras.h" 13 14 #include <future> 15 16 using namespace llvm; 17 using namespace llvm::orc; 18 19 namespace llvm { 20 namespace orc { 21 22 class EPCIndirectionUtilsAccess { 23 public: 24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; 25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; 26 27 static Expected<IndirectStubInfoVector> 28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { 29 return EPCIU.getIndirectStubs(NumStubs); 30 }; 31 }; 32 33 } // end namespace orc 34 } // end namespace llvm 35 36 namespace { 37 38 class EPCTrampolinePool : public TrampolinePool { 39 public: 40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU); 41 Error deallocatePool(); 42 43 protected: 44 Error grow() override; 45 46 using Allocation = jitlink::JITLinkMemoryManager::Allocation; 47 48 EPCIndirectionUtils &EPCIU; 49 unsigned TrampolineSize = 0; 50 unsigned TrampolinesPerPage = 0; 51 std::vector<std::unique_ptr<Allocation>> TrampolineBlocks; 52 }; 53 54 class EPCIndirectStubsManager : public IndirectStubsManager, 55 private EPCIndirectionUtilsAccess { 56 public: 57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} 58 59 Error deallocateStubs(); 60 61 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 62 JITSymbolFlags StubFlags) override; 63 64 Error createStubs(const StubInitsMap &StubInits) override; 65 66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; 67 68 JITEvaluatedSymbol findPointer(StringRef Name) override; 69 70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; 71 72 private: 73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; 74 75 std::mutex ISMMutex; 76 EPCIndirectionUtils &EPCIU; 77 StringMap<StubInfo> StubInfos; 78 }; 79 80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) 81 : EPCIU(EPCIU) { 82 auto &EPC = EPCIU.getExecutorProcessControl(); 83 auto &ABI = EPCIU.getABISupport(); 84 85 TrampolineSize = ABI.getTrampolineSize(); 86 TrampolinesPerPage = 87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; 88 } 89 90 Error EPCTrampolinePool::deallocatePool() { 91 Error Err = Error::success(); 92 for (auto &Alloc : TrampolineBlocks) 93 Err = joinErrors(std::move(Err), Alloc->deallocate()); 94 return Err; 95 } 96 97 Error EPCTrampolinePool::grow() { 98 assert(AvailableTrampolines.empty() && 99 "Grow called with trampolines still available"); 100 101 auto ResolverAddress = EPCIU.getResolverBlockAddress(); 102 assert(ResolverAddress && "Resolver address can not be null"); 103 104 auto &EPC = EPCIU.getExecutorProcessControl(); 105 constexpr auto TrampolinePagePermissions = 106 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 107 sys::Memory::MF_EXEC); 108 auto PageSize = EPC.getPageSize(); 109 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; 110 Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize), 111 0}; 112 auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); 113 114 if (!Alloc) 115 return Alloc.takeError(); 116 117 unsigned NumTrampolines = TrampolinesPerPage; 118 119 auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions); 120 auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions); 121 122 EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, 123 ResolverAddress, NumTrampolines); 124 125 auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions); 126 for (unsigned I = 0; I < NumTrampolines; ++I) 127 AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize)); 128 129 if (auto Err = (*Alloc)->finalize()) 130 return Err; 131 132 TrampolineBlocks.push_back(std::move(*Alloc)); 133 134 return Error::success(); 135 } 136 137 Error EPCIndirectStubsManager::createStub(StringRef StubName, 138 JITTargetAddress StubAddr, 139 JITSymbolFlags StubFlags) { 140 StubInitsMap SIM; 141 SIM[StubName] = std::make_pair(StubAddr, StubFlags); 142 return createStubs(SIM); 143 } 144 145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { 146 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); 147 if (!AvailableStubInfos) 148 return AvailableStubInfos.takeError(); 149 150 { 151 std::lock_guard<std::mutex> Lock(ISMMutex); 152 unsigned ASIdx = 0; 153 for (auto &SI : StubInits) { 154 auto &A = (*AvailableStubInfos)[ASIdx++]; 155 StubInfos[SI.first()] = std::make_pair(A, SI.second.second); 156 } 157 } 158 159 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 160 switch (EPCIU.getABISupport().getPointerSize()) { 161 case 4: { 162 unsigned ASIdx = 0; 163 std::vector<tpctypes::UInt32Write> PtrUpdates; 164 for (auto &SI : StubInits) 165 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, 166 static_cast<uint32_t>(SI.second.first)}); 167 return MemAccess.writeUInt32s(PtrUpdates); 168 } 169 case 8: { 170 unsigned ASIdx = 0; 171 std::vector<tpctypes::UInt64Write> PtrUpdates; 172 for (auto &SI : StubInits) 173 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, 174 static_cast<uint64_t>(SI.second.first)}); 175 return MemAccess.writeUInt64s(PtrUpdates); 176 } 177 default: 178 return make_error<StringError>("Unsupported pointer size", 179 inconvertibleErrorCode()); 180 } 181 } 182 183 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, 184 bool ExportedStubsOnly) { 185 std::lock_guard<std::mutex> Lock(ISMMutex); 186 auto I = StubInfos.find(Name); 187 if (I == StubInfos.end()) 188 return nullptr; 189 return {I->second.first.StubAddress, I->second.second}; 190 } 191 192 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { 193 std::lock_guard<std::mutex> Lock(ISMMutex); 194 auto I = StubInfos.find(Name); 195 if (I == StubInfos.end()) 196 return nullptr; 197 return {I->second.first.PointerAddress, I->second.second}; 198 } 199 200 Error EPCIndirectStubsManager::updatePointer(StringRef Name, 201 JITTargetAddress NewAddr) { 202 203 JITTargetAddress PtrAddr = 0; 204 { 205 std::lock_guard<std::mutex> Lock(ISMMutex); 206 auto I = StubInfos.find(Name); 207 if (I == StubInfos.end()) 208 return make_error<StringError>("Unknown stub name", 209 inconvertibleErrorCode()); 210 PtrAddr = I->second.first.PointerAddress; 211 } 212 213 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); 214 switch (EPCIU.getABISupport().getPointerSize()) { 215 case 4: { 216 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr); 217 return MemAccess.writeUInt32s(PUpdate); 218 } 219 case 8: { 220 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr); 221 return MemAccess.writeUInt64s(PUpdate); 222 } 223 default: 224 return make_error<StringError>("Unsupported pointer size", 225 inconvertibleErrorCode()); 226 } 227 } 228 229 } // end anonymous namespace. 230 231 namespace llvm { 232 namespace orc { 233 234 EPCIndirectionUtils::ABISupport::~ABISupport() {} 235 236 Expected<std::unique_ptr<EPCIndirectionUtils>> 237 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { 238 const auto &TT = EPC.getTargetTriple(); 239 switch (TT.getArch()) { 240 default: 241 return make_error<StringError>( 242 std::string("No EPCIndirectionUtils available for ") + TT.str(), 243 inconvertibleErrorCode()); 244 case Triple::aarch64: 245 case Triple::aarch64_32: 246 return CreateWithABI<OrcAArch64>(EPC); 247 248 case Triple::x86: 249 return CreateWithABI<OrcI386>(EPC); 250 251 case Triple::mips: 252 return CreateWithABI<OrcMips32Be>(EPC); 253 254 case Triple::mipsel: 255 return CreateWithABI<OrcMips32Le>(EPC); 256 257 case Triple::mips64: 258 case Triple::mips64el: 259 return CreateWithABI<OrcMips64>(EPC); 260 261 case Triple::x86_64: 262 if (TT.getOS() == Triple::OSType::Win32) 263 return CreateWithABI<OrcX86_64_Win32>(EPC); 264 else 265 return CreateWithABI<OrcX86_64_SysV>(EPC); 266 } 267 } 268 269 Error EPCIndirectionUtils::cleanup() { 270 Error Err = Error::success(); 271 272 for (auto &A : IndirectStubAllocs) 273 Err = joinErrors(std::move(Err), A->deallocate()); 274 275 if (TP) 276 Err = joinErrors(std::move(Err), 277 static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); 278 279 if (ResolverBlock) 280 Err = joinErrors(std::move(Err), ResolverBlock->deallocate()); 281 282 return Err; 283 } 284 285 Expected<JITTargetAddress> 286 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, 287 JITTargetAddress ReentryCtxAddr) { 288 assert(ABI && "ABI can not be null"); 289 constexpr auto ResolverBlockPermissions = 290 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 291 sys::Memory::MF_EXEC); 292 auto ResolverSize = ABI->getResolverCodeSize(); 293 294 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; 295 Request[ResolverBlockPermissions] = {EPC.getPageSize(), 296 static_cast<size_t>(ResolverSize), 0}; 297 auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); 298 if (!Alloc) 299 return Alloc.takeError(); 300 301 auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions); 302 ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions); 303 ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr, 304 ReentryCtxAddr); 305 306 if (auto Err = (*Alloc)->finalize()) 307 return std::move(Err); 308 309 ResolverBlock = std::move(*Alloc); 310 return ResolverBlockAddr; 311 } 312 313 std::unique_ptr<IndirectStubsManager> 314 EPCIndirectionUtils::createIndirectStubsManager() { 315 return std::make_unique<EPCIndirectStubsManager>(*this); 316 } 317 318 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { 319 if (!TP) 320 TP = std::make_unique<EPCTrampolinePool>(*this); 321 return *TP; 322 } 323 324 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( 325 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { 326 assert(!LCTM && 327 "createLazyCallThroughManager can not have been called before"); 328 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, 329 &getTrampolinePool()); 330 return *LCTM; 331 } 332 333 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, 334 std::unique_ptr<ABISupport> ABI) 335 : EPC(EPC), ABI(std::move(ABI)) { 336 assert(this->ABI && "ABI can not be null"); 337 338 assert(EPC.getPageSize() > getABISupport().getStubSize() && 339 "Stubs larger than one page are not supported"); 340 } 341 342 Expected<EPCIndirectionUtils::IndirectStubInfoVector> 343 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { 344 345 std::lock_guard<std::mutex> Lock(EPCUIMutex); 346 347 // If there aren't enough stubs available then allocate some more. 348 if (NumStubs > AvailableIndirectStubs.size()) { 349 auto NumStubsToAllocate = NumStubs; 350 auto PageSize = EPC.getPageSize(); 351 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); 352 NumStubsToAllocate = StubBytes / ABI->getStubSize(); 353 auto PointerBytes = 354 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); 355 356 constexpr auto StubPagePermissions = 357 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 358 sys::Memory::MF_EXEC); 359 constexpr auto PointerPagePermissions = 360 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 361 sys::Memory::MF_WRITE); 362 363 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; 364 Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes), 365 0}; 366 Request[PointerPagePermissions] = {PageSize, 0, PointerBytes}; 367 auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); 368 if (!Alloc) 369 return Alloc.takeError(); 370 371 auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions); 372 auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions); 373 374 ABI->writeIndirectStubsBlock( 375 (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr, 376 PointerTargetAddr, NumStubsToAllocate); 377 378 if (auto Err = (*Alloc)->finalize()) 379 return std::move(Err); 380 381 for (unsigned I = 0; I != NumStubsToAllocate; ++I) { 382 AvailableIndirectStubs.push_back( 383 IndirectStubInfo(StubTargetAddr, PointerTargetAddr)); 384 StubTargetAddr += ABI->getStubSize(); 385 PointerTargetAddr += ABI->getPointerSize(); 386 } 387 388 IndirectStubAllocs.push_back(std::move(*Alloc)); 389 } 390 391 assert(NumStubs <= AvailableIndirectStubs.size() && 392 "Sufficient stubs should have been allocated above"); 393 394 IndirectStubInfoVector Result; 395 while (NumStubs--) { 396 Result.push_back(AvailableIndirectStubs.back()); 397 AvailableIndirectStubs.pop_back(); 398 } 399 400 return std::move(Result); 401 } 402 403 static JITTargetAddress reentry(JITTargetAddress LCTMAddr, 404 JITTargetAddress TrampolineAddr) { 405 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); 406 std::promise<JITTargetAddress> LandingAddrP; 407 auto LandingAddrF = LandingAddrP.get_future(); 408 LCTM.resolveTrampolineLandingAddress( 409 TrampolineAddr, 410 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); 411 return LandingAddrF.get(); 412 } 413 414 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { 415 auto &LCTM = EPCIU.getLazyCallThroughManager(); 416 return EPCIU 417 .writeResolverBlock(pointerToJITTargetAddress(&reentry), 418 pointerToJITTargetAddress(&LCTM)) 419 .takeError(); 420 } 421 422 } // end namespace orc 423 } // end namespace llvm 424