1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10
11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13
14 #include <future>
15
16 using namespace llvm;
17 using namespace llvm::orc;
18
19 namespace llvm {
20 namespace orc {
21
22 class EPCIndirectionUtilsAccess {
23 public:
24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26
27 static Expected<IndirectStubInfoVector>
getIndirectStubs(EPCIndirectionUtils & EPCIU,unsigned NumStubs)28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29 return EPCIU.getIndirectStubs(NumStubs);
30 };
31 };
32
33 } // end namespace orc
34 } // end namespace llvm
35
36 namespace {
37
38 class EPCTrampolinePool : public TrampolinePool {
39 public:
40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41 Error deallocatePool();
42
43 protected:
44 Error grow() override;
45
46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
47
48 EPCIndirectionUtils &EPCIU;
49 unsigned TrampolineSize = 0;
50 unsigned TrampolinesPerPage = 0;
51 std::vector<FinalizedAlloc> TrampolineBlocks;
52 };
53
54 class EPCIndirectStubsManager : public IndirectStubsManager,
55 private EPCIndirectionUtilsAccess {
56 public:
EPCIndirectStubsManager(EPCIndirectionUtils & EPCIU)57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58
59 Error deallocateStubs();
60
61 Error createStub(StringRef StubName, ExecutorAddr StubAddr,
62 JITSymbolFlags StubFlags) override;
63
64 Error createStubs(const StubInitsMap &StubInits) override;
65
66 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;
67
68 ExecutorSymbolDef findPointer(StringRef Name) override;
69
70 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;
71
72 private:
73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74
75 std::mutex ISMMutex;
76 EPCIndirectionUtils &EPCIU;
77 StringMap<StubInfo> StubInfos;
78 };
79
EPCTrampolinePool(EPCIndirectionUtils & EPCIU)80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81 : EPCIU(EPCIU) {
82 auto &EPC = EPCIU.getExecutorProcessControl();
83 auto &ABI = EPCIU.getABISupport();
84
85 TrampolineSize = ABI.getTrampolineSize();
86 TrampolinesPerPage =
87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89
deallocatePool()90 Error EPCTrampolinePool::deallocatePool() {
91 std::promise<MSVCPError> DeallocResultP;
92 auto DeallocResultF = DeallocResultP.get_future();
93
94 EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
95 std::move(TrampolineBlocks),
96 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
97
98 return DeallocResultF.get();
99 }
100
grow()101 Error EPCTrampolinePool::grow() {
102 using namespace jitlink;
103
104 assert(AvailableTrampolines.empty() &&
105 "Grow called with trampolines still available");
106
107 auto ResolverAddress = EPCIU.getResolverBlockAddress();
108 assert(ResolverAddress && "Resolver address can not be null");
109
110 auto &EPC = EPCIU.getExecutorProcessControl();
111 auto PageSize = EPC.getPageSize();
112 auto Alloc = SimpleSegmentAlloc::Create(
113 EPC.getMemMgr(), nullptr,
114 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
115 if (!Alloc)
116 return Alloc.takeError();
117
118 unsigned NumTrampolines = TrampolinesPerPage;
119
120 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
121 EPCIU.getABISupport().writeTrampolines(
122 SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines);
123 for (unsigned I = 0; I < NumTrampolines; ++I)
124 AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize));
125
126 auto FA = Alloc->finalize();
127 if (!FA)
128 return FA.takeError();
129
130 TrampolineBlocks.push_back(std::move(*FA));
131
132 return Error::success();
133 }
134
createStub(StringRef StubName,ExecutorAddr StubAddr,JITSymbolFlags StubFlags)135 Error EPCIndirectStubsManager::createStub(StringRef StubName,
136 ExecutorAddr StubAddr,
137 JITSymbolFlags StubFlags) {
138 StubInitsMap SIM;
139 SIM[StubName] = std::make_pair(StubAddr, StubFlags);
140 return createStubs(SIM);
141 }
142
createStubs(const StubInitsMap & StubInits)143 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
144 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
145 if (!AvailableStubInfos)
146 return AvailableStubInfos.takeError();
147
148 {
149 std::lock_guard<std::mutex> Lock(ISMMutex);
150 unsigned ASIdx = 0;
151 for (auto &SI : StubInits) {
152 auto &A = (*AvailableStubInfos)[ASIdx++];
153 StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
154 }
155 }
156
157 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
158 switch (EPCIU.getABISupport().getPointerSize()) {
159 case 4: {
160 unsigned ASIdx = 0;
161 std::vector<tpctypes::UInt32Write> PtrUpdates;
162 for (auto &SI : StubInits)
163 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
164 static_cast<uint32_t>(SI.second.first.getValue())});
165 return MemAccess.writeUInt32s(PtrUpdates);
166 }
167 case 8: {
168 unsigned ASIdx = 0;
169 std::vector<tpctypes::UInt64Write> PtrUpdates;
170 for (auto &SI : StubInits)
171 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
172 static_cast<uint64_t>(SI.second.first.getValue())});
173 return MemAccess.writeUInt64s(PtrUpdates);
174 }
175 default:
176 return make_error<StringError>("Unsupported pointer size",
177 inconvertibleErrorCode());
178 }
179 }
180
findStub(StringRef Name,bool ExportedStubsOnly)181 ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,
182 bool ExportedStubsOnly) {
183 std::lock_guard<std::mutex> Lock(ISMMutex);
184 auto I = StubInfos.find(Name);
185 if (I == StubInfos.end())
186 return ExecutorSymbolDef();
187 return {I->second.first.StubAddress, I->second.second};
188 }
189
findPointer(StringRef Name)190 ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {
191 std::lock_guard<std::mutex> Lock(ISMMutex);
192 auto I = StubInfos.find(Name);
193 if (I == StubInfos.end())
194 return ExecutorSymbolDef();
195 return {I->second.first.PointerAddress, I->second.second};
196 }
197
updatePointer(StringRef Name,ExecutorAddr NewAddr)198 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
199 ExecutorAddr NewAddr) {
200
201 ExecutorAddr PtrAddr;
202 {
203 std::lock_guard<std::mutex> Lock(ISMMutex);
204 auto I = StubInfos.find(Name);
205 if (I == StubInfos.end())
206 return make_error<StringError>("Unknown stub name",
207 inconvertibleErrorCode());
208 PtrAddr = I->second.first.PointerAddress;
209 }
210
211 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
212 switch (EPCIU.getABISupport().getPointerSize()) {
213 case 4: {
214 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());
215 return MemAccess.writeUInt32s(PUpdate);
216 }
217 case 8: {
218 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());
219 return MemAccess.writeUInt64s(PUpdate);
220 }
221 default:
222 return make_error<StringError>("Unsupported pointer size",
223 inconvertibleErrorCode());
224 }
225 }
226
227 } // end anonymous namespace.
228
229 namespace llvm {
230 namespace orc {
231
232 EPCIndirectionUtils::ABISupport::~ABISupport() = default;
233
234 Expected<std::unique_ptr<EPCIndirectionUtils>>
Create(ExecutorProcessControl & EPC)235 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
236 const auto &TT = EPC.getTargetTriple();
237 switch (TT.getArch()) {
238 default:
239 return make_error<StringError>(
240 std::string("No EPCIndirectionUtils available for ") + TT.str(),
241 inconvertibleErrorCode());
242 case Triple::aarch64:
243 case Triple::aarch64_32:
244 return CreateWithABI<OrcAArch64>(EPC);
245
246 case Triple::x86:
247 return CreateWithABI<OrcI386>(EPC);
248
249 case Triple::loongarch64:
250 return CreateWithABI<OrcLoongArch64>(EPC);
251
252 case Triple::mips:
253 return CreateWithABI<OrcMips32Be>(EPC);
254
255 case Triple::mipsel:
256 return CreateWithABI<OrcMips32Le>(EPC);
257
258 case Triple::mips64:
259 case Triple::mips64el:
260 return CreateWithABI<OrcMips64>(EPC);
261
262 case Triple::riscv64:
263 return CreateWithABI<OrcRiscv64>(EPC);
264
265 case Triple::x86_64:
266 if (TT.getOS() == Triple::OSType::Win32)
267 return CreateWithABI<OrcX86_64_Win32>(EPC);
268 else
269 return CreateWithABI<OrcX86_64_SysV>(EPC);
270 }
271 }
272
cleanup()273 Error EPCIndirectionUtils::cleanup() {
274
275 auto &MemMgr = EPC.getMemMgr();
276 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
277
278 if (TP)
279 Err = joinErrors(std::move(Err),
280 static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
281
282 if (ResolverBlock)
283 Err =
284 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
285
286 return Err;
287 }
288
289 Expected<ExecutorAddr>
writeResolverBlock(ExecutorAddr ReentryFnAddr,ExecutorAddr ReentryCtxAddr)290 EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
291 ExecutorAddr ReentryCtxAddr) {
292 using namespace jitlink;
293
294 assert(ABI && "ABI can not be null");
295 auto ResolverSize = ABI->getResolverCodeSize();
296
297 auto Alloc =
298 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
299 {{MemProt::Read | MemProt::Exec,
300 {ResolverSize, Align(EPC.getPageSize())}}});
301
302 if (!Alloc)
303 return Alloc.takeError();
304
305 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
306 ResolverBlockAddr = SegInfo.Addr;
307 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
308 ReentryFnAddr, ReentryCtxAddr);
309
310 auto FA = Alloc->finalize();
311 if (!FA)
312 return FA.takeError();
313
314 ResolverBlock = std::move(*FA);
315 return ResolverBlockAddr;
316 }
317
318 std::unique_ptr<IndirectStubsManager>
createIndirectStubsManager()319 EPCIndirectionUtils::createIndirectStubsManager() {
320 return std::make_unique<EPCIndirectStubsManager>(*this);
321 }
322
getTrampolinePool()323 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
324 if (!TP)
325 TP = std::make_unique<EPCTrampolinePool>(*this);
326 return *TP;
327 }
328
createLazyCallThroughManager(ExecutionSession & ES,ExecutorAddr ErrorHandlerAddr)329 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
330 ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {
331 assert(!LCTM &&
332 "createLazyCallThroughManager can not have been called before");
333 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
334 &getTrampolinePool());
335 return *LCTM;
336 }
337
EPCIndirectionUtils(ExecutorProcessControl & EPC,std::unique_ptr<ABISupport> ABI)338 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
339 std::unique_ptr<ABISupport> ABI)
340 : EPC(EPC), ABI(std::move(ABI)) {
341 assert(this->ABI && "ABI can not be null");
342
343 assert(EPC.getPageSize() > getABISupport().getStubSize() &&
344 "Stubs larger than one page are not supported");
345 }
346
347 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
getIndirectStubs(unsigned NumStubs)348 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
349 using namespace jitlink;
350
351 std::lock_guard<std::mutex> Lock(EPCUIMutex);
352
353 // If there aren't enough stubs available then allocate some more.
354 if (NumStubs > AvailableIndirectStubs.size()) {
355 auto NumStubsToAllocate = NumStubs;
356 auto PageSize = EPC.getPageSize();
357 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
358 NumStubsToAllocate = StubBytes / ABI->getStubSize();
359 auto PtrBytes =
360 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
361
362 auto StubProt = MemProt::Read | MemProt::Exec;
363 auto PtrProt = MemProt::Read | MemProt::Write;
364
365 auto Alloc = SimpleSegmentAlloc::Create(
366 EPC.getMemMgr(), nullptr,
367 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
368 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
369
370 if (!Alloc)
371 return Alloc.takeError();
372
373 auto StubSeg = Alloc->getSegInfo(StubProt);
374 auto PtrSeg = Alloc->getSegInfo(PtrProt);
375
376 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr,
377 PtrSeg.Addr, NumStubsToAllocate);
378
379 auto FA = Alloc->finalize();
380 if (!FA)
381 return FA.takeError();
382
383 IndirectStubAllocs.push_back(std::move(*FA));
384
385 auto StubExecutorAddr = StubSeg.Addr;
386 auto PtrExecutorAddr = PtrSeg.Addr;
387 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
388 AvailableIndirectStubs.push_back(
389 IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
390 StubExecutorAddr += ABI->getStubSize();
391 PtrExecutorAddr += ABI->getPointerSize();
392 }
393 }
394
395 assert(NumStubs <= AvailableIndirectStubs.size() &&
396 "Sufficient stubs should have been allocated above");
397
398 IndirectStubInfoVector Result;
399 while (NumStubs--) {
400 Result.push_back(AvailableIndirectStubs.back());
401 AvailableIndirectStubs.pop_back();
402 }
403
404 return std::move(Result);
405 }
406
reentry(JITTargetAddress LCTMAddr,JITTargetAddress TrampolineAddr)407 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
408 JITTargetAddress TrampolineAddr) {
409 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
410 std::promise<ExecutorAddr> LandingAddrP;
411 auto LandingAddrF = LandingAddrP.get_future();
412 LCTM.resolveTrampolineLandingAddress(
413 ExecutorAddr(TrampolineAddr),
414 [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });
415 return LandingAddrF.get().getValue();
416 }
417
setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils & EPCIU)418 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
419 auto &LCTM = EPCIU.getLazyCallThroughManager();
420 return EPCIU
421 .writeResolverBlock(ExecutorAddr::fromPtr(&reentry),
422 ExecutorAddr::fromPtr(&LCTM))
423 .takeError();
424 }
425
426 } // end namespace orc
427 } // end namespace llvm
428