1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
11 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
12 #include "llvm/IR/IRBuilder.h"
13 #include "llvm/IR/Module.h"
14 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
15 #include "llvm/MC/MCInstrAnalysis.h"
16 #include "llvm/TargetParser/Triple.h"
17 #include "llvm/Transforms/Utils/Cloning.h"
18
19 #define DEBUG_TYPE "orc"
20
21 using namespace llvm;
22 using namespace llvm::orc;
23
24 namespace {
25
26 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
27 public:
28 using CompileFunction = JITCompileCallbackManager::CompileFunction;
29
CompileCallbackMaterializationUnit(SymbolStringPtr Name,CompileFunction Compile)30 CompileCallbackMaterializationUnit(SymbolStringPtr Name,
31 CompileFunction Compile)
32 : MaterializationUnit(Interface(
33 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
34 Name(std::move(Name)), Compile(std::move(Compile)) {}
35
getName() const36 StringRef getName() const override { return "<Compile Callbacks>"; }
37
38 private:
materialize(std::unique_ptr<MaterializationResponsibility> R)39 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
40 SymbolMap Result;
41 Result[Name] = {Compile(), JITSymbolFlags::Exported};
42 // No dependencies, so these calls cannot fail.
43 cantFail(R->notifyResolved(Result));
44 cantFail(R->notifyEmitted({}));
45 }
46
discard(const JITDylib & JD,const SymbolStringPtr & Name)47 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
48 llvm_unreachable("Discard should never occur on a LMU?");
49 }
50
51 SymbolStringPtr Name;
52 CompileFunction Compile;
53 };
54
55 } // namespace
56
57 namespace llvm {
58 namespace orc {
59
60 TrampolinePool::~TrampolinePool() = default;
anchor()61 void IndirectStubsManager::anchor() {}
62
63 Expected<ExecutorAddr>
getCompileCallback(CompileFunction Compile)64 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
65 if (auto TrampolineAddr = TP->getTrampoline()) {
66 auto CallbackName =
67 ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
68
69 std::lock_guard<std::mutex> Lock(CCMgrMutex);
70 AddrToSymbol[*TrampolineAddr] = CallbackName;
71 cantFail(
72 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
73 std::move(CallbackName), std::move(Compile))));
74 return *TrampolineAddr;
75 } else
76 return TrampolineAddr.takeError();
77 }
78
79 ExecutorAddr
executeCompileCallback(ExecutorAddr TrampolineAddr)80 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {
81 SymbolStringPtr Name;
82
83 {
84 std::unique_lock<std::mutex> Lock(CCMgrMutex);
85 auto I = AddrToSymbol.find(TrampolineAddr);
86
87 // If this address is not associated with a compile callback then report an
88 // error to the execution session and return ErrorHandlerAddress to the
89 // callee.
90 if (I == AddrToSymbol.end()) {
91 Lock.unlock();
92 ES.reportError(
93 make_error<StringError>("No compile callback for trampoline at " +
94 formatv("{0:x}", TrampolineAddr),
95 inconvertibleErrorCode()));
96 return ErrorHandlerAddress;
97 } else
98 Name = I->second;
99 }
100
101 if (auto Sym =
102 ES.lookup(makeJITDylibSearchOrder(
103 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
104 Name))
105 return Sym->getAddress();
106 else {
107 llvm::dbgs() << "Didn't find callback.\n";
108 // If anything goes wrong materializing Sym then report it to the session
109 // and return the ErrorHandlerAddress;
110 ES.reportError(Sym.takeError());
111 return ErrorHandlerAddress;
112 }
113 }
114
redirect(JITDylib & JD,const SymbolMap & NewDests)115 Error IndirectStubsManager::redirect(JITDylib &JD, const SymbolMap &NewDests) {
116 for (auto &[Name, Dest] : NewDests)
117 if (auto Err = updatePointer(*Name, Dest.getAddress()))
118 return Err;
119 return Error::success();
120 }
121
emitRedirectableSymbols(std::unique_ptr<MaterializationResponsibility> MR,SymbolMap InitialDests)122 void IndirectStubsManager::emitRedirectableSymbols(
123 std::unique_ptr<MaterializationResponsibility> MR, SymbolMap InitialDests) {
124 StubInitsMap StubInits;
125 for (auto &[Name, Dest] : InitialDests)
126 StubInits[*Name] = {Dest.getAddress(), Dest.getFlags()};
127 if (auto Err = createStubs(StubInits)) {
128 MR->getExecutionSession().reportError(std::move(Err));
129 return MR->failMaterialization();
130 }
131 SymbolMap Stubs;
132 for (auto &[Name, Dest] : InitialDests) {
133 auto StubSym = findStub(*Name, false);
134 assert(StubSym.getAddress() && "Stub symbol should be present");
135 Stubs[Name] = StubSym;
136 }
137 if (auto Err = MR->notifyResolved(Stubs)) {
138 MR->getExecutionSession().reportError(std::move(Err));
139 return MR->failMaterialization();
140 }
141 if (auto Err = MR->notifyEmitted({})) {
142 MR->getExecutionSession().reportError(std::move(Err));
143 return MR->failMaterialization();
144 }
145 }
146
147 Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple & T,ExecutionSession & ES,ExecutorAddr ErrorHandlerAddress)148 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
149 ExecutorAddr ErrorHandlerAddress) {
150 switch (T.getArch()) {
151 default:
152 return make_error<StringError>(
153 std::string("No callback manager available for ") + T.str(),
154 inconvertibleErrorCode());
155 case Triple::aarch64:
156 case Triple::aarch64_32: {
157 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
158 return CCMgrT::Create(ES, ErrorHandlerAddress);
159 }
160
161 case Triple::x86: {
162 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
163 return CCMgrT::Create(ES, ErrorHandlerAddress);
164 }
165
166 case Triple::loongarch64: {
167 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
168 return CCMgrT::Create(ES, ErrorHandlerAddress);
169 }
170
171 case Triple::mips: {
172 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
173 return CCMgrT::Create(ES, ErrorHandlerAddress);
174 }
175 case Triple::mipsel: {
176 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
177 return CCMgrT::Create(ES, ErrorHandlerAddress);
178 }
179
180 case Triple::mips64:
181 case Triple::mips64el: {
182 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
183 return CCMgrT::Create(ES, ErrorHandlerAddress);
184 }
185
186 case Triple::riscv64: {
187 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
188 return CCMgrT::Create(ES, ErrorHandlerAddress);
189 }
190
191 case Triple::x86_64: {
192 if (T.getOS() == Triple::OSType::Win32) {
193 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
194 return CCMgrT::Create(ES, ErrorHandlerAddress);
195 } else {
196 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
197 return CCMgrT::Create(ES, ErrorHandlerAddress);
198 }
199 }
200
201 }
202 }
203
204 std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple & T)205 createLocalIndirectStubsManagerBuilder(const Triple &T) {
206 switch (T.getArch()) {
207 default:
208 return [](){
209 return std::make_unique<
210 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
211 };
212
213 case Triple::aarch64:
214 case Triple::aarch64_32:
215 return [](){
216 return std::make_unique<
217 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
218 };
219
220 case Triple::x86:
221 return [](){
222 return std::make_unique<
223 orc::LocalIndirectStubsManager<orc::OrcI386>>();
224 };
225
226 case Triple::loongarch64:
227 return []() {
228 return std::make_unique<
229 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
230 };
231
232 case Triple::mips:
233 return [](){
234 return std::make_unique<
235 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
236 };
237
238 case Triple::mipsel:
239 return [](){
240 return std::make_unique<
241 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
242 };
243
244 case Triple::mips64:
245 case Triple::mips64el:
246 return [](){
247 return std::make_unique<
248 orc::LocalIndirectStubsManager<orc::OrcMips64>>();
249 };
250
251 case Triple::riscv64:
252 return []() {
253 return std::make_unique<
254 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
255 };
256
257 case Triple::x86_64:
258 if (T.getOS() == Triple::OSType::Win32) {
259 return [](){
260 return std::make_unique<
261 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
262 };
263 } else {
264 return [](){
265 return std::make_unique<
266 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
267 };
268 }
269
270 }
271 }
272
createIRTypedAddress(FunctionType & FT,ExecutorAddr Addr)273 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {
274 Constant *AddrIntVal =
275 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue());
276 Constant *AddrPtrVal = ConstantExpr::getIntToPtr(
277 AddrIntVal, PointerType::get(FT.getContext(), 0));
278 return AddrPtrVal;
279 }
280
createImplPointer(PointerType & PT,Module & M,const Twine & Name,Constant * Initializer)281 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
282 const Twine &Name, Constant *Initializer) {
283 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
284 Initializer, Name, nullptr,
285 GlobalValue::NotThreadLocal, 0, true);
286 IP->setVisibility(GlobalValue::HiddenVisibility);
287 return IP;
288 }
289
makeStub(Function & F,Value & ImplPointer)290 void makeStub(Function &F, Value &ImplPointer) {
291 assert(F.isDeclaration() && "Can't turn a definition into a stub.");
292 assert(F.getParent() && "Function isn't in a module.");
293 Module &M = *F.getParent();
294 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
295 IRBuilder<> Builder(EntryBlock);
296 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
297 std::vector<Value*> CallArgs;
298 for (auto &A : F.args())
299 CallArgs.push_back(&A);
300 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
301 Call->setTailCall();
302 Call->setAttributes(F.getAttributes());
303 if (F.getReturnType()->isVoidTy())
304 Builder.CreateRetVoid();
305 else
306 Builder.CreateRet(Call);
307 }
308
operator ()(Module & M)309 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
310 std::vector<GlobalValue *> PromotedGlobals;
311
312 for (auto &GV : M.global_values()) {
313 bool Promoted = true;
314
315 // Rename if necessary.
316 if (!GV.hasName())
317 GV.setName("__orc_anon." + Twine(NextId++));
318 else if (GV.getName().starts_with("\01L"))
319 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
320 else if (GV.hasLocalLinkage())
321 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
322 else
323 Promoted = false;
324
325 if (GV.hasLocalLinkage()) {
326 GV.setLinkage(GlobalValue::ExternalLinkage);
327 GV.setVisibility(GlobalValue::HiddenVisibility);
328 Promoted = true;
329 }
330 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
331
332 if (Promoted)
333 PromotedGlobals.push_back(&GV);
334 }
335
336 return PromotedGlobals;
337 }
338
cloneFunctionDecl(Module & Dst,const Function & F,ValueToValueMapTy * VMap)339 Function* cloneFunctionDecl(Module &Dst, const Function &F,
340 ValueToValueMapTy *VMap) {
341 Function *NewF =
342 Function::Create(cast<FunctionType>(F.getValueType()),
343 F.getLinkage(), F.getName(), &Dst);
344 NewF->copyAttributesFrom(&F);
345
346 if (VMap) {
347 (*VMap)[&F] = NewF;
348 auto NewArgI = NewF->arg_begin();
349 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
350 ++ArgI, ++NewArgI)
351 (*VMap)[&*ArgI] = &*NewArgI;
352 }
353
354 return NewF;
355 }
356
cloneGlobalVariableDecl(Module & Dst,const GlobalVariable & GV,ValueToValueMapTy * VMap)357 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
358 ValueToValueMapTy *VMap) {
359 GlobalVariable *NewGV = new GlobalVariable(
360 Dst, GV.getValueType(), GV.isConstant(),
361 GV.getLinkage(), nullptr, GV.getName(), nullptr,
362 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
363 NewGV->copyAttributesFrom(&GV);
364 if (VMap)
365 (*VMap)[&GV] = NewGV;
366 return NewGV;
367 }
368
cloneGlobalAliasDecl(Module & Dst,const GlobalAlias & OrigA,ValueToValueMapTy & VMap)369 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
370 ValueToValueMapTy &VMap) {
371 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
372 auto *NewA = GlobalAlias::create(OrigA.getValueType(),
373 OrigA.getType()->getPointerAddressSpace(),
374 OrigA.getLinkage(), OrigA.getName(), &Dst);
375 NewA->copyAttributesFrom(&OrigA);
376 VMap[&OrigA] = NewA;
377 return NewA;
378 }
379
addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol & Sym,jitlink::LinkGraph & G,MCDisassembler & Disassembler,MCInstrAnalysis & MIA)380 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
381 jitlink::LinkGraph &G,
382 MCDisassembler &Disassembler,
383 MCInstrAnalysis &MIA) {
384 // AArch64 appears to already come with the necessary relocations. Among other
385 // architectures, only x86_64 is currently implemented here.
386 if (G.getTargetTriple().getArch() != Triple::x86_64)
387 return Error::success();
388
389 raw_null_ostream CommentStream;
390 auto &STI = Disassembler.getSubtargetInfo();
391
392 // Determine the function bounds
393 auto &B = Sym.getBlock();
394 assert(!B.isZeroFill() && "expected content block");
395 auto SymAddress = Sym.getAddress();
396 auto SymStartInBlock =
397 (const uint8_t *)B.getContent().data() + Sym.getOffset();
398 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
399 auto Content = ArrayRef(SymStartInBlock, SymSize);
400
401 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
402
403 SmallDenseSet<uintptr_t, 8> ExistingRelocations;
404 for (auto &E : B.edges()) {
405 if (E.isRelocation())
406 ExistingRelocations.insert(E.getOffset());
407 }
408
409 size_t I = 0;
410 while (I < Content.size()) {
411 MCInst Instr;
412 uint64_t InstrSize = 0;
413 uint64_t InstrStart = SymAddress.getValue() + I;
414 auto DecodeStatus = Disassembler.getInstruction(
415 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
416 if (DecodeStatus != MCDisassembler::Success) {
417 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
418 << InstrStart);
419 return make_error<StringError>(
420 formatv("failed to disassemble at address {0:x16}", InstrStart),
421 inconvertibleErrorCode());
422 }
423 // Advance to the next instruction.
424 I += InstrSize;
425
426 // Check for a PC-relative address equal to the symbol itself.
427 auto PCRelAddr =
428 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
429 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
430 continue;
431
432 auto RelocOffInInstr =
433 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
434 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
435 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
436 << InstrStart);
437 continue;
438 }
439
440 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
441 SymAddress + Sym.getOffset();
442 if (ExistingRelocations.contains(RelocOffInBlock))
443 continue;
444
445 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
446 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
447 }
448 return Error::success();
449 }
450
451 } // End namespace orc.
452 } // End namespace llvm.
453