1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ADT/STLExtras.h"
11 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/Module.h"
15 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
16 #include "llvm/MC/MCInstrAnalysis.h"
17 #include "llvm/Support/Format.h"
18 #include "llvm/TargetParser/Triple.h"
19 #include "llvm/Transforms/Utils/Cloning.h"
20 #include <sstream>
21
22 #define DEBUG_TYPE "orc"
23
24 using namespace llvm;
25 using namespace llvm::orc;
26
27 namespace {
28
29 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
30 public:
31 using CompileFunction = JITCompileCallbackManager::CompileFunction;
32
CompileCallbackMaterializationUnit(SymbolStringPtr Name,CompileFunction Compile)33 CompileCallbackMaterializationUnit(SymbolStringPtr Name,
34 CompileFunction Compile)
35 : MaterializationUnit(Interface(
36 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
37 Name(std::move(Name)), Compile(std::move(Compile)) {}
38
getName() const39 StringRef getName() const override { return "<Compile Callbacks>"; }
40
41 private:
materialize(std::unique_ptr<MaterializationResponsibility> R)42 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
43 SymbolMap Result;
44 Result[Name] = {Compile(), JITSymbolFlags::Exported};
45 // No dependencies, so these calls cannot fail.
46 cantFail(R->notifyResolved(Result));
47 cantFail(R->notifyEmitted({}));
48 }
49
discard(const JITDylib & JD,const SymbolStringPtr & Name)50 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
51 llvm_unreachable("Discard should never occur on a LMU?");
52 }
53
54 SymbolStringPtr Name;
55 CompileFunction Compile;
56 };
57
58 } // namespace
59
60 namespace llvm {
61 namespace orc {
62
63 TrampolinePool::~TrampolinePool() = default;
anchor()64 void IndirectStubsManager::anchor() {}
65
66 Expected<ExecutorAddr>
getCompileCallback(CompileFunction Compile)67 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
68 if (auto TrampolineAddr = TP->getTrampoline()) {
69 auto CallbackName =
70 ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
71
72 std::lock_guard<std::mutex> Lock(CCMgrMutex);
73 AddrToSymbol[*TrampolineAddr] = CallbackName;
74 cantFail(
75 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
76 std::move(CallbackName), std::move(Compile))));
77 return *TrampolineAddr;
78 } else
79 return TrampolineAddr.takeError();
80 }
81
82 ExecutorAddr
executeCompileCallback(ExecutorAddr TrampolineAddr)83 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {
84 SymbolStringPtr Name;
85
86 {
87 std::unique_lock<std::mutex> Lock(CCMgrMutex);
88 auto I = AddrToSymbol.find(TrampolineAddr);
89
90 // If this address is not associated with a compile callback then report an
91 // error to the execution session and return ErrorHandlerAddress to the
92 // callee.
93 if (I == AddrToSymbol.end()) {
94 Lock.unlock();
95 ES.reportError(
96 make_error<StringError>("No compile callback for trampoline at " +
97 formatv("{0:x}", TrampolineAddr),
98 inconvertibleErrorCode()));
99 return ErrorHandlerAddress;
100 } else
101 Name = I->second;
102 }
103
104 if (auto Sym =
105 ES.lookup(makeJITDylibSearchOrder(
106 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
107 Name))
108 return Sym->getAddress();
109 else {
110 llvm::dbgs() << "Didn't find callback.\n";
111 // If anything goes wrong materializing Sym then report it to the session
112 // and return the ErrorHandlerAddress;
113 ES.reportError(Sym.takeError());
114 return ErrorHandlerAddress;
115 }
116 }
117
118 Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple & T,ExecutionSession & ES,ExecutorAddr ErrorHandlerAddress)119 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
120 ExecutorAddr ErrorHandlerAddress) {
121 switch (T.getArch()) {
122 default:
123 return make_error<StringError>(
124 std::string("No callback manager available for ") + T.str(),
125 inconvertibleErrorCode());
126 case Triple::aarch64:
127 case Triple::aarch64_32: {
128 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
129 return CCMgrT::Create(ES, ErrorHandlerAddress);
130 }
131
132 case Triple::x86: {
133 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
134 return CCMgrT::Create(ES, ErrorHandlerAddress);
135 }
136
137 case Triple::loongarch64: {
138 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
139 return CCMgrT::Create(ES, ErrorHandlerAddress);
140 }
141
142 case Triple::mips: {
143 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
144 return CCMgrT::Create(ES, ErrorHandlerAddress);
145 }
146 case Triple::mipsel: {
147 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
148 return CCMgrT::Create(ES, ErrorHandlerAddress);
149 }
150
151 case Triple::mips64:
152 case Triple::mips64el: {
153 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
154 return CCMgrT::Create(ES, ErrorHandlerAddress);
155 }
156
157 case Triple::riscv64: {
158 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
159 return CCMgrT::Create(ES, ErrorHandlerAddress);
160 }
161
162 case Triple::x86_64: {
163 if (T.getOS() == Triple::OSType::Win32) {
164 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
165 return CCMgrT::Create(ES, ErrorHandlerAddress);
166 } else {
167 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
168 return CCMgrT::Create(ES, ErrorHandlerAddress);
169 }
170 }
171
172 }
173 }
174
175 std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple & T)176 createLocalIndirectStubsManagerBuilder(const Triple &T) {
177 switch (T.getArch()) {
178 default:
179 return [](){
180 return std::make_unique<
181 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
182 };
183
184 case Triple::aarch64:
185 case Triple::aarch64_32:
186 return [](){
187 return std::make_unique<
188 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
189 };
190
191 case Triple::x86:
192 return [](){
193 return std::make_unique<
194 orc::LocalIndirectStubsManager<orc::OrcI386>>();
195 };
196
197 case Triple::loongarch64:
198 return []() {
199 return std::make_unique<
200 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
201 };
202
203 case Triple::mips:
204 return [](){
205 return std::make_unique<
206 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
207 };
208
209 case Triple::mipsel:
210 return [](){
211 return std::make_unique<
212 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
213 };
214
215 case Triple::mips64:
216 case Triple::mips64el:
217 return [](){
218 return std::make_unique<
219 orc::LocalIndirectStubsManager<orc::OrcMips64>>();
220 };
221
222 case Triple::riscv64:
223 return []() {
224 return std::make_unique<
225 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
226 };
227
228 case Triple::x86_64:
229 if (T.getOS() == Triple::OSType::Win32) {
230 return [](){
231 return std::make_unique<
232 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
233 };
234 } else {
235 return [](){
236 return std::make_unique<
237 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
238 };
239 }
240
241 }
242 }
243
createIRTypedAddress(FunctionType & FT,ExecutorAddr Addr)244 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {
245 Constant *AddrIntVal =
246 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue());
247 Constant *AddrPtrVal =
248 ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0));
249 return AddrPtrVal;
250 }
251
createImplPointer(PointerType & PT,Module & M,const Twine & Name,Constant * Initializer)252 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
253 const Twine &Name, Constant *Initializer) {
254 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
255 Initializer, Name, nullptr,
256 GlobalValue::NotThreadLocal, 0, true);
257 IP->setVisibility(GlobalValue::HiddenVisibility);
258 return IP;
259 }
260
makeStub(Function & F,Value & ImplPointer)261 void makeStub(Function &F, Value &ImplPointer) {
262 assert(F.isDeclaration() && "Can't turn a definition into a stub.");
263 assert(F.getParent() && "Function isn't in a module.");
264 Module &M = *F.getParent();
265 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
266 IRBuilder<> Builder(EntryBlock);
267 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
268 std::vector<Value*> CallArgs;
269 for (auto &A : F.args())
270 CallArgs.push_back(&A);
271 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
272 Call->setTailCall();
273 Call->setAttributes(F.getAttributes());
274 if (F.getReturnType()->isVoidTy())
275 Builder.CreateRetVoid();
276 else
277 Builder.CreateRet(Call);
278 }
279
operator ()(Module & M)280 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
281 std::vector<GlobalValue *> PromotedGlobals;
282
283 for (auto &GV : M.global_values()) {
284 bool Promoted = true;
285
286 // Rename if necessary.
287 if (!GV.hasName())
288 GV.setName("__orc_anon." + Twine(NextId++));
289 else if (GV.getName().starts_with("\01L"))
290 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
291 else if (GV.hasLocalLinkage())
292 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
293 else
294 Promoted = false;
295
296 if (GV.hasLocalLinkage()) {
297 GV.setLinkage(GlobalValue::ExternalLinkage);
298 GV.setVisibility(GlobalValue::HiddenVisibility);
299 Promoted = true;
300 }
301 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
302
303 if (Promoted)
304 PromotedGlobals.push_back(&GV);
305 }
306
307 return PromotedGlobals;
308 }
309
cloneFunctionDecl(Module & Dst,const Function & F,ValueToValueMapTy * VMap)310 Function* cloneFunctionDecl(Module &Dst, const Function &F,
311 ValueToValueMapTy *VMap) {
312 Function *NewF =
313 Function::Create(cast<FunctionType>(F.getValueType()),
314 F.getLinkage(), F.getName(), &Dst);
315 NewF->copyAttributesFrom(&F);
316
317 if (VMap) {
318 (*VMap)[&F] = NewF;
319 auto NewArgI = NewF->arg_begin();
320 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
321 ++ArgI, ++NewArgI)
322 (*VMap)[&*ArgI] = &*NewArgI;
323 }
324
325 return NewF;
326 }
327
cloneGlobalVariableDecl(Module & Dst,const GlobalVariable & GV,ValueToValueMapTy * VMap)328 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
329 ValueToValueMapTy *VMap) {
330 GlobalVariable *NewGV = new GlobalVariable(
331 Dst, GV.getValueType(), GV.isConstant(),
332 GV.getLinkage(), nullptr, GV.getName(), nullptr,
333 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
334 NewGV->copyAttributesFrom(&GV);
335 if (VMap)
336 (*VMap)[&GV] = NewGV;
337 return NewGV;
338 }
339
cloneGlobalAliasDecl(Module & Dst,const GlobalAlias & OrigA,ValueToValueMapTy & VMap)340 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
341 ValueToValueMapTy &VMap) {
342 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
343 auto *NewA = GlobalAlias::create(OrigA.getValueType(),
344 OrigA.getType()->getPointerAddressSpace(),
345 OrigA.getLinkage(), OrigA.getName(), &Dst);
346 NewA->copyAttributesFrom(&OrigA);
347 VMap[&OrigA] = NewA;
348 return NewA;
349 }
350
addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol & Sym,jitlink::LinkGraph & G,MCDisassembler & Disassembler,MCInstrAnalysis & MIA)351 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
352 jitlink::LinkGraph &G,
353 MCDisassembler &Disassembler,
354 MCInstrAnalysis &MIA) {
355 // AArch64 appears to already come with the necessary relocations. Among other
356 // architectures, only x86_64 is currently implemented here.
357 if (G.getTargetTriple().getArch() != Triple::x86_64)
358 return Error::success();
359
360 raw_null_ostream CommentStream;
361 auto &STI = Disassembler.getSubtargetInfo();
362
363 // Determine the function bounds
364 auto &B = Sym.getBlock();
365 assert(!B.isZeroFill() && "expected content block");
366 auto SymAddress = Sym.getAddress();
367 auto SymStartInBlock =
368 (const uint8_t *)B.getContent().data() + Sym.getOffset();
369 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
370 auto Content = ArrayRef(SymStartInBlock, SymSize);
371
372 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
373
374 SmallDenseSet<uintptr_t, 8> ExistingRelocations;
375 for (auto &E : B.edges()) {
376 if (E.isRelocation())
377 ExistingRelocations.insert(E.getOffset());
378 }
379
380 size_t I = 0;
381 while (I < Content.size()) {
382 MCInst Instr;
383 uint64_t InstrSize = 0;
384 uint64_t InstrStart = SymAddress.getValue() + I;
385 auto DecodeStatus = Disassembler.getInstruction(
386 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
387 if (DecodeStatus != MCDisassembler::Success) {
388 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
389 << InstrStart);
390 return make_error<StringError>(
391 formatv("failed to disassemble at address {0:x16}", InstrStart),
392 inconvertibleErrorCode());
393 }
394 // Advance to the next instruction.
395 I += InstrSize;
396
397 // Check for a PC-relative address equal to the symbol itself.
398 auto PCRelAddr =
399 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
400 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
401 continue;
402
403 auto RelocOffInInstr =
404 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
405 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
406 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
407 << InstrStart);
408 continue;
409 }
410
411 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
412 SymAddress + Sym.getOffset();
413 if (ExistingRelocations.contains(RelocOffInBlock))
414 continue;
415
416 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
417 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
418 }
419 return Error::success();
420 }
421
422 } // End namespace orc.
423 } // End namespace llvm.
424