xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp (revision 9e5787d2284e187abb5b654d924394a65772e004)
1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ADT/STLExtras.h"
11 #include "llvm/ADT/Triple.h"
12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/Support/Format.h"
15 #include "llvm/Transforms/Utils/Cloning.h"
16 #include <sstream>
17 
18 using namespace llvm;
19 using namespace llvm::orc;
20 
21 namespace {
22 
23 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
24 public:
25   using CompileFunction = JITCompileCallbackManager::CompileFunction;
26 
27   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
28                                      CompileFunction Compile, VModuleKey K)
29       : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}),
30                             nullptr, std::move(K)),
31         Name(std::move(Name)), Compile(std::move(Compile)) {}
32 
33   StringRef getName() const override { return "<Compile Callbacks>"; }
34 
35 private:
36   void materialize(MaterializationResponsibility R) override {
37     SymbolMap Result;
38     Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
39     // No dependencies, so these calls cannot fail.
40     cantFail(R.notifyResolved(Result));
41     cantFail(R.notifyEmitted());
42   }
43 
44   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
45     llvm_unreachable("Discard should never occur on a LMU?");
46   }
47 
48   SymbolStringPtr Name;
49   CompileFunction Compile;
50 };
51 
52 } // namespace
53 
54 namespace llvm {
55 namespace orc {
56 
57 void IndirectStubsManager::anchor() {}
58 void TrampolinePool::anchor() {}
59 
60 Expected<JITTargetAddress>
61 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
62   if (auto TrampolineAddr = TP->getTrampoline()) {
63     auto CallbackName =
64         ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
65 
66     std::lock_guard<std::mutex> Lock(CCMgrMutex);
67     AddrToSymbol[*TrampolineAddr] = CallbackName;
68     cantFail(CallbacksJD.define(
69         std::make_unique<CompileCallbackMaterializationUnit>(
70             std::move(CallbackName), std::move(Compile),
71             ES.allocateVModule())));
72     return *TrampolineAddr;
73   } else
74     return TrampolineAddr.takeError();
75 }
76 
77 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
78     JITTargetAddress TrampolineAddr) {
79   SymbolStringPtr Name;
80 
81   {
82     std::unique_lock<std::mutex> Lock(CCMgrMutex);
83     auto I = AddrToSymbol.find(TrampolineAddr);
84 
85     // If this address is not associated with a compile callback then report an
86     // error to the execution session and return ErrorHandlerAddress to the
87     // callee.
88     if (I == AddrToSymbol.end()) {
89       Lock.unlock();
90       std::string ErrMsg;
91       {
92         raw_string_ostream ErrMsgStream(ErrMsg);
93         ErrMsgStream << "No compile callback for trampoline at "
94                      << format("0x%016" PRIx64, TrampolineAddr);
95       }
96       ES.reportError(
97           make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
98       return ErrorHandlerAddress;
99     } else
100       Name = I->second;
101   }
102 
103   if (auto Sym =
104           ES.lookup(makeJITDylibSearchOrder(
105                         &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
106                     Name))
107     return Sym->getAddress();
108   else {
109     llvm::dbgs() << "Didn't find callback.\n";
110     // If anything goes wrong materializing Sym then report it to the session
111     // and return the ErrorHandlerAddress;
112     ES.reportError(Sym.takeError());
113     return ErrorHandlerAddress;
114   }
115 }
116 
117 Expected<std::unique_ptr<JITCompileCallbackManager>>
118 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
119                                   JITTargetAddress ErrorHandlerAddress) {
120   switch (T.getArch()) {
121   default:
122     return make_error<StringError>(
123         std::string("No callback manager available for ") + T.str(),
124         inconvertibleErrorCode());
125   case Triple::aarch64:
126   case Triple::aarch64_32: {
127     typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
128     return CCMgrT::Create(ES, ErrorHandlerAddress);
129     }
130 
131     case Triple::x86: {
132       typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
133       return CCMgrT::Create(ES, ErrorHandlerAddress);
134     }
135 
136     case Triple::mips: {
137       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
138       return CCMgrT::Create(ES, ErrorHandlerAddress);
139     }
140     case Triple::mipsel: {
141       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
142       return CCMgrT::Create(ES, ErrorHandlerAddress);
143     }
144 
145     case Triple::mips64:
146     case Triple::mips64el: {
147       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
148       return CCMgrT::Create(ES, ErrorHandlerAddress);
149     }
150 
151     case Triple::x86_64: {
152       if ( T.getOS() == Triple::OSType::Win32 ) {
153         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
154         return CCMgrT::Create(ES, ErrorHandlerAddress);
155       } else {
156         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
157         return CCMgrT::Create(ES, ErrorHandlerAddress);
158       }
159     }
160 
161   }
162 }
163 
164 std::function<std::unique_ptr<IndirectStubsManager>()>
165 createLocalIndirectStubsManagerBuilder(const Triple &T) {
166   switch (T.getArch()) {
167     default:
168       return [](){
169         return std::make_unique<
170                        orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
171       };
172 
173     case Triple::aarch64:
174     case Triple::aarch64_32:
175       return [](){
176         return std::make_unique<
177                        orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
178       };
179 
180     case Triple::x86:
181       return [](){
182         return std::make_unique<
183                        orc::LocalIndirectStubsManager<orc::OrcI386>>();
184       };
185 
186     case Triple::mips:
187       return [](){
188           return std::make_unique<
189                       orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
190       };
191 
192     case Triple::mipsel:
193       return [](){
194           return std::make_unique<
195                       orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
196       };
197 
198     case Triple::mips64:
199     case Triple::mips64el:
200       return [](){
201           return std::make_unique<
202                       orc::LocalIndirectStubsManager<orc::OrcMips64>>();
203       };
204 
205     case Triple::x86_64:
206       if (T.getOS() == Triple::OSType::Win32) {
207         return [](){
208           return std::make_unique<
209                      orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
210         };
211       } else {
212         return [](){
213           return std::make_unique<
214                      orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
215         };
216       }
217 
218   }
219 }
220 
221 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
222   Constant *AddrIntVal =
223     ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
224   Constant *AddrPtrVal =
225     ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
226                           PointerType::get(&FT, 0));
227   return AddrPtrVal;
228 }
229 
230 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
231                                   const Twine &Name, Constant *Initializer) {
232   auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
233                                Initializer, Name, nullptr,
234                                GlobalValue::NotThreadLocal, 0, true);
235   IP->setVisibility(GlobalValue::HiddenVisibility);
236   return IP;
237 }
238 
239 void makeStub(Function &F, Value &ImplPointer) {
240   assert(F.isDeclaration() && "Can't turn a definition into a stub.");
241   assert(F.getParent() && "Function isn't in a module.");
242   Module &M = *F.getParent();
243   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
244   IRBuilder<> Builder(EntryBlock);
245   LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
246   std::vector<Value*> CallArgs;
247   for (auto &A : F.args())
248     CallArgs.push_back(&A);
249   CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
250   Call->setTailCall();
251   Call->setAttributes(F.getAttributes());
252   if (F.getReturnType()->isVoidTy())
253     Builder.CreateRetVoid();
254   else
255     Builder.CreateRet(Call);
256 }
257 
258 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
259   std::vector<GlobalValue *> PromotedGlobals;
260 
261   for (auto &GV : M.global_values()) {
262     bool Promoted = true;
263 
264     // Rename if necessary.
265     if (!GV.hasName())
266       GV.setName("__orc_anon." + Twine(NextId++));
267     else if (GV.getName().startswith("\01L"))
268       GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
269     else if (GV.hasLocalLinkage())
270       GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
271     else
272       Promoted = false;
273 
274     if (GV.hasLocalLinkage()) {
275       GV.setLinkage(GlobalValue::ExternalLinkage);
276       GV.setVisibility(GlobalValue::HiddenVisibility);
277       Promoted = true;
278     }
279     GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
280 
281     if (Promoted)
282       PromotedGlobals.push_back(&GV);
283   }
284 
285   return PromotedGlobals;
286 }
287 
288 Function* cloneFunctionDecl(Module &Dst, const Function &F,
289                             ValueToValueMapTy *VMap) {
290   Function *NewF =
291     Function::Create(cast<FunctionType>(F.getValueType()),
292                      F.getLinkage(), F.getName(), &Dst);
293   NewF->copyAttributesFrom(&F);
294 
295   if (VMap) {
296     (*VMap)[&F] = NewF;
297     auto NewArgI = NewF->arg_begin();
298     for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
299          ++ArgI, ++NewArgI)
300       (*VMap)[&*ArgI] = &*NewArgI;
301   }
302 
303   return NewF;
304 }
305 
306 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
307                       ValueMaterializer *Materializer,
308                       Function *NewF) {
309   assert(!OrigF.isDeclaration() && "Nothing to move");
310   if (!NewF)
311     NewF = cast<Function>(VMap[&OrigF]);
312   else
313     assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
314   assert(NewF && "Function mapping missing from VMap.");
315   assert(NewF->getParent() != OrigF.getParent() &&
316          "moveFunctionBody should only be used to move bodies between "
317          "modules.");
318 
319   SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
320   CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns,
321                     "", nullptr, nullptr, Materializer);
322   OrigF.deleteBody();
323 }
324 
325 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
326                                         ValueToValueMapTy *VMap) {
327   GlobalVariable *NewGV = new GlobalVariable(
328       Dst, GV.getValueType(), GV.isConstant(),
329       GV.getLinkage(), nullptr, GV.getName(), nullptr,
330       GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
331   NewGV->copyAttributesFrom(&GV);
332   if (VMap)
333     (*VMap)[&GV] = NewGV;
334   return NewGV;
335 }
336 
337 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
338                                    ValueToValueMapTy &VMap,
339                                    ValueMaterializer *Materializer,
340                                    GlobalVariable *NewGV) {
341   assert(OrigGV.hasInitializer() && "Nothing to move");
342   if (!NewGV)
343     NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
344   else
345     assert(VMap[&OrigGV] == NewGV &&
346            "Incorrect global variable mapping in VMap.");
347   assert(NewGV->getParent() != OrigGV.getParent() &&
348          "moveGlobalVariableInitializer should only be used to move "
349          "initializers between modules");
350 
351   NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
352                                  nullptr, Materializer));
353 }
354 
355 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
356                                   ValueToValueMapTy &VMap) {
357   assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
358   auto *NewA = GlobalAlias::create(OrigA.getValueType(),
359                                    OrigA.getType()->getPointerAddressSpace(),
360                                    OrigA.getLinkage(), OrigA.getName(), &Dst);
361   NewA->copyAttributesFrom(&OrigA);
362   VMap[&OrigA] = NewA;
363   return NewA;
364 }
365 
366 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
367                               ValueToValueMapTy &VMap) {
368   auto *MFs = Src.getModuleFlagsMetadata();
369   if (!MFs)
370     return;
371   for (auto *MF : MFs->operands())
372     Dst.addModuleFlag(MapMetadata(MF, VMap));
373 }
374 
375 } // End namespace orc.
376 } // End namespace llvm.
377