1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ADT/STLExtras.h" 11 #include "llvm/ADT/Triple.h" 12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 13 #include "llvm/IR/CallSite.h" 14 #include "llvm/IR/IRBuilder.h" 15 #include "llvm/Support/Format.h" 16 #include "llvm/Transforms/Utils/Cloning.h" 17 #include <sstream> 18 19 using namespace llvm; 20 using namespace llvm::orc; 21 22 namespace { 23 24 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 25 public: 26 using CompileFunction = JITCompileCallbackManager::CompileFunction; 27 28 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 29 CompileFunction Compile, VModuleKey K) 30 : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), 31 std::move(K)), 32 Name(std::move(Name)), Compile(std::move(Compile)) {} 33 34 StringRef getName() const override { return "<Compile Callbacks>"; } 35 36 private: 37 void materialize(MaterializationResponsibility R) override { 38 SymbolMap Result; 39 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); 40 R.notifyResolved(Result); 41 R.notifyEmitted(); 42 } 43 44 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 45 llvm_unreachable("Discard should never occur on a LMU?"); 46 } 47 48 SymbolStringPtr Name; 49 CompileFunction Compile; 50 }; 51 52 } // namespace 53 54 namespace llvm { 55 namespace orc { 56 57 void IndirectStubsManager::anchor() {} 58 void TrampolinePool::anchor() {} 59 60 Expected<JITTargetAddress> 61 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 62 if (auto TrampolineAddr = TP->getTrampoline()) { 63 auto CallbackName = 64 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 65 66 std::lock_guard<std::mutex> Lock(CCMgrMutex); 67 AddrToSymbol[*TrampolineAddr] = CallbackName; 68 cantFail(CallbacksJD.define( 69 llvm::make_unique<CompileCallbackMaterializationUnit>( 70 std::move(CallbackName), std::move(Compile), 71 ES.allocateVModule()))); 72 return *TrampolineAddr; 73 } else 74 return TrampolineAddr.takeError(); 75 } 76 77 JITTargetAddress JITCompileCallbackManager::executeCompileCallback( 78 JITTargetAddress TrampolineAddr) { 79 SymbolStringPtr Name; 80 81 { 82 std::unique_lock<std::mutex> Lock(CCMgrMutex); 83 auto I = AddrToSymbol.find(TrampolineAddr); 84 85 // If this address is not associated with a compile callback then report an 86 // error to the execution session and return ErrorHandlerAddress to the 87 // callee. 88 if (I == AddrToSymbol.end()) { 89 Lock.unlock(); 90 std::string ErrMsg; 91 { 92 raw_string_ostream ErrMsgStream(ErrMsg); 93 ErrMsgStream << "No compile callback for trampoline at " 94 << format("0x%016" PRIx64, TrampolineAddr); 95 } 96 ES.reportError( 97 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); 98 return ErrorHandlerAddress; 99 } else 100 Name = I->second; 101 } 102 103 if (auto Sym = ES.lookup(JITDylibSearchList({{&CallbacksJD, true}}), Name)) 104 return Sym->getAddress(); 105 else { 106 llvm::dbgs() << "Didn't find callback.\n"; 107 // If anything goes wrong materializing Sym then report it to the session 108 // and return the ErrorHandlerAddress; 109 ES.reportError(Sym.takeError()); 110 return ErrorHandlerAddress; 111 } 112 } 113 114 Expected<std::unique_ptr<JITCompileCallbackManager>> 115 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 116 JITTargetAddress ErrorHandlerAddress) { 117 switch (T.getArch()) { 118 default: 119 return make_error<StringError>( 120 std::string("No callback manager available for ") + T.str(), 121 inconvertibleErrorCode()); 122 case Triple::aarch64: { 123 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 124 return CCMgrT::Create(ES, ErrorHandlerAddress); 125 } 126 127 case Triple::x86: { 128 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 129 return CCMgrT::Create(ES, ErrorHandlerAddress); 130 } 131 132 case Triple::mips: { 133 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 134 return CCMgrT::Create(ES, ErrorHandlerAddress); 135 } 136 case Triple::mipsel: { 137 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 138 return CCMgrT::Create(ES, ErrorHandlerAddress); 139 } 140 141 case Triple::mips64: 142 case Triple::mips64el: { 143 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 144 return CCMgrT::Create(ES, ErrorHandlerAddress); 145 } 146 147 case Triple::x86_64: { 148 if ( T.getOS() == Triple::OSType::Win32 ) { 149 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 150 return CCMgrT::Create(ES, ErrorHandlerAddress); 151 } else { 152 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 153 return CCMgrT::Create(ES, ErrorHandlerAddress); 154 } 155 } 156 157 } 158 } 159 160 std::function<std::unique_ptr<IndirectStubsManager>()> 161 createLocalIndirectStubsManagerBuilder(const Triple &T) { 162 switch (T.getArch()) { 163 default: 164 return [](){ 165 return llvm::make_unique< 166 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 167 }; 168 169 case Triple::aarch64: 170 return [](){ 171 return llvm::make_unique< 172 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 173 }; 174 175 case Triple::x86: 176 return [](){ 177 return llvm::make_unique< 178 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 179 }; 180 181 case Triple::mips: 182 return [](){ 183 return llvm::make_unique< 184 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 185 }; 186 187 case Triple::mipsel: 188 return [](){ 189 return llvm::make_unique< 190 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 191 }; 192 193 case Triple::mips64: 194 case Triple::mips64el: 195 return [](){ 196 return llvm::make_unique< 197 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 198 }; 199 200 case Triple::x86_64: 201 if (T.getOS() == Triple::OSType::Win32) { 202 return [](){ 203 return llvm::make_unique< 204 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 205 }; 206 } else { 207 return [](){ 208 return llvm::make_unique< 209 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 210 }; 211 } 212 213 } 214 } 215 216 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { 217 Constant *AddrIntVal = 218 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); 219 Constant *AddrPtrVal = 220 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, 221 PointerType::get(&FT, 0)); 222 return AddrPtrVal; 223 } 224 225 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 226 const Twine &Name, Constant *Initializer) { 227 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 228 Initializer, Name, nullptr, 229 GlobalValue::NotThreadLocal, 0, true); 230 IP->setVisibility(GlobalValue::HiddenVisibility); 231 return IP; 232 } 233 234 void makeStub(Function &F, Value &ImplPointer) { 235 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 236 assert(F.getParent() && "Function isn't in a module."); 237 Module &M = *F.getParent(); 238 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 239 IRBuilder<> Builder(EntryBlock); 240 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 241 std::vector<Value*> CallArgs; 242 for (auto &A : F.args()) 243 CallArgs.push_back(&A); 244 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 245 Call->setTailCall(); 246 Call->setAttributes(F.getAttributes()); 247 if (F.getReturnType()->isVoidTy()) 248 Builder.CreateRetVoid(); 249 else 250 Builder.CreateRet(Call); 251 } 252 253 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 254 std::vector<GlobalValue *> PromotedGlobals; 255 256 for (auto &GV : M.global_values()) { 257 bool Promoted = true; 258 259 // Rename if necessary. 260 if (!GV.hasName()) 261 GV.setName("__orc_anon." + Twine(NextId++)); 262 else if (GV.getName().startswith("\01L")) 263 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 264 else if (GV.hasLocalLinkage()) 265 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 266 else 267 Promoted = false; 268 269 if (GV.hasLocalLinkage()) { 270 GV.setLinkage(GlobalValue::ExternalLinkage); 271 GV.setVisibility(GlobalValue::HiddenVisibility); 272 Promoted = true; 273 } 274 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 275 276 if (Promoted) 277 PromotedGlobals.push_back(&GV); 278 } 279 280 return PromotedGlobals; 281 } 282 283 Function* cloneFunctionDecl(Module &Dst, const Function &F, 284 ValueToValueMapTy *VMap) { 285 Function *NewF = 286 Function::Create(cast<FunctionType>(F.getValueType()), 287 F.getLinkage(), F.getName(), &Dst); 288 NewF->copyAttributesFrom(&F); 289 290 if (VMap) { 291 (*VMap)[&F] = NewF; 292 auto NewArgI = NewF->arg_begin(); 293 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 294 ++ArgI, ++NewArgI) 295 (*VMap)[&*ArgI] = &*NewArgI; 296 } 297 298 return NewF; 299 } 300 301 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, 302 ValueMaterializer *Materializer, 303 Function *NewF) { 304 assert(!OrigF.isDeclaration() && "Nothing to move"); 305 if (!NewF) 306 NewF = cast<Function>(VMap[&OrigF]); 307 else 308 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); 309 assert(NewF && "Function mapping missing from VMap."); 310 assert(NewF->getParent() != OrigF.getParent() && 311 "moveFunctionBody should only be used to move bodies between " 312 "modules."); 313 314 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. 315 CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, 316 "", nullptr, nullptr, Materializer); 317 OrigF.deleteBody(); 318 } 319 320 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 321 ValueToValueMapTy *VMap) { 322 GlobalVariable *NewGV = new GlobalVariable( 323 Dst, GV.getValueType(), GV.isConstant(), 324 GV.getLinkage(), nullptr, GV.getName(), nullptr, 325 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 326 NewGV->copyAttributesFrom(&GV); 327 if (VMap) 328 (*VMap)[&GV] = NewGV; 329 return NewGV; 330 } 331 332 void moveGlobalVariableInitializer(GlobalVariable &OrigGV, 333 ValueToValueMapTy &VMap, 334 ValueMaterializer *Materializer, 335 GlobalVariable *NewGV) { 336 assert(OrigGV.hasInitializer() && "Nothing to move"); 337 if (!NewGV) 338 NewGV = cast<GlobalVariable>(VMap[&OrigGV]); 339 else 340 assert(VMap[&OrigGV] == NewGV && 341 "Incorrect global variable mapping in VMap."); 342 assert(NewGV->getParent() != OrigGV.getParent() && 343 "moveGlobalVariableInitializer should only be used to move " 344 "initializers between modules"); 345 346 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, 347 nullptr, Materializer)); 348 } 349 350 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 351 ValueToValueMapTy &VMap) { 352 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 353 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 354 OrigA.getType()->getPointerAddressSpace(), 355 OrigA.getLinkage(), OrigA.getName(), &Dst); 356 NewA->copyAttributesFrom(&OrigA); 357 VMap[&OrigA] = NewA; 358 return NewA; 359 } 360 361 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, 362 ValueToValueMapTy &VMap) { 363 auto *MFs = Src.getModuleFlagsMetadata(); 364 if (!MFs) 365 return; 366 for (auto *MF : MFs->operands()) 367 Dst.addModuleFlag(MapMetadata(MF, VMap)); 368 } 369 370 } // End namespace orc. 371 } // End namespace llvm. 372