1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ADT/STLExtras.h" 11 #include "llvm/ADT/Triple.h" 12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 13 #include "llvm/IR/CallSite.h" 14 #include "llvm/IR/IRBuilder.h" 15 #include "llvm/Support/Format.h" 16 #include "llvm/Transforms/Utils/Cloning.h" 17 #include <sstream> 18 19 using namespace llvm; 20 using namespace llvm::orc; 21 22 namespace { 23 24 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 25 public: 26 using CompileFunction = JITCompileCallbackManager::CompileFunction; 27 28 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 29 CompileFunction Compile, VModuleKey K) 30 : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), 31 std::move(K)), 32 Name(std::move(Name)), Compile(std::move(Compile)) {} 33 34 StringRef getName() const override { return "<Compile Callbacks>"; } 35 36 private: 37 void materialize(MaterializationResponsibility R) override { 38 SymbolMap Result; 39 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); 40 // No dependencies, so these calls cannot fail. 41 cantFail(R.notifyResolved(Result)); 42 cantFail(R.notifyEmitted()); 43 } 44 45 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 46 llvm_unreachable("Discard should never occur on a LMU?"); 47 } 48 49 SymbolStringPtr Name; 50 CompileFunction Compile; 51 }; 52 53 } // namespace 54 55 namespace llvm { 56 namespace orc { 57 58 void IndirectStubsManager::anchor() {} 59 void TrampolinePool::anchor() {} 60 61 Expected<JITTargetAddress> 62 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 63 if (auto TrampolineAddr = TP->getTrampoline()) { 64 auto CallbackName = 65 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 66 67 std::lock_guard<std::mutex> Lock(CCMgrMutex); 68 AddrToSymbol[*TrampolineAddr] = CallbackName; 69 cantFail(CallbacksJD.define( 70 std::make_unique<CompileCallbackMaterializationUnit>( 71 std::move(CallbackName), std::move(Compile), 72 ES.allocateVModule()))); 73 return *TrampolineAddr; 74 } else 75 return TrampolineAddr.takeError(); 76 } 77 78 JITTargetAddress JITCompileCallbackManager::executeCompileCallback( 79 JITTargetAddress TrampolineAddr) { 80 SymbolStringPtr Name; 81 82 { 83 std::unique_lock<std::mutex> Lock(CCMgrMutex); 84 auto I = AddrToSymbol.find(TrampolineAddr); 85 86 // If this address is not associated with a compile callback then report an 87 // error to the execution session and return ErrorHandlerAddress to the 88 // callee. 89 if (I == AddrToSymbol.end()) { 90 Lock.unlock(); 91 std::string ErrMsg; 92 { 93 raw_string_ostream ErrMsgStream(ErrMsg); 94 ErrMsgStream << "No compile callback for trampoline at " 95 << format("0x%016" PRIx64, TrampolineAddr); 96 } 97 ES.reportError( 98 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); 99 return ErrorHandlerAddress; 100 } else 101 Name = I->second; 102 } 103 104 if (auto Sym = 105 ES.lookup(makeJITDylibSearchOrder( 106 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), 107 Name)) 108 return Sym->getAddress(); 109 else { 110 llvm::dbgs() << "Didn't find callback.\n"; 111 // If anything goes wrong materializing Sym then report it to the session 112 // and return the ErrorHandlerAddress; 113 ES.reportError(Sym.takeError()); 114 return ErrorHandlerAddress; 115 } 116 } 117 118 Expected<std::unique_ptr<JITCompileCallbackManager>> 119 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 120 JITTargetAddress ErrorHandlerAddress) { 121 switch (T.getArch()) { 122 default: 123 return make_error<StringError>( 124 std::string("No callback manager available for ") + T.str(), 125 inconvertibleErrorCode()); 126 case Triple::aarch64: 127 case Triple::aarch64_32: { 128 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 129 return CCMgrT::Create(ES, ErrorHandlerAddress); 130 } 131 132 case Triple::x86: { 133 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 134 return CCMgrT::Create(ES, ErrorHandlerAddress); 135 } 136 137 case Triple::mips: { 138 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 139 return CCMgrT::Create(ES, ErrorHandlerAddress); 140 } 141 case Triple::mipsel: { 142 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 143 return CCMgrT::Create(ES, ErrorHandlerAddress); 144 } 145 146 case Triple::mips64: 147 case Triple::mips64el: { 148 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 149 return CCMgrT::Create(ES, ErrorHandlerAddress); 150 } 151 152 case Triple::x86_64: { 153 if ( T.getOS() == Triple::OSType::Win32 ) { 154 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 155 return CCMgrT::Create(ES, ErrorHandlerAddress); 156 } else { 157 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 158 return CCMgrT::Create(ES, ErrorHandlerAddress); 159 } 160 } 161 162 } 163 } 164 165 std::function<std::unique_ptr<IndirectStubsManager>()> 166 createLocalIndirectStubsManagerBuilder(const Triple &T) { 167 switch (T.getArch()) { 168 default: 169 return [](){ 170 return std::make_unique< 171 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 172 }; 173 174 case Triple::aarch64: 175 case Triple::aarch64_32: 176 return [](){ 177 return std::make_unique< 178 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 179 }; 180 181 case Triple::x86: 182 return [](){ 183 return std::make_unique< 184 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 185 }; 186 187 case Triple::mips: 188 return [](){ 189 return std::make_unique< 190 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 191 }; 192 193 case Triple::mipsel: 194 return [](){ 195 return std::make_unique< 196 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 197 }; 198 199 case Triple::mips64: 200 case Triple::mips64el: 201 return [](){ 202 return std::make_unique< 203 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 204 }; 205 206 case Triple::x86_64: 207 if (T.getOS() == Triple::OSType::Win32) { 208 return [](){ 209 return std::make_unique< 210 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 211 }; 212 } else { 213 return [](){ 214 return std::make_unique< 215 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 216 }; 217 } 218 219 } 220 } 221 222 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { 223 Constant *AddrIntVal = 224 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); 225 Constant *AddrPtrVal = 226 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, 227 PointerType::get(&FT, 0)); 228 return AddrPtrVal; 229 } 230 231 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 232 const Twine &Name, Constant *Initializer) { 233 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 234 Initializer, Name, nullptr, 235 GlobalValue::NotThreadLocal, 0, true); 236 IP->setVisibility(GlobalValue::HiddenVisibility); 237 return IP; 238 } 239 240 void makeStub(Function &F, Value &ImplPointer) { 241 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 242 assert(F.getParent() && "Function isn't in a module."); 243 Module &M = *F.getParent(); 244 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 245 IRBuilder<> Builder(EntryBlock); 246 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 247 std::vector<Value*> CallArgs; 248 for (auto &A : F.args()) 249 CallArgs.push_back(&A); 250 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 251 Call->setTailCall(); 252 Call->setAttributes(F.getAttributes()); 253 if (F.getReturnType()->isVoidTy()) 254 Builder.CreateRetVoid(); 255 else 256 Builder.CreateRet(Call); 257 } 258 259 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 260 std::vector<GlobalValue *> PromotedGlobals; 261 262 for (auto &GV : M.global_values()) { 263 bool Promoted = true; 264 265 // Rename if necessary. 266 if (!GV.hasName()) 267 GV.setName("__orc_anon." + Twine(NextId++)); 268 else if (GV.getName().startswith("\01L")) 269 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 270 else if (GV.hasLocalLinkage()) 271 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 272 else 273 Promoted = false; 274 275 if (GV.hasLocalLinkage()) { 276 GV.setLinkage(GlobalValue::ExternalLinkage); 277 GV.setVisibility(GlobalValue::HiddenVisibility); 278 Promoted = true; 279 } 280 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 281 282 if (Promoted) 283 PromotedGlobals.push_back(&GV); 284 } 285 286 return PromotedGlobals; 287 } 288 289 Function* cloneFunctionDecl(Module &Dst, const Function &F, 290 ValueToValueMapTy *VMap) { 291 Function *NewF = 292 Function::Create(cast<FunctionType>(F.getValueType()), 293 F.getLinkage(), F.getName(), &Dst); 294 NewF->copyAttributesFrom(&F); 295 296 if (VMap) { 297 (*VMap)[&F] = NewF; 298 auto NewArgI = NewF->arg_begin(); 299 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 300 ++ArgI, ++NewArgI) 301 (*VMap)[&*ArgI] = &*NewArgI; 302 } 303 304 return NewF; 305 } 306 307 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, 308 ValueMaterializer *Materializer, 309 Function *NewF) { 310 assert(!OrigF.isDeclaration() && "Nothing to move"); 311 if (!NewF) 312 NewF = cast<Function>(VMap[&OrigF]); 313 else 314 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); 315 assert(NewF && "Function mapping missing from VMap."); 316 assert(NewF->getParent() != OrigF.getParent() && 317 "moveFunctionBody should only be used to move bodies between " 318 "modules."); 319 320 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. 321 CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, 322 "", nullptr, nullptr, Materializer); 323 OrigF.deleteBody(); 324 } 325 326 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 327 ValueToValueMapTy *VMap) { 328 GlobalVariable *NewGV = new GlobalVariable( 329 Dst, GV.getValueType(), GV.isConstant(), 330 GV.getLinkage(), nullptr, GV.getName(), nullptr, 331 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 332 NewGV->copyAttributesFrom(&GV); 333 if (VMap) 334 (*VMap)[&GV] = NewGV; 335 return NewGV; 336 } 337 338 void moveGlobalVariableInitializer(GlobalVariable &OrigGV, 339 ValueToValueMapTy &VMap, 340 ValueMaterializer *Materializer, 341 GlobalVariable *NewGV) { 342 assert(OrigGV.hasInitializer() && "Nothing to move"); 343 if (!NewGV) 344 NewGV = cast<GlobalVariable>(VMap[&OrigGV]); 345 else 346 assert(VMap[&OrigGV] == NewGV && 347 "Incorrect global variable mapping in VMap."); 348 assert(NewGV->getParent() != OrigGV.getParent() && 349 "moveGlobalVariableInitializer should only be used to move " 350 "initializers between modules"); 351 352 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, 353 nullptr, Materializer)); 354 } 355 356 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 357 ValueToValueMapTy &VMap) { 358 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 359 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 360 OrigA.getType()->getPointerAddressSpace(), 361 OrigA.getLinkage(), OrigA.getName(), &Dst); 362 NewA->copyAttributesFrom(&OrigA); 363 VMap[&OrigA] = NewA; 364 return NewA; 365 } 366 367 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, 368 ValueToValueMapTy &VMap) { 369 auto *MFs = Src.getModuleFlagsMetadata(); 370 if (!MFs) 371 return; 372 for (auto *MF : MFs->operands()) 373 Dst.addModuleFlag(MapMetadata(MF, VMap)); 374 } 375 376 } // End namespace orc. 377 } // End namespace llvm. 378