1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ADT/STLExtras.h" 11 #include "llvm/ADT/Triple.h" 12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 13 #include "llvm/IR/IRBuilder.h" 14 #include "llvm/Support/Format.h" 15 #include "llvm/Transforms/Utils/Cloning.h" 16 #include <sstream> 17 18 using namespace llvm; 19 using namespace llvm::orc; 20 21 namespace { 22 23 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 24 public: 25 using CompileFunction = JITCompileCallbackManager::CompileFunction; 26 27 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 28 CompileFunction Compile, VModuleKey K) 29 : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), 30 nullptr, std::move(K)), 31 Name(std::move(Name)), Compile(std::move(Compile)) {} 32 33 StringRef getName() const override { return "<Compile Callbacks>"; } 34 35 private: 36 void materialize(MaterializationResponsibility R) override { 37 SymbolMap Result; 38 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); 39 // No dependencies, so these calls cannot fail. 40 cantFail(R.notifyResolved(Result)); 41 cantFail(R.notifyEmitted()); 42 } 43 44 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 45 llvm_unreachable("Discard should never occur on a LMU?"); 46 } 47 48 SymbolStringPtr Name; 49 CompileFunction Compile; 50 }; 51 52 } // namespace 53 54 namespace llvm { 55 namespace orc { 56 57 void IndirectStubsManager::anchor() {} 58 void TrampolinePool::anchor() {} 59 60 Expected<JITTargetAddress> 61 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 62 if (auto TrampolineAddr = TP->getTrampoline()) { 63 auto CallbackName = 64 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 65 66 std::lock_guard<std::mutex> Lock(CCMgrMutex); 67 AddrToSymbol[*TrampolineAddr] = CallbackName; 68 cantFail(CallbacksJD.define( 69 std::make_unique<CompileCallbackMaterializationUnit>( 70 std::move(CallbackName), std::move(Compile), 71 ES.allocateVModule()))); 72 return *TrampolineAddr; 73 } else 74 return TrampolineAddr.takeError(); 75 } 76 77 JITTargetAddress JITCompileCallbackManager::executeCompileCallback( 78 JITTargetAddress TrampolineAddr) { 79 SymbolStringPtr Name; 80 81 { 82 std::unique_lock<std::mutex> Lock(CCMgrMutex); 83 auto I = AddrToSymbol.find(TrampolineAddr); 84 85 // If this address is not associated with a compile callback then report an 86 // error to the execution session and return ErrorHandlerAddress to the 87 // callee. 88 if (I == AddrToSymbol.end()) { 89 Lock.unlock(); 90 std::string ErrMsg; 91 { 92 raw_string_ostream ErrMsgStream(ErrMsg); 93 ErrMsgStream << "No compile callback for trampoline at " 94 << format("0x%016" PRIx64, TrampolineAddr); 95 } 96 ES.reportError( 97 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); 98 return ErrorHandlerAddress; 99 } else 100 Name = I->second; 101 } 102 103 if (auto Sym = 104 ES.lookup(makeJITDylibSearchOrder( 105 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), 106 Name)) 107 return Sym->getAddress(); 108 else { 109 llvm::dbgs() << "Didn't find callback.\n"; 110 // If anything goes wrong materializing Sym then report it to the session 111 // and return the ErrorHandlerAddress; 112 ES.reportError(Sym.takeError()); 113 return ErrorHandlerAddress; 114 } 115 } 116 117 Expected<std::unique_ptr<JITCompileCallbackManager>> 118 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 119 JITTargetAddress ErrorHandlerAddress) { 120 switch (T.getArch()) { 121 default: 122 return make_error<StringError>( 123 std::string("No callback manager available for ") + T.str(), 124 inconvertibleErrorCode()); 125 case Triple::aarch64: 126 case Triple::aarch64_32: { 127 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 128 return CCMgrT::Create(ES, ErrorHandlerAddress); 129 } 130 131 case Triple::x86: { 132 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 133 return CCMgrT::Create(ES, ErrorHandlerAddress); 134 } 135 136 case Triple::mips: { 137 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 138 return CCMgrT::Create(ES, ErrorHandlerAddress); 139 } 140 case Triple::mipsel: { 141 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 142 return CCMgrT::Create(ES, ErrorHandlerAddress); 143 } 144 145 case Triple::mips64: 146 case Triple::mips64el: { 147 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 148 return CCMgrT::Create(ES, ErrorHandlerAddress); 149 } 150 151 case Triple::x86_64: { 152 if ( T.getOS() == Triple::OSType::Win32 ) { 153 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 154 return CCMgrT::Create(ES, ErrorHandlerAddress); 155 } else { 156 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 157 return CCMgrT::Create(ES, ErrorHandlerAddress); 158 } 159 } 160 161 } 162 } 163 164 std::function<std::unique_ptr<IndirectStubsManager>()> 165 createLocalIndirectStubsManagerBuilder(const Triple &T) { 166 switch (T.getArch()) { 167 default: 168 return [](){ 169 return std::make_unique< 170 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 171 }; 172 173 case Triple::aarch64: 174 case Triple::aarch64_32: 175 return [](){ 176 return std::make_unique< 177 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 178 }; 179 180 case Triple::x86: 181 return [](){ 182 return std::make_unique< 183 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 184 }; 185 186 case Triple::mips: 187 return [](){ 188 return std::make_unique< 189 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 190 }; 191 192 case Triple::mipsel: 193 return [](){ 194 return std::make_unique< 195 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 196 }; 197 198 case Triple::mips64: 199 case Triple::mips64el: 200 return [](){ 201 return std::make_unique< 202 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 203 }; 204 205 case Triple::x86_64: 206 if (T.getOS() == Triple::OSType::Win32) { 207 return [](){ 208 return std::make_unique< 209 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 210 }; 211 } else { 212 return [](){ 213 return std::make_unique< 214 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 215 }; 216 } 217 218 } 219 } 220 221 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { 222 Constant *AddrIntVal = 223 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); 224 Constant *AddrPtrVal = 225 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, 226 PointerType::get(&FT, 0)); 227 return AddrPtrVal; 228 } 229 230 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 231 const Twine &Name, Constant *Initializer) { 232 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 233 Initializer, Name, nullptr, 234 GlobalValue::NotThreadLocal, 0, true); 235 IP->setVisibility(GlobalValue::HiddenVisibility); 236 return IP; 237 } 238 239 void makeStub(Function &F, Value &ImplPointer) { 240 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 241 assert(F.getParent() && "Function isn't in a module."); 242 Module &M = *F.getParent(); 243 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 244 IRBuilder<> Builder(EntryBlock); 245 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 246 std::vector<Value*> CallArgs; 247 for (auto &A : F.args()) 248 CallArgs.push_back(&A); 249 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 250 Call->setTailCall(); 251 Call->setAttributes(F.getAttributes()); 252 if (F.getReturnType()->isVoidTy()) 253 Builder.CreateRetVoid(); 254 else 255 Builder.CreateRet(Call); 256 } 257 258 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 259 std::vector<GlobalValue *> PromotedGlobals; 260 261 for (auto &GV : M.global_values()) { 262 bool Promoted = true; 263 264 // Rename if necessary. 265 if (!GV.hasName()) 266 GV.setName("__orc_anon." + Twine(NextId++)); 267 else if (GV.getName().startswith("\01L")) 268 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 269 else if (GV.hasLocalLinkage()) 270 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 271 else 272 Promoted = false; 273 274 if (GV.hasLocalLinkage()) { 275 GV.setLinkage(GlobalValue::ExternalLinkage); 276 GV.setVisibility(GlobalValue::HiddenVisibility); 277 Promoted = true; 278 } 279 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 280 281 if (Promoted) 282 PromotedGlobals.push_back(&GV); 283 } 284 285 return PromotedGlobals; 286 } 287 288 Function* cloneFunctionDecl(Module &Dst, const Function &F, 289 ValueToValueMapTy *VMap) { 290 Function *NewF = 291 Function::Create(cast<FunctionType>(F.getValueType()), 292 F.getLinkage(), F.getName(), &Dst); 293 NewF->copyAttributesFrom(&F); 294 295 if (VMap) { 296 (*VMap)[&F] = NewF; 297 auto NewArgI = NewF->arg_begin(); 298 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 299 ++ArgI, ++NewArgI) 300 (*VMap)[&*ArgI] = &*NewArgI; 301 } 302 303 return NewF; 304 } 305 306 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, 307 ValueMaterializer *Materializer, 308 Function *NewF) { 309 assert(!OrigF.isDeclaration() && "Nothing to move"); 310 if (!NewF) 311 NewF = cast<Function>(VMap[&OrigF]); 312 else 313 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); 314 assert(NewF && "Function mapping missing from VMap."); 315 assert(NewF->getParent() != OrigF.getParent() && 316 "moveFunctionBody should only be used to move bodies between " 317 "modules."); 318 319 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. 320 CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, 321 "", nullptr, nullptr, Materializer); 322 OrigF.deleteBody(); 323 } 324 325 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 326 ValueToValueMapTy *VMap) { 327 GlobalVariable *NewGV = new GlobalVariable( 328 Dst, GV.getValueType(), GV.isConstant(), 329 GV.getLinkage(), nullptr, GV.getName(), nullptr, 330 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 331 NewGV->copyAttributesFrom(&GV); 332 if (VMap) 333 (*VMap)[&GV] = NewGV; 334 return NewGV; 335 } 336 337 void moveGlobalVariableInitializer(GlobalVariable &OrigGV, 338 ValueToValueMapTy &VMap, 339 ValueMaterializer *Materializer, 340 GlobalVariable *NewGV) { 341 assert(OrigGV.hasInitializer() && "Nothing to move"); 342 if (!NewGV) 343 NewGV = cast<GlobalVariable>(VMap[&OrigGV]); 344 else 345 assert(VMap[&OrigGV] == NewGV && 346 "Incorrect global variable mapping in VMap."); 347 assert(NewGV->getParent() != OrigGV.getParent() && 348 "moveGlobalVariableInitializer should only be used to move " 349 "initializers between modules"); 350 351 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, 352 nullptr, Materializer)); 353 } 354 355 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 356 ValueToValueMapTy &VMap) { 357 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 358 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 359 OrigA.getType()->getPointerAddressSpace(), 360 OrigA.getLinkage(), OrigA.getName(), &Dst); 361 NewA->copyAttributesFrom(&OrigA); 362 VMap[&OrigA] = NewA; 363 return NewA; 364 } 365 366 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, 367 ValueToValueMapTy &VMap) { 368 auto *MFs = Src.getModuleFlagsMetadata(); 369 if (!MFs) 370 return; 371 for (auto *MF : MFs->operands()) 372 Dst.addModuleFlag(MapMetadata(MF, VMap)); 373 } 374 375 } // End namespace orc. 376 } // End namespace llvm. 377