1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ADT/STLExtras.h" 11 #include "llvm/ADT/Triple.h" 12 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 13 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 14 #include "llvm/IR/IRBuilder.h" 15 #include "llvm/MC/MCDisassembler/MCDisassembler.h" 16 #include "llvm/MC/MCInstrAnalysis.h" 17 #include "llvm/Support/Format.h" 18 #include "llvm/Transforms/Utils/Cloning.h" 19 #include <sstream> 20 21 #define DEBUG_TYPE "orc" 22 23 using namespace llvm; 24 using namespace llvm::orc; 25 26 namespace { 27 28 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 29 public: 30 using CompileFunction = JITCompileCallbackManager::CompileFunction; 31 32 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 33 CompileFunction Compile) 34 : MaterializationUnit(Interface( 35 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)), 36 Name(std::move(Name)), Compile(std::move(Compile)) {} 37 38 StringRef getName() const override { return "<Compile Callbacks>"; } 39 40 private: 41 void materialize(std::unique_ptr<MaterializationResponsibility> R) override { 42 SymbolMap Result; 43 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); 44 // No dependencies, so these calls cannot fail. 45 cantFail(R->notifyResolved(Result)); 46 cantFail(R->notifyEmitted()); 47 } 48 49 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 50 llvm_unreachable("Discard should never occur on a LMU?"); 51 } 52 53 SymbolStringPtr Name; 54 CompileFunction Compile; 55 }; 56 57 } // namespace 58 59 namespace llvm { 60 namespace orc { 61 62 TrampolinePool::~TrampolinePool() = default; 63 void IndirectStubsManager::anchor() {} 64 65 Expected<JITTargetAddress> 66 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 67 if (auto TrampolineAddr = TP->getTrampoline()) { 68 auto CallbackName = 69 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 70 71 std::lock_guard<std::mutex> Lock(CCMgrMutex); 72 AddrToSymbol[*TrampolineAddr] = CallbackName; 73 cantFail( 74 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>( 75 std::move(CallbackName), std::move(Compile)))); 76 return *TrampolineAddr; 77 } else 78 return TrampolineAddr.takeError(); 79 } 80 81 JITTargetAddress JITCompileCallbackManager::executeCompileCallback( 82 JITTargetAddress TrampolineAddr) { 83 SymbolStringPtr Name; 84 85 { 86 std::unique_lock<std::mutex> Lock(CCMgrMutex); 87 auto I = AddrToSymbol.find(TrampolineAddr); 88 89 // If this address is not associated with a compile callback then report an 90 // error to the execution session and return ErrorHandlerAddress to the 91 // callee. 92 if (I == AddrToSymbol.end()) { 93 Lock.unlock(); 94 std::string ErrMsg; 95 { 96 raw_string_ostream ErrMsgStream(ErrMsg); 97 ErrMsgStream << "No compile callback for trampoline at " 98 << format("0x%016" PRIx64, TrampolineAddr); 99 } 100 ES.reportError( 101 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); 102 return ErrorHandlerAddress; 103 } else 104 Name = I->second; 105 } 106 107 if (auto Sym = 108 ES.lookup(makeJITDylibSearchOrder( 109 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), 110 Name)) 111 return Sym->getAddress(); 112 else { 113 llvm::dbgs() << "Didn't find callback.\n"; 114 // If anything goes wrong materializing Sym then report it to the session 115 // and return the ErrorHandlerAddress; 116 ES.reportError(Sym.takeError()); 117 return ErrorHandlerAddress; 118 } 119 } 120 121 Expected<std::unique_ptr<JITCompileCallbackManager>> 122 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 123 JITTargetAddress ErrorHandlerAddress) { 124 switch (T.getArch()) { 125 default: 126 return make_error<StringError>( 127 std::string("No callback manager available for ") + T.str(), 128 inconvertibleErrorCode()); 129 case Triple::aarch64: 130 case Triple::aarch64_32: { 131 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 132 return CCMgrT::Create(ES, ErrorHandlerAddress); 133 } 134 135 case Triple::x86: { 136 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 137 return CCMgrT::Create(ES, ErrorHandlerAddress); 138 } 139 140 case Triple::loongarch64: { 141 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT; 142 return CCMgrT::Create(ES, ErrorHandlerAddress); 143 } 144 145 case Triple::mips: { 146 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 147 return CCMgrT::Create(ES, ErrorHandlerAddress); 148 } 149 case Triple::mipsel: { 150 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 151 return CCMgrT::Create(ES, ErrorHandlerAddress); 152 } 153 154 case Triple::mips64: 155 case Triple::mips64el: { 156 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 157 return CCMgrT::Create(ES, ErrorHandlerAddress); 158 } 159 160 case Triple::riscv64: { 161 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT; 162 return CCMgrT::Create(ES, ErrorHandlerAddress); 163 } 164 165 case Triple::x86_64: { 166 if (T.getOS() == Triple::OSType::Win32) { 167 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 168 return CCMgrT::Create(ES, ErrorHandlerAddress); 169 } else { 170 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 171 return CCMgrT::Create(ES, ErrorHandlerAddress); 172 } 173 } 174 175 } 176 } 177 178 std::function<std::unique_ptr<IndirectStubsManager>()> 179 createLocalIndirectStubsManagerBuilder(const Triple &T) { 180 switch (T.getArch()) { 181 default: 182 return [](){ 183 return std::make_unique< 184 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 185 }; 186 187 case Triple::aarch64: 188 case Triple::aarch64_32: 189 return [](){ 190 return std::make_unique< 191 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 192 }; 193 194 case Triple::x86: 195 return [](){ 196 return std::make_unique< 197 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 198 }; 199 200 case Triple::loongarch64: 201 return []() { 202 return std::make_unique< 203 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>(); 204 }; 205 206 case Triple::mips: 207 return [](){ 208 return std::make_unique< 209 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 210 }; 211 212 case Triple::mipsel: 213 return [](){ 214 return std::make_unique< 215 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 216 }; 217 218 case Triple::mips64: 219 case Triple::mips64el: 220 return [](){ 221 return std::make_unique< 222 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 223 }; 224 225 case Triple::riscv64: 226 return []() { 227 return std::make_unique< 228 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>(); 229 }; 230 231 case Triple::x86_64: 232 if (T.getOS() == Triple::OSType::Win32) { 233 return [](){ 234 return std::make_unique< 235 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 236 }; 237 } else { 238 return [](){ 239 return std::make_unique< 240 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 241 }; 242 } 243 244 } 245 } 246 247 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { 248 Constant *AddrIntVal = 249 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); 250 Constant *AddrPtrVal = 251 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, 252 PointerType::get(&FT, 0)); 253 return AddrPtrVal; 254 } 255 256 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 257 const Twine &Name, Constant *Initializer) { 258 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 259 Initializer, Name, nullptr, 260 GlobalValue::NotThreadLocal, 0, true); 261 IP->setVisibility(GlobalValue::HiddenVisibility); 262 return IP; 263 } 264 265 void makeStub(Function &F, Value &ImplPointer) { 266 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 267 assert(F.getParent() && "Function isn't in a module."); 268 Module &M = *F.getParent(); 269 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 270 IRBuilder<> Builder(EntryBlock); 271 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 272 std::vector<Value*> CallArgs; 273 for (auto &A : F.args()) 274 CallArgs.push_back(&A); 275 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 276 Call->setTailCall(); 277 Call->setAttributes(F.getAttributes()); 278 if (F.getReturnType()->isVoidTy()) 279 Builder.CreateRetVoid(); 280 else 281 Builder.CreateRet(Call); 282 } 283 284 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 285 std::vector<GlobalValue *> PromotedGlobals; 286 287 for (auto &GV : M.global_values()) { 288 bool Promoted = true; 289 290 // Rename if necessary. 291 if (!GV.hasName()) 292 GV.setName("__orc_anon." + Twine(NextId++)); 293 else if (GV.getName().startswith("\01L")) 294 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 295 else if (GV.hasLocalLinkage()) 296 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 297 else 298 Promoted = false; 299 300 if (GV.hasLocalLinkage()) { 301 GV.setLinkage(GlobalValue::ExternalLinkage); 302 GV.setVisibility(GlobalValue::HiddenVisibility); 303 Promoted = true; 304 } 305 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 306 307 if (Promoted) 308 PromotedGlobals.push_back(&GV); 309 } 310 311 return PromotedGlobals; 312 } 313 314 Function* cloneFunctionDecl(Module &Dst, const Function &F, 315 ValueToValueMapTy *VMap) { 316 Function *NewF = 317 Function::Create(cast<FunctionType>(F.getValueType()), 318 F.getLinkage(), F.getName(), &Dst); 319 NewF->copyAttributesFrom(&F); 320 321 if (VMap) { 322 (*VMap)[&F] = NewF; 323 auto NewArgI = NewF->arg_begin(); 324 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 325 ++ArgI, ++NewArgI) 326 (*VMap)[&*ArgI] = &*NewArgI; 327 } 328 329 return NewF; 330 } 331 332 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, 333 ValueMaterializer *Materializer, 334 Function *NewF) { 335 assert(!OrigF.isDeclaration() && "Nothing to move"); 336 if (!NewF) 337 NewF = cast<Function>(VMap[&OrigF]); 338 else 339 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); 340 assert(NewF && "Function mapping missing from VMap."); 341 assert(NewF->getParent() != OrigF.getParent() && 342 "moveFunctionBody should only be used to move bodies between " 343 "modules."); 344 345 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. 346 CloneFunctionInto(NewF, &OrigF, VMap, 347 CloneFunctionChangeType::DifferentModule, Returns, "", 348 nullptr, nullptr, Materializer); 349 OrigF.deleteBody(); 350 } 351 352 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 353 ValueToValueMapTy *VMap) { 354 GlobalVariable *NewGV = new GlobalVariable( 355 Dst, GV.getValueType(), GV.isConstant(), 356 GV.getLinkage(), nullptr, GV.getName(), nullptr, 357 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 358 NewGV->copyAttributesFrom(&GV); 359 if (VMap) 360 (*VMap)[&GV] = NewGV; 361 return NewGV; 362 } 363 364 void moveGlobalVariableInitializer(GlobalVariable &OrigGV, 365 ValueToValueMapTy &VMap, 366 ValueMaterializer *Materializer, 367 GlobalVariable *NewGV) { 368 assert(OrigGV.hasInitializer() && "Nothing to move"); 369 if (!NewGV) 370 NewGV = cast<GlobalVariable>(VMap[&OrigGV]); 371 else 372 assert(VMap[&OrigGV] == NewGV && 373 "Incorrect global variable mapping in VMap."); 374 assert(NewGV->getParent() != OrigGV.getParent() && 375 "moveGlobalVariableInitializer should only be used to move " 376 "initializers between modules"); 377 378 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, 379 nullptr, Materializer)); 380 } 381 382 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 383 ValueToValueMapTy &VMap) { 384 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 385 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 386 OrigA.getType()->getPointerAddressSpace(), 387 OrigA.getLinkage(), OrigA.getName(), &Dst); 388 NewA->copyAttributesFrom(&OrigA); 389 VMap[&OrigA] = NewA; 390 return NewA; 391 } 392 393 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, 394 ValueToValueMapTy &VMap) { 395 auto *MFs = Src.getModuleFlagsMetadata(); 396 if (!MFs) 397 return; 398 for (auto *MF : MFs->operands()) 399 Dst.addModuleFlag(MapMetadata(MF, VMap)); 400 } 401 402 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, 403 jitlink::LinkGraph &G, 404 MCDisassembler &Disassembler, 405 MCInstrAnalysis &MIA) { 406 // AArch64 appears to already come with the necessary relocations. Among other 407 // architectures, only x86_64 is currently implemented here. 408 if (G.getTargetTriple().getArch() != Triple::x86_64) 409 return Error::success(); 410 411 raw_null_ostream CommentStream; 412 auto &STI = Disassembler.getSubtargetInfo(); 413 414 // Determine the function bounds 415 auto &B = Sym.getBlock(); 416 assert(!B.isZeroFill() && "expected content block"); 417 auto SymAddress = Sym.getAddress(); 418 auto SymStartInBlock = 419 (const uint8_t *)B.getContent().data() + Sym.getOffset(); 420 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); 421 auto Content = ArrayRef(SymStartInBlock, SymSize); 422 423 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n"); 424 425 SmallDenseSet<uintptr_t, 8> ExistingRelocations; 426 for (auto &E : B.edges()) { 427 if (E.isRelocation()) 428 ExistingRelocations.insert(E.getOffset()); 429 } 430 431 size_t I = 0; 432 while (I < Content.size()) { 433 MCInst Instr; 434 uint64_t InstrSize = 0; 435 uint64_t InstrStart = SymAddress.getValue() + I; 436 auto DecodeStatus = Disassembler.getInstruction( 437 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream); 438 if (DecodeStatus != MCDisassembler::Success) { 439 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address " 440 << InstrStart); 441 return make_error<StringError>( 442 formatv("failed to disassemble at address {0:x16}", InstrStart), 443 inconvertibleErrorCode()); 444 } 445 // Advance to the next instruction. 446 I += InstrSize; 447 448 // Check for a PC-relative address equal to the symbol itself. 449 auto PCRelAddr = 450 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize); 451 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue()) 452 continue; 453 454 auto RelocOffInInstr = 455 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize); 456 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) { 457 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at " 458 << InstrStart); 459 continue; 460 } 461 462 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr - 463 SymAddress + Sym.getOffset(); 464 if (ExistingRelocations.contains(RelocOffInBlock)) 465 continue; 466 467 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart); 468 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4); 469 } 470 return Error::success(); 471 } 472 473 } // End namespace orc. 474 } // End namespace llvm. 475