1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 10 #include "llvm/ADT/STLExtras.h" 11 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 13 #include "llvm/IR/IRBuilder.h" 14 #include "llvm/MC/MCDisassembler/MCDisassembler.h" 15 #include "llvm/MC/MCInstrAnalysis.h" 16 #include "llvm/Support/Format.h" 17 #include "llvm/TargetParser/Triple.h" 18 #include "llvm/Transforms/Utils/Cloning.h" 19 #include <sstream> 20 21 #define DEBUG_TYPE "orc" 22 23 using namespace llvm; 24 using namespace llvm::orc; 25 26 namespace { 27 28 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { 29 public: 30 using CompileFunction = JITCompileCallbackManager::CompileFunction; 31 32 CompileCallbackMaterializationUnit(SymbolStringPtr Name, 33 CompileFunction Compile) 34 : MaterializationUnit(Interface( 35 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)), 36 Name(std::move(Name)), Compile(std::move(Compile)) {} 37 38 StringRef getName() const override { return "<Compile Callbacks>"; } 39 40 private: 41 void materialize(std::unique_ptr<MaterializationResponsibility> R) override { 42 SymbolMap Result; 43 Result[Name] = {Compile(), JITSymbolFlags::Exported}; 44 // No dependencies, so these calls cannot fail. 45 cantFail(R->notifyResolved(Result)); 46 cantFail(R->notifyEmitted()); 47 } 48 49 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { 50 llvm_unreachable("Discard should never occur on a LMU?"); 51 } 52 53 SymbolStringPtr Name; 54 CompileFunction Compile; 55 }; 56 57 } // namespace 58 59 namespace llvm { 60 namespace orc { 61 62 TrampolinePool::~TrampolinePool() = default; 63 void IndirectStubsManager::anchor() {} 64 65 Expected<ExecutorAddr> 66 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { 67 if (auto TrampolineAddr = TP->getTrampoline()) { 68 auto CallbackName = 69 ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); 70 71 std::lock_guard<std::mutex> Lock(CCMgrMutex); 72 AddrToSymbol[*TrampolineAddr] = CallbackName; 73 cantFail( 74 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>( 75 std::move(CallbackName), std::move(Compile)))); 76 return *TrampolineAddr; 77 } else 78 return TrampolineAddr.takeError(); 79 } 80 81 ExecutorAddr 82 JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) { 83 SymbolStringPtr Name; 84 85 { 86 std::unique_lock<std::mutex> Lock(CCMgrMutex); 87 auto I = AddrToSymbol.find(TrampolineAddr); 88 89 // If this address is not associated with a compile callback then report an 90 // error to the execution session and return ErrorHandlerAddress to the 91 // callee. 92 if (I == AddrToSymbol.end()) { 93 Lock.unlock(); 94 ES.reportError( 95 make_error<StringError>("No compile callback for trampoline at " + 96 formatv("{0:x}", TrampolineAddr), 97 inconvertibleErrorCode())); 98 return ErrorHandlerAddress; 99 } else 100 Name = I->second; 101 } 102 103 if (auto Sym = 104 ES.lookup(makeJITDylibSearchOrder( 105 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), 106 Name)) 107 return Sym->getAddress(); 108 else { 109 llvm::dbgs() << "Didn't find callback.\n"; 110 // If anything goes wrong materializing Sym then report it to the session 111 // and return the ErrorHandlerAddress; 112 ES.reportError(Sym.takeError()); 113 return ErrorHandlerAddress; 114 } 115 } 116 117 Expected<std::unique_ptr<JITCompileCallbackManager>> 118 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 119 ExecutorAddr ErrorHandlerAddress) { 120 switch (T.getArch()) { 121 default: 122 return make_error<StringError>( 123 std::string("No callback manager available for ") + T.str(), 124 inconvertibleErrorCode()); 125 case Triple::aarch64: 126 case Triple::aarch64_32: { 127 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; 128 return CCMgrT::Create(ES, ErrorHandlerAddress); 129 } 130 131 case Triple::x86: { 132 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; 133 return CCMgrT::Create(ES, ErrorHandlerAddress); 134 } 135 136 case Triple::loongarch64: { 137 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT; 138 return CCMgrT::Create(ES, ErrorHandlerAddress); 139 } 140 141 case Triple::mips: { 142 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; 143 return CCMgrT::Create(ES, ErrorHandlerAddress); 144 } 145 case Triple::mipsel: { 146 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; 147 return CCMgrT::Create(ES, ErrorHandlerAddress); 148 } 149 150 case Triple::mips64: 151 case Triple::mips64el: { 152 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; 153 return CCMgrT::Create(ES, ErrorHandlerAddress); 154 } 155 156 case Triple::riscv64: { 157 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT; 158 return CCMgrT::Create(ES, ErrorHandlerAddress); 159 } 160 161 case Triple::x86_64: { 162 if (T.getOS() == Triple::OSType::Win32) { 163 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; 164 return CCMgrT::Create(ES, ErrorHandlerAddress); 165 } else { 166 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; 167 return CCMgrT::Create(ES, ErrorHandlerAddress); 168 } 169 } 170 171 } 172 } 173 174 std::function<std::unique_ptr<IndirectStubsManager>()> 175 createLocalIndirectStubsManagerBuilder(const Triple &T) { 176 switch (T.getArch()) { 177 default: 178 return [](){ 179 return std::make_unique< 180 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); 181 }; 182 183 case Triple::aarch64: 184 case Triple::aarch64_32: 185 return [](){ 186 return std::make_unique< 187 orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); 188 }; 189 190 case Triple::x86: 191 return [](){ 192 return std::make_unique< 193 orc::LocalIndirectStubsManager<orc::OrcI386>>(); 194 }; 195 196 case Triple::loongarch64: 197 return []() { 198 return std::make_unique< 199 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>(); 200 }; 201 202 case Triple::mips: 203 return [](){ 204 return std::make_unique< 205 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); 206 }; 207 208 case Triple::mipsel: 209 return [](){ 210 return std::make_unique< 211 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); 212 }; 213 214 case Triple::mips64: 215 case Triple::mips64el: 216 return [](){ 217 return std::make_unique< 218 orc::LocalIndirectStubsManager<orc::OrcMips64>>(); 219 }; 220 221 case Triple::riscv64: 222 return []() { 223 return std::make_unique< 224 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>(); 225 }; 226 227 case Triple::x86_64: 228 if (T.getOS() == Triple::OSType::Win32) { 229 return [](){ 230 return std::make_unique< 231 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); 232 }; 233 } else { 234 return [](){ 235 return std::make_unique< 236 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); 237 }; 238 } 239 240 } 241 } 242 243 Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { 244 Constant *AddrIntVal = 245 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); 246 Constant *AddrPtrVal = 247 ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0)); 248 return AddrPtrVal; 249 } 250 251 GlobalVariable* createImplPointer(PointerType &PT, Module &M, 252 const Twine &Name, Constant *Initializer) { 253 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, 254 Initializer, Name, nullptr, 255 GlobalValue::NotThreadLocal, 0, true); 256 IP->setVisibility(GlobalValue::HiddenVisibility); 257 return IP; 258 } 259 260 void makeStub(Function &F, Value &ImplPointer) { 261 assert(F.isDeclaration() && "Can't turn a definition into a stub."); 262 assert(F.getParent() && "Function isn't in a module."); 263 Module &M = *F.getParent(); 264 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); 265 IRBuilder<> Builder(EntryBlock); 266 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); 267 std::vector<Value*> CallArgs; 268 for (auto &A : F.args()) 269 CallArgs.push_back(&A); 270 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); 271 Call->setTailCall(); 272 Call->setAttributes(F.getAttributes()); 273 if (F.getReturnType()->isVoidTy()) 274 Builder.CreateRetVoid(); 275 else 276 Builder.CreateRet(Call); 277 } 278 279 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { 280 std::vector<GlobalValue *> PromotedGlobals; 281 282 for (auto &GV : M.global_values()) { 283 bool Promoted = true; 284 285 // Rename if necessary. 286 if (!GV.hasName()) 287 GV.setName("__orc_anon." + Twine(NextId++)); 288 else if (GV.getName().starts_with("\01L")) 289 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); 290 else if (GV.hasLocalLinkage()) 291 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); 292 else 293 Promoted = false; 294 295 if (GV.hasLocalLinkage()) { 296 GV.setLinkage(GlobalValue::ExternalLinkage); 297 GV.setVisibility(GlobalValue::HiddenVisibility); 298 Promoted = true; 299 } 300 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); 301 302 if (Promoted) 303 PromotedGlobals.push_back(&GV); 304 } 305 306 return PromotedGlobals; 307 } 308 309 Function* cloneFunctionDecl(Module &Dst, const Function &F, 310 ValueToValueMapTy *VMap) { 311 Function *NewF = 312 Function::Create(cast<FunctionType>(F.getValueType()), 313 F.getLinkage(), F.getName(), &Dst); 314 NewF->copyAttributesFrom(&F); 315 316 if (VMap) { 317 (*VMap)[&F] = NewF; 318 auto NewArgI = NewF->arg_begin(); 319 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; 320 ++ArgI, ++NewArgI) 321 (*VMap)[&*ArgI] = &*NewArgI; 322 } 323 324 return NewF; 325 } 326 327 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 328 ValueToValueMapTy *VMap) { 329 GlobalVariable *NewGV = new GlobalVariable( 330 Dst, GV.getValueType(), GV.isConstant(), 331 GV.getLinkage(), nullptr, GV.getName(), nullptr, 332 GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); 333 NewGV->copyAttributesFrom(&GV); 334 if (VMap) 335 (*VMap)[&GV] = NewGV; 336 return NewGV; 337 } 338 339 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 340 ValueToValueMapTy &VMap) { 341 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); 342 auto *NewA = GlobalAlias::create(OrigA.getValueType(), 343 OrigA.getType()->getPointerAddressSpace(), 344 OrigA.getLinkage(), OrigA.getName(), &Dst); 345 NewA->copyAttributesFrom(&OrigA); 346 VMap[&OrigA] = NewA; 347 return NewA; 348 } 349 350 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, 351 jitlink::LinkGraph &G, 352 MCDisassembler &Disassembler, 353 MCInstrAnalysis &MIA) { 354 // AArch64 appears to already come with the necessary relocations. Among other 355 // architectures, only x86_64 is currently implemented here. 356 if (G.getTargetTriple().getArch() != Triple::x86_64) 357 return Error::success(); 358 359 raw_null_ostream CommentStream; 360 auto &STI = Disassembler.getSubtargetInfo(); 361 362 // Determine the function bounds 363 auto &B = Sym.getBlock(); 364 assert(!B.isZeroFill() && "expected content block"); 365 auto SymAddress = Sym.getAddress(); 366 auto SymStartInBlock = 367 (const uint8_t *)B.getContent().data() + Sym.getOffset(); 368 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); 369 auto Content = ArrayRef(SymStartInBlock, SymSize); 370 371 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n"); 372 373 SmallDenseSet<uintptr_t, 8> ExistingRelocations; 374 for (auto &E : B.edges()) { 375 if (E.isRelocation()) 376 ExistingRelocations.insert(E.getOffset()); 377 } 378 379 size_t I = 0; 380 while (I < Content.size()) { 381 MCInst Instr; 382 uint64_t InstrSize = 0; 383 uint64_t InstrStart = SymAddress.getValue() + I; 384 auto DecodeStatus = Disassembler.getInstruction( 385 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream); 386 if (DecodeStatus != MCDisassembler::Success) { 387 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address " 388 << InstrStart); 389 return make_error<StringError>( 390 formatv("failed to disassemble at address {0:x16}", InstrStart), 391 inconvertibleErrorCode()); 392 } 393 // Advance to the next instruction. 394 I += InstrSize; 395 396 // Check for a PC-relative address equal to the symbol itself. 397 auto PCRelAddr = 398 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize); 399 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue()) 400 continue; 401 402 auto RelocOffInInstr = 403 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize); 404 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) { 405 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at " 406 << InstrStart); 407 continue; 408 } 409 410 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr - 411 SymAddress + Sym.getOffset(); 412 if (ExistingRelocations.contains(RelocOffInBlock)) 413 continue; 414 415 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart); 416 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4); 417 } 418 return Error::success(); 419 } 420 421 } // End namespace orc. 422 } // End namespace llvm. 423