1 //===- StackMaps.cpp ------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/CodeGen/StackMaps.h" 10 #include "llvm/ADT/DenseMapInfo.h" 11 #include "llvm/ADT/STLExtras.h" 12 #include "llvm/ADT/Twine.h" 13 #include "llvm/CodeGen/AsmPrinter.h" 14 #include "llvm/CodeGen/MachineFrameInfo.h" 15 #include "llvm/CodeGen/MachineFunction.h" 16 #include "llvm/CodeGen/MachineInstr.h" 17 #include "llvm/CodeGen/MachineOperand.h" 18 #include "llvm/CodeGen/TargetOpcodes.h" 19 #include "llvm/CodeGen/TargetRegisterInfo.h" 20 #include "llvm/CodeGen/TargetSubtargetInfo.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/MC/MCContext.h" 23 #include "llvm/MC/MCExpr.h" 24 #include "llvm/MC/MCObjectFileInfo.h" 25 #include "llvm/MC/MCRegisterInfo.h" 26 #include "llvm/MC/MCStreamer.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/Debug.h" 29 #include "llvm/Support/ErrorHandling.h" 30 #include "llvm/Support/MathExtras.h" 31 #include "llvm/Support/raw_ostream.h" 32 #include <algorithm> 33 #include <cassert> 34 #include <cstdint> 35 #include <iterator> 36 #include <utility> 37 38 using namespace llvm; 39 40 #define DEBUG_TYPE "stackmaps" 41 42 static cl::opt<int> StackMapVersion( 43 "stackmap-version", cl::init(3), cl::Hidden, 44 cl::desc("Specify the stackmap encoding version (default = 3)")); 45 46 const char *StackMaps::WSMP = "Stack Maps: "; 47 48 static uint64_t getConstMetaVal(const MachineInstr &MI, unsigned Idx) { 49 assert(MI.getOperand(Idx).isImm() && 50 MI.getOperand(Idx).getImm() == StackMaps::ConstantOp); 51 const auto &MO = MI.getOperand(Idx + 1); 52 assert(MO.isImm()); 53 return MO.getImm(); 54 } 55 56 StackMapOpers::StackMapOpers(const MachineInstr *MI) 57 : MI(MI) { 58 assert(getVarIdx() <= MI->getNumOperands() && 59 "invalid stackmap definition"); 60 } 61 62 PatchPointOpers::PatchPointOpers(const MachineInstr *MI) 63 : MI(MI), HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() && 64 !MI->getOperand(0).isImplicit()) { 65 #ifndef NDEBUG 66 unsigned CheckStartIdx = 0, e = MI->getNumOperands(); 67 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() && 68 MI->getOperand(CheckStartIdx).isDef() && 69 !MI->getOperand(CheckStartIdx).isImplicit()) 70 ++CheckStartIdx; 71 72 assert(getMetaIdx() == CheckStartIdx && 73 "Unexpected additional definition in Patchpoint intrinsic."); 74 #endif 75 } 76 77 unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const { 78 if (!StartIdx) 79 StartIdx = getVarIdx(); 80 81 // Find the next scratch register (implicit def and early clobber) 82 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands(); 83 while (ScratchIdx < e && 84 !(MI->getOperand(ScratchIdx).isReg() && 85 MI->getOperand(ScratchIdx).isDef() && 86 MI->getOperand(ScratchIdx).isImplicit() && 87 MI->getOperand(ScratchIdx).isEarlyClobber())) 88 ++ScratchIdx; 89 90 assert(ScratchIdx != e && "No scratch register available"); 91 return ScratchIdx; 92 } 93 94 unsigned StatepointOpers::getNumGcMapEntriesIdx() { 95 // Take index of num of allocas and skip all allocas records. 96 unsigned CurIdx = getNumAllocaIdx(); 97 unsigned NumAllocas = getConstMetaVal(*MI, CurIdx - 1); 98 CurIdx++; 99 while (NumAllocas--) 100 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); 101 return CurIdx + 1; // skip <StackMaps::ConstantOp> 102 } 103 104 unsigned StatepointOpers::getNumAllocaIdx() { 105 // Take index of num of gc ptrs and skip all gc ptr records. 106 unsigned CurIdx = getNumGCPtrIdx(); 107 unsigned NumGCPtrs = getConstMetaVal(*MI, CurIdx - 1); 108 CurIdx++; 109 while (NumGCPtrs--) 110 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); 111 return CurIdx + 1; // skip <StackMaps::ConstantOp> 112 } 113 114 unsigned StatepointOpers::getNumGCPtrIdx() { 115 // Take index of num of deopt args and skip all deopt records. 116 unsigned CurIdx = getNumDeoptArgsIdx(); 117 unsigned NumDeoptArgs = getConstMetaVal(*MI, CurIdx - 1); 118 CurIdx++; 119 while (NumDeoptArgs--) { 120 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx); 121 } 122 return CurIdx + 1; // skip <StackMaps::ConstantOp> 123 } 124 125 int StatepointOpers::getFirstGCPtrIdx() { 126 unsigned NumGCPtrsIdx = getNumGCPtrIdx(); 127 unsigned NumGCPtrs = getConstMetaVal(*MI, NumGCPtrsIdx - 1); 128 if (NumGCPtrs == 0) 129 return -1; 130 ++NumGCPtrsIdx; // skip <num gc ptrs> 131 assert(NumGCPtrsIdx < MI->getNumOperands()); 132 return (int)NumGCPtrsIdx; 133 } 134 135 unsigned StatepointOpers::getGCPointerMap( 136 SmallVectorImpl<std::pair<unsigned, unsigned>> &GCMap) { 137 unsigned CurIdx = getNumGcMapEntriesIdx(); 138 unsigned GCMapSize = getConstMetaVal(*MI, CurIdx - 1); 139 CurIdx++; 140 for (unsigned N = 0; N < GCMapSize; ++N) { 141 unsigned B = MI->getOperand(CurIdx++).getImm(); 142 unsigned D = MI->getOperand(CurIdx++).getImm(); 143 GCMap.push_back(std::make_pair(B, D)); 144 } 145 146 return GCMapSize; 147 } 148 149 StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) { 150 if (StackMapVersion != 3) 151 llvm_unreachable("Unsupported stackmap version!"); 152 } 153 154 unsigned StackMaps::getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx) { 155 assert(CurIdx < MI->getNumOperands() && "Bad meta arg index"); 156 const auto &MO = MI->getOperand(CurIdx); 157 if (MO.isImm()) { 158 switch (MO.getImm()) { 159 default: 160 llvm_unreachable("Unrecognized operand type."); 161 case StackMaps::DirectMemRefOp: 162 CurIdx += 2; 163 break; 164 case StackMaps::IndirectMemRefOp: 165 CurIdx += 3; 166 break; 167 case StackMaps::ConstantOp: 168 ++CurIdx; 169 break; 170 } 171 } 172 ++CurIdx; 173 assert(CurIdx < MI->getNumOperands() && "points past operand list"); 174 return CurIdx; 175 } 176 177 /// Go up the super-register chain until we hit a valid dwarf register number. 178 static unsigned getDwarfRegNum(unsigned Reg, const TargetRegisterInfo *TRI) { 179 int RegNum = TRI->getDwarfRegNum(Reg, false); 180 for (MCSuperRegIterator SR(Reg, TRI); SR.isValid() && RegNum < 0; ++SR) 181 RegNum = TRI->getDwarfRegNum(*SR, false); 182 183 assert(RegNum >= 0 && "Invalid Dwarf register number."); 184 return (unsigned)RegNum; 185 } 186 187 MachineInstr::const_mop_iterator 188 StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, 189 MachineInstr::const_mop_iterator MOE, LocationVec &Locs, 190 LiveOutVec &LiveOuts) const { 191 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo(); 192 if (MOI->isImm()) { 193 switch (MOI->getImm()) { 194 default: 195 llvm_unreachable("Unrecognized operand type."); 196 case StackMaps::DirectMemRefOp: { 197 auto &DL = AP.MF->getDataLayout(); 198 199 unsigned Size = DL.getPointerSizeInBits(); 200 assert((Size % 8) == 0 && "Need pointer size in bytes."); 201 Size /= 8; 202 Register Reg = (++MOI)->getReg(); 203 int64_t Imm = (++MOI)->getImm(); 204 Locs.emplace_back(StackMaps::Location::Direct, Size, 205 getDwarfRegNum(Reg, TRI), Imm); 206 break; 207 } 208 case StackMaps::IndirectMemRefOp: { 209 int64_t Size = (++MOI)->getImm(); 210 assert(Size > 0 && "Need a valid size for indirect memory locations."); 211 Register Reg = (++MOI)->getReg(); 212 int64_t Imm = (++MOI)->getImm(); 213 Locs.emplace_back(StackMaps::Location::Indirect, Size, 214 getDwarfRegNum(Reg, TRI), Imm); 215 break; 216 } 217 case StackMaps::ConstantOp: { 218 ++MOI; 219 assert(MOI->isImm() && "Expected constant operand."); 220 int64_t Imm = MOI->getImm(); 221 Locs.emplace_back(Location::Constant, sizeof(int64_t), 0, Imm); 222 break; 223 } 224 } 225 return ++MOI; 226 } 227 228 // The physical register number will ultimately be encoded as a DWARF regno. 229 // The stack map also records the size of a spill slot that can hold the 230 // register content. (The runtime can track the actual size of the data type 231 // if it needs to.) 232 if (MOI->isReg()) { 233 // Skip implicit registers (this includes our scratch registers) 234 if (MOI->isImplicit()) 235 return ++MOI; 236 237 if (MOI->isUndef()) { 238 // Record `undef` register as constant. Use same value as ISel uses. 239 Locs.emplace_back(Location::Constant, sizeof(int64_t), 0, 0xFEFEFEFE); 240 return ++MOI; 241 } 242 243 assert(Register::isPhysicalRegister(MOI->getReg()) && 244 "Virtreg operands should have been rewritten before now."); 245 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(MOI->getReg()); 246 assert(!MOI->getSubReg() && "Physical subreg still around."); 247 248 unsigned Offset = 0; 249 unsigned DwarfRegNum = getDwarfRegNum(MOI->getReg(), TRI); 250 unsigned LLVMRegNum = *TRI->getLLVMRegNum(DwarfRegNum, false); 251 unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNum, MOI->getReg()); 252 if (SubRegIdx) 253 Offset = TRI->getSubRegIdxOffset(SubRegIdx); 254 255 Locs.emplace_back(Location::Register, TRI->getSpillSize(*RC), 256 DwarfRegNum, Offset); 257 return ++MOI; 258 } 259 260 if (MOI->isRegLiveOut()) 261 LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut()); 262 263 return ++MOI; 264 } 265 266 void StackMaps::print(raw_ostream &OS) { 267 const TargetRegisterInfo *TRI = 268 AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr; 269 OS << WSMP << "callsites:\n"; 270 for (const auto &CSI : CSInfos) { 271 const LocationVec &CSLocs = CSI.Locations; 272 const LiveOutVec &LiveOuts = CSI.LiveOuts; 273 274 OS << WSMP << "callsite " << CSI.ID << "\n"; 275 OS << WSMP << " has " << CSLocs.size() << " locations\n"; 276 277 unsigned Idx = 0; 278 for (const auto &Loc : CSLocs) { 279 OS << WSMP << "\t\tLoc " << Idx << ": "; 280 switch (Loc.Type) { 281 case Location::Unprocessed: 282 OS << "<Unprocessed operand>"; 283 break; 284 case Location::Register: 285 OS << "Register "; 286 if (TRI) 287 OS << printReg(Loc.Reg, TRI); 288 else 289 OS << Loc.Reg; 290 break; 291 case Location::Direct: 292 OS << "Direct "; 293 if (TRI) 294 OS << printReg(Loc.Reg, TRI); 295 else 296 OS << Loc.Reg; 297 if (Loc.Offset) 298 OS << " + " << Loc.Offset; 299 break; 300 case Location::Indirect: 301 OS << "Indirect "; 302 if (TRI) 303 OS << printReg(Loc.Reg, TRI); 304 else 305 OS << Loc.Reg; 306 OS << "+" << Loc.Offset; 307 break; 308 case Location::Constant: 309 OS << "Constant " << Loc.Offset; 310 break; 311 case Location::ConstantIndex: 312 OS << "Constant Index " << Loc.Offset; 313 break; 314 } 315 OS << "\t[encoding: .byte " << Loc.Type << ", .byte 0" 316 << ", .short " << Loc.Size << ", .short " << Loc.Reg << ", .short 0" 317 << ", .int " << Loc.Offset << "]\n"; 318 Idx++; 319 } 320 321 OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n"; 322 323 Idx = 0; 324 for (const auto &LO : LiveOuts) { 325 OS << WSMP << "\t\tLO " << Idx << ": "; 326 if (TRI) 327 OS << printReg(LO.Reg, TRI); 328 else 329 OS << LO.Reg; 330 OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte " 331 << LO.Size << "]\n"; 332 Idx++; 333 } 334 } 335 } 336 337 /// Create a live-out register record for the given register Reg. 338 StackMaps::LiveOutReg 339 StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const { 340 unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI); 341 unsigned Size = TRI->getSpillSize(*TRI->getMinimalPhysRegClass(Reg)); 342 return LiveOutReg(Reg, DwarfRegNum, Size); 343 } 344 345 /// Parse the register live-out mask and return a vector of live-out registers 346 /// that need to be recorded in the stackmap. 347 StackMaps::LiveOutVec 348 StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const { 349 assert(Mask && "No register mask specified"); 350 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo(); 351 LiveOutVec LiveOuts; 352 353 // Create a LiveOutReg for each bit that is set in the register mask. 354 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg) 355 if ((Mask[Reg / 32] >> (Reg % 32)) & 1) 356 LiveOuts.push_back(createLiveOutReg(Reg, TRI)); 357 358 // We don't need to keep track of a register if its super-register is already 359 // in the list. Merge entries that refer to the same dwarf register and use 360 // the maximum size that needs to be spilled. 361 362 llvm::sort(LiveOuts, [](const LiveOutReg &LHS, const LiveOutReg &RHS) { 363 // Only sort by the dwarf register number. 364 return LHS.DwarfRegNum < RHS.DwarfRegNum; 365 }); 366 367 for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) { 368 for (auto II = std::next(I); II != E; ++II) { 369 if (I->DwarfRegNum != II->DwarfRegNum) { 370 // Skip all the now invalid entries. 371 I = --II; 372 break; 373 } 374 I->Size = std::max(I->Size, II->Size); 375 if (TRI->isSuperRegister(I->Reg, II->Reg)) 376 I->Reg = II->Reg; 377 II->Reg = 0; // mark for deletion. 378 } 379 } 380 381 llvm::erase_if(LiveOuts, [](const LiveOutReg &LO) { return LO.Reg == 0; }); 382 383 return LiveOuts; 384 } 385 386 // See statepoint MI format description in StatepointOpers' class comment 387 // in include/llvm/CodeGen/StackMaps.h 388 void StackMaps::parseStatepointOpers(const MachineInstr &MI, 389 MachineInstr::const_mop_iterator MOI, 390 MachineInstr::const_mop_iterator MOE, 391 LocationVec &Locations, 392 LiveOutVec &LiveOuts) { 393 LLVM_DEBUG(dbgs() << "record statepoint : " << MI << "\n"); 394 StatepointOpers SO(&MI); 395 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); // CC 396 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); // Flags 397 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); // Num Deopts 398 399 // Record Deopt Args. 400 unsigned NumDeoptArgs = Locations.back().Offset; 401 assert(Locations.back().Type == Location::Constant); 402 assert(NumDeoptArgs == SO.getNumDeoptArgs()); 403 404 while (NumDeoptArgs--) 405 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); 406 407 // Record gc base/derived pairs 408 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp); 409 ++MOI; 410 assert(MOI->isImm()); 411 unsigned NumGCPointers = MOI->getImm(); 412 ++MOI; 413 if (NumGCPointers) { 414 // Map logical index of GC ptr to MI operand index. 415 SmallVector<unsigned, 8> GCPtrIndices; 416 unsigned GCPtrIdx = (unsigned)SO.getFirstGCPtrIdx(); 417 assert((int)GCPtrIdx != -1); 418 assert(MOI - MI.operands_begin() == GCPtrIdx + 0LL); 419 while (NumGCPointers--) { 420 GCPtrIndices.push_back(GCPtrIdx); 421 GCPtrIdx = StackMaps::getNextMetaArgIdx(&MI, GCPtrIdx); 422 } 423 424 SmallVector<std::pair<unsigned, unsigned>, 8> GCPairs; 425 unsigned NumGCPairs = SO.getGCPointerMap(GCPairs); 426 (void)NumGCPairs; 427 LLVM_DEBUG(dbgs() << "NumGCPairs = " << NumGCPairs << "\n"); 428 429 auto MOB = MI.operands_begin(); 430 for (auto &P : GCPairs) { 431 assert(P.first < GCPtrIndices.size() && "base pointer index not found"); 432 assert(P.second < GCPtrIndices.size() && 433 "derived pointer index not found"); 434 unsigned BaseIdx = GCPtrIndices[P.first]; 435 unsigned DerivedIdx = GCPtrIndices[P.second]; 436 LLVM_DEBUG(dbgs() << "Base : " << BaseIdx << " Derived : " << DerivedIdx 437 << "\n"); 438 (void)parseOperand(MOB + BaseIdx, MOE, Locations, LiveOuts); 439 (void)parseOperand(MOB + DerivedIdx, MOE, Locations, LiveOuts); 440 } 441 442 MOI = MOB + GCPtrIdx; 443 } 444 445 // Record gc allocas 446 assert(MOI < MOE); 447 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp); 448 ++MOI; 449 unsigned NumAllocas = MOI->getImm(); 450 ++MOI; 451 while (NumAllocas--) { 452 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); 453 assert(MOI < MOE); 454 } 455 } 456 457 void StackMaps::recordStackMapOpers(const MCSymbol &MILabel, 458 const MachineInstr &MI, uint64_t ID, 459 MachineInstr::const_mop_iterator MOI, 460 MachineInstr::const_mop_iterator MOE, 461 bool recordResult) { 462 MCContext &OutContext = AP.OutStreamer->getContext(); 463 464 LocationVec Locations; 465 LiveOutVec LiveOuts; 466 467 if (recordResult) { 468 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value."); 469 parseOperand(MI.operands_begin(), std::next(MI.operands_begin()), Locations, 470 LiveOuts); 471 } 472 473 // Parse operands. 474 if (MI.getOpcode() == TargetOpcode::STATEPOINT) 475 parseStatepointOpers(MI, MOI, MOE, Locations, LiveOuts); 476 else 477 while (MOI != MOE) 478 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); 479 480 // Move large constants into the constant pool. 481 for (auto &Loc : Locations) { 482 // Constants are encoded as sign-extended integers. 483 // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool. 484 if (Loc.Type == Location::Constant && !isInt<32>(Loc.Offset)) { 485 Loc.Type = Location::ConstantIndex; 486 // ConstPool is intentionally a MapVector of 'uint64_t's (as 487 // opposed to 'int64_t's). We should never be in a situation 488 // where we have to insert either the tombstone or the empty 489 // keys into a map, and for a DenseMap<uint64_t, T> these are 490 // (uint64_t)0 and (uint64_t)-1. They can be and are 491 // represented using 32 bit integers. 492 assert((uint64_t)Loc.Offset != DenseMapInfo<uint64_t>::getEmptyKey() && 493 (uint64_t)Loc.Offset != 494 DenseMapInfo<uint64_t>::getTombstoneKey() && 495 "empty and tombstone keys should fit in 32 bits!"); 496 auto Result = ConstPool.insert(std::make_pair(Loc.Offset, Loc.Offset)); 497 Loc.Offset = Result.first - ConstPool.begin(); 498 } 499 } 500 501 // Create an expression to calculate the offset of the callsite from function 502 // entry. 503 const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub( 504 MCSymbolRefExpr::create(&MILabel, OutContext), 505 MCSymbolRefExpr::create(AP.CurrentFnSymForSize, OutContext), OutContext); 506 507 CSInfos.emplace_back(CSOffsetExpr, ID, std::move(Locations), 508 std::move(LiveOuts)); 509 510 // Record the stack size of the current function and update callsite count. 511 const MachineFrameInfo &MFI = AP.MF->getFrameInfo(); 512 const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo(); 513 bool HasDynamicFrameSize = 514 MFI.hasVarSizedObjects() || RegInfo->needsStackRealignment(*(AP.MF)); 515 uint64_t FrameSize = HasDynamicFrameSize ? UINT64_MAX : MFI.getStackSize(); 516 517 auto CurrentIt = FnInfos.find(AP.CurrentFnSym); 518 if (CurrentIt != FnInfos.end()) 519 CurrentIt->second.RecordCount++; 520 else 521 FnInfos.insert(std::make_pair(AP.CurrentFnSym, FunctionInfo(FrameSize))); 522 } 523 524 void StackMaps::recordStackMap(const MCSymbol &L, const MachineInstr &MI) { 525 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap"); 526 527 StackMapOpers opers(&MI); 528 const int64_t ID = MI.getOperand(PatchPointOpers::IDPos).getImm(); 529 recordStackMapOpers(L, MI, ID, std::next(MI.operands_begin(), 530 opers.getVarIdx()), 531 MI.operands_end()); 532 } 533 534 void StackMaps::recordPatchPoint(const MCSymbol &L, const MachineInstr &MI) { 535 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint"); 536 537 PatchPointOpers opers(&MI); 538 const int64_t ID = opers.getID(); 539 auto MOI = std::next(MI.operands_begin(), opers.getStackMapStartIdx()); 540 recordStackMapOpers(L, MI, ID, MOI, MI.operands_end(), 541 opers.isAnyReg() && opers.hasDef()); 542 543 #ifndef NDEBUG 544 // verify anyregcc 545 auto &Locations = CSInfos.back().Locations; 546 if (opers.isAnyReg()) { 547 unsigned NArgs = opers.getNumCallArgs(); 548 for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i) 549 assert(Locations[i].Type == Location::Register && 550 "anyreg arg must be in reg."); 551 } 552 #endif 553 } 554 555 void StackMaps::recordStatepoint(const MCSymbol &L, const MachineInstr &MI) { 556 assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint"); 557 558 StatepointOpers opers(&MI); 559 const unsigned StartIdx = opers.getVarIdx(); 560 recordStackMapOpers(L, MI, opers.getID(), MI.operands_begin() + StartIdx, 561 MI.operands_end(), false); 562 } 563 564 /// Emit the stackmap header. 565 /// 566 /// Header { 567 /// uint8 : Stack Map Version (currently 3) 568 /// uint8 : Reserved (expected to be 0) 569 /// uint16 : Reserved (expected to be 0) 570 /// } 571 /// uint32 : NumFunctions 572 /// uint32 : NumConstants 573 /// uint32 : NumRecords 574 void StackMaps::emitStackmapHeader(MCStreamer &OS) { 575 // Header. 576 OS.emitIntValue(StackMapVersion, 1); // Version. 577 OS.emitIntValue(0, 1); // Reserved. 578 OS.emitInt16(0); // Reserved. 579 580 // Num functions. 581 LLVM_DEBUG(dbgs() << WSMP << "#functions = " << FnInfos.size() << '\n'); 582 OS.emitInt32(FnInfos.size()); 583 // Num constants. 584 LLVM_DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n'); 585 OS.emitInt32(ConstPool.size()); 586 // Num callsites. 587 LLVM_DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n'); 588 OS.emitInt32(CSInfos.size()); 589 } 590 591 /// Emit the function frame record for each function. 592 /// 593 /// StkSizeRecord[NumFunctions] { 594 /// uint64 : Function Address 595 /// uint64 : Stack Size 596 /// uint64 : Record Count 597 /// } 598 void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) { 599 // Function Frame records. 600 LLVM_DEBUG(dbgs() << WSMP << "functions:\n"); 601 for (auto const &FR : FnInfos) { 602 LLVM_DEBUG(dbgs() << WSMP << "function addr: " << FR.first 603 << " frame size: " << FR.second.StackSize 604 << " callsite count: " << FR.second.RecordCount << '\n'); 605 OS.emitSymbolValue(FR.first, 8); 606 OS.emitIntValue(FR.second.StackSize, 8); 607 OS.emitIntValue(FR.second.RecordCount, 8); 608 } 609 } 610 611 /// Emit the constant pool. 612 /// 613 /// int64 : Constants[NumConstants] 614 void StackMaps::emitConstantPoolEntries(MCStreamer &OS) { 615 // Constant pool entries. 616 LLVM_DEBUG(dbgs() << WSMP << "constants:\n"); 617 for (const auto &ConstEntry : ConstPool) { 618 LLVM_DEBUG(dbgs() << WSMP << ConstEntry.second << '\n'); 619 OS.emitIntValue(ConstEntry.second, 8); 620 } 621 } 622 623 /// Emit the callsite info for each callsite. 624 /// 625 /// StkMapRecord[NumRecords] { 626 /// uint64 : PatchPoint ID 627 /// uint32 : Instruction Offset 628 /// uint16 : Reserved (record flags) 629 /// uint16 : NumLocations 630 /// Location[NumLocations] { 631 /// uint8 : Register | Direct | Indirect | Constant | ConstantIndex 632 /// uint8 : Size in Bytes 633 /// uint16 : Dwarf RegNum 634 /// int32 : Offset 635 /// } 636 /// uint16 : Padding 637 /// uint16 : NumLiveOuts 638 /// LiveOuts[NumLiveOuts] { 639 /// uint16 : Dwarf RegNum 640 /// uint8 : Reserved 641 /// uint8 : Size in Bytes 642 /// } 643 /// uint32 : Padding (only if required to align to 8 byte) 644 /// } 645 /// 646 /// Location Encoding, Type, Value: 647 /// 0x1, Register, Reg (value in register) 648 /// 0x2, Direct, Reg + Offset (frame index) 649 /// 0x3, Indirect, [Reg + Offset] (spilled value) 650 /// 0x4, Constant, Offset (small constant) 651 /// 0x5, ConstIndex, Constants[Offset] (large constant) 652 void StackMaps::emitCallsiteEntries(MCStreamer &OS) { 653 LLVM_DEBUG(print(dbgs())); 654 // Callsite entries. 655 for (const auto &CSI : CSInfos) { 656 const LocationVec &CSLocs = CSI.Locations; 657 const LiveOutVec &LiveOuts = CSI.LiveOuts; 658 659 // Verify stack map entry. It's better to communicate a problem to the 660 // runtime than crash in case of in-process compilation. Currently, we do 661 // simple overflow checks, but we may eventually communicate other 662 // compilation errors this way. 663 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) { 664 OS.emitIntValue(UINT64_MAX, 8); // Invalid ID. 665 OS.emitValue(CSI.CSOffsetExpr, 4); 666 OS.emitInt16(0); // Reserved. 667 OS.emitInt16(0); // 0 locations. 668 OS.emitInt16(0); // padding. 669 OS.emitInt16(0); // 0 live-out registers. 670 OS.emitInt32(0); // padding. 671 continue; 672 } 673 674 OS.emitIntValue(CSI.ID, 8); 675 OS.emitValue(CSI.CSOffsetExpr, 4); 676 677 // Reserved for flags. 678 OS.emitInt16(0); 679 OS.emitInt16(CSLocs.size()); 680 681 for (const auto &Loc : CSLocs) { 682 OS.emitIntValue(Loc.Type, 1); 683 OS.emitIntValue(0, 1); // Reserved 684 OS.emitInt16(Loc.Size); 685 OS.emitInt16(Loc.Reg); 686 OS.emitInt16(0); // Reserved 687 OS.emitInt32(Loc.Offset); 688 } 689 690 // Emit alignment to 8 byte. 691 OS.emitValueToAlignment(8); 692 693 // Num live-out registers and padding to align to 4 byte. 694 OS.emitInt16(0); 695 OS.emitInt16(LiveOuts.size()); 696 697 for (const auto &LO : LiveOuts) { 698 OS.emitInt16(LO.DwarfRegNum); 699 OS.emitIntValue(0, 1); 700 OS.emitIntValue(LO.Size, 1); 701 } 702 // Emit alignment to 8 byte. 703 OS.emitValueToAlignment(8); 704 } 705 } 706 707 /// Serialize the stackmap data. 708 void StackMaps::serializeToStackMapSection() { 709 (void)WSMP; 710 // Bail out if there's no stack map data. 711 assert((!CSInfos.empty() || ConstPool.empty()) && 712 "Expected empty constant pool too!"); 713 assert((!CSInfos.empty() || FnInfos.empty()) && 714 "Expected empty function record too!"); 715 if (CSInfos.empty()) 716 return; 717 718 MCContext &OutContext = AP.OutStreamer->getContext(); 719 MCStreamer &OS = *AP.OutStreamer; 720 721 // Create the section. 722 MCSection *StackMapSection = 723 OutContext.getObjectFileInfo()->getStackMapSection(); 724 OS.SwitchSection(StackMapSection); 725 726 // Emit a dummy symbol to force section inclusion. 727 OS.emitLabel(OutContext.getOrCreateSymbol(Twine("__LLVM_StackMaps"))); 728 729 // Serialize data. 730 LLVM_DEBUG(dbgs() << "********** Stack Map Output **********\n"); 731 emitStackmapHeader(OS); 732 emitFunctionFrameRecords(OS); 733 emitConstantPoolEntries(OS); 734 emitCallsiteEntries(OS); 735 OS.AddBlankLine(); 736 737 // Clean up. 738 CSInfos.clear(); 739 ConstPool.clear(); 740 } 741