1 //===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// Statepoint instruction in deopt parameters contains values which are 11 /// meaningful to the runtime and should be able to be read at the moment the 12 /// call returns. So we can say that we need to encode the fact that these 13 /// values are "late read" by runtime. If we could express this notion for 14 /// register allocator it would produce the right form for us. 15 /// The need to fixup (i.e this pass) is specifically handling the fact that 16 /// we cannot describe such a late read for the register allocator. 17 /// Register allocator may put the value on a register clobbered by the call. 18 /// This pass forces the spill of such registers and replaces corresponding 19 /// statepoint operands to added spill slots. 20 /// 21 //===----------------------------------------------------------------------===// 22 23 #include "llvm/ADT/SmallSet.h" 24 #include "llvm/ADT/Statistic.h" 25 #include "llvm/CodeGen/MachineFrameInfo.h" 26 #include "llvm/CodeGen/MachineFunctionPass.h" 27 #include "llvm/CodeGen/StackMaps.h" 28 #include "llvm/CodeGen/TargetInstrInfo.h" 29 #include "llvm/IR/Statepoint.h" 30 #include "llvm/InitializePasses.h" 31 #include "llvm/Support/Debug.h" 32 33 using namespace llvm; 34 35 #define DEBUG_TYPE "fixup-statepoint-caller-saved" 36 STATISTIC(NumSpilledRegisters, "Number of spilled register"); 37 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated"); 38 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended"); 39 40 static cl::opt<bool> FixupSCSExtendSlotSize( 41 "fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), 42 cl::desc("Allow spill in spill slot of greater size than register size"), 43 cl::Hidden); 44 45 static cl::opt<bool> PassGCPtrInCSR( 46 "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), 47 cl::desc("Allow passing GC Pointer arguments in callee saved registers")); 48 49 static cl::opt<bool> EnableCopyProp( 50 "fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), 51 cl::desc("Enable simple copy propagation during register reloading")); 52 53 // This is purely debugging option. 54 // It may be handy for investigating statepoint spilling issues. 55 static cl::opt<unsigned> MaxStatepointsWithRegs( 56 "fixup-max-csr-statepoints", cl::Hidden, 57 cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers")); 58 59 namespace { 60 61 class FixupStatepointCallerSaved : public MachineFunctionPass { 62 public: 63 static char ID; 64 65 FixupStatepointCallerSaved() : MachineFunctionPass(ID) { 66 initializeFixupStatepointCallerSavedPass(*PassRegistry::getPassRegistry()); 67 } 68 69 void getAnalysisUsage(AnalysisUsage &AU) const override { 70 AU.setPreservesCFG(); 71 MachineFunctionPass::getAnalysisUsage(AU); 72 } 73 74 StringRef getPassName() const override { 75 return "Fixup Statepoint Caller Saved"; 76 } 77 78 bool runOnMachineFunction(MachineFunction &MF) override; 79 }; 80 81 } // End anonymous namespace. 82 83 char FixupStatepointCallerSaved::ID = 0; 84 char &llvm::FixupStatepointCallerSavedID = FixupStatepointCallerSaved::ID; 85 86 INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE, 87 "Fixup Statepoint Caller Saved", false, false) 88 INITIALIZE_PASS_END(FixupStatepointCallerSaved, DEBUG_TYPE, 89 "Fixup Statepoint Caller Saved", false, false) 90 91 // Utility function to get size of the register. 92 static unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg) { 93 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg); 94 return TRI.getSpillSize(*RC); 95 } 96 97 // Try to eliminate redundant copy to register which we're going to 98 // spill, i.e. try to change: 99 // X = COPY Y 100 // SPILL X 101 // to 102 // SPILL Y 103 // If there are no uses of X between copy and STATEPOINT, that COPY 104 // may be eliminated. 105 // Reg - register we're about to spill 106 // RI - On entry points to statepoint. 107 // On successful copy propagation set to new spill point. 108 // IsKill - set to true if COPY is Kill (there are no uses of Y) 109 // Returns either found source copy register or original one. 110 static Register performCopyPropagation(Register Reg, 111 MachineBasicBlock::iterator &RI, 112 bool &IsKill, const TargetInstrInfo &TII, 113 const TargetRegisterInfo &TRI) { 114 // First check if statepoint itself uses Reg in non-meta operands. 115 int Idx = RI->findRegisterUseOperandIdx(Reg, false, &TRI); 116 if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) { 117 IsKill = false; 118 return Reg; 119 } 120 121 if (!EnableCopyProp) 122 return Reg; 123 124 MachineBasicBlock *MBB = RI->getParent(); 125 MachineBasicBlock::reverse_iterator E = MBB->rend(); 126 MachineInstr *Def = nullptr, *Use = nullptr; 127 for (auto It = ++(RI.getReverse()); It != E; ++It) { 128 if (It->readsRegister(Reg, &TRI) && !Use) 129 Use = &*It; 130 if (It->modifiesRegister(Reg, &TRI)) { 131 Def = &*It; 132 break; 133 } 134 } 135 136 if (!Def) 137 return Reg; 138 139 auto DestSrc = TII.isCopyInstr(*Def); 140 if (!DestSrc || DestSrc->Destination->getReg() != Reg) 141 return Reg; 142 143 Register SrcReg = DestSrc->Source->getReg(); 144 145 if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, SrcReg)) 146 return Reg; 147 148 LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation " 149 << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI) 150 << "\n"); 151 152 // Insert spill immediately after Def 153 RI = ++MachineBasicBlock::iterator(Def); 154 IsKill = DestSrc->Source->isKill(); 155 156 if (!Use) { 157 // There are no uses of original register between COPY and STATEPOINT. 158 // There can't be any after STATEPOINT, so we can eliminate Def. 159 LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def); 160 Def->eraseFromParent(); 161 } else if (IsKill) { 162 // COPY will remain in place, spill will be inserted *after* it, so it is 163 // not a kill of source anymore. 164 const_cast<MachineOperand *>(DestSrc->Source)->setIsKill(false); 165 } 166 167 return SrcReg; 168 } 169 170 namespace { 171 // Pair {Register, FrameIndex} 172 using RegSlotPair = std::pair<Register, int>; 173 174 // Keeps track of what reloads were inserted in MBB. 175 class RegReloadCache { 176 using ReloadSet = SmallSet<RegSlotPair, 8>; 177 DenseMap<const MachineBasicBlock *, ReloadSet> Reloads; 178 179 public: 180 RegReloadCache() = default; 181 182 // Record reload of Reg from FI in block MBB 183 void recordReload(Register Reg, int FI, const MachineBasicBlock *MBB) { 184 RegSlotPair RSP(Reg, FI); 185 auto Res = Reloads[MBB].insert(RSP); 186 (void)Res; 187 assert(Res.second && "reload already exists"); 188 } 189 190 // Does basic block MBB contains reload of Reg from FI? 191 bool hasReload(Register Reg, int FI, const MachineBasicBlock *MBB) { 192 RegSlotPair RSP(Reg, FI); 193 return Reloads.count(MBB) && Reloads[MBB].count(RSP); 194 } 195 }; 196 197 // Cache used frame indexes during statepoint re-write to re-use them in 198 // processing next statepoint instruction. 199 // Two strategies. One is to preserve the size of spill slot while another one 200 // extends the size of spill slots to reduce the number of them, causing 201 // the less total frame size. But unspill will have "implicit" any extend. 202 class FrameIndexesCache { 203 private: 204 struct FrameIndexesPerSize { 205 // List of used frame indexes during processing previous statepoints. 206 SmallVector<int, 8> Slots; 207 // Current index of un-used yet frame index. 208 unsigned Index = 0; 209 }; 210 MachineFrameInfo &MFI; 211 const TargetRegisterInfo &TRI; 212 // Map size to list of frame indexes of this size. If the mode is 213 // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes. 214 // If the size of required spill slot is greater than in a cache then the 215 // size will be increased. 216 DenseMap<unsigned, FrameIndexesPerSize> Cache; 217 218 // Keeps track of slots reserved for the shared landing pad processing. 219 // Initialized from GlobalIndices for the current EHPad. 220 SmallSet<int, 8> ReservedSlots; 221 222 // Landing pad can be destination of several statepoints. Every register 223 // defined by such statepoints must be spilled to the same stack slot. 224 // This map keeps that information. 225 DenseMap<const MachineBasicBlock *, SmallVector<RegSlotPair, 8>> 226 GlobalIndices; 227 228 FrameIndexesPerSize &getCacheBucket(unsigned Size) { 229 // In FixupSCSExtendSlotSize mode the bucket with 0 index is used 230 // for all sizes. 231 return Cache[FixupSCSExtendSlotSize ? 0 : Size]; 232 } 233 234 public: 235 FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI) 236 : MFI(MFI), TRI(TRI) {} 237 // Reset the current state of used frame indexes. After invocation of 238 // this function all frame indexes are available for allocation with 239 // the exception of slots reserved for landing pad processing (if any). 240 void reset(const MachineBasicBlock *EHPad) { 241 for (auto &It : Cache) 242 It.second.Index = 0; 243 244 ReservedSlots.clear(); 245 if (EHPad && GlobalIndices.count(EHPad)) 246 for (auto &RSP : GlobalIndices[EHPad]) 247 ReservedSlots.insert(RSP.second); 248 } 249 250 // Get frame index to spill the register. 251 int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) { 252 // Check if slot for Reg is already reserved at EHPad. 253 auto It = GlobalIndices.find(EHPad); 254 if (It != GlobalIndices.end()) { 255 auto &Vec = It->second; 256 auto Idx = llvm::find_if( 257 Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; }); 258 if (Idx != Vec.end()) { 259 int FI = Idx->second; 260 LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register " 261 << printReg(Reg, &TRI) << " at " 262 << printMBBReference(*EHPad) << "\n"); 263 assert(ReservedSlots.count(FI) && "using unreserved slot"); 264 return FI; 265 } 266 } 267 268 unsigned Size = getRegisterSize(TRI, Reg); 269 FrameIndexesPerSize &Line = getCacheBucket(Size); 270 while (Line.Index < Line.Slots.size()) { 271 int FI = Line.Slots[Line.Index++]; 272 if (ReservedSlots.count(FI)) 273 continue; 274 // If all sizes are kept together we probably need to extend the 275 // spill slot size. 276 if (MFI.getObjectSize(FI) < Size) { 277 MFI.setObjectSize(FI, Size); 278 MFI.setObjectAlignment(FI, Align(Size)); 279 NumSpillSlotsExtended++; 280 } 281 return FI; 282 } 283 int FI = MFI.CreateSpillStackObject(Size, Align(Size)); 284 NumSpillSlotsAllocated++; 285 Line.Slots.push_back(FI); 286 ++Line.Index; 287 288 // Remember assignment {Reg, FI} for EHPad 289 if (EHPad) { 290 GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI)); 291 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg " 292 << printReg(Reg, &TRI) << " at landing pad " 293 << printMBBReference(*EHPad) << "\n"); 294 } 295 296 return FI; 297 } 298 299 // Sort all registers to spill in descendent order. In the 300 // FixupSCSExtendSlotSize mode it will minimize the total frame size. 301 // In non FixupSCSExtendSlotSize mode we can skip this step. 302 void sortRegisters(SmallVectorImpl<Register> &Regs) { 303 if (!FixupSCSExtendSlotSize) 304 return; 305 llvm::sort(Regs, [&](Register &A, Register &B) { 306 return getRegisterSize(TRI, A) > getRegisterSize(TRI, B); 307 }); 308 } 309 }; 310 311 // Describes the state of the current processing statepoint instruction. 312 class StatepointState { 313 private: 314 // statepoint instruction. 315 MachineInstr &MI; 316 MachineFunction &MF; 317 // If non-null then statepoint is invoke, and this points to the landing pad. 318 MachineBasicBlock *EHPad; 319 const TargetRegisterInfo &TRI; 320 const TargetInstrInfo &TII; 321 MachineFrameInfo &MFI; 322 // Mask with callee saved registers. 323 const uint32_t *Mask; 324 // Cache of frame indexes used on previous instruction processing. 325 FrameIndexesCache &CacheFI; 326 bool AllowGCPtrInCSR; 327 // Operands with physical registers requiring spilling. 328 SmallVector<unsigned, 8> OpsToSpill; 329 // Set of register to spill. 330 SmallVector<Register, 8> RegsToSpill; 331 // Set of registers to reload after statepoint. 332 SmallVector<Register, 8> RegsToReload; 333 // Map Register to Frame Slot index. 334 DenseMap<Register, int> RegToSlotIdx; 335 336 public: 337 StatepointState(MachineInstr &MI, const uint32_t *Mask, 338 FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR) 339 : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()), 340 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()), 341 Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) { 342 343 // Find statepoint's landing pad, if any. 344 EHPad = nullptr; 345 MachineBasicBlock *MBB = MI.getParent(); 346 // Invoke statepoint must be last one in block. 347 bool Last = std::none_of(++MI.getIterator(), MBB->end().getInstrIterator(), 348 [](MachineInstr &I) { 349 return I.getOpcode() == TargetOpcode::STATEPOINT; 350 }); 351 352 if (!Last) 353 return; 354 355 auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); }; 356 357 assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads"); 358 359 auto It = llvm::find_if(MBB->successors(), IsEHPad); 360 if (It != MBB->succ_end()) 361 EHPad = *It; 362 } 363 364 MachineBasicBlock *getEHPad() const { return EHPad; } 365 366 // Return true if register is callee saved. 367 bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; } 368 369 // Iterates over statepoint meta args to find caller saver registers. 370 // Also cache the size of found registers. 371 // Returns true if caller save registers found. 372 bool findRegistersToSpill() { 373 SmallSet<Register, 8> GCRegs; 374 // All GC pointer operands assigned to registers produce new value. 375 // Since they're tied to their defs, it is enough to collect def registers. 376 for (const auto &Def : MI.defs()) 377 GCRegs.insert(Def.getReg()); 378 379 SmallSet<Register, 8> VisitedRegs; 380 for (unsigned Idx = StatepointOpers(&MI).getVarIdx(), 381 EndIdx = MI.getNumOperands(); 382 Idx < EndIdx; ++Idx) { 383 MachineOperand &MO = MI.getOperand(Idx); 384 // Leave `undef` operands as is, StackMaps will rewrite them 385 // into a constant. 386 if (!MO.isReg() || MO.isImplicit() || MO.isUndef()) 387 continue; 388 Register Reg = MO.getReg(); 389 assert(Reg.isPhysical() && "Only physical regs are expected"); 390 391 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !GCRegs.contains(Reg))) 392 continue; 393 394 LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index " 395 << Idx << "\n"); 396 397 if (VisitedRegs.insert(Reg).second) 398 RegsToSpill.push_back(Reg); 399 OpsToSpill.push_back(Idx); 400 } 401 CacheFI.sortRegisters(RegsToSpill); 402 return !RegsToSpill.empty(); 403 } 404 405 // Spill all caller saved registers right before statepoint instruction. 406 // Remember frame index where register is spilled. 407 void spillRegisters() { 408 for (Register Reg : RegsToSpill) { 409 int FI = CacheFI.getFrameIndex(Reg, EHPad); 410 411 NumSpilledRegisters++; 412 RegToSlotIdx[Reg] = FI; 413 414 LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI 415 << "\n"); 416 417 // Perform trivial copy propagation 418 bool IsKill = true; 419 MachineBasicBlock::iterator InsertBefore(MI); 420 Reg = performCopyPropagation(Reg, InsertBefore, IsKill, TII, TRI); 421 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg); 422 423 LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore); 424 TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI, 425 RC, &TRI, Register()); 426 } 427 } 428 429 void insertReloadBefore(unsigned Reg, MachineBasicBlock::iterator It, 430 MachineBasicBlock *MBB) { 431 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg); 432 int FI = RegToSlotIdx[Reg]; 433 if (It != MBB->end()) { 434 TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI, Register()); 435 return; 436 } 437 438 // To insert reload at the end of MBB, insert it before last instruction 439 // and then swap them. 440 assert(!MBB->empty() && "Empty block"); 441 --It; 442 TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI, Register()); 443 MachineInstr *Reload = It->getPrevNode(); 444 int Dummy = 0; 445 (void)Dummy; 446 assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg); 447 assert(Dummy == FI); 448 MBB->remove(Reload); 449 MBB->insertAfter(It, Reload); 450 } 451 452 // Insert reloads of (relocated) registers spilled in statepoint. 453 void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) { 454 MachineBasicBlock *MBB = NewStatepoint->getParent(); 455 auto InsertPoint = std::next(NewStatepoint->getIterator()); 456 457 for (auto Reg : RegsToReload) { 458 insertReloadBefore(Reg, InsertPoint, MBB); 459 LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI " 460 << RegToSlotIdx[Reg] << " after statepoint\n"); 461 462 if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) { 463 RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad); 464 auto EHPadInsertPoint = 465 EHPad->SkipPHIsLabelsAndDebug(EHPad->begin(), Reg); 466 insertReloadBefore(Reg, EHPadInsertPoint, EHPad); 467 LLVM_DEBUG(dbgs() << "...also reload at EHPad " 468 << printMBBReference(*EHPad) << "\n"); 469 } 470 } 471 } 472 473 // Re-write statepoint machine instruction to replace caller saved operands 474 // with indirect memory location (frame index). 475 MachineInstr *rewriteStatepoint() { 476 MachineInstr *NewMI = 477 MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true); 478 MachineInstrBuilder MIB(MF, NewMI); 479 480 unsigned NumOps = MI.getNumOperands(); 481 482 // New indices for the remaining defs. 483 SmallVector<unsigned, 8> NewIndices; 484 unsigned NumDefs = MI.getNumDefs(); 485 for (unsigned I = 0; I < NumDefs; ++I) { 486 MachineOperand &DefMO = MI.getOperand(I); 487 assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand"); 488 Register Reg = DefMO.getReg(); 489 assert(DefMO.isTied() && "Def is expected to be tied"); 490 // We skipped undef uses and did not spill them, so we should not 491 // proceed with defs here. 492 if (MI.getOperand(MI.findTiedOperandIdx(I)).isUndef()) { 493 if (AllowGCPtrInCSR) { 494 NewIndices.push_back(NewMI->getNumOperands()); 495 MIB.addReg(Reg, RegState::Define); 496 } 497 continue; 498 } 499 if (!AllowGCPtrInCSR) { 500 assert(is_contained(RegsToSpill, Reg)); 501 RegsToReload.push_back(Reg); 502 } else { 503 if (isCalleeSaved(Reg)) { 504 NewIndices.push_back(NewMI->getNumOperands()); 505 MIB.addReg(Reg, RegState::Define); 506 } else { 507 NewIndices.push_back(NumOps); 508 RegsToReload.push_back(Reg); 509 } 510 } 511 } 512 513 // Add End marker. 514 OpsToSpill.push_back(MI.getNumOperands()); 515 unsigned CurOpIdx = 0; 516 517 for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) { 518 MachineOperand &MO = MI.getOperand(I); 519 if (I == OpsToSpill[CurOpIdx]) { 520 int FI = RegToSlotIdx[MO.getReg()]; 521 MIB.addImm(StackMaps::IndirectMemRefOp); 522 MIB.addImm(getRegisterSize(TRI, MO.getReg())); 523 assert(MO.isReg() && "Should be register"); 524 assert(MO.getReg().isPhysical() && "Should be physical register"); 525 MIB.addFrameIndex(FI); 526 MIB.addImm(0); 527 ++CurOpIdx; 528 } else { 529 MIB.add(MO); 530 unsigned OldDef; 531 if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) { 532 assert(OldDef < NumDefs); 533 assert(NewIndices[OldDef] < NumOps); 534 MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1); 535 } 536 } 537 } 538 assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed"); 539 // Add mem operands. 540 NewMI->setMemRefs(MF, MI.memoperands()); 541 for (auto It : RegToSlotIdx) { 542 Register R = It.first; 543 int FrameIndex = It.second; 544 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 545 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad; 546 if (is_contained(RegsToReload, R)) 547 Flags |= MachineMemOperand::MOStore; 548 auto *MMO = 549 MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, R), 550 MFI.getObjectAlign(FrameIndex)); 551 NewMI->addMemOperand(MF, MMO); 552 } 553 554 // Insert new statepoint and erase old one. 555 MI.getParent()->insert(MI, NewMI); 556 557 LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n"); 558 MI.eraseFromParent(); 559 return NewMI; 560 } 561 }; 562 563 class StatepointProcessor { 564 private: 565 MachineFunction &MF; 566 const TargetRegisterInfo &TRI; 567 FrameIndexesCache CacheFI; 568 RegReloadCache ReloadCache; 569 570 public: 571 StatepointProcessor(MachineFunction &MF) 572 : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()), 573 CacheFI(MF.getFrameInfo(), TRI) {} 574 575 bool process(MachineInstr &MI, bool AllowGCPtrInCSR) { 576 StatepointOpers SO(&MI); 577 uint64_t Flags = SO.getFlags(); 578 // Do nothing for LiveIn, it supports all registers. 579 if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn) 580 return false; 581 LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " " 582 << MI.getParent()->getName() << " : process statepoint " 583 << MI); 584 CallingConv::ID CC = SO.getCallingConv(); 585 const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC); 586 StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR); 587 CacheFI.reset(SS.getEHPad()); 588 589 if (!SS.findRegistersToSpill()) 590 return false; 591 592 SS.spillRegisters(); 593 auto *NewStatepoint = SS.rewriteStatepoint(); 594 SS.insertReloads(NewStatepoint, ReloadCache); 595 return true; 596 } 597 }; 598 } // namespace 599 600 bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) { 601 if (skipFunction(MF.getFunction())) 602 return false; 603 604 const Function &F = MF.getFunction(); 605 if (!F.hasGC()) 606 return false; 607 608 SmallVector<MachineInstr *, 16> Statepoints; 609 for (MachineBasicBlock &BB : MF) 610 for (MachineInstr &I : BB) 611 if (I.getOpcode() == TargetOpcode::STATEPOINT) 612 Statepoints.push_back(&I); 613 614 if (Statepoints.empty()) 615 return false; 616 617 bool Changed = false; 618 StatepointProcessor SPP(MF); 619 unsigned NumStatepoints = 0; 620 bool AllowGCPtrInCSR = PassGCPtrInCSR; 621 for (MachineInstr *I : Statepoints) { 622 ++NumStatepoints; 623 if (MaxStatepointsWithRegs.getNumOccurrences() && 624 NumStatepoints >= MaxStatepointsWithRegs) 625 AllowGCPtrInCSR = false; 626 Changed |= SPP.process(*I, AllowGCPtrInCSR); 627 } 628 return Changed; 629 } 630