xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/FixupStatepointCallerSaved.cpp (revision acb1f1269c6f4ff89a0d28ba742f6687e9ef779d)
1 //===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers  ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// Statepoint instruction in deopt parameters contains values which are
12 /// meaningful to the runtime and should be able to be read at the moment the
13 /// call returns. So we can say that we need to encode the fact that these
14 /// values are "late read" by runtime. If we could express this notion for
15 /// register allocator it would produce the right form for us.
16 /// The need to fixup (i.e this pass) is specifically handling the fact that
17 /// we cannot describe such a late read for the register allocator.
18 /// Register allocator may put the value on a register clobbered by the call.
19 /// This pass forces the spill of such registers and replaces corresponding
20 /// statepoint operands to added spill slots.
21 ///
22 //===----------------------------------------------------------------------===//
23 
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/CodeGen/MachineFrameInfo.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/CodeGen/StackMaps.h"
31 #include "llvm/CodeGen/TargetFrameLowering.h"
32 #include "llvm/CodeGen/TargetInstrInfo.h"
33 #include "llvm/IR/Statepoint.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "fixup-statepoint-caller-saved"
40 STATISTIC(NumSpilledRegisters, "Number of spilled register");
41 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");
42 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");
43 
44 static cl::opt<bool> FixupSCSExtendSlotSize(
45     "fixup-scs-extend-slot-size", cl::Hidden, cl::init(false),
46     cl::desc("Allow spill in spill slot of greater size than register size"),
47     cl::Hidden);
48 
49 static cl::opt<bool> PassGCPtrInCSR(
50     "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false),
51     cl::desc("Allow passing GC Pointer arguments in callee saved registers"));
52 
53 static cl::opt<bool> EnableCopyProp(
54     "fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true),
55     cl::desc("Enable simple copy propagation during register reloading"));
56 
57 // This is purely debugging option.
58 // It may be handy for investigating statepoint spilling issues.
59 static cl::opt<unsigned> MaxStatepointsWithRegs(
60     "fixup-max-csr-statepoints", cl::Hidden,
61     cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));
62 
63 namespace {
64 
65 class FixupStatepointCallerSaved : public MachineFunctionPass {
66 public:
67   static char ID;
68 
69   FixupStatepointCallerSaved() : MachineFunctionPass(ID) {
70     initializeFixupStatepointCallerSavedPass(*PassRegistry::getPassRegistry());
71   }
72 
73   void getAnalysisUsage(AnalysisUsage &AU) const override {
74     AU.setPreservesCFG();
75     MachineFunctionPass::getAnalysisUsage(AU);
76   }
77 
78   StringRef getPassName() const override {
79     return "Fixup Statepoint Caller Saved";
80   }
81 
82   bool runOnMachineFunction(MachineFunction &MF) override;
83 };
84 
85 } // End anonymous namespace.
86 
87 char FixupStatepointCallerSaved::ID = 0;
88 char &llvm::FixupStatepointCallerSavedID = FixupStatepointCallerSaved::ID;
89 
90 INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE,
91                       "Fixup Statepoint Caller Saved", false, false)
92 INITIALIZE_PASS_END(FixupStatepointCallerSaved, DEBUG_TYPE,
93                     "Fixup Statepoint Caller Saved", false, false)
94 
95 // Utility function to get size of the register.
96 static unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg) {
97   const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
98   return TRI.getSpillSize(*RC);
99 }
100 
101 // Try to eliminate redundant copy to register which we're going to
102 // spill, i.e. try to change:
103 //    X = COPY Y
104 //    SPILL X
105 //  to
106 //    SPILL Y
107 //  If there are no uses of X between copy and STATEPOINT, that COPY
108 //  may be eliminated.
109 //  Reg - register we're about to spill
110 //  RI - On entry points to statepoint.
111 //       On successful copy propagation set to new spill point.
112 //  IsKill - set to true if COPY is Kill (there are no uses of Y)
113 //  Returns either found source copy register or original one.
114 static Register performCopyPropagation(Register Reg,
115                                        MachineBasicBlock::iterator &RI,
116                                        bool &IsKill, const TargetInstrInfo &TII,
117                                        const TargetRegisterInfo &TRI) {
118   // First check if statepoint itself uses Reg in non-meta operands.
119   int Idx = RI->findRegisterUseOperandIdx(Reg, false, &TRI);
120   if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
121     IsKill = false;
122     return Reg;
123   }
124 
125   if (!EnableCopyProp)
126     return Reg;
127 
128   MachineBasicBlock *MBB = RI->getParent();
129   MachineBasicBlock::reverse_iterator E = MBB->rend();
130   MachineInstr *Def = nullptr, *Use = nullptr;
131   for (auto It = ++(RI.getReverse()); It != E; ++It) {
132     if (It->readsRegister(Reg, &TRI) && !Use)
133       Use = &*It;
134     if (It->modifiesRegister(Reg, &TRI)) {
135       Def = &*It;
136       break;
137     }
138   }
139 
140   if (!Def)
141     return Reg;
142 
143   auto DestSrc = TII.isCopyInstr(*Def);
144   if (!DestSrc || DestSrc->Destination->getReg() != Reg)
145     return Reg;
146 
147   Register SrcReg = DestSrc->Source->getReg();
148 
149   if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, SrcReg))
150     return Reg;
151 
152   LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "
153                     << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI)
154                     << "\n");
155 
156   // Insert spill immediately after Def
157   RI = ++MachineBasicBlock::iterator(Def);
158   IsKill = DestSrc->Source->isKill();
159 
160   // There are no uses of original register between COPY and STATEPOINT.
161   // There can't be any after STATEPOINT, so we can eliminate Def.
162   if (!Use) {
163     LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);
164     Def->eraseFromParent();
165   }
166   return SrcReg;
167 }
168 
169 namespace {
170 // Pair {Register, FrameIndex}
171 using RegSlotPair = std::pair<Register, int>;
172 
173 // Keeps track of what reloads were inserted in MBB.
174 class RegReloadCache {
175   using ReloadSet = SmallSet<RegSlotPair, 8>;
176   DenseMap<const MachineBasicBlock *, ReloadSet> Reloads;
177 
178 public:
179   RegReloadCache() = default;
180 
181   // Record reload of Reg from FI in block MBB
182   void recordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
183     RegSlotPair RSP(Reg, FI);
184     auto Res = Reloads[MBB].insert(RSP);
185     (void)Res;
186     assert(Res.second && "reload already exists");
187   }
188 
189   // Does basic block MBB contains reload of Reg from FI?
190   bool hasReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
191     RegSlotPair RSP(Reg, FI);
192     return Reloads.count(MBB) && Reloads[MBB].count(RSP);
193   }
194 };
195 
196 // Cache used frame indexes during statepoint re-write to re-use them in
197 // processing next statepoint instruction.
198 // Two strategies. One is to preserve the size of spill slot while another one
199 // extends the size of spill slots to reduce the number of them, causing
200 // the less total frame size. But unspill will have "implicit" any extend.
201 class FrameIndexesCache {
202 private:
203   struct FrameIndexesPerSize {
204     // List of used frame indexes during processing previous statepoints.
205     SmallVector<int, 8> Slots;
206     // Current index of un-used yet frame index.
207     unsigned Index = 0;
208   };
209   MachineFrameInfo &MFI;
210   const TargetRegisterInfo &TRI;
211   // Map size to list of frame indexes of this size. If the mode is
212   // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes.
213   // If the size of required spill slot is greater than in a cache then the
214   // size will be increased.
215   DenseMap<unsigned, FrameIndexesPerSize> Cache;
216 
217   // Keeps track of slots reserved for the shared landing pad processing.
218   // Initialized from GlobalIndices for the current EHPad.
219   SmallSet<int, 8> ReservedSlots;
220 
221   // Landing pad can be destination of several statepoints. Every register
222   // defined by such statepoints must be spilled to the same stack slot.
223   // This map keeps that information.
224   DenseMap<const MachineBasicBlock *, SmallVector<RegSlotPair, 8>>
225       GlobalIndices;
226 
227   FrameIndexesPerSize &getCacheBucket(unsigned Size) {
228     // In FixupSCSExtendSlotSize mode the bucket with 0 index is used
229     // for all sizes.
230     return Cache[FixupSCSExtendSlotSize ? 0 : Size];
231   }
232 
233 public:
234   FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI)
235       : MFI(MFI), TRI(TRI) {}
236   // Reset the current state of used frame indexes. After invocation of
237   // this function all frame indexes are available for allocation with
238   // the exception of slots reserved for landing pad processing (if any).
239   void reset(const MachineBasicBlock *EHPad) {
240     for (auto &It : Cache)
241       It.second.Index = 0;
242 
243     ReservedSlots.clear();
244     if (EHPad && GlobalIndices.count(EHPad))
245       for (auto &RSP : GlobalIndices[EHPad])
246         ReservedSlots.insert(RSP.second);
247   }
248 
249   // Get frame index to spill the register.
250   int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {
251     // Check if slot for Reg is already reserved at EHPad.
252     auto It = GlobalIndices.find(EHPad);
253     if (It != GlobalIndices.end()) {
254       auto &Vec = It->second;
255       auto Idx = llvm::find_if(
256           Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });
257       if (Idx != Vec.end()) {
258         int FI = Idx->second;
259         LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "
260                           << printReg(Reg, &TRI) << " at "
261                           << printMBBReference(*EHPad) << "\n");
262         assert(ReservedSlots.count(FI) && "using unreserved slot");
263         return FI;
264       }
265     }
266 
267     unsigned Size = getRegisterSize(TRI, Reg);
268     FrameIndexesPerSize &Line = getCacheBucket(Size);
269     while (Line.Index < Line.Slots.size()) {
270       int FI = Line.Slots[Line.Index++];
271       if (ReservedSlots.count(FI))
272         continue;
273       // If all sizes are kept together we probably need to extend the
274       // spill slot size.
275       if (MFI.getObjectSize(FI) < Size) {
276         MFI.setObjectSize(FI, Size);
277         MFI.setObjectAlignment(FI, Align(Size));
278         NumSpillSlotsExtended++;
279       }
280       return FI;
281     }
282     int FI = MFI.CreateSpillStackObject(Size, Align(Size));
283     NumSpillSlotsAllocated++;
284     Line.Slots.push_back(FI);
285     ++Line.Index;
286 
287     // Remember assignment {Reg, FI} for EHPad
288     if (EHPad) {
289       GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));
290       LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "
291                         << printReg(Reg, &TRI) << " at landing pad "
292                         << printMBBReference(*EHPad) << "\n");
293     }
294 
295     return FI;
296   }
297 
298   // Sort all registers to spill in descendent order. In the
299   // FixupSCSExtendSlotSize mode it will minimize the total frame size.
300   // In non FixupSCSExtendSlotSize mode we can skip this step.
301   void sortRegisters(SmallVectorImpl<Register> &Regs) {
302     if (!FixupSCSExtendSlotSize)
303       return;
304     llvm::sort(Regs, [&](Register &A, Register &B) {
305       return getRegisterSize(TRI, A) > getRegisterSize(TRI, B);
306     });
307   }
308 };
309 
310 // Describes the state of the current processing statepoint instruction.
311 class StatepointState {
312 private:
313   // statepoint instruction.
314   MachineInstr &MI;
315   MachineFunction &MF;
316   // If non-null then statepoint is invoke, and this points to the landing pad.
317   MachineBasicBlock *EHPad;
318   const TargetRegisterInfo &TRI;
319   const TargetInstrInfo &TII;
320   MachineFrameInfo &MFI;
321   // Mask with callee saved registers.
322   const uint32_t *Mask;
323   // Cache of frame indexes used on previous instruction processing.
324   FrameIndexesCache &CacheFI;
325   bool AllowGCPtrInCSR;
326   // Operands with physical registers requiring spilling.
327   SmallVector<unsigned, 8> OpsToSpill;
328   // Set of register to spill.
329   SmallVector<Register, 8> RegsToSpill;
330   // Set of registers to reload after statepoint.
331   SmallVector<Register, 8> RegsToReload;
332   // Map Register to Frame Slot index.
333   DenseMap<Register, int> RegToSlotIdx;
334 
335 public:
336   StatepointState(MachineInstr &MI, const uint32_t *Mask,
337                   FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)
338       : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),
339         TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
340         Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
341 
342     // Find statepoint's landing pad, if any.
343     EHPad = nullptr;
344     MachineBasicBlock *MBB = MI.getParent();
345     // Invoke statepoint must be last one in block.
346     bool Last = std::none_of(++MI.getIterator(), MBB->end().getInstrIterator(),
347                              [](MachineInstr &I) {
348                                return I.getOpcode() == TargetOpcode::STATEPOINT;
349                              });
350 
351     if (!Last)
352       return;
353 
354     auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };
355 
356     assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads");
357 
358     auto It = llvm::find_if(MBB->successors(), IsEHPad);
359     if (It != MBB->succ_end())
360       EHPad = *It;
361   }
362 
363   MachineBasicBlock *getEHPad() const { return EHPad; }
364 
365   // Return true if register is callee saved.
366   bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; }
367 
368   // Iterates over statepoint meta args to find caller saver registers.
369   // Also cache the size of found registers.
370   // Returns true if caller save registers found.
371   bool findRegistersToSpill() {
372     SmallSet<Register, 8> GCRegs;
373     // All GC pointer operands assigned to registers produce new value.
374     // Since they're tied to their defs, it is enough to collect def registers.
375     for (const auto &Def : MI.defs())
376       GCRegs.insert(Def.getReg());
377 
378     SmallSet<Register, 8> VisitedRegs;
379     for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),
380                   EndIdx = MI.getNumOperands();
381          Idx < EndIdx; ++Idx) {
382       MachineOperand &MO = MI.getOperand(Idx);
383       // Leave `undef` operands as is, StackMaps will rewrite them
384       // into a constant.
385       if (!MO.isReg() || MO.isImplicit() || MO.isUndef())
386         continue;
387       Register Reg = MO.getReg();
388       assert(Reg.isPhysical() && "Only physical regs are expected");
389 
390       if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !is_contained(GCRegs, Reg)))
391         continue;
392 
393       LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index "
394                         << Idx << "\n");
395 
396       if (VisitedRegs.insert(Reg).second)
397         RegsToSpill.push_back(Reg);
398       OpsToSpill.push_back(Idx);
399     }
400     CacheFI.sortRegisters(RegsToSpill);
401     return !RegsToSpill.empty();
402   }
403 
404   // Spill all caller saved registers right before statepoint instruction.
405   // Remember frame index where register is spilled.
406   void spillRegisters() {
407     for (Register Reg : RegsToSpill) {
408       int FI = CacheFI.getFrameIndex(Reg, EHPad);
409       const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
410 
411       NumSpilledRegisters++;
412       RegToSlotIdx[Reg] = FI;
413 
414       LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI
415                         << "\n");
416 
417       // Perform trivial copy propagation
418       bool IsKill = true;
419       MachineBasicBlock::iterator InsertBefore(MI);
420       Reg = performCopyPropagation(Reg, InsertBefore, IsKill, TII, TRI);
421 
422       LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);
423       TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI,
424                               RC, &TRI);
425     }
426   }
427 
428   void insertReloadBefore(unsigned Reg, MachineBasicBlock::iterator It,
429                           MachineBasicBlock *MBB) {
430     const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
431     int FI = RegToSlotIdx[Reg];
432     if (It != MBB->end()) {
433       TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
434       return;
435     }
436 
437     // To insert reload at the end of MBB, insert it before last instruction
438     // and then swap them.
439     assert(!MBB->empty() && "Empty block");
440     --It;
441     TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
442     MachineInstr *Reload = It->getPrevNode();
443     int Dummy = 0;
444     (void)Dummy;
445     assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);
446     assert(Dummy == FI);
447     MBB->remove(Reload);
448     MBB->insertAfter(It, Reload);
449   }
450 
451   // Insert reloads of (relocated) registers spilled in statepoint.
452   void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {
453     MachineBasicBlock *MBB = NewStatepoint->getParent();
454     auto InsertPoint = std::next(NewStatepoint->getIterator());
455 
456     for (auto Reg : RegsToReload) {
457       insertReloadBefore(Reg, InsertPoint, MBB);
458       LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI "
459                         << RegToSlotIdx[Reg] << " after statepoint\n");
460 
461       if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) {
462         RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad);
463         auto EHPadInsertPoint = EHPad->SkipPHIsLabelsAndDebug(EHPad->begin());
464         insertReloadBefore(Reg, EHPadInsertPoint, EHPad);
465         LLVM_DEBUG(dbgs() << "...also reload at EHPad "
466                           << printMBBReference(*EHPad) << "\n");
467       }
468     }
469   }
470 
471   // Re-write statepoint machine instruction to replace caller saved operands
472   // with indirect memory location (frame index).
473   MachineInstr *rewriteStatepoint() {
474     MachineInstr *NewMI =
475         MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
476     MachineInstrBuilder MIB(MF, NewMI);
477 
478     unsigned NumOps = MI.getNumOperands();
479 
480     // New indices for the remaining defs.
481     SmallVector<unsigned, 8> NewIndices;
482     unsigned NumDefs = MI.getNumDefs();
483     for (unsigned I = 0; I < NumDefs; ++I) {
484       MachineOperand &DefMO = MI.getOperand(I);
485       assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");
486       Register Reg = DefMO.getReg();
487       if (!AllowGCPtrInCSR) {
488         assert(is_contained(RegsToSpill, Reg));
489         RegsToReload.push_back(Reg);
490       } else {
491         if (isCalleeSaved(Reg)) {
492           NewIndices.push_back(NewMI->getNumOperands());
493           MIB.addReg(Reg, RegState::Define);
494         } else {
495           NewIndices.push_back(NumOps);
496           RegsToReload.push_back(Reg);
497         }
498       }
499     }
500 
501     // Add End marker.
502     OpsToSpill.push_back(MI.getNumOperands());
503     unsigned CurOpIdx = 0;
504 
505     for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {
506       MachineOperand &MO = MI.getOperand(I);
507       if (I == OpsToSpill[CurOpIdx]) {
508         int FI = RegToSlotIdx[MO.getReg()];
509         MIB.addImm(StackMaps::IndirectMemRefOp);
510         MIB.addImm(getRegisterSize(TRI, MO.getReg()));
511         assert(MO.isReg() && "Should be register");
512         assert(MO.getReg().isPhysical() && "Should be physical register");
513         MIB.addFrameIndex(FI);
514         MIB.addImm(0);
515         ++CurOpIdx;
516       } else {
517         MIB.add(MO);
518         unsigned OldDef;
519         if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) {
520           assert(OldDef < NumDefs);
521           assert(NewIndices[OldDef] < NumOps);
522           MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);
523         }
524       }
525     }
526     assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");
527     // Add mem operands.
528     NewMI->setMemRefs(MF, MI.memoperands());
529     for (auto It : RegToSlotIdx) {
530       Register R = It.first;
531       int FrameIndex = It.second;
532       auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
533       MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
534       if (is_contained(RegsToReload, R))
535         Flags |= MachineMemOperand::MOStore;
536       auto *MMO =
537           MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, R),
538                                   MFI.getObjectAlign(FrameIndex));
539       NewMI->addMemOperand(MF, MMO);
540     }
541 
542     // Insert new statepoint and erase old one.
543     MI.getParent()->insert(MI, NewMI);
544 
545     LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");
546     MI.eraseFromParent();
547     return NewMI;
548   }
549 };
550 
551 class StatepointProcessor {
552 private:
553   MachineFunction &MF;
554   const TargetRegisterInfo &TRI;
555   FrameIndexesCache CacheFI;
556   RegReloadCache ReloadCache;
557 
558 public:
559   StatepointProcessor(MachineFunction &MF)
560       : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),
561         CacheFI(MF.getFrameInfo(), TRI) {}
562 
563   bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {
564     StatepointOpers SO(&MI);
565     uint64_t Flags = SO.getFlags();
566     // Do nothing for LiveIn, it supports all registers.
567     if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)
568       return false;
569     LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "
570                       << MI.getParent()->getName() << " : process statepoint "
571                       << MI);
572     CallingConv::ID CC = SO.getCallingConv();
573     const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);
574     StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);
575     CacheFI.reset(SS.getEHPad());
576 
577     if (!SS.findRegistersToSpill())
578       return false;
579 
580     SS.spillRegisters();
581     auto *NewStatepoint = SS.rewriteStatepoint();
582     SS.insertReloads(NewStatepoint, ReloadCache);
583     return true;
584   }
585 };
586 } // namespace
587 
588 bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) {
589   if (skipFunction(MF.getFunction()))
590     return false;
591 
592   const Function &F = MF.getFunction();
593   if (!F.hasGC())
594     return false;
595 
596   SmallVector<MachineInstr *, 16> Statepoints;
597   for (MachineBasicBlock &BB : MF)
598     for (MachineInstr &I : BB)
599       if (I.getOpcode() == TargetOpcode::STATEPOINT)
600         Statepoints.push_back(&I);
601 
602   if (Statepoints.empty())
603     return false;
604 
605   bool Changed = false;
606   StatepointProcessor SPP(MF);
607   unsigned NumStatepoints = 0;
608   bool AllowGCPtrInCSR = PassGCPtrInCSR;
609   for (MachineInstr *I : Statepoints) {
610     ++NumStatepoints;
611     if (MaxStatepointsWithRegs.getNumOccurrences() &&
612         NumStatepoints >= MaxStatepointsWithRegs)
613       AllowGCPtrInCSR = false;
614     Changed |= SPP.process(*I, AllowGCPtrInCSR);
615   }
616   return Changed;
617 }
618