1 //===-- AMDGPUMarkLastScratchLoad.cpp -------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Mark scratch load/spill instructions which are guaranteed to be the last time 10 // this scratch slot is used so it can be evicted from caches. 11 // 12 // TODO: Handle general stack accesses not just spilling. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #include "AMDGPU.h" 17 #include "GCNSubtarget.h" 18 #include "llvm/CodeGen/LiveIntervals.h" 19 #include "llvm/CodeGen/LiveStacks.h" 20 #include "llvm/CodeGen/MachineOperand.h" 21 22 using namespace llvm; 23 24 #define DEBUG_TYPE "amdgpu-mark-last-scratch-load" 25 26 namespace { 27 28 class AMDGPUMarkLastScratchLoad { 29 private: 30 LiveStacks *LS = nullptr; 31 LiveIntervals *LIS = nullptr; 32 SlotIndexes *SI = nullptr; 33 const SIInstrInfo *SII = nullptr; 34 35 public: 36 AMDGPUMarkLastScratchLoad(LiveStacks *LS, LiveIntervals *LIS, SlotIndexes *SI) 37 : LS(LS), LIS(LIS), SI(SI) {} 38 bool run(MachineFunction &MF); 39 }; 40 41 class AMDGPUMarkLastScratchLoadLegacy : public MachineFunctionPass { 42 public: 43 static char ID; 44 45 AMDGPUMarkLastScratchLoadLegacy() : MachineFunctionPass(ID) {} 46 47 bool runOnMachineFunction(MachineFunction &MF) override; 48 49 void getAnalysisUsage(AnalysisUsage &AU) const override { 50 AU.addRequired<SlotIndexesWrapperPass>(); 51 AU.addRequired<LiveIntervalsWrapperPass>(); 52 AU.addRequired<LiveStacksWrapperLegacy>(); 53 AU.setPreservesAll(); 54 MachineFunctionPass::getAnalysisUsage(AU); 55 } 56 57 StringRef getPassName() const override { 58 return "AMDGPU Mark Last Scratch Load"; 59 } 60 }; 61 62 } // end anonymous namespace 63 64 bool AMDGPUMarkLastScratchLoadLegacy::runOnMachineFunction( 65 MachineFunction &MF) { 66 if (skipFunction(MF.getFunction())) 67 return false; 68 69 auto &LS = getAnalysis<LiveStacksWrapperLegacy>().getLS(); 70 auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS(); 71 auto &SI = getAnalysis<SlotIndexesWrapperPass>().getSI(); 72 73 return AMDGPUMarkLastScratchLoad(&LS, &LIS, &SI).run(MF); 74 } 75 76 PreservedAnalyses 77 AMDGPUMarkLastScratchLoadPass::run(MachineFunction &MF, 78 MachineFunctionAnalysisManager &MFAM) { 79 auto &LS = MFAM.getResult<LiveStacksAnalysis>(MF); 80 auto &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF); 81 auto &SI = MFAM.getResult<SlotIndexesAnalysis>(MF); 82 83 AMDGPUMarkLastScratchLoad(&LS, &LIS, &SI).run(MF); 84 return PreservedAnalyses::all(); 85 } 86 87 bool AMDGPUMarkLastScratchLoad::run(MachineFunction &MF) { 88 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 89 if (ST.getGeneration() < AMDGPUSubtarget::GFX12) 90 return false; 91 92 SII = ST.getInstrInfo(); 93 SlotIndexes &Slots = *LIS->getSlotIndexes(); 94 95 const unsigned NumSlots = LS->getNumIntervals(); 96 if (NumSlots == 0) { 97 LLVM_DEBUG(dbgs() << "No live slots, skipping\n"); 98 return false; 99 } 100 101 LLVM_DEBUG(dbgs() << LS->getNumIntervals() << " intervals\n"); 102 103 bool Changed = false; 104 105 for (auto &[SS, LI] : *LS) { 106 for (const LiveRange::Segment &Segment : LI.segments) { 107 108 // Ignore segments that run to the end of basic block because in this case 109 // slot is still live at the end of it. 110 if (Segment.end.isBlock()) 111 continue; 112 113 const int FrameIndex = LI.reg().stackSlotIndex(); 114 MachineInstr *LastLoad = nullptr; 115 116 MachineInstr *MISegmentEnd = SI->getInstructionFromIndex(Segment.end); 117 118 // If there is no instruction at this slot because it was deleted take the 119 // instruction from the next slot. 120 if (!MISegmentEnd) { 121 SlotIndex NextSlot = Slots.getNextNonNullIndex(Segment.end); 122 MISegmentEnd = SI->getInstructionFromIndex(NextSlot); 123 } 124 125 MachineInstr *MISegmentStart = SI->getInstructionFromIndex(Segment.start); 126 MachineBasicBlock *BB = MISegmentEnd->getParent(); 127 128 // Start iteration backwards from segment end until the start of basic 129 // block or start of segment if it is in the same basic block. 130 auto End = BB->rend(); 131 if (MISegmentStart && MISegmentStart->getParent() == BB) 132 End = MISegmentStart->getReverseIterator(); 133 134 for (auto MI = MISegmentEnd->getReverseIterator(); MI != End; ++MI) { 135 int LoadFI = 0; 136 137 if (SII->isLoadFromStackSlot(*MI, LoadFI) && LoadFI == FrameIndex) { 138 LastLoad = &*MI; 139 break; 140 } 141 } 142 143 if (LastLoad && !LastLoad->memoperands_empty()) { 144 MachineMemOperand *MMO = *LastLoad->memoperands_begin(); 145 MMO->setFlags(MOLastUse); 146 Changed = true; 147 LLVM_DEBUG(dbgs() << " Found last load: " << *LastLoad); 148 } 149 } 150 } 151 152 return Changed; 153 } 154 155 char AMDGPUMarkLastScratchLoadLegacy::ID = 0; 156 157 char &llvm::AMDGPUMarkLastScratchLoadID = AMDGPUMarkLastScratchLoadLegacy::ID; 158 159 INITIALIZE_PASS_BEGIN(AMDGPUMarkLastScratchLoadLegacy, DEBUG_TYPE, 160 "AMDGPU Mark last scratch load", false, false) 161 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass) 162 INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy) 163 INITIALIZE_PASS_END(AMDGPUMarkLastScratchLoadLegacy, DEBUG_TYPE, 164 "AMDGPU Mark last scratch load", false, false) 165