1 //===-- AMDGPUMarkLastScratchLoad.cpp -------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Mark scratch load/spill instructions which are guaranteed to be the last time 10 // this scratch slot is used so it can be evicted from caches. 11 // 12 // TODO: Handle general stack accesses not just spilling. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #include "AMDGPU.h" 17 #include "GCNSubtarget.h" 18 #include "llvm/CodeGen/LiveIntervals.h" 19 #include "llvm/CodeGen/LiveStacks.h" 20 #include "llvm/CodeGen/MachineOperand.h" 21 22 using namespace llvm; 23 24 #define DEBUG_TYPE "amdgpu-mark-last-scratch-load" 25 26 namespace { 27 28 class AMDGPUMarkLastScratchLoad : public MachineFunctionPass { 29 private: 30 LiveStacks *LS = nullptr; 31 LiveIntervals *LIS = nullptr; 32 SlotIndexes *SI = nullptr; 33 const SIInstrInfo *SII = nullptr; 34 35 public: 36 static char ID; 37 38 AMDGPUMarkLastScratchLoad() : MachineFunctionPass(ID) { 39 initializeAMDGPUMarkLastScratchLoadPass(*PassRegistry::getPassRegistry()); 40 } 41 42 bool runOnMachineFunction(MachineFunction &MF) override; 43 44 void getAnalysisUsage(AnalysisUsage &AU) const override { 45 AU.addRequired<SlotIndexes>(); 46 AU.addRequired<LiveIntervals>(); 47 AU.addRequired<LiveStacks>(); 48 AU.setPreservesAll(); 49 MachineFunctionPass::getAnalysisUsage(AU); 50 } 51 52 StringRef getPassName() const override { 53 return "AMDGPU Mark Last Scratch Load"; 54 } 55 }; 56 57 } // end anonymous namespace 58 59 bool AMDGPUMarkLastScratchLoad::runOnMachineFunction(MachineFunction &MF) { 60 if (skipFunction(MF.getFunction())) 61 return false; 62 63 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 64 if (ST.getGeneration() < AMDGPUSubtarget::GFX12) 65 return false; 66 67 LS = &getAnalysis<LiveStacks>(); 68 LIS = &getAnalysis<LiveIntervals>(); 69 SI = &getAnalysis<SlotIndexes>(); 70 SII = ST.getInstrInfo(); 71 SlotIndexes &Slots = *LIS->getSlotIndexes(); 72 73 const unsigned NumSlots = LS->getNumIntervals(); 74 if (NumSlots == 0) { 75 LLVM_DEBUG(dbgs() << "No live slots, skipping\n"); 76 return false; 77 } 78 79 LLVM_DEBUG(dbgs() << LS->getNumIntervals() << " intervals\n"); 80 81 bool Changed = false; 82 83 for (auto &[SS, LI] : *LS) { 84 for (const LiveRange::Segment &Segment : LI.segments) { 85 86 // Ignore segments that run to the end of basic block because in this case 87 // slot is still live at the end of it. 88 if (Segment.end.isBlock()) 89 continue; 90 91 const int FrameIndex = Register::stackSlot2Index(LI.reg()); 92 MachineInstr *LastLoad = nullptr; 93 94 MachineInstr *MISegmentEnd = SI->getInstructionFromIndex(Segment.end); 95 96 // If there is no instruction at this slot because it was deleted take the 97 // instruction from the next slot. 98 if (!MISegmentEnd) { 99 SlotIndex NextSlot = Slots.getNextNonNullIndex(Segment.end); 100 MISegmentEnd = SI->getInstructionFromIndex(NextSlot); 101 } 102 103 MachineInstr *MISegmentStart = SI->getInstructionFromIndex(Segment.start); 104 MachineBasicBlock *BB = MISegmentEnd->getParent(); 105 106 // Start iteration backwards from segment end until the start of basic 107 // block or start of segment if it is in the same basic block. 108 auto End = BB->rend(); 109 if (MISegmentStart && MISegmentStart->getParent() == BB) 110 End = MISegmentStart->getReverseIterator(); 111 112 for (auto MI = MISegmentEnd->getReverseIterator(); MI != End; ++MI) { 113 int LoadFI = 0; 114 115 if (SII->isLoadFromStackSlot(*MI, LoadFI) && LoadFI == FrameIndex) { 116 LastLoad = &*MI; 117 break; 118 } 119 } 120 121 if (LastLoad && !LastLoad->memoperands_empty()) { 122 MachineMemOperand *MMO = *LastLoad->memoperands_begin(); 123 MMO->setFlags(MOLastUse); 124 Changed = true; 125 LLVM_DEBUG(dbgs() << " Found last load: " << *LastLoad); 126 } 127 } 128 } 129 130 return Changed; 131 } 132 133 char AMDGPUMarkLastScratchLoad::ID = 0; 134 135 char &llvm::AMDGPUMarkLastScratchLoadID = AMDGPUMarkLastScratchLoad::ID; 136 137 INITIALIZE_PASS_BEGIN(AMDGPUMarkLastScratchLoad, DEBUG_TYPE, 138 "AMDGPU Mark last scratch load", false, false) 139 INITIALIZE_PASS_DEPENDENCY(SlotIndexes) 140 INITIALIZE_PASS_DEPENDENCY(LiveStacks) 141 INITIALIZE_PASS_END(AMDGPUMarkLastScratchLoad, DEBUG_TYPE, 142 "AMDGPU Mark last scratch load", false, false) 143