1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 10 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 11 12 #include "AMDGPUSubtarget.h" 13 #include "llvm/ADT/DenseMap.h" 14 #include "llvm/CodeGen/LiveIntervals.h" 15 #include "llvm/CodeGen/MachineBasicBlock.h" 16 #include "llvm/CodeGen/MachineInstr.h" 17 #include "llvm/CodeGen/SlotIndexes.h" 18 #include "llvm/MC/LaneBitmask.h" 19 #include "llvm/Support/Debug.h" 20 #include <algorithm> 21 #include <limits> 22 23 namespace llvm { 24 25 class MachineRegisterInfo; 26 class raw_ostream; 27 28 struct GCNRegPressure { 29 enum RegKind { 30 SGPR32, 31 SGPR_TUPLE, 32 VGPR32, 33 VGPR_TUPLE, 34 AGPR32, 35 AGPR_TUPLE, 36 TOTAL_KINDS 37 }; 38 39 GCNRegPressure() { 40 clear(); 41 } 42 43 bool empty() const { return getSGPRNum() == 0 && getVGPRNum() == 0; } 44 45 void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); } 46 47 unsigned getSGPRNum() const { return Value[SGPR32]; } 48 unsigned getVGPRNum() const { return std::max(Value[VGPR32], Value[AGPR32]); } 49 50 unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE], 51 Value[AGPR_TUPLE]); } 52 unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; } 53 54 unsigned getOccupancy(const GCNSubtarget &ST) const { 55 return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()), 56 ST.getOccupancyWithNumVGPRs(getVGPRNum())); 57 } 58 59 void inc(unsigned Reg, 60 LaneBitmask PrevMask, 61 LaneBitmask NewMask, 62 const MachineRegisterInfo &MRI); 63 64 bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const { 65 return getOccupancy(ST) > O.getOccupancy(ST); 66 } 67 68 bool less(const GCNSubtarget &ST, const GCNRegPressure& O, 69 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const; 70 71 bool operator==(const GCNRegPressure &O) const { 72 return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value); 73 } 74 75 bool operator!=(const GCNRegPressure &O) const { 76 return !(*this == O); 77 } 78 79 void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const; 80 void dump() const { print(dbgs()); } 81 82 private: 83 unsigned Value[TOTAL_KINDS]; 84 85 static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI); 86 87 friend GCNRegPressure max(const GCNRegPressure &P1, 88 const GCNRegPressure &P2); 89 }; 90 91 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) { 92 GCNRegPressure Res; 93 for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I) 94 Res.Value[I] = std::max(P1.Value[I], P2.Value[I]); 95 return Res; 96 } 97 98 class GCNRPTracker { 99 public: 100 using LiveRegSet = DenseMap<unsigned, LaneBitmask>; 101 102 protected: 103 const LiveIntervals &LIS; 104 LiveRegSet LiveRegs; 105 GCNRegPressure CurPressure, MaxPressure; 106 const MachineInstr *LastTrackedMI = nullptr; 107 mutable const MachineRegisterInfo *MRI = nullptr; 108 109 GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {} 110 111 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy, 112 bool After); 113 114 public: 115 // live regs for the current state 116 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; } 117 const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; } 118 119 void clearMaxPressure() { MaxPressure.clear(); } 120 121 // returns MaxPressure, resetting it 122 decltype(MaxPressure) moveMaxPressure() { 123 auto Res = MaxPressure; 124 MaxPressure.clear(); 125 return Res; 126 } 127 128 decltype(LiveRegs) moveLiveRegs() { 129 return std::move(LiveRegs); 130 } 131 132 static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs, 133 const MachineRegisterInfo &MRI); 134 }; 135 136 class GCNUpwardRPTracker : public GCNRPTracker { 137 public: 138 GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} 139 140 // reset tracker to the point just below MI 141 // filling live regs upon this point using LIS 142 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr); 143 144 // move to the state just above the MI 145 void recede(const MachineInstr &MI); 146 147 // checks whether the tracker's state after receding MI corresponds 148 // to reported by LIS 149 bool isValid() const; 150 }; 151 152 class GCNDownwardRPTracker : public GCNRPTracker { 153 // Last position of reset or advanceBeforeNext 154 MachineBasicBlock::const_iterator NextMI; 155 156 MachineBasicBlock::const_iterator MBBEnd; 157 158 public: 159 GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} 160 161 const MachineBasicBlock::const_iterator getNext() const { return NextMI; } 162 163 // Reset tracker to the point before the MI 164 // filling live regs upon this point using LIS. 165 // Returns false if block is empty except debug values. 166 bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr); 167 168 // Move to the state right before the next MI. Returns false if reached 169 // end of the block. 170 bool advanceBeforeNext(); 171 172 // Move to the state at the MI, advanceBeforeNext has to be called first. 173 void advanceToNext(); 174 175 // Move to the state at the next MI. Returns false if reached end of block. 176 bool advance(); 177 178 // Advance instructions until before End. 179 bool advance(MachineBasicBlock::const_iterator End); 180 181 // Reset to Begin and advance to End. 182 bool advance(MachineBasicBlock::const_iterator Begin, 183 MachineBasicBlock::const_iterator End, 184 const LiveRegSet *LiveRegsCopy = nullptr); 185 }; 186 187 LaneBitmask getLiveLaneMask(unsigned Reg, 188 SlotIndex SI, 189 const LiveIntervals &LIS, 190 const MachineRegisterInfo &MRI); 191 192 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, 193 const LiveIntervals &LIS, 194 const MachineRegisterInfo &MRI); 195 196 /// creates a map MachineInstr -> LiveRegSet 197 /// R - range of iterators on instructions 198 /// After - upon entry or exit of every instruction 199 /// Note: there is no entry in the map for instructions with empty live reg set 200 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R)) 201 template <typename Range> 202 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet> 203 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) { 204 std::vector<SlotIndex> Indexes; 205 Indexes.reserve(std::distance(R.begin(), R.end())); 206 auto &SII = *LIS.getSlotIndexes(); 207 for (MachineInstr *I : R) { 208 auto SI = SII.getInstructionIndex(*I); 209 Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex()); 210 } 211 std::sort(Indexes.begin(), Indexes.end()); 212 213 auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo(); 214 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap; 215 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs; 216 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 217 auto Reg = Register::index2VirtReg(I); 218 if (!LIS.hasInterval(Reg)) 219 continue; 220 auto &LI = LIS.getInterval(Reg); 221 LiveIdxs.clear(); 222 if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs))) 223 continue; 224 if (!LI.hasSubRanges()) { 225 for (auto SI : LiveIdxs) 226 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] = 227 MRI.getMaxLaneMaskForVReg(Reg); 228 } else 229 for (const auto &S : LI.subranges()) { 230 // constrain search for subranges by indexes live at main range 231 SRLiveIdxs.clear(); 232 S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs)); 233 for (auto SI : SRLiveIdxs) 234 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask; 235 } 236 } 237 return LiveRegMap; 238 } 239 240 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, 241 const LiveIntervals &LIS) { 242 return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS, 243 MI.getParent()->getParent()->getRegInfo()); 244 } 245 246 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, 247 const LiveIntervals &LIS) { 248 return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS, 249 MI.getParent()->getParent()->getRegInfo()); 250 } 251 252 template <typename Range> 253 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, 254 Range &&LiveRegs) { 255 GCNRegPressure Res; 256 for (const auto &RM : LiveRegs) 257 Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI); 258 return Res; 259 } 260 261 bool isEqual(const GCNRPTracker::LiveRegSet &S1, 262 const GCNRPTracker::LiveRegSet &S2); 263 264 void printLivesAt(SlotIndex SI, 265 const LiveIntervals &LIS, 266 const MachineRegisterInfo &MRI); 267 268 } // end namespace llvm 269 270 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 271