1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file defines the GCNRegPressure class, which tracks registry pressure 11 /// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It 12 /// also implements a compare function, which compares different register 13 /// pressures, and declares one with max occupancy as winner. 14 /// 15 //===----------------------------------------------------------------------===// 16 17 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 18 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 19 20 #include "GCNSubtarget.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include <algorithm> 23 24 namespace llvm { 25 26 class MachineRegisterInfo; 27 class raw_ostream; 28 class SlotIndex; 29 30 struct GCNRegPressure { 31 enum RegKind { 32 SGPR32, 33 SGPR_TUPLE, 34 VGPR32, 35 VGPR_TUPLE, 36 AGPR32, 37 AGPR_TUPLE, 38 TOTAL_KINDS 39 }; 40 41 GCNRegPressure() { 42 clear(); 43 } 44 45 bool empty() const { return getSGPRNum() == 0 && getVGPRNum(false) == 0; } 46 47 void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); } 48 49 unsigned getSGPRNum() const { return Value[SGPR32]; } 50 unsigned getVGPRNum(bool UnifiedVGPRFile) const { 51 if (UnifiedVGPRFile) { 52 return Value[AGPR32] ? alignTo(Value[VGPR32], 4) + Value[AGPR32] 53 : Value[VGPR32] + Value[AGPR32]; 54 } 55 return std::max(Value[VGPR32], Value[AGPR32]); 56 } 57 unsigned getAGPRNum() const { return Value[AGPR32]; } 58 59 unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE], 60 Value[AGPR_TUPLE]); } 61 unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; } 62 63 unsigned getOccupancy(const GCNSubtarget &ST) const { 64 return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()), 65 ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts()))); 66 } 67 68 void inc(unsigned Reg, 69 LaneBitmask PrevMask, 70 LaneBitmask NewMask, 71 const MachineRegisterInfo &MRI); 72 73 bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const { 74 return getOccupancy(ST) > O.getOccupancy(ST); 75 } 76 77 bool less(const GCNSubtarget &ST, const GCNRegPressure& O, 78 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const; 79 80 bool operator==(const GCNRegPressure &O) const { 81 return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value); 82 } 83 84 bool operator!=(const GCNRegPressure &O) const { 85 return !(*this == O); 86 } 87 88 void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const; 89 void dump() const { print(dbgs()); } 90 91 private: 92 unsigned Value[TOTAL_KINDS]; 93 94 static unsigned getRegKind(Register Reg, const MachineRegisterInfo &MRI); 95 96 friend GCNRegPressure max(const GCNRegPressure &P1, 97 const GCNRegPressure &P2); 98 }; 99 100 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) { 101 GCNRegPressure Res; 102 for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I) 103 Res.Value[I] = std::max(P1.Value[I], P2.Value[I]); 104 return Res; 105 } 106 107 class GCNRPTracker { 108 public: 109 using LiveRegSet = DenseMap<unsigned, LaneBitmask>; 110 111 protected: 112 const LiveIntervals &LIS; 113 LiveRegSet LiveRegs; 114 GCNRegPressure CurPressure, MaxPressure; 115 const MachineInstr *LastTrackedMI = nullptr; 116 mutable const MachineRegisterInfo *MRI = nullptr; 117 118 GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {} 119 120 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy, 121 bool After); 122 123 public: 124 // live regs for the current state 125 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; } 126 const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; } 127 128 void clearMaxPressure() { MaxPressure.clear(); } 129 130 // returns MaxPressure, resetting it 131 decltype(MaxPressure) moveMaxPressure() { 132 auto Res = MaxPressure; 133 MaxPressure.clear(); 134 return Res; 135 } 136 137 decltype(LiveRegs) moveLiveRegs() { 138 return std::move(LiveRegs); 139 } 140 141 static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs, 142 const MachineRegisterInfo &MRI); 143 }; 144 145 class GCNUpwardRPTracker : public GCNRPTracker { 146 public: 147 GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} 148 149 // reset tracker to the point just below MI 150 // filling live regs upon this point using LIS 151 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr); 152 153 // move to the state just above the MI 154 void recede(const MachineInstr &MI); 155 156 // checks whether the tracker's state after receding MI corresponds 157 // to reported by LIS 158 bool isValid() const; 159 }; 160 161 class GCNDownwardRPTracker : public GCNRPTracker { 162 // Last position of reset or advanceBeforeNext 163 MachineBasicBlock::const_iterator NextMI; 164 165 MachineBasicBlock::const_iterator MBBEnd; 166 167 public: 168 GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} 169 170 MachineBasicBlock::const_iterator getNext() const { return NextMI; } 171 172 // Reset tracker to the point before the MI 173 // filling live regs upon this point using LIS. 174 // Returns false if block is empty except debug values. 175 bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr); 176 177 // Move to the state right before the next MI. Returns false if reached 178 // end of the block. 179 bool advanceBeforeNext(); 180 181 // Move to the state at the MI, advanceBeforeNext has to be called first. 182 void advanceToNext(); 183 184 // Move to the state at the next MI. Returns false if reached end of block. 185 bool advance(); 186 187 // Advance instructions until before End. 188 bool advance(MachineBasicBlock::const_iterator End); 189 190 // Reset to Begin and advance to End. 191 bool advance(MachineBasicBlock::const_iterator Begin, 192 MachineBasicBlock::const_iterator End, 193 const LiveRegSet *LiveRegsCopy = nullptr); 194 }; 195 196 LaneBitmask getLiveLaneMask(unsigned Reg, 197 SlotIndex SI, 198 const LiveIntervals &LIS, 199 const MachineRegisterInfo &MRI); 200 201 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, 202 const LiveIntervals &LIS, 203 const MachineRegisterInfo &MRI); 204 205 /// creates a map MachineInstr -> LiveRegSet 206 /// R - range of iterators on instructions 207 /// After - upon entry or exit of every instruction 208 /// Note: there is no entry in the map for instructions with empty live reg set 209 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R)) 210 template <typename Range> 211 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet> 212 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) { 213 std::vector<SlotIndex> Indexes; 214 Indexes.reserve(std::distance(R.begin(), R.end())); 215 auto &SII = *LIS.getSlotIndexes(); 216 for (MachineInstr *I : R) { 217 auto SI = SII.getInstructionIndex(*I); 218 Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex()); 219 } 220 llvm::sort(Indexes); 221 222 auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo(); 223 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap; 224 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs; 225 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 226 auto Reg = Register::index2VirtReg(I); 227 if (!LIS.hasInterval(Reg)) 228 continue; 229 auto &LI = LIS.getInterval(Reg); 230 LiveIdxs.clear(); 231 if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs))) 232 continue; 233 if (!LI.hasSubRanges()) { 234 for (auto SI : LiveIdxs) 235 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] = 236 MRI.getMaxLaneMaskForVReg(Reg); 237 } else 238 for (const auto &S : LI.subranges()) { 239 // constrain search for subranges by indexes live at main range 240 SRLiveIdxs.clear(); 241 S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs)); 242 for (auto SI : SRLiveIdxs) 243 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask; 244 } 245 } 246 return LiveRegMap; 247 } 248 249 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, 250 const LiveIntervals &LIS) { 251 return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS, 252 MI.getParent()->getParent()->getRegInfo()); 253 } 254 255 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, 256 const LiveIntervals &LIS) { 257 return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS, 258 MI.getParent()->getParent()->getRegInfo()); 259 } 260 261 template <typename Range> 262 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, 263 Range &&LiveRegs) { 264 GCNRegPressure Res; 265 for (const auto &RM : LiveRegs) 266 Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI); 267 return Res; 268 } 269 270 bool isEqual(const GCNRPTracker::LiveRegSet &S1, 271 const GCNRPTracker::LiveRegSet &S2); 272 273 void printLivesAt(SlotIndex SI, 274 const LiveIntervals &LIS, 275 const MachineRegisterInfo &MRI); 276 277 } // end namespace llvm 278 279 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 280