1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file defines the GCNRegPressure class, which tracks registry pressure 11 /// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It 12 /// also implements a compare function, which compares different register 13 /// pressures, and declares one with max occupance as winner. 14 /// 15 //===----------------------------------------------------------------------===// 16 17 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 18 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 19 20 #include "AMDGPUSubtarget.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/CodeGen/LiveIntervals.h" 23 #include "llvm/CodeGen/MachineBasicBlock.h" 24 #include "llvm/CodeGen/MachineInstr.h" 25 #include "llvm/CodeGen/SlotIndexes.h" 26 #include "llvm/MC/LaneBitmask.h" 27 #include "llvm/Support/Debug.h" 28 #include <algorithm> 29 #include <limits> 30 31 namespace llvm { 32 33 class MachineRegisterInfo; 34 class raw_ostream; 35 36 struct GCNRegPressure { 37 enum RegKind { 38 SGPR32, 39 SGPR_TUPLE, 40 VGPR32, 41 VGPR_TUPLE, 42 AGPR32, 43 AGPR_TUPLE, 44 TOTAL_KINDS 45 }; 46 47 GCNRegPressure() { 48 clear(); 49 } 50 51 bool empty() const { return getSGPRNum() == 0 && getVGPRNum() == 0; } 52 53 void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); } 54 55 unsigned getSGPRNum() const { return Value[SGPR32]; } 56 unsigned getVGPRNum() const { return std::max(Value[VGPR32], Value[AGPR32]); } 57 58 unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE], 59 Value[AGPR_TUPLE]); } 60 unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; } 61 62 unsigned getOccupancy(const GCNSubtarget &ST) const { 63 return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()), 64 ST.getOccupancyWithNumVGPRs(getVGPRNum())); 65 } 66 67 void inc(unsigned Reg, 68 LaneBitmask PrevMask, 69 LaneBitmask NewMask, 70 const MachineRegisterInfo &MRI); 71 72 bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const { 73 return getOccupancy(ST) > O.getOccupancy(ST); 74 } 75 76 bool less(const GCNSubtarget &ST, const GCNRegPressure& O, 77 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const; 78 79 bool operator==(const GCNRegPressure &O) const { 80 return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value); 81 } 82 83 bool operator!=(const GCNRegPressure &O) const { 84 return !(*this == O); 85 } 86 87 void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const; 88 void dump() const { print(dbgs()); } 89 90 private: 91 unsigned Value[TOTAL_KINDS]; 92 93 static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI); 94 95 friend GCNRegPressure max(const GCNRegPressure &P1, 96 const GCNRegPressure &P2); 97 }; 98 99 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) { 100 GCNRegPressure Res; 101 for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I) 102 Res.Value[I] = std::max(P1.Value[I], P2.Value[I]); 103 return Res; 104 } 105 106 class GCNRPTracker { 107 public: 108 using LiveRegSet = DenseMap<unsigned, LaneBitmask>; 109 110 protected: 111 const LiveIntervals &LIS; 112 LiveRegSet LiveRegs; 113 GCNRegPressure CurPressure, MaxPressure; 114 const MachineInstr *LastTrackedMI = nullptr; 115 mutable const MachineRegisterInfo *MRI = nullptr; 116 117 GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {} 118 119 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy, 120 bool After); 121 122 public: 123 // live regs for the current state 124 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; } 125 const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; } 126 127 void clearMaxPressure() { MaxPressure.clear(); } 128 129 // returns MaxPressure, resetting it 130 decltype(MaxPressure) moveMaxPressure() { 131 auto Res = MaxPressure; 132 MaxPressure.clear(); 133 return Res; 134 } 135 136 decltype(LiveRegs) moveLiveRegs() { 137 return std::move(LiveRegs); 138 } 139 140 static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs, 141 const MachineRegisterInfo &MRI); 142 }; 143 144 class GCNUpwardRPTracker : public GCNRPTracker { 145 public: 146 GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} 147 148 // reset tracker to the point just below MI 149 // filling live regs upon this point using LIS 150 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr); 151 152 // move to the state just above the MI 153 void recede(const MachineInstr &MI); 154 155 // checks whether the tracker's state after receding MI corresponds 156 // to reported by LIS 157 bool isValid() const; 158 }; 159 160 class GCNDownwardRPTracker : public GCNRPTracker { 161 // Last position of reset or advanceBeforeNext 162 MachineBasicBlock::const_iterator NextMI; 163 164 MachineBasicBlock::const_iterator MBBEnd; 165 166 public: 167 GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {} 168 169 const MachineBasicBlock::const_iterator getNext() const { return NextMI; } 170 171 // Reset tracker to the point before the MI 172 // filling live regs upon this point using LIS. 173 // Returns false if block is empty except debug values. 174 bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr); 175 176 // Move to the state right before the next MI. Returns false if reached 177 // end of the block. 178 bool advanceBeforeNext(); 179 180 // Move to the state at the MI, advanceBeforeNext has to be called first. 181 void advanceToNext(); 182 183 // Move to the state at the next MI. Returns false if reached end of block. 184 bool advance(); 185 186 // Advance instructions until before End. 187 bool advance(MachineBasicBlock::const_iterator End); 188 189 // Reset to Begin and advance to End. 190 bool advance(MachineBasicBlock::const_iterator Begin, 191 MachineBasicBlock::const_iterator End, 192 const LiveRegSet *LiveRegsCopy = nullptr); 193 }; 194 195 LaneBitmask getLiveLaneMask(unsigned Reg, 196 SlotIndex SI, 197 const LiveIntervals &LIS, 198 const MachineRegisterInfo &MRI); 199 200 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI, 201 const LiveIntervals &LIS, 202 const MachineRegisterInfo &MRI); 203 204 /// creates a map MachineInstr -> LiveRegSet 205 /// R - range of iterators on instructions 206 /// After - upon entry or exit of every instruction 207 /// Note: there is no entry in the map for instructions with empty live reg set 208 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R)) 209 template <typename Range> 210 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet> 211 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) { 212 std::vector<SlotIndex> Indexes; 213 Indexes.reserve(std::distance(R.begin(), R.end())); 214 auto &SII = *LIS.getSlotIndexes(); 215 for (MachineInstr *I : R) { 216 auto SI = SII.getInstructionIndex(*I); 217 Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex()); 218 } 219 llvm::sort(Indexes); 220 221 auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo(); 222 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap; 223 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs; 224 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 225 auto Reg = Register::index2VirtReg(I); 226 if (!LIS.hasInterval(Reg)) 227 continue; 228 auto &LI = LIS.getInterval(Reg); 229 LiveIdxs.clear(); 230 if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs))) 231 continue; 232 if (!LI.hasSubRanges()) { 233 for (auto SI : LiveIdxs) 234 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] = 235 MRI.getMaxLaneMaskForVReg(Reg); 236 } else 237 for (const auto &S : LI.subranges()) { 238 // constrain search for subranges by indexes live at main range 239 SRLiveIdxs.clear(); 240 S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs)); 241 for (auto SI : SRLiveIdxs) 242 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask; 243 } 244 } 245 return LiveRegMap; 246 } 247 248 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, 249 const LiveIntervals &LIS) { 250 return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS, 251 MI.getParent()->getParent()->getRegInfo()); 252 } 253 254 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, 255 const LiveIntervals &LIS) { 256 return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS, 257 MI.getParent()->getParent()->getRegInfo()); 258 } 259 260 template <typename Range> 261 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, 262 Range &&LiveRegs) { 263 GCNRegPressure Res; 264 for (const auto &RM : LiveRegs) 265 Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI); 266 return Res; 267 } 268 269 bool isEqual(const GCNRPTracker::LiveRegSet &S1, 270 const GCNRPTracker::LiveRegSet &S2); 271 272 void printLivesAt(SlotIndex SI, 273 const LiveIntervals &LIS, 274 const MachineRegisterInfo &MRI); 275 276 } // end namespace llvm 277 278 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H 279