1 //===-- SIFormMemoryClauses.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass creates bundles of SMEM and VMEM instructions forming memory 11 /// clauses if XNACK is enabled. Def operands of clauses are marked as early 12 /// clobber to make sure we will not override any source within a clause. 13 /// 14 //===----------------------------------------------------------------------===// 15 16 #include "AMDGPU.h" 17 #include "AMDGPUSubtarget.h" 18 #include "GCNRegPressure.h" 19 #include "SIInstrInfo.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "SIRegisterInfo.h" 22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 23 #include "llvm/ADT/DenseMap.h" 24 #include "llvm/CodeGen/LiveIntervals.h" 25 #include "llvm/CodeGen/MachineFunctionPass.h" 26 27 using namespace llvm; 28 29 #define DEBUG_TYPE "si-form-memory-clauses" 30 31 // Clauses longer then 15 instructions would overflow one of the counters 32 // and stall. They can stall even earlier if there are outstanding counters. 33 static cl::opt<unsigned> 34 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15), 35 cl::desc("Maximum length of a memory clause, instructions")); 36 37 namespace { 38 39 class SIFormMemoryClauses : public MachineFunctionPass { 40 typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse; 41 42 public: 43 static char ID; 44 45 public: 46 SIFormMemoryClauses() : MachineFunctionPass(ID) { 47 initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry()); 48 } 49 50 bool runOnMachineFunction(MachineFunction &MF) override; 51 52 StringRef getPassName() const override { 53 return "SI Form memory clauses"; 54 } 55 56 void getAnalysisUsage(AnalysisUsage &AU) const override { 57 AU.addRequired<LiveIntervals>(); 58 AU.setPreservesAll(); 59 MachineFunctionPass::getAnalysisUsage(AU); 60 } 61 62 private: 63 template <typename Callable> 64 void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const; 65 66 bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const; 67 bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT); 68 void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const; 69 bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses, 70 GCNDownwardRPTracker &RPT); 71 72 const GCNSubtarget *ST; 73 const SIRegisterInfo *TRI; 74 const MachineRegisterInfo *MRI; 75 SIMachineFunctionInfo *MFI; 76 77 unsigned LastRecordedOccupancy; 78 unsigned MaxVGPRs; 79 unsigned MaxSGPRs; 80 }; 81 82 } // End anonymous namespace. 83 84 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE, 85 "SI Form memory clauses", false, false) 86 INITIALIZE_PASS_DEPENDENCY(LiveIntervals) 87 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE, 88 "SI Form memory clauses", false, false) 89 90 91 char SIFormMemoryClauses::ID = 0; 92 93 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID; 94 95 FunctionPass *llvm::createSIFormMemoryClausesPass() { 96 return new SIFormMemoryClauses(); 97 } 98 99 static bool isVMEMClauseInst(const MachineInstr &MI) { 100 return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI); 101 } 102 103 static bool isSMEMClauseInst(const MachineInstr &MI) { 104 return SIInstrInfo::isSMRD(MI); 105 } 106 107 // There no sense to create store clauses, they do not define anything, 108 // thus there is nothing to set early-clobber. 109 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) { 110 if (MI.isDebugValue() || MI.isBundled()) 111 return false; 112 if (!MI.mayLoad() || MI.mayStore()) 113 return false; 114 if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 || 115 AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1) 116 return false; 117 if (IsVMEMClause && !isVMEMClauseInst(MI)) 118 return false; 119 if (!IsVMEMClause && !isSMEMClauseInst(MI)) 120 return false; 121 // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it. 122 for (const MachineOperand &ResMO : MI.defs()) { 123 unsigned ResReg = ResMO.getReg(); 124 for (const MachineOperand &MO : MI.uses()) { 125 if (!MO.isReg() || MO.isDef()) 126 continue; 127 if (MO.getReg() == ResReg) 128 return false; 129 } 130 break; // Only check the first def. 131 } 132 return true; 133 } 134 135 static unsigned getMopState(const MachineOperand &MO) { 136 unsigned S = 0; 137 if (MO.isImplicit()) 138 S |= RegState::Implicit; 139 if (MO.isDead()) 140 S |= RegState::Dead; 141 if (MO.isUndef()) 142 S |= RegState::Undef; 143 if (MO.isKill()) 144 S |= RegState::Kill; 145 if (MO.isEarlyClobber()) 146 S |= RegState::EarlyClobber; 147 if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()) && MO.isRenamable()) 148 S |= RegState::Renamable; 149 return S; 150 } 151 152 template <typename Callable> 153 void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask, 154 Callable Func) const { 155 if (LaneMask.all() || TargetRegisterInfo::isPhysicalRegister(Reg) || 156 LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) { 157 Func(0); 158 return; 159 } 160 161 const TargetRegisterClass *RC = MRI->getRegClass(Reg); 162 unsigned E = TRI->getNumSubRegIndices(); 163 SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs; 164 for (unsigned Idx = 1; Idx < E; ++Idx) { 165 // Is this index even compatible with the given class? 166 if (TRI->getSubClassWithSubReg(RC, Idx) != RC) 167 continue; 168 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx); 169 // Early exit if we found a perfect match. 170 if (SubRegMask == LaneMask) { 171 Func(Idx); 172 return; 173 } 174 175 if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none()) 176 continue; 177 178 CoveringSubregs.push_back(Idx); 179 } 180 181 llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) { 182 LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A); 183 LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B); 184 unsigned NA = MaskA.getNumLanes(); 185 unsigned NB = MaskB.getNumLanes(); 186 if (NA != NB) 187 return NA > NB; 188 return MaskA.getHighestLane() > MaskB.getHighestLane(); 189 }); 190 191 for (unsigned Idx : CoveringSubregs) { 192 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx); 193 if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none()) 194 continue; 195 196 Func(Idx); 197 LaneMask &= ~SubRegMask; 198 if (LaneMask.none()) 199 return; 200 } 201 202 llvm_unreachable("Failed to find all subregs to cover lane mask"); 203 } 204 205 // Returns false if there is a use of a def already in the map. 206 // In this case we must break the clause. 207 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI, 208 RegUse &Defs, RegUse &Uses) const { 209 // Check interference with defs. 210 for (const MachineOperand &MO : MI.operands()) { 211 // TODO: Prologue/Epilogue Insertion pass does not process bundled 212 // instructions. 213 if (MO.isFI()) 214 return false; 215 216 if (!MO.isReg()) 217 continue; 218 219 unsigned Reg = MO.getReg(); 220 221 // If it is tied we will need to write same register as we read. 222 if (MO.isTied()) 223 return false; 224 225 RegUse &Map = MO.isDef() ? Uses : Defs; 226 auto Conflict = Map.find(Reg); 227 if (Conflict == Map.end()) 228 continue; 229 230 if (TargetRegisterInfo::isPhysicalRegister(Reg)) 231 return false; 232 233 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg()); 234 if ((Conflict->second.second & Mask).any()) 235 return false; 236 } 237 238 return true; 239 } 240 241 // Since all defs in the clause are early clobber we can run out of registers. 242 // Function returns false if pressure would hit the limit if instruction is 243 // bundled into a memory clause. 244 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI, 245 GCNDownwardRPTracker &RPT) { 246 // NB: skip advanceBeforeNext() call. Since all defs will be marked 247 // early-clobber they will all stay alive at least to the end of the 248 // clause. Therefor we should not decrease pressure even if load 249 // pointer becomes dead and could otherwise be reused for destination. 250 RPT.advanceToNext(); 251 GCNRegPressure MaxPressure = RPT.moveMaxPressure(); 252 unsigned Occupancy = MaxPressure.getOccupancy(*ST); 253 if (Occupancy >= MFI->getMinAllowedOccupancy() && 254 MaxPressure.getVGPRNum() <= MaxVGPRs && 255 MaxPressure.getSGPRNum() <= MaxSGPRs) { 256 LastRecordedOccupancy = Occupancy; 257 return true; 258 } 259 return false; 260 } 261 262 // Collect register defs and uses along with their lane masks and states. 263 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI, 264 RegUse &Defs, RegUse &Uses) const { 265 for (const MachineOperand &MO : MI.operands()) { 266 if (!MO.isReg()) 267 continue; 268 unsigned Reg = MO.getReg(); 269 if (!Reg) 270 continue; 271 272 LaneBitmask Mask = TargetRegisterInfo::isVirtualRegister(Reg) ? 273 TRI->getSubRegIndexLaneMask(MO.getSubReg()) : 274 LaneBitmask::getAll(); 275 RegUse &Map = MO.isDef() ? Defs : Uses; 276 277 auto Loc = Map.find(Reg); 278 unsigned State = getMopState(MO); 279 if (Loc == Map.end()) { 280 Map[Reg] = std::make_pair(State, Mask); 281 } else { 282 Loc->second.first |= State; 283 Loc->second.second |= Mask; 284 } 285 } 286 } 287 288 // Check register def/use conflicts, occupancy limits and collect def/use maps. 289 // Return true if instruction can be bundled with previous. It it cannot 290 // def/use maps are not updated. 291 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI, 292 RegUse &Defs, RegUse &Uses, 293 GCNDownwardRPTracker &RPT) { 294 if (!canBundle(MI, Defs, Uses)) 295 return false; 296 297 if (!checkPressure(MI, RPT)) 298 return false; 299 300 collectRegUses(MI, Defs, Uses); 301 return true; 302 } 303 304 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) { 305 if (skipFunction(MF.getFunction())) 306 return false; 307 308 ST = &MF.getSubtarget<GCNSubtarget>(); 309 if (!ST->isXNACKEnabled()) 310 return false; 311 312 const SIInstrInfo *TII = ST->getInstrInfo(); 313 TRI = ST->getRegisterInfo(); 314 MRI = &MF.getRegInfo(); 315 MFI = MF.getInfo<SIMachineFunctionInfo>(); 316 LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); 317 SlotIndexes *Ind = LIS->getSlotIndexes(); 318 bool Changed = false; 319 320 MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count(); 321 MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count(); 322 unsigned FuncMaxClause = AMDGPU::getIntegerAttribute( 323 MF.getFunction(), "amdgpu-max-memory-clause", MaxClause); 324 325 for (MachineBasicBlock &MBB : MF) { 326 MachineBasicBlock::instr_iterator Next; 327 for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) { 328 MachineInstr &MI = *I; 329 Next = std::next(I); 330 331 bool IsVMEM = isVMEMClauseInst(MI); 332 333 if (!isValidClauseInst(MI, IsVMEM)) 334 continue; 335 336 RegUse Defs, Uses; 337 GCNDownwardRPTracker RPT(*LIS); 338 RPT.reset(MI); 339 340 if (!processRegUses(MI, Defs, Uses, RPT)) 341 continue; 342 343 unsigned Length = 1; 344 for ( ; Next != E && Length < FuncMaxClause; ++Next) { 345 if (!isValidClauseInst(*Next, IsVMEM)) 346 break; 347 348 // A load from pointer which was loaded inside the same bundle is an 349 // impossible clause because we will need to write and read the same 350 // register inside. In this case processRegUses will return false. 351 if (!processRegUses(*Next, Defs, Uses, RPT)) 352 break; 353 354 ++Length; 355 } 356 if (Length < 2) 357 continue; 358 359 Changed = true; 360 MFI->limitOccupancy(LastRecordedOccupancy); 361 362 auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE)); 363 Ind->insertMachineInstrInMaps(*B); 364 365 for (auto BI = I; BI != Next; ++BI) { 366 BI->bundleWithPred(); 367 Ind->removeSingleMachineInstrFromMaps(*BI); 368 369 for (MachineOperand &MO : BI->defs()) 370 if (MO.readsReg()) 371 MO.setIsInternalRead(true); 372 } 373 374 for (auto &&R : Defs) { 375 forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) { 376 unsigned S = R.second.first | RegState::EarlyClobber; 377 if (!SubReg) 378 S &= ~(RegState::Undef | RegState::Dead); 379 B.addDef(R.first, S, SubReg); 380 }); 381 } 382 383 for (auto &&R : Uses) { 384 forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) { 385 B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg); 386 }); 387 } 388 389 for (auto &&R : Defs) { 390 unsigned Reg = R.first; 391 Uses.erase(Reg); 392 if (TargetRegisterInfo::isPhysicalRegister(Reg)) 393 continue; 394 LIS->removeInterval(Reg); 395 LIS->createAndComputeVirtRegInterval(Reg); 396 } 397 398 for (auto &&R : Uses) { 399 unsigned Reg = R.first; 400 if (TargetRegisterInfo::isPhysicalRegister(Reg)) 401 continue; 402 LIS->removeInterval(Reg); 403 LIS->createAndComputeVirtRegInterval(Reg); 404 } 405 } 406 } 407 408 return Changed; 409 } 410