1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "GCNHazardRecognizer.h" 14 #include "GCNSubtarget.h" 15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 16 #include "llvm/CodeGen/MachineFunction.h" 17 #include "llvm/CodeGen/ScheduleDAG.h" 18 #include "llvm/Support/TargetParser.h" 19 20 using namespace llvm; 21 22 //===----------------------------------------------------------------------===// 23 // Hazard Recoginizer Implementation 24 //===----------------------------------------------------------------------===// 25 26 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 27 const GCNSubtarget &ST); 28 29 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 30 IsHazardRecognizerMode(false), 31 CurrCycleInstr(nullptr), 32 MF(MF), 33 ST(MF.getSubtarget<GCNSubtarget>()), 34 TII(*ST.getInstrInfo()), 35 TRI(TII.getRegisterInfo()), 36 ClauseUses(TRI.getNumRegUnits()), 37 ClauseDefs(TRI.getNumRegUnits()) { 38 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; 39 TSchedModel.init(&ST); 40 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); 41 } 42 43 void GCNHazardRecognizer::Reset() { 44 EmittedInstrs.clear(); 45 } 46 47 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 48 EmitInstruction(SU->getInstr()); 49 } 50 51 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 52 CurrCycleInstr = MI; 53 } 54 55 static bool isDivFMas(unsigned Opcode) { 56 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; 57 } 58 59 static bool isSGetReg(unsigned Opcode) { 60 return Opcode == AMDGPU::S_GETREG_B32; 61 } 62 63 static bool isSSetReg(unsigned Opcode) { 64 switch (Opcode) { 65 case AMDGPU::S_SETREG_B32: 66 case AMDGPU::S_SETREG_B32_mode: 67 case AMDGPU::S_SETREG_IMM32_B32: 68 case AMDGPU::S_SETREG_IMM32_B32_mode: 69 return true; 70 } 71 return false; 72 } 73 74 static bool isRWLane(unsigned Opcode) { 75 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; 76 } 77 78 static bool isRFE(unsigned Opcode) { 79 return Opcode == AMDGPU::S_RFE_B64; 80 } 81 82 static bool isSMovRel(unsigned Opcode) { 83 switch (Opcode) { 84 case AMDGPU::S_MOVRELS_B32: 85 case AMDGPU::S_MOVRELS_B64: 86 case AMDGPU::S_MOVRELD_B32: 87 case AMDGPU::S_MOVRELD_B64: 88 return true; 89 default: 90 return false; 91 } 92 } 93 94 static bool isDGEMM(unsigned Opcode) { 95 return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 96 Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 || 97 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 || 98 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64 || 99 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64 || 100 Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64; 101 } 102 103 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { 104 unsigned Opcode = MI.getOpcode(); 105 106 if (!SIInstrInfo::isMAI(MI) || 107 isDGEMM(Opcode) || 108 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 109 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) 110 return false; 111 112 return true; 113 } 114 115 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, 116 const MachineInstr &MI) { 117 if (TII.isAlwaysGDS(MI.getOpcode())) 118 return true; 119 120 switch (MI.getOpcode()) { 121 case AMDGPU::S_SENDMSG: 122 case AMDGPU::S_SENDMSGHALT: 123 case AMDGPU::S_TTRACEDATA: 124 return true; 125 // These DS opcodes don't support GDS. 126 case AMDGPU::DS_NOP: 127 case AMDGPU::DS_PERMUTE_B32: 128 case AMDGPU::DS_BPERMUTE_B32: 129 return false; 130 default: 131 if (TII.isDS(MI.getOpcode())) { 132 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 133 AMDGPU::OpName::gds); 134 if (MI.getOperand(GDS).getImm()) 135 return true; 136 } 137 return false; 138 } 139 } 140 141 static bool isPermlane(const MachineInstr &MI) { 142 unsigned Opcode = MI.getOpcode(); 143 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || 144 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; 145 } 146 147 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { 148 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, 149 AMDGPU::OpName::simm16); 150 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; 151 } 152 153 ScheduleHazardRecognizer::HazardType 154 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 155 MachineInstr *MI = SU->getInstr(); 156 // If we are not in "HazardRecognizerMode" and therefore not being run from 157 // the scheduler, track possible stalls from hazards but don't insert noops. 158 auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; 159 160 if (MI->isBundle()) 161 return NoHazard; 162 163 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 164 return HazardType; 165 166 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) 167 return HazardType; 168 169 if (checkFPAtomicToDenormModeHazard(MI) > 0) 170 return HazardType; 171 172 if (ST.hasNoDataDepHazard()) 173 return NoHazard; 174 175 // FIXME: Should flat be considered vmem? 176 if ((SIInstrInfo::isVMEM(*MI) || 177 SIInstrInfo::isFLAT(*MI)) 178 && checkVMEMHazards(MI) > 0) 179 return HazardType; 180 181 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) 182 return HazardType; 183 184 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 185 return HazardType; 186 187 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) 188 return HazardType; 189 190 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) 191 return HazardType; 192 193 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 194 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 195 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 196 return HazardType; 197 198 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) 199 return HazardType; 200 201 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) 202 return HazardType; 203 204 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) 205 return HazardType; 206 207 if (ST.hasReadM0MovRelInterpHazard() && 208 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) && 209 checkReadM0Hazards(MI) > 0) 210 return HazardType; 211 212 if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) && 213 checkReadM0Hazards(MI) > 0) 214 return HazardType; 215 216 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) 217 return HazardType; 218 219 if ((SIInstrInfo::isVMEM(*MI) || 220 SIInstrInfo::isFLAT(*MI) || 221 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) 222 return HazardType; 223 224 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) 225 return HazardType; 226 227 return NoHazard; 228 } 229 230 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, 231 unsigned Quantity) { 232 while (Quantity > 0) { 233 unsigned Arg = std::min(Quantity, 8u); 234 Quantity -= Arg; 235 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) 236 .addImm(Arg - 1); 237 } 238 } 239 240 void GCNHazardRecognizer::processBundle() { 241 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); 242 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); 243 // Check bundled MachineInstr's for hazards. 244 for (; MI != E && MI->isInsideBundle(); ++MI) { 245 CurrCycleInstr = &*MI; 246 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); 247 248 if (IsHazardRecognizerMode) { 249 fixHazards(CurrCycleInstr); 250 251 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); 252 } 253 254 // It’s unnecessary to track more than MaxLookAhead instructions. Since we 255 // include the bundled MI directly after, only add a maximum of 256 // (MaxLookAhead - 1) noops to EmittedInstrs. 257 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) 258 EmittedInstrs.push_front(nullptr); 259 260 EmittedInstrs.push_front(CurrCycleInstr); 261 EmittedInstrs.resize(MaxLookAhead); 262 } 263 CurrCycleInstr = nullptr; 264 } 265 266 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 267 IsHazardRecognizerMode = true; 268 CurrCycleInstr = MI; 269 unsigned W = PreEmitNoopsCommon(MI); 270 fixHazards(MI); 271 CurrCycleInstr = nullptr; 272 return W; 273 } 274 275 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { 276 if (MI->isBundle()) 277 return 0; 278 279 int WaitStates = 0; 280 281 if (SIInstrInfo::isSMRD(*MI)) 282 return std::max(WaitStates, checkSMRDHazards(MI)); 283 284 if (ST.hasNSAtoVMEMBug()) 285 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); 286 287 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); 288 289 if (ST.hasNoDataDepHazard()) 290 return WaitStates; 291 292 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) 293 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); 294 295 if (SIInstrInfo::isVALU(*MI)) 296 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); 297 298 if (SIInstrInfo::isDPP(*MI)) 299 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); 300 301 if (isDivFMas(MI->getOpcode())) 302 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); 303 304 if (isRWLane(MI->getOpcode())) 305 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); 306 307 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 308 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 309 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 310 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); 311 312 if (MI->isInlineAsm()) 313 return std::max(WaitStates, checkInlineAsmHazards(MI)); 314 315 if (isSGetReg(MI->getOpcode())) 316 return std::max(WaitStates, checkGetRegHazards(MI)); 317 318 if (isSSetReg(MI->getOpcode())) 319 return std::max(WaitStates, checkSetRegHazards(MI)); 320 321 if (isRFE(MI->getOpcode())) 322 return std::max(WaitStates, checkRFEHazards(MI)); 323 324 if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) || 325 isSMovRel(MI->getOpcode()))) 326 return std::max(WaitStates, checkReadM0Hazards(MI)); 327 328 if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) 329 return std::max(WaitStates, checkReadM0Hazards(MI)); 330 331 if (SIInstrInfo::isMAI(*MI)) 332 return std::max(WaitStates, checkMAIHazards(MI)); 333 334 if (SIInstrInfo::isVMEM(*MI) || 335 SIInstrInfo::isFLAT(*MI) || 336 SIInstrInfo::isDS(*MI)) 337 return std::max(WaitStates, checkMAILdStHazards(MI)); 338 339 return WaitStates; 340 } 341 342 void GCNHazardRecognizer::EmitNoop() { 343 EmittedInstrs.push_front(nullptr); 344 } 345 346 void GCNHazardRecognizer::AdvanceCycle() { 347 // When the scheduler detects a stall, it will call AdvanceCycle() without 348 // emitting any instructions. 349 if (!CurrCycleInstr) { 350 EmittedInstrs.push_front(nullptr); 351 return; 352 } 353 354 if (CurrCycleInstr->isBundle()) { 355 processBundle(); 356 return; 357 } 358 359 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); 360 if (!NumWaitStates) { 361 CurrCycleInstr = nullptr; 362 return; 363 } 364 365 // Keep track of emitted instructions 366 EmittedInstrs.push_front(CurrCycleInstr); 367 368 // Add a nullptr for each additional wait state after the first. Make sure 369 // not to add more than getMaxLookAhead() items to the list, since we 370 // truncate the list to that size right after this loop. 371 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 372 i < e; ++i) { 373 EmittedInstrs.push_front(nullptr); 374 } 375 376 // getMaxLookahead() is the largest number of wait states we will ever need 377 // to insert, so there is no point in keeping track of more than that many 378 // wait states. 379 EmittedInstrs.resize(getMaxLookAhead()); 380 381 CurrCycleInstr = nullptr; 382 } 383 384 void GCNHazardRecognizer::RecedeCycle() { 385 llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 386 } 387 388 //===----------------------------------------------------------------------===// 389 // Helper Functions 390 //===----------------------------------------------------------------------===// 391 392 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn; 393 394 // Returns a minimum wait states since \p I walking all predecessors. 395 // Only scans until \p IsExpired does not return true. 396 // Can only be run in a hazard recognizer mode. 397 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 398 const MachineBasicBlock *MBB, 399 MachineBasicBlock::const_reverse_instr_iterator I, 400 int WaitStates, IsExpiredFn IsExpired, 401 DenseSet<const MachineBasicBlock *> &Visited) { 402 for (auto E = MBB->instr_rend(); I != E; ++I) { 403 // Don't add WaitStates for parent BUNDLE instructions. 404 if (I->isBundle()) 405 continue; 406 407 if (IsHazard(*I)) 408 return WaitStates; 409 410 if (I->isInlineAsm()) 411 continue; 412 413 WaitStates += SIInstrInfo::getNumWaitStates(*I); 414 415 if (IsExpired(*I, WaitStates)) 416 return std::numeric_limits<int>::max(); 417 } 418 419 int MinWaitStates = std::numeric_limits<int>::max(); 420 for (MachineBasicBlock *Pred : MBB->predecessors()) { 421 if (!Visited.insert(Pred).second) 422 continue; 423 424 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), 425 WaitStates, IsExpired, Visited); 426 427 MinWaitStates = std::min(MinWaitStates, W); 428 } 429 430 return MinWaitStates; 431 } 432 433 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 434 const MachineInstr *MI, IsExpiredFn IsExpired) { 435 DenseSet<const MachineBasicBlock *> Visited; 436 return getWaitStatesSince(IsHazard, MI->getParent(), 437 std::next(MI->getReverseIterator()), 438 0, IsExpired, Visited); 439 } 440 441 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { 442 if (IsHazardRecognizerMode) { 443 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) { 444 return WaitStates >= Limit; 445 }; 446 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); 447 } 448 449 int WaitStates = 0; 450 for (MachineInstr *MI : EmittedInstrs) { 451 if (MI) { 452 if (IsHazard(*MI)) 453 return WaitStates; 454 455 if (MI->isInlineAsm()) 456 continue; 457 } 458 ++WaitStates; 459 460 if (WaitStates >= Limit) 461 break; 462 } 463 return std::numeric_limits<int>::max(); 464 } 465 466 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, 467 IsHazardFn IsHazardDef, 468 int Limit) { 469 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 470 471 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { 472 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); 473 }; 474 475 return getWaitStatesSince(IsHazardFn, Limit); 476 } 477 478 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, 479 int Limit) { 480 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { 481 return isSSetReg(MI.getOpcode()) && IsHazard(MI); 482 }; 483 484 return getWaitStatesSince(IsHazardFn, Limit); 485 } 486 487 //===----------------------------------------------------------------------===// 488 // No-op Hazard Detection 489 //===----------------------------------------------------------------------===// 490 491 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, 492 MCRegister Reg) { 493 for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) 494 BV.set(*RUI); 495 } 496 497 static void addRegsToSet(const SIRegisterInfo &TRI, 498 iterator_range<MachineInstr::const_mop_iterator> Ops, 499 BitVector &Set) { 500 for (const MachineOperand &Op : Ops) { 501 if (Op.isReg()) 502 addRegUnits(TRI, Set, Op.getReg().asMCReg()); 503 } 504 } 505 506 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { 507 // XXX: Do we need to worry about implicit operands 508 addRegsToSet(TRI, MI.defs(), ClauseDefs); 509 addRegsToSet(TRI, MI.uses(), ClauseUses); 510 } 511 512 static bool breaksSMEMSoftClause(MachineInstr *MI) { 513 return !SIInstrInfo::isSMRD(*MI); 514 } 515 516 static bool breaksVMEMSoftClause(MachineInstr *MI) { 517 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); 518 } 519 520 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { 521 // SMEM soft clause are only present on VI+, and only matter if xnack is 522 // enabled. 523 if (!ST.isXNACKEnabled()) 524 return 0; 525 526 bool IsSMRD = TII.isSMRD(*MEM); 527 528 resetClause(); 529 530 // A soft-clause is any group of consecutive SMEM instructions. The 531 // instructions in this group may return out of order and/or may be 532 // replayed (i.e. the same instruction issued more than once). 533 // 534 // In order to handle these situations correctly we need to make sure that 535 // when a clause has more than one instruction, no instruction in the clause 536 // writes to a register that is read by another instruction in the clause 537 // (including itself). If we encounter this situaion, we need to break the 538 // clause by inserting a non SMEM instruction. 539 540 for (MachineInstr *MI : EmittedInstrs) { 541 // When we hit a non-SMEM instruction then we have passed the start of the 542 // clause and we can stop. 543 if (!MI) 544 break; 545 546 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) 547 break; 548 549 addClauseInst(*MI); 550 } 551 552 if (ClauseDefs.none()) 553 return 0; 554 555 // We need to make sure not to put loads and stores in the same clause if they 556 // use the same address. For now, just start a new clause whenever we see a 557 // store. 558 if (MEM->mayStore()) 559 return 1; 560 561 addClauseInst(*MEM); 562 563 // If the set of defs and uses intersect then we cannot add this instruction 564 // to the clause, so we have a hazard. 565 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; 566 } 567 568 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 569 int WaitStatesNeeded = 0; 570 571 WaitStatesNeeded = checkSoftClauseHazards(SMRD); 572 573 // This SMRD hazard only affects SI. 574 if (!ST.hasSMRDReadVALUDefHazard()) 575 return WaitStatesNeeded; 576 577 // A read of an SGPR by SMRD instruction requires 4 wait states when the 578 // SGPR was written by a VALU instruction. 579 int SmrdSgprWaitStates = 4; 580 auto IsHazardDefFn = [this](const MachineInstr &MI) { 581 return TII.isVALU(MI); 582 }; 583 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { 584 return TII.isSALU(MI); 585 }; 586 587 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); 588 589 for (const MachineOperand &Use : SMRD->uses()) { 590 if (!Use.isReg()) 591 continue; 592 int WaitStatesNeededForUse = 593 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 594 SmrdSgprWaitStates); 595 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 596 597 // This fixes what appears to be undocumented hardware behavior in SI where 598 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor 599 // needs some number of nops in between. We don't know how many we need, but 600 // let's use 4. This wasn't discovered before probably because the only 601 // case when this happens is when we expand a 64-bit pointer into a full 602 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was 603 // probably never encountered in the closed-source land. 604 if (IsBufferSMRD) { 605 int WaitStatesNeededForUse = 606 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 607 IsBufferHazardDefFn, 608 SmrdSgprWaitStates); 609 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 610 } 611 } 612 613 return WaitStatesNeeded; 614 } 615 616 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 617 if (!ST.hasVMEMReadSGPRVALUDefHazard()) 618 return 0; 619 620 int WaitStatesNeeded = checkSoftClauseHazards(VMEM); 621 622 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 623 // SGPR was written by a VALU Instruction. 624 const int VmemSgprWaitStates = 5; 625 auto IsHazardDefFn = [this](const MachineInstr &MI) { 626 return TII.isVALU(MI); 627 }; 628 for (const MachineOperand &Use : VMEM->uses()) { 629 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) 630 continue; 631 632 int WaitStatesNeededForUse = 633 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 634 VmemSgprWaitStates); 635 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 636 } 637 return WaitStatesNeeded; 638 } 639 640 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 641 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 642 const SIInstrInfo *TII = ST.getInstrInfo(); 643 644 // Check for DPP VGPR read after VALU VGPR write and EXEC write. 645 int DppVgprWaitStates = 2; 646 int DppExecWaitStates = 5; 647 int WaitStatesNeeded = 0; 648 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 649 return TII->isVALU(MI); 650 }; 651 652 for (const MachineOperand &Use : DPP->uses()) { 653 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 654 continue; 655 int WaitStatesNeededForUse = 656 DppVgprWaitStates - getWaitStatesSinceDef( 657 Use.getReg(), 658 [](const MachineInstr &) { return true; }, 659 DppVgprWaitStates); 660 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 661 } 662 663 WaitStatesNeeded = std::max( 664 WaitStatesNeeded, 665 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, 666 DppExecWaitStates)); 667 668 return WaitStatesNeeded; 669 } 670 671 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { 672 const SIInstrInfo *TII = ST.getInstrInfo(); 673 674 // v_div_fmas requires 4 wait states after a write to vcc from a VALU 675 // instruction. 676 const int DivFMasWaitStates = 4; 677 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 678 return TII->isVALU(MI); 679 }; 680 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, 681 DivFMasWaitStates); 682 683 return DivFMasWaitStates - WaitStatesNeeded; 684 } 685 686 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { 687 const SIInstrInfo *TII = ST.getInstrInfo(); 688 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); 689 690 const int GetRegWaitStates = 2; 691 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { 692 return GetRegHWReg == getHWReg(TII, MI); 693 }; 694 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); 695 696 return GetRegWaitStates - WaitStatesNeeded; 697 } 698 699 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { 700 const SIInstrInfo *TII = ST.getInstrInfo(); 701 unsigned HWReg = getHWReg(TII, *SetRegInstr); 702 703 const int SetRegWaitStates = ST.getSetRegWaitStates(); 704 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { 705 return HWReg == getHWReg(TII, MI); 706 }; 707 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); 708 return SetRegWaitStates - WaitStatesNeeded; 709 } 710 711 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { 712 if (!MI.mayStore()) 713 return -1; 714 715 const SIInstrInfo *TII = ST.getInstrInfo(); 716 unsigned Opcode = MI.getOpcode(); 717 const MCInstrDesc &Desc = MI.getDesc(); 718 719 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 720 int VDataRCID = -1; 721 if (VDataIdx != -1) 722 VDataRCID = Desc.OpInfo[VDataIdx].RegClass; 723 724 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { 725 // There is no hazard if the instruction does not use vector regs 726 // (like wbinvl1) 727 if (VDataIdx == -1) 728 return -1; 729 // For MUBUF/MTBUF instructions this hazard only exists if the 730 // instruction is not using a register in the soffset field. 731 const MachineOperand *SOffset = 732 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 733 // If we have no soffset operand, then assume this field has been 734 // hardcoded to zero. 735 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && 736 (!SOffset || !SOffset->isReg())) 737 return VDataIdx; 738 } 739 740 // MIMG instructions create a hazard if they don't use a 256-bit T# and 741 // the store size is greater than 8 bytes and they have more than two bits 742 // of their dmask set. 743 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. 744 if (TII->isMIMG(MI)) { 745 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 746 assert(SRsrcIdx != -1 && 747 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); 748 (void)SRsrcIdx; 749 } 750 751 if (TII->isFLAT(MI)) { 752 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 753 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) 754 return DataIdx; 755 } 756 757 return -1; 758 } 759 760 int 761 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, 762 const MachineRegisterInfo &MRI) { 763 // Helper to check for the hazard where VMEM instructions that store more than 764 // 8 bytes can have there store data over written by the next instruction. 765 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 766 767 const int VALUWaitStates = 1; 768 int WaitStatesNeeded = 0; 769 770 if (!TRI->isVectorRegister(MRI, Def.getReg())) 771 return WaitStatesNeeded; 772 Register Reg = Def.getReg(); 773 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { 774 int DataIdx = createsVALUHazard(MI); 775 return DataIdx >= 0 && 776 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); 777 }; 778 int WaitStatesNeededForDef = 779 VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); 780 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 781 782 return WaitStatesNeeded; 783 } 784 785 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { 786 // This checks for the hazard where VMEM instructions that store more than 787 // 8 bytes can have there store data over written by the next instruction. 788 if (!ST.has12DWordStoreHazard()) 789 return 0; 790 791 const MachineRegisterInfo &MRI = MF.getRegInfo(); 792 int WaitStatesNeeded = 0; 793 794 for (const MachineOperand &Def : VALU->defs()) { 795 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); 796 } 797 798 return WaitStatesNeeded; 799 } 800 801 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { 802 // This checks for hazards associated with inline asm statements. 803 // Since inline asms can contain just about anything, we use this 804 // to call/leverage other check*Hazard routines. Note that 805 // this function doesn't attempt to address all possible inline asm 806 // hazards (good luck), but is a collection of what has been 807 // problematic thus far. 808 809 // see checkVALUHazards() 810 if (!ST.has12DWordStoreHazard()) 811 return 0; 812 813 const MachineRegisterInfo &MRI = MF.getRegInfo(); 814 int WaitStatesNeeded = 0; 815 816 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands(); 817 I != E; ++I) { 818 const MachineOperand &Op = IA->getOperand(I); 819 if (Op.isReg() && Op.isDef()) { 820 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); 821 } 822 } 823 824 return WaitStatesNeeded; 825 } 826 827 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { 828 const SIInstrInfo *TII = ST.getInstrInfo(); 829 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 830 const MachineRegisterInfo &MRI = MF.getRegInfo(); 831 832 const MachineOperand *LaneSelectOp = 833 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); 834 835 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) 836 return 0; 837 838 Register LaneSelectReg = LaneSelectOp->getReg(); 839 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; 840 841 const int RWLaneWaitStates = 4; 842 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, 843 RWLaneWaitStates); 844 return RWLaneWaitStates - WaitStatesSince; 845 } 846 847 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { 848 if (!ST.hasRFEHazards()) 849 return 0; 850 851 const SIInstrInfo *TII = ST.getInstrInfo(); 852 853 const int RFEWaitStates = 1; 854 855 auto IsHazardFn = [TII](const MachineInstr &MI) { 856 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; 857 }; 858 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); 859 return RFEWaitStates - WaitStatesNeeded; 860 } 861 862 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { 863 const SIInstrInfo *TII = ST.getInstrInfo(); 864 const int SMovRelWaitStates = 1; 865 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; 866 return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, 867 SMovRelWaitStates); 868 } 869 870 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { 871 fixVMEMtoScalarWriteHazards(MI); 872 fixVcmpxPermlaneHazards(MI); 873 fixSMEMtoVectorWriteHazards(MI); 874 fixVcmpxExecWARHazard(MI); 875 fixLdsBranchVmemWARHazard(MI); 876 } 877 878 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { 879 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) 880 return false; 881 882 const SIInstrInfo *TII = ST.getInstrInfo(); 883 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVOPC(MI); }; 884 885 auto IsExpiredFn = [](const MachineInstr &MI, int) { 886 unsigned Opc = MI.getOpcode(); 887 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && 888 Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; 889 }; 890 891 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 892 std::numeric_limits<int>::max()) 893 return false; 894 895 // V_NOP will be discarded by SQ. 896 // Use V_MOB_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* 897 // which is always a VGPR and available. 898 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); 899 Register Reg = Src0->getReg(); 900 bool IsUndef = Src0->isUndef(); 901 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 902 TII->get(AMDGPU::V_MOV_B32_e32)) 903 .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) 904 .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); 905 906 return true; 907 } 908 909 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { 910 if (!ST.hasVMEMtoScalarWriteHazard()) 911 return false; 912 913 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) 914 return false; 915 916 if (MI->getNumDefs() == 0) 917 return false; 918 919 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 920 921 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { 922 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) && 923 !SIInstrInfo::isFLAT(I)) 924 return false; 925 926 for (const MachineOperand &Def : MI->defs()) { 927 const MachineOperand *Op = 928 I.findRegisterUseOperand(Def.getReg(), false, TRI); 929 if (!Op) 930 continue; 931 return true; 932 } 933 return false; 934 }; 935 936 auto IsExpiredFn = [](const MachineInstr &MI, int) { 937 return SIInstrInfo::isVALU(MI) || 938 (MI.getOpcode() == AMDGPU::S_WAITCNT && 939 !MI.getOperand(0).getImm()) || 940 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 941 MI.getOperand(0).getImm() == 0xffe3); 942 }; 943 944 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 945 std::numeric_limits<int>::max()) 946 return false; 947 948 const SIInstrInfo *TII = ST.getInstrInfo(); 949 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 950 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 951 .addImm(0xffe3); 952 return true; 953 } 954 955 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { 956 if (!ST.hasSMEMtoVectorWriteHazard()) 957 return false; 958 959 if (!SIInstrInfo::isVALU(*MI)) 960 return false; 961 962 unsigned SDSTName; 963 switch (MI->getOpcode()) { 964 case AMDGPU::V_READLANE_B32: 965 case AMDGPU::V_READFIRSTLANE_B32: 966 SDSTName = AMDGPU::OpName::vdst; 967 break; 968 default: 969 SDSTName = AMDGPU::OpName::sdst; 970 break; 971 } 972 973 const SIInstrInfo *TII = ST.getInstrInfo(); 974 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 975 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); 976 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); 977 if (!SDST) { 978 for (const auto &MO : MI->implicit_operands()) { 979 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) { 980 SDST = &MO; 981 break; 982 } 983 } 984 } 985 986 if (!SDST) 987 return false; 988 989 const Register SDSTReg = SDST->getReg(); 990 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) { 991 return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI); 992 }; 993 994 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { 995 if (TII->isSALU(MI)) { 996 switch (MI.getOpcode()) { 997 case AMDGPU::S_SETVSKIP: 998 case AMDGPU::S_VERSION: 999 case AMDGPU::S_WAITCNT_VSCNT: 1000 case AMDGPU::S_WAITCNT_VMCNT: 1001 case AMDGPU::S_WAITCNT_EXPCNT: 1002 // These instructions cannot not mitigate the hazard. 1003 return false; 1004 case AMDGPU::S_WAITCNT_LGKMCNT: 1005 // Reducing lgkmcnt count to 0 always mitigates the hazard. 1006 return (MI.getOperand(1).getImm() == 0) && 1007 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); 1008 case AMDGPU::S_WAITCNT: { 1009 const int64_t Imm = MI.getOperand(0).getImm(); 1010 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); 1011 return (Decoded.LgkmCnt == 0); 1012 } 1013 default: 1014 // SOPP instructions cannot mitigate the hazard. 1015 if (TII->isSOPP(MI)) 1016 return false; 1017 // At this point the SALU can be assumed to mitigate the hazard 1018 // because either: 1019 // (a) it is independent of the at risk SMEM (breaking chain), 1020 // or 1021 // (b) it is dependent on the SMEM, in which case an appropriate 1022 // s_waitcnt lgkmcnt _must_ exist between it and the at risk 1023 // SMEM instruction. 1024 return true; 1025 } 1026 } 1027 return false; 1028 }; 1029 1030 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1031 std::numeric_limits<int>::max()) 1032 return false; 1033 1034 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1035 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) 1036 .addImm(0); 1037 return true; 1038 } 1039 1040 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { 1041 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) 1042 return false; 1043 1044 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1045 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) 1046 return false; 1047 1048 auto IsHazardFn = [TRI](const MachineInstr &I) { 1049 if (SIInstrInfo::isVALU(I)) 1050 return false; 1051 return I.readsRegister(AMDGPU::EXEC, TRI); 1052 }; 1053 1054 const SIInstrInfo *TII = ST.getInstrInfo(); 1055 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { 1056 if (SIInstrInfo::isVALU(MI)) { 1057 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) 1058 return true; 1059 for (auto MO : MI.implicit_operands()) 1060 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) 1061 return true; 1062 } 1063 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1064 (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe) 1065 return true; 1066 return false; 1067 }; 1068 1069 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1070 std::numeric_limits<int>::max()) 1071 return false; 1072 1073 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1074 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1075 .addImm(0xfffe); 1076 return true; 1077 } 1078 1079 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 1080 const GCNSubtarget &ST) { 1081 if (!ST.hasLdsBranchVmemWARHazard()) 1082 return false; 1083 1084 // Check if the necessary condition for the hazard is met: both LDS and VMEM 1085 // instructions need to appear in the same function. 1086 bool HasLds = false; 1087 bool HasVmem = false; 1088 for (auto &MBB : MF) { 1089 for (auto &MI : MBB) { 1090 HasLds |= SIInstrInfo::isDS(MI); 1091 HasVmem |= 1092 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); 1093 if (HasLds && HasVmem) 1094 return true; 1095 } 1096 } 1097 return false; 1098 } 1099 1100 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { 1101 if (!RunLdsBranchVmemWARHazardFixup) 1102 return false; 1103 1104 assert(ST.hasLdsBranchVmemWARHazard()); 1105 1106 auto IsHazardInst = [](const MachineInstr &MI) { 1107 if (SIInstrInfo::isDS(MI)) 1108 return 1; 1109 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) 1110 return 2; 1111 return 0; 1112 }; 1113 1114 auto InstType = IsHazardInst(*MI); 1115 if (!InstType) 1116 return false; 1117 1118 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) { 1119 return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1120 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1121 !I.getOperand(1).getImm()); 1122 }; 1123 1124 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) { 1125 if (!I.isBranch()) 1126 return false; 1127 1128 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) { 1129 auto InstType2 = IsHazardInst(I); 1130 return InstType2 && InstType != InstType2; 1131 }; 1132 1133 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) { 1134 auto InstType2 = IsHazardInst(I); 1135 if (InstType == InstType2) 1136 return true; 1137 1138 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1139 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1140 !I.getOperand(1).getImm(); 1141 }; 1142 1143 return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) != 1144 std::numeric_limits<int>::max(); 1145 }; 1146 1147 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1148 std::numeric_limits<int>::max()) 1149 return false; 1150 1151 const SIInstrInfo *TII = ST.getInstrInfo(); 1152 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1153 TII->get(AMDGPU::S_WAITCNT_VSCNT)) 1154 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) 1155 .addImm(0); 1156 1157 return true; 1158 } 1159 1160 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { 1161 int NSAtoVMEMWaitStates = 1; 1162 1163 if (!ST.hasNSAtoVMEMBug()) 1164 return 0; 1165 1166 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) 1167 return 0; 1168 1169 const SIInstrInfo *TII = ST.getInstrInfo(); 1170 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 1171 if (!Offset || (Offset->getImm() & 6) == 0) 1172 return 0; 1173 1174 auto IsHazardFn = [TII](const MachineInstr &I) { 1175 if (!SIInstrInfo::isMIMG(I)) 1176 return false; 1177 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); 1178 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && 1179 TII->getInstSizeInBytes(I) >= 16; 1180 }; 1181 1182 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); 1183 } 1184 1185 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { 1186 int FPAtomicToDenormModeWaitStates = 3; 1187 1188 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) 1189 return 0; 1190 1191 auto IsHazardFn = [](const MachineInstr &I) { 1192 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I)) 1193 return false; 1194 return SIInstrInfo::isFPAtomic(I); 1195 }; 1196 1197 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { 1198 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) 1199 return true; 1200 1201 switch (MI.getOpcode()) { 1202 case AMDGPU::S_WAITCNT: 1203 case AMDGPU::S_WAITCNT_VSCNT: 1204 case AMDGPU::S_WAITCNT_VMCNT: 1205 case AMDGPU::S_WAITCNT_EXPCNT: 1206 case AMDGPU::S_WAITCNT_LGKMCNT: 1207 case AMDGPU::S_WAIT_IDLE: 1208 return true; 1209 default: 1210 break; 1211 } 1212 1213 return false; 1214 }; 1215 1216 return FPAtomicToDenormModeWaitStates - 1217 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); 1218 } 1219 1220 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { 1221 assert(SIInstrInfo::isMAI(*MI)); 1222 1223 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); 1224 } 1225 1226 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { 1227 int WaitStatesNeeded = 0; 1228 unsigned Opc = MI->getOpcode(); 1229 1230 auto IsVALUFn = [](const MachineInstr &MI) { 1231 return SIInstrInfo::isVALU(MI); 1232 }; 1233 1234 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write 1235 const int LegacyVALUWritesVGPRWaitStates = 2; 1236 const int VALUWritesExecWaitStates = 4; 1237 const int MaxWaitStates = 4; 1238 1239 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1240 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); 1241 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1242 1243 if (WaitStatesNeeded < MaxWaitStates) { 1244 for (const MachineOperand &Use : MI->explicit_uses()) { 1245 const int MaxWaitStates = 2; 1246 1247 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1248 continue; 1249 1250 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - 1251 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); 1252 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1253 1254 if (WaitStatesNeeded == MaxWaitStates) 1255 break; 1256 } 1257 } 1258 } 1259 1260 auto IsMFMAFn = [](const MachineInstr &MI) { 1261 return SIInstrInfo::isMAI(MI) && 1262 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1263 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1264 }; 1265 1266 for (const MachineOperand &Op : MI->explicit_operands()) { 1267 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) 1268 continue; 1269 1270 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1271 continue; 1272 1273 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; 1274 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; 1275 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; 1276 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; 1277 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; 1278 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; 1279 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; 1280 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; 1281 const int MaxWaitStates = 18; 1282 Register Reg = Op.getReg(); 1283 unsigned HazardDefLatency = 0; 1284 1285 auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, 1286 this](const MachineInstr &MI) { 1287 if (!IsMFMAFn(MI)) 1288 return false; 1289 Register DstReg = MI.getOperand(0).getReg(); 1290 if (DstReg == Reg) 1291 return false; 1292 HazardDefLatency = 1293 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 1294 return TRI.regsOverlap(DstReg, Reg); 1295 }; 1296 1297 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, 1298 MaxWaitStates); 1299 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; 1300 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1301 int OpNo = MI->getOperandNo(&Op); 1302 if (OpNo == SrcCIdx) { 1303 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; 1304 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { 1305 switch (HazardDefLatency) { 1306 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; 1307 break; 1308 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; 1309 break; 1310 case 16: LLVM_FALLTHROUGH; 1311 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; 1312 break; 1313 } 1314 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1315 switch (HazardDefLatency) { 1316 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; 1317 break; 1318 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; 1319 break; 1320 case 16: LLVM_FALLTHROUGH; 1321 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; 1322 break; 1323 } 1324 } 1325 1326 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1327 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1328 1329 if (WaitStatesNeeded == MaxWaitStates) 1330 return WaitStatesNeeded; // Early exit. 1331 1332 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { 1333 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1334 return false; 1335 Register DstReg = MI.getOperand(0).getReg(); 1336 return TRI.regsOverlap(Reg, DstReg); 1337 }; 1338 1339 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; 1340 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; 1341 const int AccVGPRWriteAccVgprReadWaitStates = 3; 1342 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; 1343 if (OpNo == SrcCIdx) 1344 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; 1345 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) 1346 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; 1347 1348 WaitStatesNeededForUse = NeedWaitStates - 1349 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); 1350 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1351 1352 if (WaitStatesNeeded == MaxWaitStates) 1353 return WaitStatesNeeded; // Early exit. 1354 } 1355 1356 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1357 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; 1358 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; 1359 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; 1360 const int MaxWaitStates = 13; 1361 Register DstReg = MI->getOperand(0).getReg(); 1362 unsigned HazardDefLatency = 0; 1363 1364 auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, 1365 this](const MachineInstr &MI) { 1366 if (!IsMFMAFn(MI)) 1367 return false; 1368 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); 1369 HazardDefLatency = 1370 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 1371 return TRI.regsOverlap(Reg, DstReg); 1372 }; 1373 1374 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); 1375 int NeedWaitStates; 1376 switch (HazardDefLatency) { 1377 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; 1378 break; 1379 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; 1380 break; 1381 case 16: LLVM_FALLTHROUGH; 1382 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; 1383 break; 1384 } 1385 1386 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; 1387 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1388 } 1389 1390 return WaitStatesNeeded; 1391 } 1392 1393 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { 1394 int WaitStatesNeeded = 0; 1395 unsigned Opc = MI->getOpcode(); 1396 1397 auto IsMFMAFn = [](const MachineInstr &MI) { 1398 return SIInstrInfo::isMAI(MI) && 1399 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1400 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1401 }; 1402 1403 auto IsLegacyVALUFn = [&IsMFMAFn](const MachineInstr &MI) { 1404 return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI); 1405 }; 1406 1407 auto IsLegacyVALUNotDotFn = [&IsMFMAFn](const MachineInstr &MI) { 1408 return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI) && !SIInstrInfo::isDOT(MI); 1409 }; 1410 1411 if (!IsMFMAFn(*MI)) 1412 return WaitStatesNeeded; 1413 1414 const int VALUWritesExecWaitStates = 4; 1415 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1416 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, 1417 VALUWritesExecWaitStates); 1418 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1419 1420 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1421 1422 // Loop for both DGEMM and S/HGEMM 2nd instruction. 1423 for (const MachineOperand &Use : MI->explicit_uses()) { 1424 const int LegacyVALUNotDotWritesVGPRWaitStates = 2; 1425 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; 1426 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; 1427 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; 1428 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; 1429 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; 1430 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; 1431 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; 1432 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; 1433 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; 1434 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; 1435 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; 1436 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; 1437 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; 1438 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; 1439 const int MaxWaitStates = 19; 1440 1441 if (!Use.isReg()) 1442 continue; 1443 Register Reg = Use.getReg(); 1444 bool FullReg; 1445 const MachineInstr *MI1; 1446 1447 auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &FullReg, &MI1, 1448 this](const MachineInstr &MI) { 1449 if (!IsMFMAFn(MI)) 1450 return false; 1451 Register DstReg = MI.getOperand(0).getReg(); 1452 FullReg = (DstReg == Reg); 1453 MI1 = &MI; 1454 return TRI.regsOverlap(DstReg, Reg); 1455 }; 1456 1457 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - 1458 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); 1459 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1460 1461 int NumWaitStates = 1462 getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); 1463 if (NumWaitStates == std::numeric_limits<int>::max()) 1464 continue; 1465 1466 int OpNo = MI->getOperandNo(&Use); 1467 unsigned Opc1 = MI1->getOpcode(); 1468 int NeedWaitStates = 0; 1469 if (OpNo == SrcCIdx) { 1470 if (!isDGEMM(Opc) && isDGEMM(Opc1)) { 1471 NeedWaitStates = 0; 1472 } else if (FullReg) { 1473 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1474 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && 1475 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 1476 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) 1477 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; 1478 } else { 1479 switch (Opc1) { 1480 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1481 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1482 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 1483 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 1484 if (!isXDL(ST, *MI)) 1485 NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates; 1486 break; 1487 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1488 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1489 if (!isXDL(ST, *MI)) 1490 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; 1491 break; 1492 default: 1493 switch (TSchedModel.computeInstrLatency(MI1)) { 1494 case 2: 1495 NeedWaitStates = isDGEMM(Opc) 1496 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates 1497 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; 1498 break; 1499 case 8: 1500 NeedWaitStates = isDGEMM(Opc) 1501 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates 1502 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; 1503 break; 1504 case 16: LLVM_FALLTHROUGH; 1505 default: 1506 NeedWaitStates = isDGEMM(Opc) 1507 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates 1508 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; 1509 } 1510 } 1511 } 1512 } else { 1513 switch (Opc1) { 1514 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 1515 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 1516 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 1517 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 1518 NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; 1519 break; 1520 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 1521 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 1522 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; 1523 break; 1524 default: 1525 switch (TSchedModel.computeInstrLatency(MI1)) { 1526 case 2: 1527 NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates; 1528 break; 1529 case 8: 1530 NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates; 1531 break; 1532 case 16: LLVM_FALLTHROUGH; 1533 default: 1534 NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates; 1535 } 1536 } 1537 } 1538 if (WaitStatesNeeded >= NeedWaitStates) 1539 continue; 1540 1541 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; 1542 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1543 1544 if (WaitStatesNeeded == MaxWaitStates) 1545 break; 1546 } 1547 1548 return WaitStatesNeeded; 1549 } 1550 1551 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { 1552 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards() 1553 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) 1554 return 0; 1555 1556 int WaitStatesNeeded = 0; 1557 1558 auto IsAccVgprReadFn = [](const MachineInstr &MI) { 1559 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; 1560 }; 1561 1562 for (const MachineOperand &Op : MI->explicit_uses()) { 1563 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) 1564 continue; 1565 1566 Register Reg = Op.getReg(); 1567 1568 const int AccVgprReadLdStWaitStates = 2; 1569 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; 1570 const int MaxWaitStates = 2; 1571 1572 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - 1573 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); 1574 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1575 1576 if (WaitStatesNeeded == MaxWaitStates) 1577 return WaitStatesNeeded; // Early exit. 1578 1579 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { 1580 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && 1581 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1582 return false; 1583 auto IsVALUFn = [](const MachineInstr &MI) { 1584 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); 1585 }; 1586 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < 1587 std::numeric_limits<int>::max(); 1588 }; 1589 1590 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - 1591 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); 1592 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1593 } 1594 1595 return WaitStatesNeeded; 1596 } 1597 1598 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { 1599 if (!ST.hasGFX90AInsts()) 1600 return 0; 1601 1602 auto IsMFMAFn = [](const MachineInstr &MI) -> bool { 1603 return SIInstrInfo::isMAI(MI) && 1604 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1605 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; 1606 }; 1607 1608 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { 1609 return isDGEMM(MI.getOpcode()); 1610 }; 1611 1612 // This is checked in checkMAIHazards90A() 1613 if (IsMFMAFn(*MI)) 1614 return 0; 1615 1616 int WaitStatesNeeded = 0; 1617 1618 bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) || 1619 SIInstrInfo::isFLAT(*MI) || 1620 SIInstrInfo::isDS(*MI) || 1621 SIInstrInfo::isEXP(*MI); 1622 bool IsVALU = SIInstrInfo::isVALU(*MI); 1623 1624 const MachineInstr *MFMA = nullptr; 1625 unsigned Reg; 1626 auto IsMFMAWriteFn = [&Reg, &IsMFMAFn, &MFMA, this](const MachineInstr &MI) { 1627 if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 1628 return false; 1629 MFMA = &MI; 1630 return true; 1631 }; 1632 1633 const MachineInstr *DOT = nullptr; 1634 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { 1635 if (!SIInstrInfo::isDOT(MI) || 1636 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 1637 return false; 1638 DOT = &MI; 1639 return true; 1640 }; 1641 1642 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 1643 AMDGPU::OpName::src2); 1644 1645 if (IsMemOrExport || IsVALU) { 1646 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; 1647 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; 1648 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; 1649 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; 1650 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; 1651 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; 1652 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; 1653 const int DotWriteSameDotReadSrcAB = 3; 1654 const int DotWriteDifferentVALURead = 3; 1655 const int MaxWaitStates = 19; 1656 1657 for (const MachineOperand &Use : MI->explicit_uses()) { 1658 if (!Use.isReg()) 1659 continue; 1660 Reg = Use.getReg(); 1661 1662 DOT = nullptr; 1663 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1664 MaxWaitStates); 1665 if (DOT) { 1666 int NeedWaitStates = 0; 1667 if (DOT->getOpcode() == MI->getOpcode()) { 1668 if (&Use - &MI->getOperand(0) != SrcCIdx) 1669 NeedWaitStates = DotWriteSameDotReadSrcAB; 1670 } else { 1671 NeedWaitStates = DotWriteDifferentVALURead; 1672 } 1673 1674 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1675 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1676 } 1677 1678 MFMA = nullptr; 1679 WaitStatesSinceDef = 1680 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 1681 if (!MFMA) 1682 continue; 1683 1684 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1685 int NeedWaitStates = MaxWaitStates; 1686 switch (HazardDefLatency) { 1687 case 2: 1688 NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates; 1689 break; 1690 case 4: 1691 assert(isDGEMM(MFMA->getOpcode())); 1692 NeedWaitStates = 1693 IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates 1694 : DMFMA4x4WriteVgprVALUReadWaitStates; 1695 break; 1696 case 8: 1697 NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates; 1698 break; 1699 case 16: LLVM_FALLTHROUGH; 1700 default: 1701 NeedWaitStates = 1702 isDGEMM(MFMA->getOpcode()) 1703 ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates 1704 : DMFMA16x16WriteVgprVALUReadWaitStates 1705 : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; 1706 break; 1707 } 1708 1709 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1710 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1711 1712 if (WaitStatesNeeded == MaxWaitStates) 1713 break; 1714 } 1715 } 1716 1717 unsigned Opc = MI->getOpcode(); 1718 const int DMFMAToFMA64WaitStates = 2; 1719 if ((Opc == AMDGPU::V_FMA_F64_e64 || 1720 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || 1721 Opc == AMDGPU::V_FMAC_F64_dpp) && 1722 WaitStatesNeeded < DMFMAToFMA64WaitStates) { 1723 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - 1724 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); 1725 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1726 } 1727 1728 if (!IsVALU && !IsMemOrExport) 1729 return WaitStatesNeeded; 1730 1731 for (const MachineOperand &Def : MI->defs()) { 1732 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; 1733 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; 1734 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; 1735 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; 1736 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; 1737 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; 1738 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; 1739 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; 1740 const int DotWriteDifferentVALUWrite = 3; 1741 const int MaxWaitStates = 19; 1742 const int MaxWarWaitStates = 15; 1743 1744 Reg = Def.getReg(); 1745 1746 DOT = nullptr; 1747 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 1748 MaxWaitStates); 1749 if (DOT && DOT->getOpcode() != MI->getOpcode()) 1750 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - 1751 WaitStatesSinceDef); 1752 1753 MFMA = nullptr; 1754 WaitStatesSinceDef = 1755 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 1756 if (MFMA) { 1757 int NeedWaitStates = MaxWaitStates; 1758 switch (TSchedModel.computeInstrLatency(MFMA)) { 1759 case 2: 1760 NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates; 1761 break; 1762 case 4: 1763 assert(isDGEMM(MFMA->getOpcode())); 1764 NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; 1765 break; 1766 case 8: 1767 NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates; 1768 break; 1769 case 16: LLVM_FALLTHROUGH; 1770 default: 1771 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 1772 ? DMFMA16x16WriteVgprVALUWriteWaitStates 1773 : SMFMA32x32WriteVgprVALUWawWaitStates; 1774 break; 1775 } 1776 1777 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1778 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1779 1780 if (WaitStatesNeeded == MaxWaitStates) 1781 break; 1782 } 1783 1784 auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, 1785 this](const MachineInstr &MI) { 1786 if (!IsMFMAFn(MI) || isDGEMM(MI.getOpcode()) || 1787 !MI.readsRegister(Reg, &TRI)) 1788 return false; 1789 1790 const MachineOperand *SrcC = 1791 TII.getNamedOperand(MI, AMDGPU::OpName::src2); 1792 assert(SrcC); 1793 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) 1794 return false; 1795 1796 MFMA = &MI; 1797 return true; 1798 }; 1799 1800 MFMA = nullptr; 1801 int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, 1802 MaxWarWaitStates); 1803 if (!MFMA) 1804 continue; 1805 1806 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 1807 int NeedWaitStates = MaxWaitStates; 1808 switch (HazardDefLatency) { 1809 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; 1810 break; 1811 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; 1812 break; 1813 case 16: LLVM_FALLTHROUGH; 1814 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; 1815 break; 1816 } 1817 1818 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; 1819 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1820 } 1821 1822 return WaitStatesNeeded; 1823 } 1824 1825 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { 1826 if (!SU->isInstr()) 1827 return false; 1828 1829 const MachineInstr *MAI = nullptr; 1830 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { 1831 MAI = nullptr; 1832 if (SIInstrInfo::isMAI(MI) && 1833 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 1834 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64) 1835 MAI = &MI; 1836 return MAI != nullptr; 1837 }; 1838 1839 MachineInstr *MI = SU->getInstr(); 1840 if (IsMFMAFn(*MI)) { 1841 int W = getWaitStatesSince(IsMFMAFn, 16); 1842 if (MAI) 1843 return W < (int)TSchedModel.computeInstrLatency(MAI); 1844 } 1845 1846 return false; 1847 } 1848