1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements hazard recognizers for scheduling on GCN processors. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "GCNHazardRecognizer.h" 14 #include "GCNSubtarget.h" 15 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 16 #include "SIMachineFunctionInfo.h" 17 #include "llvm/CodeGen/MachineFunction.h" 18 #include "llvm/CodeGen/ScheduleDAG.h" 19 #include "llvm/Support/TargetParser.h" 20 21 using namespace llvm; 22 23 namespace { 24 25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> { 26 MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {} 27 28 bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) { 29 if (Arg.getAsInteger(0, Value)) 30 return O.error("'" + Arg + "' value invalid for uint argument!"); 31 32 if (Value > 100) 33 return O.error("'" + Arg + "' value must be in the range [0, 100]!"); 34 35 return false; 36 } 37 }; 38 39 } // end anonymous namespace 40 41 static cl::opt<unsigned, false, MFMAPaddingRatioParser> 42 MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, 43 cl::desc("Fill a percentage of the latency between " 44 "neighboring MFMA with s_nops.")); 45 46 //===----------------------------------------------------------------------===// 47 // Hazard Recognizer Implementation 48 //===----------------------------------------------------------------------===// 49 50 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 51 const GCNSubtarget &ST); 52 53 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : 54 IsHazardRecognizerMode(false), 55 CurrCycleInstr(nullptr), 56 MF(MF), 57 ST(MF.getSubtarget<GCNSubtarget>()), 58 TII(*ST.getInstrInfo()), 59 TRI(TII.getRegisterInfo()), 60 ClauseUses(TRI.getNumRegUnits()), 61 ClauseDefs(TRI.getNumRegUnits()) { 62 MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5; 63 TSchedModel.init(&ST); 64 RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST); 65 } 66 67 void GCNHazardRecognizer::Reset() { 68 EmittedInstrs.clear(); 69 } 70 71 void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { 72 EmitInstruction(SU->getInstr()); 73 } 74 75 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { 76 CurrCycleInstr = MI; 77 } 78 79 static bool isDivFMas(unsigned Opcode) { 80 return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64; 81 } 82 83 static bool isSGetReg(unsigned Opcode) { 84 return Opcode == AMDGPU::S_GETREG_B32; 85 } 86 87 static bool isSSetReg(unsigned Opcode) { 88 switch (Opcode) { 89 case AMDGPU::S_SETREG_B32: 90 case AMDGPU::S_SETREG_B32_mode: 91 case AMDGPU::S_SETREG_IMM32_B32: 92 case AMDGPU::S_SETREG_IMM32_B32_mode: 93 return true; 94 } 95 return false; 96 } 97 98 static bool isRWLane(unsigned Opcode) { 99 return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32; 100 } 101 102 static bool isRFE(unsigned Opcode) { 103 return Opcode == AMDGPU::S_RFE_B64; 104 } 105 106 static bool isSMovRel(unsigned Opcode) { 107 switch (Opcode) { 108 case AMDGPU::S_MOVRELS_B32: 109 case AMDGPU::S_MOVRELS_B64: 110 case AMDGPU::S_MOVRELD_B32: 111 case AMDGPU::S_MOVRELD_B64: 112 return true; 113 default: 114 return false; 115 } 116 } 117 118 static bool isDGEMM(unsigned Opcode) { 119 return AMDGPU::getMAIIsDGEMM(Opcode); 120 } 121 122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { 123 unsigned Opcode = MI.getOpcode(); 124 125 if (!SIInstrInfo::isMAI(MI) || 126 isDGEMM(Opcode) || 127 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 128 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) 129 return false; 130 131 if (!ST.hasGFX940Insts()) 132 return true; 133 134 return AMDGPU::getMAIIsGFX940XDL(Opcode); 135 } 136 137 static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, 138 const MachineInstr &MI) { 139 if (TII.isAlwaysGDS(MI.getOpcode())) 140 return true; 141 142 switch (MI.getOpcode()) { 143 case AMDGPU::S_SENDMSG: 144 case AMDGPU::S_SENDMSGHALT: 145 case AMDGPU::S_TTRACEDATA: 146 return true; 147 // These DS opcodes don't support GDS. 148 case AMDGPU::DS_NOP: 149 case AMDGPU::DS_PERMUTE_B32: 150 case AMDGPU::DS_BPERMUTE_B32: 151 return false; 152 default: 153 if (TII.isDS(MI.getOpcode())) { 154 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 155 AMDGPU::OpName::gds); 156 if (MI.getOperand(GDS).getImm()) 157 return true; 158 } 159 return false; 160 } 161 } 162 163 static bool isPermlane(const MachineInstr &MI) { 164 unsigned Opcode = MI.getOpcode(); 165 return Opcode == AMDGPU::V_PERMLANE16_B32_e64 || 166 Opcode == AMDGPU::V_PERMLANEX16_B32_e64; 167 } 168 169 static bool isLdsDma(const MachineInstr &MI) { 170 return SIInstrInfo::isVALU(MI) && 171 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); 172 } 173 174 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { 175 const MachineOperand *RegOp = TII->getNamedOperand(RegInstr, 176 AMDGPU::OpName::simm16); 177 return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_; 178 } 179 180 ScheduleHazardRecognizer::HazardType 181 GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { 182 MachineInstr *MI = SU->getInstr(); 183 // If we are not in "HazardRecognizerMode" and therefore not being run from 184 // the scheduler, track possible stalls from hazards but don't insert noops. 185 auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard; 186 187 if (MI->isBundle()) 188 return NoHazard; 189 190 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) 191 return HazardType; 192 193 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) 194 return HazardType; 195 196 if (checkFPAtomicToDenormModeHazard(MI) > 0) 197 return HazardType; 198 199 if (ST.hasNoDataDepHazard()) 200 return NoHazard; 201 202 // FIXME: Should flat be considered vmem? 203 if ((SIInstrInfo::isVMEM(*MI) || 204 SIInstrInfo::isFLAT(*MI)) 205 && checkVMEMHazards(MI) > 0) 206 return HazardType; 207 208 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) 209 return HazardType; 210 211 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) 212 return HazardType; 213 214 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) 215 return HazardType; 216 217 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) 218 return HazardType; 219 220 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 221 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 222 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 223 return HazardType; 224 225 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) 226 return HazardType; 227 228 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) 229 return HazardType; 230 231 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) 232 return HazardType; 233 234 if (((ST.hasReadM0MovRelInterpHazard() && 235 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) || 236 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || 237 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || 238 (ST.hasReadM0LdsDirectHazard() && 239 MI->readsRegister(AMDGPU::LDS_DIRECT))) && 240 checkReadM0Hazards(MI) > 0) 241 return HazardType; 242 243 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) 244 return HazardType; 245 246 if ((SIInstrInfo::isVMEM(*MI) || 247 SIInstrInfo::isFLAT(*MI) || 248 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) 249 return HazardType; 250 251 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) 252 return HazardType; 253 254 return NoHazard; 255 } 256 257 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, 258 unsigned Quantity) { 259 while (Quantity > 0) { 260 unsigned Arg = std::min(Quantity, 8u); 261 Quantity -= Arg; 262 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) 263 .addImm(Arg - 1); 264 } 265 } 266 267 unsigned 268 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { 269 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); 270 assert(TSchedModel.getWriteProcResBegin(SC) != 271 TSchedModel.getWriteProcResEnd(SC)); 272 return TSchedModel.getWriteProcResBegin(SC)->Cycles; 273 } 274 275 void GCNHazardRecognizer::processBundle() { 276 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); 277 MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end(); 278 // Check bundled MachineInstr's for hazards. 279 for (; MI != E && MI->isInsideBundle(); ++MI) { 280 CurrCycleInstr = &*MI; 281 unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr); 282 283 if (IsHazardRecognizerMode) { 284 fixHazards(CurrCycleInstr); 285 286 insertNoopsInBundle(CurrCycleInstr, TII, WaitStates); 287 } 288 289 // It’s unnecessary to track more than MaxLookAhead instructions. Since we 290 // include the bundled MI directly after, only add a maximum of 291 // (MaxLookAhead - 1) noops to EmittedInstrs. 292 for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i) 293 EmittedInstrs.push_front(nullptr); 294 295 EmittedInstrs.push_front(CurrCycleInstr); 296 EmittedInstrs.resize(MaxLookAhead); 297 } 298 CurrCycleInstr = nullptr; 299 } 300 301 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { 302 IsHazardRecognizerMode = true; 303 CurrCycleInstr = MI; 304 unsigned W = PreEmitNoopsCommon(MI); 305 fixHazards(MI); 306 CurrCycleInstr = nullptr; 307 return W; 308 } 309 310 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { 311 if (MI->isBundle()) 312 return 0; 313 314 int WaitStates = 0; 315 316 if (SIInstrInfo::isSMRD(*MI)) 317 return std::max(WaitStates, checkSMRDHazards(MI)); 318 319 if (ST.hasNSAtoVMEMBug()) 320 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); 321 322 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); 323 324 if (ST.hasNoDataDepHazard()) 325 return WaitStates; 326 327 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) 328 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); 329 330 if (SIInstrInfo::isVALU(*MI)) 331 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); 332 333 if (SIInstrInfo::isDPP(*MI)) 334 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); 335 336 if (isDivFMas(MI->getOpcode())) 337 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); 338 339 if (isRWLane(MI->getOpcode())) 340 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); 341 342 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || 343 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || 344 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) 345 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); 346 347 if (MI->isInlineAsm()) 348 return std::max(WaitStates, checkInlineAsmHazards(MI)); 349 350 if (isSGetReg(MI->getOpcode())) 351 return std::max(WaitStates, checkGetRegHazards(MI)); 352 353 if (isSSetReg(MI->getOpcode())) 354 return std::max(WaitStates, checkSetRegHazards(MI)); 355 356 if (isRFE(MI->getOpcode())) 357 return std::max(WaitStates, checkRFEHazards(MI)); 358 359 if ((ST.hasReadM0MovRelInterpHazard() && 360 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) || 361 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || 362 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || 363 (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT))) 364 return std::max(WaitStates, checkReadM0Hazards(MI)); 365 366 if (SIInstrInfo::isMAI(*MI)) 367 return std::max(WaitStates, checkMAIHazards(MI)); 368 369 if (SIInstrInfo::isVMEM(*MI) || 370 SIInstrInfo::isFLAT(*MI) || 371 SIInstrInfo::isDS(*MI)) 372 return std::max(WaitStates, checkMAILdStHazards(MI)); 373 374 return WaitStates; 375 } 376 377 void GCNHazardRecognizer::EmitNoop() { 378 EmittedInstrs.push_front(nullptr); 379 } 380 381 void GCNHazardRecognizer::AdvanceCycle() { 382 // When the scheduler detects a stall, it will call AdvanceCycle() without 383 // emitting any instructions. 384 if (!CurrCycleInstr) { 385 EmittedInstrs.push_front(nullptr); 386 return; 387 } 388 389 if (CurrCycleInstr->isBundle()) { 390 processBundle(); 391 return; 392 } 393 394 unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); 395 if (!NumWaitStates) { 396 CurrCycleInstr = nullptr; 397 return; 398 } 399 400 // Keep track of emitted instructions 401 EmittedInstrs.push_front(CurrCycleInstr); 402 403 // Add a nullptr for each additional wait state after the first. Make sure 404 // not to add more than getMaxLookAhead() items to the list, since we 405 // truncate the list to that size right after this loop. 406 for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead()); 407 i < e; ++i) { 408 EmittedInstrs.push_front(nullptr); 409 } 410 411 // getMaxLookahead() is the largest number of wait states we will ever need 412 // to insert, so there is no point in keeping track of more than that many 413 // wait states. 414 EmittedInstrs.resize(getMaxLookAhead()); 415 416 CurrCycleInstr = nullptr; 417 } 418 419 void GCNHazardRecognizer::RecedeCycle() { 420 llvm_unreachable("hazard recognizer does not support bottom-up scheduling."); 421 } 422 423 //===----------------------------------------------------------------------===// 424 // Helper Functions 425 //===----------------------------------------------------------------------===// 426 427 typedef enum { HazardFound, HazardExpired, NoHazardFound } HazardFnResult; 428 429 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn; 430 typedef function_ref<unsigned int(const MachineInstr &)> GetNumWaitStatesFn; 431 432 // Search for a hazard in a block and its predecessors. 433 template <typename StateT> 434 static bool 435 hasHazard(StateT State, 436 function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard, 437 function_ref<void(StateT &, const MachineInstr &)> UpdateState, 438 const MachineBasicBlock *MBB, 439 MachineBasicBlock::const_reverse_instr_iterator I, 440 DenseSet<const MachineBasicBlock *> &Visited) { 441 for (auto E = MBB->instr_rend(); I != E; ++I) { 442 // No need to look at parent BUNDLE instructions. 443 if (I->isBundle()) 444 continue; 445 446 switch (IsHazard(State, *I)) { 447 case HazardFound: 448 return true; 449 case HazardExpired: 450 return false; 451 default: 452 // Continue search 453 break; 454 } 455 456 if (I->isInlineAsm() || I->isMetaInstruction()) 457 continue; 458 459 UpdateState(State, *I); 460 } 461 462 for (MachineBasicBlock *Pred : MBB->predecessors()) { 463 if (!Visited.insert(Pred).second) 464 continue; 465 466 if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(), 467 Visited)) 468 return true; 469 } 470 471 return false; 472 } 473 474 // Returns a minimum wait states since \p I walking all predecessors. 475 // Only scans until \p IsExpired does not return true. 476 // Can only be run in a hazard recognizer mode. 477 static int getWaitStatesSince( 478 GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, 479 MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, 480 IsExpiredFn IsExpired, DenseSet<const MachineBasicBlock *> &Visited, 481 GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates) { 482 for (auto E = MBB->instr_rend(); I != E; ++I) { 483 // Don't add WaitStates for parent BUNDLE instructions. 484 if (I->isBundle()) 485 continue; 486 487 if (IsHazard(*I)) 488 return WaitStates; 489 490 if (I->isInlineAsm()) 491 continue; 492 493 WaitStates += GetNumWaitStates(*I); 494 495 if (IsExpired(*I, WaitStates)) 496 return std::numeric_limits<int>::max(); 497 } 498 499 int MinWaitStates = std::numeric_limits<int>::max(); 500 for (MachineBasicBlock *Pred : MBB->predecessors()) { 501 if (!Visited.insert(Pred).second) 502 continue; 503 504 int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates, 505 IsExpired, Visited, GetNumWaitStates); 506 507 MinWaitStates = std::min(MinWaitStates, W); 508 } 509 510 return MinWaitStates; 511 } 512 513 static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, 514 const MachineInstr *MI, IsExpiredFn IsExpired) { 515 DenseSet<const MachineBasicBlock *> Visited; 516 return getWaitStatesSince(IsHazard, MI->getParent(), 517 std::next(MI->getReverseIterator()), 518 0, IsExpired, Visited); 519 } 520 521 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { 522 if (IsHazardRecognizerMode) { 523 auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) { 524 return WaitStates >= Limit; 525 }; 526 return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn); 527 } 528 529 int WaitStates = 0; 530 for (MachineInstr *MI : EmittedInstrs) { 531 if (MI) { 532 if (IsHazard(*MI)) 533 return WaitStates; 534 535 if (MI->isInlineAsm()) 536 continue; 537 } 538 ++WaitStates; 539 540 if (WaitStates >= Limit) 541 break; 542 } 543 return std::numeric_limits<int>::max(); 544 } 545 546 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg, 547 IsHazardFn IsHazardDef, 548 int Limit) { 549 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 550 551 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { 552 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); 553 }; 554 555 return getWaitStatesSince(IsHazardFn, Limit); 556 } 557 558 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard, 559 int Limit) { 560 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { 561 return isSSetReg(MI.getOpcode()) && IsHazard(MI); 562 }; 563 564 return getWaitStatesSince(IsHazardFn, Limit); 565 } 566 567 //===----------------------------------------------------------------------===// 568 // No-op Hazard Detection 569 //===----------------------------------------------------------------------===// 570 571 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, 572 MCRegister Reg) { 573 for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) 574 BV.set(*RUI); 575 } 576 577 static void addRegsToSet(const SIRegisterInfo &TRI, 578 iterator_range<MachineInstr::const_mop_iterator> Ops, 579 BitVector &Set) { 580 for (const MachineOperand &Op : Ops) { 581 if (Op.isReg()) 582 addRegUnits(TRI, Set, Op.getReg().asMCReg()); 583 } 584 } 585 586 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { 587 // XXX: Do we need to worry about implicit operands 588 addRegsToSet(TRI, MI.defs(), ClauseDefs); 589 addRegsToSet(TRI, MI.uses(), ClauseUses); 590 } 591 592 static bool breaksSMEMSoftClause(MachineInstr *MI) { 593 return !SIInstrInfo::isSMRD(*MI); 594 } 595 596 static bool breaksVMEMSoftClause(MachineInstr *MI) { 597 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); 598 } 599 600 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { 601 // SMEM soft clause are only present on VI+, and only matter if xnack is 602 // enabled. 603 if (!ST.isXNACKEnabled()) 604 return 0; 605 606 bool IsSMRD = TII.isSMRD(*MEM); 607 608 resetClause(); 609 610 // A soft-clause is any group of consecutive SMEM instructions. The 611 // instructions in this group may return out of order and/or may be 612 // replayed (i.e. the same instruction issued more than once). 613 // 614 // In order to handle these situations correctly we need to make sure that 615 // when a clause has more than one instruction, no instruction in the clause 616 // writes to a register that is read by another instruction in the clause 617 // (including itself). If we encounter this situation, we need to break the 618 // clause by inserting a non SMEM instruction. 619 620 for (MachineInstr *MI : EmittedInstrs) { 621 // When we hit a non-SMEM instruction then we have passed the start of the 622 // clause and we can stop. 623 if (!MI) 624 break; 625 626 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) 627 break; 628 629 addClauseInst(*MI); 630 } 631 632 if (ClauseDefs.none()) 633 return 0; 634 635 // We need to make sure not to put loads and stores in the same clause if they 636 // use the same address. For now, just start a new clause whenever we see a 637 // store. 638 if (MEM->mayStore()) 639 return 1; 640 641 addClauseInst(*MEM); 642 643 // If the set of defs and uses intersect then we cannot add this instruction 644 // to the clause, so we have a hazard. 645 return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; 646 } 647 648 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { 649 int WaitStatesNeeded = 0; 650 651 WaitStatesNeeded = checkSoftClauseHazards(SMRD); 652 653 // This SMRD hazard only affects SI. 654 if (!ST.hasSMRDReadVALUDefHazard()) 655 return WaitStatesNeeded; 656 657 // A read of an SGPR by SMRD instruction requires 4 wait states when the 658 // SGPR was written by a VALU instruction. 659 int SmrdSgprWaitStates = 4; 660 auto IsHazardDefFn = [this](const MachineInstr &MI) { 661 return TII.isVALU(MI); 662 }; 663 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { 664 return TII.isSALU(MI); 665 }; 666 667 bool IsBufferSMRD = TII.isBufferSMRD(*SMRD); 668 669 for (const MachineOperand &Use : SMRD->uses()) { 670 if (!Use.isReg()) 671 continue; 672 int WaitStatesNeededForUse = 673 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 674 SmrdSgprWaitStates); 675 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 676 677 // This fixes what appears to be undocumented hardware behavior in SI where 678 // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor 679 // needs some number of nops in between. We don't know how many we need, but 680 // let's use 4. This wasn't discovered before probably because the only 681 // case when this happens is when we expand a 64-bit pointer into a full 682 // descriptor and use s_buffer_load_dword instead of s_load_dword, which was 683 // probably never encountered in the closed-source land. 684 if (IsBufferSMRD) { 685 int WaitStatesNeededForUse = 686 SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), 687 IsBufferHazardDefFn, 688 SmrdSgprWaitStates); 689 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 690 } 691 } 692 693 return WaitStatesNeeded; 694 } 695 696 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { 697 if (!ST.hasVMEMReadSGPRVALUDefHazard()) 698 return 0; 699 700 int WaitStatesNeeded = checkSoftClauseHazards(VMEM); 701 702 // A read of an SGPR by a VMEM instruction requires 5 wait states when the 703 // SGPR was written by a VALU Instruction. 704 const int VmemSgprWaitStates = 5; 705 auto IsHazardDefFn = [this](const MachineInstr &MI) { 706 return TII.isVALU(MI); 707 }; 708 for (const MachineOperand &Use : VMEM->uses()) { 709 if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg())) 710 continue; 711 712 int WaitStatesNeededForUse = 713 VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn, 714 VmemSgprWaitStates); 715 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 716 } 717 return WaitStatesNeeded; 718 } 719 720 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { 721 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 722 const SIInstrInfo *TII = ST.getInstrInfo(); 723 724 // Check for DPP VGPR read after VALU VGPR write and EXEC write. 725 int DppVgprWaitStates = 2; 726 int DppExecWaitStates = 5; 727 int WaitStatesNeeded = 0; 728 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 729 return TII->isVALU(MI); 730 }; 731 732 for (const MachineOperand &Use : DPP->uses()) { 733 if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg())) 734 continue; 735 int WaitStatesNeededForUse = 736 DppVgprWaitStates - getWaitStatesSinceDef( 737 Use.getReg(), 738 [](const MachineInstr &) { return true; }, 739 DppVgprWaitStates); 740 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 741 } 742 743 WaitStatesNeeded = std::max( 744 WaitStatesNeeded, 745 DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn, 746 DppExecWaitStates)); 747 748 return WaitStatesNeeded; 749 } 750 751 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { 752 const SIInstrInfo *TII = ST.getInstrInfo(); 753 754 // v_div_fmas requires 4 wait states after a write to vcc from a VALU 755 // instruction. 756 const int DivFMasWaitStates = 4; 757 auto IsHazardDefFn = [TII](const MachineInstr &MI) { 758 return TII->isVALU(MI); 759 }; 760 int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn, 761 DivFMasWaitStates); 762 763 return DivFMasWaitStates - WaitStatesNeeded; 764 } 765 766 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { 767 const SIInstrInfo *TII = ST.getInstrInfo(); 768 unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr); 769 770 const int GetRegWaitStates = 2; 771 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { 772 return GetRegHWReg == getHWReg(TII, MI); 773 }; 774 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates); 775 776 return GetRegWaitStates - WaitStatesNeeded; 777 } 778 779 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { 780 const SIInstrInfo *TII = ST.getInstrInfo(); 781 unsigned HWReg = getHWReg(TII, *SetRegInstr); 782 783 const int SetRegWaitStates = ST.getSetRegWaitStates(); 784 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { 785 return HWReg == getHWReg(TII, MI); 786 }; 787 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates); 788 return SetRegWaitStates - WaitStatesNeeded; 789 } 790 791 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { 792 if (!MI.mayStore()) 793 return -1; 794 795 const SIInstrInfo *TII = ST.getInstrInfo(); 796 unsigned Opcode = MI.getOpcode(); 797 const MCInstrDesc &Desc = MI.getDesc(); 798 799 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 800 int VDataRCID = -1; 801 if (VDataIdx != -1) 802 VDataRCID = Desc.OpInfo[VDataIdx].RegClass; 803 804 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { 805 // There is no hazard if the instruction does not use vector regs 806 // (like wbinvl1) 807 if (VDataIdx == -1) 808 return -1; 809 // For MUBUF/MTBUF instructions this hazard only exists if the 810 // instruction is not using a register in the soffset field. 811 const MachineOperand *SOffset = 812 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); 813 // If we have no soffset operand, then assume this field has been 814 // hardcoded to zero. 815 if (AMDGPU::getRegBitWidth(VDataRCID) > 64 && 816 (!SOffset || !SOffset->isReg())) 817 return VDataIdx; 818 } 819 820 // MIMG instructions create a hazard if they don't use a 256-bit T# and 821 // the store size is greater than 8 bytes and they have more than two bits 822 // of their dmask set. 823 // All our MIMG definitions use a 256-bit T#, so we can skip checking for them. 824 if (TII->isMIMG(MI)) { 825 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 826 assert(SRsrcIdx != -1 && 827 AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256); 828 (void)SRsrcIdx; 829 } 830 831 if (TII->isFLAT(MI)) { 832 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata); 833 if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64) 834 return DataIdx; 835 } 836 837 return -1; 838 } 839 840 int 841 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def, 842 const MachineRegisterInfo &MRI) { 843 // Helper to check for the hazard where VMEM instructions that store more than 844 // 8 bytes can have there store data over written by the next instruction. 845 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 846 847 const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1; 848 int WaitStatesNeeded = 0; 849 850 if (!TRI->isVectorRegister(MRI, Def.getReg())) 851 return WaitStatesNeeded; 852 Register Reg = Def.getReg(); 853 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { 854 int DataIdx = createsVALUHazard(MI); 855 return DataIdx >= 0 && 856 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); 857 }; 858 int WaitStatesNeededForDef = 859 VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates); 860 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 861 862 return WaitStatesNeeded; 863 } 864 865 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { 866 int WaitStatesNeeded = 0; 867 868 if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) { 869 const int TransDefWaitstates = 1; 870 871 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { 872 if (!SIInstrInfo::isTRANS(MI)) 873 return false; 874 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 875 const SIInstrInfo *TII = ST.getInstrInfo(); 876 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); 877 878 for (const MachineOperand &Use : VALU->explicit_uses()) { 879 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) 880 return true; 881 } 882 883 return false; 884 }; 885 886 int WaitStatesNeededForDef = 887 TransDefWaitstates - 888 getWaitStatesSince(IsTransDefFn, TransDefWaitstates); 889 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 890 } 891 892 if (ST.hasDstSelForwardingHazard()) { 893 const int Shift16DefWaitstates = 1; 894 895 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) { 896 if (!SIInstrInfo::isVALU(MI)) 897 return false; 898 const SIInstrInfo *TII = ST.getInstrInfo(); 899 if (SIInstrInfo::isSDWA(MI)) { 900 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) 901 if (DstSel->getImm() == AMDGPU::SDWA::DWORD) 902 return false; 903 } else { 904 if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(), 905 AMDGPU::OpName::op_sel) == -1) || 906 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) 907 ->getImm() & 908 SISrcMods::DST_OP_SEL)) 909 return false; 910 } 911 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 912 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { 913 Register Def = Dst->getReg(); 914 915 for (const MachineOperand &Use : VALU->explicit_uses()) { 916 if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg())) 917 return true; 918 } 919 } 920 921 return false; 922 }; 923 924 int WaitStatesNeededForDef = 925 Shift16DefWaitstates - 926 getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates); 927 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 928 } 929 930 if (ST.hasVDecCoExecHazard()) { 931 const int VALUWriteSGPRVALUReadWaitstates = 2; 932 const int VALUWriteEXECRWLane = 4; 933 const int VALUWriteVGPRReadlaneRead = 1; 934 935 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 936 const MachineRegisterInfo &MRI = MF.getRegInfo(); 937 Register UseReg; 938 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { 939 if (!SIInstrInfo::isVALU(MI)) 940 return false; 941 return MI.modifiesRegister(UseReg, TRI); 942 }; 943 944 for (const MachineOperand &Use : VALU->explicit_uses()) { 945 if (!Use.isReg()) 946 continue; 947 948 UseReg = Use.getReg(); 949 if (TRI->isSGPRReg(MRI, UseReg)) { 950 int WaitStatesNeededForDef = 951 VALUWriteSGPRVALUReadWaitstates - 952 getWaitStatesSince(IsVALUDefSGPRFn, 953 VALUWriteSGPRVALUReadWaitstates); 954 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 955 } 956 } 957 958 if (VALU->readsRegister(AMDGPU::VCC, TRI)) { 959 UseReg = AMDGPU::VCC; 960 int WaitStatesNeededForDef = 961 VALUWriteSGPRVALUReadWaitstates - 962 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates); 963 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 964 } 965 966 switch (VALU->getOpcode()) { 967 case AMDGPU::V_READLANE_B32: 968 case AMDGPU::V_READFIRSTLANE_B32: { 969 MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0); 970 UseReg = Src->getReg(); 971 int WaitStatesNeededForDef = 972 VALUWriteVGPRReadlaneRead - 973 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead); 974 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 975 } 976 LLVM_FALLTHROUGH; 977 case AMDGPU::V_WRITELANE_B32: { 978 UseReg = AMDGPU::EXEC; 979 int WaitStatesNeededForDef = 980 VALUWriteEXECRWLane - 981 getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane); 982 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef); 983 break; 984 } 985 default: 986 break; 987 } 988 } 989 990 // This checks for the hazard where VMEM instructions that store more than 991 // 8 bytes can have there store data over written by the next instruction. 992 if (!ST.has12DWordStoreHazard()) 993 return WaitStatesNeeded; 994 995 const MachineRegisterInfo &MRI = MF.getRegInfo(); 996 997 for (const MachineOperand &Def : VALU->defs()) { 998 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI)); 999 } 1000 1001 return WaitStatesNeeded; 1002 } 1003 1004 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { 1005 // This checks for hazards associated with inline asm statements. 1006 // Since inline asms can contain just about anything, we use this 1007 // to call/leverage other check*Hazard routines. Note that 1008 // this function doesn't attempt to address all possible inline asm 1009 // hazards (good luck), but is a collection of what has been 1010 // problematic thus far. 1011 1012 // see checkVALUHazards() 1013 if (!ST.has12DWordStoreHazard()) 1014 return 0; 1015 1016 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1017 int WaitStatesNeeded = 0; 1018 1019 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands(); 1020 I != E; ++I) { 1021 const MachineOperand &Op = IA->getOperand(I); 1022 if (Op.isReg() && Op.isDef()) { 1023 WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI)); 1024 } 1025 } 1026 1027 return WaitStatesNeeded; 1028 } 1029 1030 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { 1031 const SIInstrInfo *TII = ST.getInstrInfo(); 1032 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1033 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1034 1035 const MachineOperand *LaneSelectOp = 1036 TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1); 1037 1038 if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg())) 1039 return 0; 1040 1041 Register LaneSelectReg = LaneSelectOp->getReg(); 1042 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; 1043 1044 const int RWLaneWaitStates = 4; 1045 int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn, 1046 RWLaneWaitStates); 1047 return RWLaneWaitStates - WaitStatesSince; 1048 } 1049 1050 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { 1051 if (!ST.hasRFEHazards()) 1052 return 0; 1053 1054 const SIInstrInfo *TII = ST.getInstrInfo(); 1055 1056 const int RFEWaitStates = 1; 1057 1058 auto IsHazardFn = [TII](const MachineInstr &MI) { 1059 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; 1060 }; 1061 int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates); 1062 return RFEWaitStates - WaitStatesNeeded; 1063 } 1064 1065 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { 1066 const SIInstrInfo *TII = ST.getInstrInfo(); 1067 const int ReadM0WaitStates = 1; 1068 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; 1069 return ReadM0WaitStates - 1070 getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates); 1071 } 1072 1073 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { 1074 fixVMEMtoScalarWriteHazards(MI); 1075 fixVcmpxPermlaneHazards(MI); 1076 fixSMEMtoVectorWriteHazards(MI); 1077 fixVcmpxExecWARHazard(MI); 1078 fixLdsBranchVmemWARHazard(MI); 1079 if (ST.hasLdsDirect()) { 1080 fixLdsDirectVALUHazard(MI); 1081 fixLdsDirectVMEMHazard(MI); 1082 } 1083 fixVALUPartialForwardingHazard(MI); 1084 fixVALUTransUseHazard(MI); 1085 fixWMMAHazards(MI); 1086 } 1087 1088 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { 1089 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) 1090 return false; 1091 1092 const SIInstrInfo *TII = ST.getInstrInfo(); 1093 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1094 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { 1095 return (TII->isVOPC(MI) || 1096 ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) && 1097 MI.modifiesRegister(AMDGPU::EXEC, TRI); 1098 }; 1099 1100 auto IsExpiredFn = [](const MachineInstr &MI, int) { 1101 unsigned Opc = MI.getOpcode(); 1102 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && 1103 Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa; 1104 }; 1105 1106 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1107 std::numeric_limits<int>::max()) 1108 return false; 1109 1110 // V_NOP will be discarded by SQ. 1111 // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE* 1112 // which is always a VGPR and available. 1113 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); 1114 Register Reg = Src0->getReg(); 1115 bool IsUndef = Src0->isUndef(); 1116 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1117 TII->get(AMDGPU::V_MOV_B32_e32)) 1118 .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0)) 1119 .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill); 1120 1121 return true; 1122 } 1123 1124 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { 1125 if (!ST.hasVMEMtoScalarWriteHazard()) 1126 return false; 1127 1128 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) 1129 return false; 1130 1131 if (MI->getNumDefs() == 0) 1132 return false; 1133 1134 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1135 1136 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { 1137 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) && 1138 !SIInstrInfo::isFLAT(I)) 1139 return false; 1140 1141 for (const MachineOperand &Def : MI->defs()) { 1142 const MachineOperand *Op = 1143 I.findRegisterUseOperand(Def.getReg(), false, TRI); 1144 if (!Op) 1145 continue; 1146 return true; 1147 } 1148 return false; 1149 }; 1150 1151 auto IsExpiredFn = [](const MachineInstr &MI, int) { 1152 return SIInstrInfo::isVALU(MI) || 1153 (MI.getOpcode() == AMDGPU::S_WAITCNT && 1154 !MI.getOperand(0).getImm()) || 1155 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1156 MI.getOperand(0).getImm() == 0xffe3); 1157 }; 1158 1159 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1160 std::numeric_limits<int>::max()) 1161 return false; 1162 1163 const SIInstrInfo *TII = ST.getInstrInfo(); 1164 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1165 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1166 .addImm(0xffe3); 1167 return true; 1168 } 1169 1170 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { 1171 if (!ST.hasSMEMtoVectorWriteHazard()) 1172 return false; 1173 1174 if (!SIInstrInfo::isVALU(*MI)) 1175 return false; 1176 1177 unsigned SDSTName; 1178 switch (MI->getOpcode()) { 1179 case AMDGPU::V_READLANE_B32: 1180 case AMDGPU::V_READFIRSTLANE_B32: 1181 SDSTName = AMDGPU::OpName::vdst; 1182 break; 1183 default: 1184 SDSTName = AMDGPU::OpName::sdst; 1185 break; 1186 } 1187 1188 const SIInstrInfo *TII = ST.getInstrInfo(); 1189 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1190 const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU()); 1191 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); 1192 if (!SDST) { 1193 for (const auto &MO : MI->implicit_operands()) { 1194 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) { 1195 SDST = &MO; 1196 break; 1197 } 1198 } 1199 } 1200 1201 if (!SDST) 1202 return false; 1203 1204 const Register SDSTReg = SDST->getReg(); 1205 auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) { 1206 return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI); 1207 }; 1208 1209 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { 1210 if (TII->isSALU(MI)) { 1211 switch (MI.getOpcode()) { 1212 case AMDGPU::S_SETVSKIP: 1213 case AMDGPU::S_VERSION: 1214 case AMDGPU::S_WAITCNT_VSCNT: 1215 case AMDGPU::S_WAITCNT_VMCNT: 1216 case AMDGPU::S_WAITCNT_EXPCNT: 1217 // These instructions cannot not mitigate the hazard. 1218 return false; 1219 case AMDGPU::S_WAITCNT_LGKMCNT: 1220 // Reducing lgkmcnt count to 0 always mitigates the hazard. 1221 return (MI.getOperand(1).getImm() == 0) && 1222 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); 1223 case AMDGPU::S_WAITCNT: { 1224 const int64_t Imm = MI.getOperand(0).getImm(); 1225 AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm); 1226 return (Decoded.LgkmCnt == 0); 1227 } 1228 default: 1229 // SOPP instructions cannot mitigate the hazard. 1230 if (TII->isSOPP(MI)) 1231 return false; 1232 // At this point the SALU can be assumed to mitigate the hazard 1233 // because either: 1234 // (a) it is independent of the at risk SMEM (breaking chain), 1235 // or 1236 // (b) it is dependent on the SMEM, in which case an appropriate 1237 // s_waitcnt lgkmcnt _must_ exist between it and the at risk 1238 // SMEM instruction. 1239 return true; 1240 } 1241 } 1242 return false; 1243 }; 1244 1245 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1246 std::numeric_limits<int>::max()) 1247 return false; 1248 1249 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1250 TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL) 1251 .addImm(0); 1252 return true; 1253 } 1254 1255 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { 1256 if (!ST.hasVcmpxExecWARHazard() || !SIInstrInfo::isVALU(*MI)) 1257 return false; 1258 1259 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1260 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) 1261 return false; 1262 1263 auto IsHazardFn = [TRI](const MachineInstr &I) { 1264 if (SIInstrInfo::isVALU(I)) 1265 return false; 1266 return I.readsRegister(AMDGPU::EXEC, TRI); 1267 }; 1268 1269 const SIInstrInfo *TII = ST.getInstrInfo(); 1270 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { 1271 if (SIInstrInfo::isVALU(MI)) { 1272 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) 1273 return true; 1274 for (auto MO : MI.implicit_operands()) 1275 if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) 1276 return true; 1277 } 1278 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1279 (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe) 1280 return true; 1281 return false; 1282 }; 1283 1284 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1285 std::numeric_limits<int>::max()) 1286 return false; 1287 1288 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1289 TII->get(AMDGPU::S_WAITCNT_DEPCTR)) 1290 .addImm(0xfffe); 1291 return true; 1292 } 1293 1294 static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, 1295 const GCNSubtarget &ST) { 1296 if (!ST.hasLdsBranchVmemWARHazard()) 1297 return false; 1298 1299 // Check if the necessary condition for the hazard is met: both LDS and VMEM 1300 // instructions need to appear in the same function. 1301 bool HasLds = false; 1302 bool HasVmem = false; 1303 for (auto &MBB : MF) { 1304 for (auto &MI : MBB) { 1305 HasLds |= SIInstrInfo::isDS(MI); 1306 HasVmem |= 1307 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); 1308 if (HasLds && HasVmem) 1309 return true; 1310 } 1311 } 1312 return false; 1313 } 1314 1315 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { 1316 if (!RunLdsBranchVmemWARHazardFixup) 1317 return false; 1318 1319 assert(ST.hasLdsBranchVmemWARHazard()); 1320 1321 auto IsHazardInst = [](const MachineInstr &MI) { 1322 if (SIInstrInfo::isDS(MI)) 1323 return 1; 1324 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) 1325 return 2; 1326 return 0; 1327 }; 1328 1329 auto InstType = IsHazardInst(*MI); 1330 if (!InstType) 1331 return false; 1332 1333 auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) { 1334 return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1335 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1336 !I.getOperand(1).getImm()); 1337 }; 1338 1339 auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) { 1340 if (!I.isBranch()) 1341 return false; 1342 1343 auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) { 1344 auto InstType2 = IsHazardInst(I); 1345 return InstType2 && InstType != InstType2; 1346 }; 1347 1348 auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) { 1349 auto InstType2 = IsHazardInst(I); 1350 if (InstType == InstType2) 1351 return true; 1352 1353 return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT && 1354 I.getOperand(0).getReg() == AMDGPU::SGPR_NULL && 1355 !I.getOperand(1).getImm(); 1356 }; 1357 1358 return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) != 1359 std::numeric_limits<int>::max(); 1360 }; 1361 1362 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1363 std::numeric_limits<int>::max()) 1364 return false; 1365 1366 const SIInstrInfo *TII = ST.getInstrInfo(); 1367 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1368 TII->get(AMDGPU::S_WAITCNT_VSCNT)) 1369 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) 1370 .addImm(0); 1371 1372 return true; 1373 } 1374 1375 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { 1376 if (!SIInstrInfo::isLDSDIR(*MI)) 1377 return false; 1378 1379 const int NoHazardWaitStates = 15; 1380 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); 1381 const Register VDSTReg = VDST->getReg(); 1382 1383 bool VisitedTrans = false; 1384 auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) { 1385 if (!SIInstrInfo::isVALU(I)) 1386 return false; 1387 VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I); 1388 // Cover both WAR and WAW 1389 return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); 1390 }; 1391 auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) { 1392 if (WaitStates >= NoHazardWaitStates) 1393 return true; 1394 // Instructions which cause va_vdst==0 expire hazard 1395 return SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || 1396 SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I); 1397 }; 1398 auto GetWaitStatesFn = [](const MachineInstr &MI) { 1399 return SIInstrInfo::isVALU(MI) ? 1 : 0; 1400 }; 1401 1402 DenseSet<const MachineBasicBlock *> Visited; 1403 auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(), 1404 std::next(MI->getReverseIterator()), 0, 1405 IsExpiredFn, Visited, GetWaitStatesFn); 1406 1407 // Transcendentals can execute in parallel to other VALUs. 1408 // This makes va_vdst count unusable with a mixture of VALU and TRANS. 1409 if (VisitedTrans) 1410 Count = 0; 1411 1412 MachineOperand *WaitVdstOp = 1413 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); 1414 WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates)); 1415 1416 return true; 1417 } 1418 1419 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { 1420 if (!SIInstrInfo::isLDSDIR(*MI)) 1421 return false; 1422 1423 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); 1424 const Register VDSTReg = VDST->getReg(); 1425 1426 auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) { 1427 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I) && 1428 !SIInstrInfo::isDS(I)) 1429 return false; 1430 return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI); 1431 }; 1432 auto IsExpiredFn = [](const MachineInstr &I, int) { 1433 return SIInstrInfo::isVALU(I) || SIInstrInfo::isEXP(I) || 1434 (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) || 1435 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1436 I.getOperand(0).getImm() == 0xffe3); 1437 }; 1438 1439 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1440 std::numeric_limits<int>::max()) 1441 return false; 1442 1443 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1444 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 1445 .addImm(0xffe3); 1446 1447 return true; 1448 } 1449 1450 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { 1451 if (!ST.isWave64()) 1452 return false; 1453 if (!ST.hasVALUPartialForwardingHazard()) 1454 return false; 1455 if (!SIInstrInfo::isVALU(*MI)) 1456 return false; 1457 1458 SmallSetVector<Register, 4> SrcVGPRs; 1459 1460 for (const MachineOperand &Use : MI->explicit_uses()) { 1461 if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1462 SrcVGPRs.insert(Use.getReg()); 1463 } 1464 1465 // Only applies with >= 2 unique VGPR sources 1466 if (SrcVGPRs.size() <= 1) 1467 return false; 1468 1469 // Look for the following pattern: 1470 // Va <- VALU [PreExecPos] 1471 // intv1 1472 // Exec <- SALU [ExecPos] 1473 // intv2 1474 // Vb <- VALU [PostExecPos] 1475 // intv3 1476 // MI Va, Vb (WaitState = 0) 1477 // 1478 // Where: 1479 // intv1 + intv2 <= 2 VALUs 1480 // intv3 <= 4 VALUs 1481 // 1482 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI. 1483 1484 const int Intv1plus2MaxVALUs = 2; 1485 const int Intv3MaxVALUs = 4; 1486 const int IntvMaxVALUs = 6; 1487 const int NoHazardVALUWaitStates = IntvMaxVALUs + 2; 1488 1489 struct StateType { 1490 SmallDenseMap<Register, int, 4> DefPos; 1491 int ExecPos = std::numeric_limits<int>::max(); 1492 int VALUs = 0; 1493 }; 1494 1495 StateType State; 1496 1497 // This overloads expiry testing with all the hazard detection 1498 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) { 1499 // Too many VALU states have passed 1500 if (State.VALUs > NoHazardVALUWaitStates) 1501 return HazardExpired; 1502 1503 // Instructions which cause va_vdst==0 expire hazard 1504 if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || 1505 SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || 1506 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1507 I.getOperand(0).getImm() == 0x0fff)) 1508 return HazardExpired; 1509 1510 // Track registers writes 1511 bool Changed = false; 1512 if (SIInstrInfo::isVALU(I)) { 1513 for (Register Src : SrcVGPRs) { 1514 if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) { 1515 State.DefPos[Src] = State.VALUs; 1516 Changed = true; 1517 } 1518 } 1519 } else if (SIInstrInfo::isSALU(I)) { 1520 if (State.ExecPos == std::numeric_limits<int>::max()) { 1521 if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) { 1522 State.ExecPos = State.VALUs; 1523 Changed = true; 1524 } 1525 } 1526 } 1527 1528 // Early expiration: too many VALUs in intv3 1529 if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty()) 1530 return HazardExpired; 1531 1532 // Only evaluate state if something changed 1533 if (!Changed) 1534 return NoHazardFound; 1535 1536 // Determine positions of VALUs pre/post exec change 1537 if (State.ExecPos == std::numeric_limits<int>::max()) 1538 return NoHazardFound; 1539 1540 int PreExecPos = std::numeric_limits<int>::max(); 1541 int PostExecPos = std::numeric_limits<int>::max(); 1542 1543 for (auto Entry : State.DefPos) { 1544 int DefVALUs = Entry.second; 1545 if (DefVALUs != std::numeric_limits<int>::max()) { 1546 if (DefVALUs >= State.ExecPos) 1547 PreExecPos = std::min(PreExecPos, DefVALUs); 1548 else if (DefVALUs < State.ExecPos) 1549 PostExecPos = std::min(PostExecPos, DefVALUs); 1550 } 1551 } 1552 1553 // Need a VALUs post exec change 1554 if (PostExecPos == std::numeric_limits<int>::max()) 1555 return NoHazardFound; 1556 1557 // Too many VALUs in intv3? 1558 int Intv3VALUs = PostExecPos; 1559 if (Intv3VALUs > Intv3MaxVALUs) 1560 return HazardExpired; 1561 1562 // Too many VALUs in intv2? 1563 int Intv2VALUs = (State.ExecPos - PostExecPos) - 1; 1564 if (Intv2VALUs > Intv1plus2MaxVALUs) 1565 return HazardExpired; 1566 1567 // Need a VALUs pre exec change 1568 if (PreExecPos == std::numeric_limits<int>::max()) 1569 return NoHazardFound; 1570 1571 // Too many VALUs in intv1? 1572 int Intv1VALUs = PreExecPos - State.ExecPos; 1573 if (Intv1VALUs > Intv1plus2MaxVALUs) 1574 return HazardExpired; 1575 1576 // Too many VALUs in intv1 + intv2 1577 if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs) 1578 return HazardExpired; 1579 1580 return HazardFound; 1581 }; 1582 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { 1583 if (SIInstrInfo::isVALU(MI)) 1584 State.VALUs += 1; 1585 }; 1586 1587 DenseSet<const MachineBasicBlock *> Visited; 1588 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), 1589 std::next(MI->getReverseIterator()), Visited)) 1590 return false; 1591 1592 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1593 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 1594 .addImm(0x0fff); 1595 1596 return true; 1597 } 1598 1599 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { 1600 if (!ST.hasVALUTransUseHazard()) 1601 return false; 1602 if (!SIInstrInfo::isVALU(*MI)) 1603 return false; 1604 1605 SmallSet<Register, 4> SrcVGPRs; 1606 1607 for (const MachineOperand &Use : MI->explicit_uses()) { 1608 if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1609 SrcVGPRs.insert(Use.getReg()); 1610 } 1611 1612 // Look for the following pattern: 1613 // Va <- TRANS VALU 1614 // intv 1615 // MI Va (WaitState = 0) 1616 // 1617 // Where: 1618 // intv <= 5 VALUs / 1 TRANS 1619 // 1620 // If found, insert an appropriate S_WAITCNT_DEPCTR before MI. 1621 1622 const int IntvMaxVALUs = 5; 1623 const int IntvMaxTRANS = 1; 1624 1625 struct StateType { 1626 int VALUs = 0; 1627 int TRANS = 0; 1628 }; 1629 1630 StateType State; 1631 1632 // This overloads expiry testing with all the hazard detection 1633 auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) { 1634 // Too many VALU states have passed 1635 if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS) 1636 return HazardExpired; 1637 1638 // Instructions which cause va_vdst==0 expire hazard 1639 if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) || 1640 SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) || 1641 (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && 1642 I.getOperand(0).getImm() == 0x0fff)) 1643 return HazardExpired; 1644 1645 // Track registers writes 1646 if (SIInstrInfo::isTRANS(I)) { 1647 for (Register Src : SrcVGPRs) { 1648 if (I.modifiesRegister(Src, &TRI)) { 1649 return HazardFound; 1650 } 1651 } 1652 } 1653 1654 return NoHazardFound; 1655 }; 1656 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { 1657 if (SIInstrInfo::isVALU(MI)) 1658 State.VALUs += 1; 1659 if (SIInstrInfo::isTRANS(MI)) 1660 State.TRANS += 1; 1661 }; 1662 1663 DenseSet<const MachineBasicBlock *> Visited; 1664 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), 1665 std::next(MI->getReverseIterator()), Visited)) 1666 return false; 1667 1668 // Hazard is observed - insert a wait on va_dst counter to ensure hazard is 1669 // avoided (mask 0x0fff achieves this). 1670 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), 1671 TII.get(AMDGPU::S_WAITCNT_DEPCTR)) 1672 .addImm(0x0fff); 1673 1674 return true; 1675 } 1676 1677 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { 1678 if (!SIInstrInfo::isWMMA(*MI)) 1679 return false; 1680 1681 const SIInstrInfo *TII = ST.getInstrInfo(); 1682 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1683 1684 auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) { 1685 if (!SIInstrInfo::isWMMA(I)) 1686 return false; 1687 1688 // Src0 or Src1 of the current wmma instruction overlaps with the dest of 1689 // the previous wmma. 1690 const Register CurSrc0Reg = 1691 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); 1692 const Register CurSrc1Reg = 1693 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); 1694 1695 const Register PrevDstReg = 1696 TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg(); 1697 1698 if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) || 1699 TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) { 1700 return true; 1701 } 1702 1703 // Src2 of the current wmma instruction overlaps with the dest of the 1704 // previous wmma. 1705 const MachineOperand *Src2 = 1706 TII->getNamedOperand(*MI, AMDGPU::OpName::src2); 1707 const Register CurSrc2Reg = Src2->isReg() ? Src2->getReg() : Register(); 1708 1709 if (CurSrc2Reg != AMDGPU::NoRegister && 1710 TRI->regsOverlap(PrevDstReg, CurSrc2Reg)) { 1711 1712 const MachineOperand *Src2Mods = 1713 TII->getNamedOperand(*MI, AMDGPU::OpName::src2_modifiers); 1714 const bool NoSrc2Mods = 1715 (Src2Mods->getImm() & (SISrcMods::NEG | SISrcMods::NEG_HI)) == 0; 1716 // Exception: there is no hazard if the wmma instructions are of the same 1717 // type and there is no input modifier on src2 of the current instruction. 1718 return !(NoSrc2Mods && (TII->pseudoToMCOpcode(I.getOpcode()) == 1719 TII->pseudoToMCOpcode(MI->getOpcode()))); 1720 } 1721 1722 return false; 1723 }; 1724 1725 auto IsExpiredFn = [](const MachineInstr &I, int) { 1726 return SIInstrInfo::isVALU(I); 1727 }; 1728 1729 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == 1730 std::numeric_limits<int>::max()) 1731 return false; 1732 1733 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); 1734 1735 return true; 1736 } 1737 1738 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { 1739 int NSAtoVMEMWaitStates = 1; 1740 1741 if (!ST.hasNSAtoVMEMBug()) 1742 return 0; 1743 1744 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) 1745 return 0; 1746 1747 const SIInstrInfo *TII = ST.getInstrInfo(); 1748 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); 1749 if (!Offset || (Offset->getImm() & 6) == 0) 1750 return 0; 1751 1752 auto IsHazardFn = [TII](const MachineInstr &I) { 1753 if (!SIInstrInfo::isMIMG(I)) 1754 return false; 1755 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode()); 1756 return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA && 1757 TII->getInstSizeInBytes(I) >= 16; 1758 }; 1759 1760 return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1); 1761 } 1762 1763 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { 1764 int FPAtomicToDenormModeWaitStates = 3; 1765 1766 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) 1767 return 0; 1768 1769 auto IsHazardFn = [](const MachineInstr &I) { 1770 if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I)) 1771 return false; 1772 return SIInstrInfo::isFPAtomic(I); 1773 }; 1774 1775 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { 1776 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) 1777 return true; 1778 1779 switch (MI.getOpcode()) { 1780 case AMDGPU::S_WAITCNT: 1781 case AMDGPU::S_WAITCNT_VSCNT: 1782 case AMDGPU::S_WAITCNT_VMCNT: 1783 case AMDGPU::S_WAITCNT_EXPCNT: 1784 case AMDGPU::S_WAITCNT_LGKMCNT: 1785 case AMDGPU::S_WAIT_IDLE: 1786 return true; 1787 default: 1788 break; 1789 } 1790 1791 return false; 1792 }; 1793 1794 return FPAtomicToDenormModeWaitStates - 1795 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); 1796 } 1797 1798 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { 1799 assert(SIInstrInfo::isMAI(*MI)); 1800 1801 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); 1802 } 1803 1804 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { 1805 // Early exit if no padding is requested. 1806 if (MFMAPaddingRatio == 0) 1807 return 0; 1808 1809 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 1810 if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) 1811 return 0; 1812 1813 int NeighborMFMALatency = 0; 1814 auto IsNeighboringMFMA = [&NeighborMFMALatency, 1815 this](const MachineInstr &MI) { 1816 if (!SIInstrInfo::isMFMA(MI)) 1817 return false; 1818 1819 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); 1820 return true; 1821 }; 1822 1823 const int MaxMFMAPipelineWaitStates = 16; 1824 int WaitStatesSinceNeighborMFMA = 1825 getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates); 1826 1827 int NeighborMFMAPaddingNeeded = 1828 (NeighborMFMALatency * MFMAPaddingRatio / 100) - 1829 WaitStatesSinceNeighborMFMA; 1830 1831 return std::max(0, NeighborMFMAPaddingNeeded); 1832 } 1833 1834 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { 1835 int WaitStatesNeeded = 0; 1836 unsigned Opc = MI->getOpcode(); 1837 1838 auto IsVALUFn = [](const MachineInstr &MI) { 1839 return SIInstrInfo::isVALU(MI); 1840 }; 1841 1842 if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write 1843 const int LegacyVALUWritesVGPRWaitStates = 2; 1844 const int VALUWritesExecWaitStates = 4; 1845 const int MaxWaitStates = 4; 1846 1847 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 1848 getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates); 1849 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1850 1851 if (WaitStatesNeeded < MaxWaitStates) { 1852 for (const MachineOperand &Use : MI->explicit_uses()) { 1853 const int MaxWaitStates = 2; 1854 1855 if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg())) 1856 continue; 1857 1858 int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates - 1859 getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates); 1860 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1861 1862 if (WaitStatesNeeded == MaxWaitStates) 1863 break; 1864 } 1865 } 1866 } 1867 1868 for (const MachineOperand &Op : MI->explicit_operands()) { 1869 if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg())) 1870 continue; 1871 1872 if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1873 continue; 1874 1875 const int MFMAWritesAGPROverlappedSrcABWaitStates = 4; 1876 const int MFMAWritesAGPROverlappedSrcCWaitStates = 2; 1877 const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4; 1878 const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10; 1879 const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18; 1880 const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1; 1881 const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7; 1882 const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15; 1883 const int MaxWaitStates = 18; 1884 Register Reg = Op.getReg(); 1885 unsigned HazardDefLatency = 0; 1886 1887 auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency, 1888 this](const MachineInstr &MI) { 1889 if (!SIInstrInfo::isMFMA(MI)) 1890 return false; 1891 Register DstReg = MI.getOperand(0).getReg(); 1892 if (DstReg == Reg) 1893 return false; 1894 HazardDefLatency = 1895 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 1896 return TRI.regsOverlap(DstReg, Reg); 1897 }; 1898 1899 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, 1900 MaxWaitStates); 1901 int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates; 1902 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1903 int OpNo = MI->getOperandNo(&Op); 1904 if (OpNo == SrcCIdx) { 1905 NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates; 1906 } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) { 1907 switch (HazardDefLatency) { 1908 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates; 1909 break; 1910 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates; 1911 break; 1912 case 16: LLVM_FALLTHROUGH; 1913 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates; 1914 break; 1915 } 1916 } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1917 switch (HazardDefLatency) { 1918 case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates; 1919 break; 1920 case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates; 1921 break; 1922 case 16: LLVM_FALLTHROUGH; 1923 default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates; 1924 break; 1925 } 1926 } 1927 1928 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 1929 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1930 1931 if (WaitStatesNeeded == MaxWaitStates) 1932 return WaitStatesNeeded; // Early exit. 1933 1934 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { 1935 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 1936 return false; 1937 Register DstReg = MI.getOperand(0).getReg(); 1938 return TRI.regsOverlap(Reg, DstReg); 1939 }; 1940 1941 const int AccVGPRWriteMFMAReadSrcCWaitStates = 1; 1942 const int AccVGPRWriteMFMAReadSrcABWaitStates = 3; 1943 const int AccVGPRWriteAccVgprReadWaitStates = 3; 1944 NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates; 1945 if (OpNo == SrcCIdx) 1946 NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates; 1947 else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) 1948 NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates; 1949 1950 WaitStatesNeededForUse = NeedWaitStates - 1951 getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates); 1952 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1953 1954 if (WaitStatesNeeded == MaxWaitStates) 1955 return WaitStatesNeeded; // Early exit. 1956 } 1957 1958 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) { 1959 const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0; 1960 const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5; 1961 const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13; 1962 const int MaxWaitStates = 13; 1963 Register DstReg = MI->getOperand(0).getReg(); 1964 unsigned HazardDefLatency = 0; 1965 1966 auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency, 1967 this](const MachineInstr &MI) { 1968 if (!SIInstrInfo::isMFMA(MI)) 1969 return false; 1970 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); 1971 HazardDefLatency = 1972 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); 1973 return TRI.regsOverlap(Reg, DstReg); 1974 }; 1975 1976 int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates); 1977 int NeedWaitStates; 1978 switch (HazardDefLatency) { 1979 case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates; 1980 break; 1981 case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates; 1982 break; 1983 case 16: LLVM_FALLTHROUGH; 1984 default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates; 1985 break; 1986 } 1987 1988 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince; 1989 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 1990 } 1991 1992 // Pad neighboring MFMA with noops for better inter-wave performance. 1993 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); 1994 1995 return WaitStatesNeeded; 1996 } 1997 1998 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { 1999 int WaitStatesNeeded = 0; 2000 unsigned Opc = MI->getOpcode(); 2001 2002 auto IsLegacyVALUFn = [](const MachineInstr &MI) { 2003 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); 2004 }; 2005 2006 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { 2007 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && 2008 !SIInstrInfo::isDOT(MI); 2009 }; 2010 2011 if (!SIInstrInfo::isMFMA(*MI)) 2012 return WaitStatesNeeded; 2013 2014 const int VALUWritesExecWaitStates = 4; 2015 int WaitStatesNeededForUse = VALUWritesExecWaitStates - 2016 getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn, 2017 VALUWritesExecWaitStates); 2018 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2019 2020 int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 2021 2022 // Loop for both DGEMM and S/HGEMM 2nd instruction. 2023 for (const MachineOperand &Use : MI->explicit_uses()) { 2024 const int LegacyVALUNotDotWritesVGPRWaitStates = 2; 2025 const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2; 2026 const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3; 2027 const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5; 2028 const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4; 2029 const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9; 2030 const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8; 2031 const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17; 2032 const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16; 2033 const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8; 2034 const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16; 2035 const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3; 2036 const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9; 2037 const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17; 2038 const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9; 2039 const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4; 2040 const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5; 2041 const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11; 2042 const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19; 2043 const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4; 2044 const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6; 2045 const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10; 2046 const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18; 2047 const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5; 2048 const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7; 2049 const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11; 2050 const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19; 2051 const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6; 2052 const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11; 2053 const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4; 2054 const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2; 2055 const int MaxWaitStates = 19; 2056 2057 if (!Use.isReg()) 2058 continue; 2059 Register Reg = Use.getReg(); 2060 bool FullReg; 2061 const MachineInstr *MI1; 2062 2063 auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1, 2064 this](const MachineInstr &MI) { 2065 if (!SIInstrInfo::isMFMA(MI)) 2066 return false; 2067 Register DstReg = MI.getOperand(0).getReg(); 2068 FullReg = (DstReg == Reg); 2069 MI1 = &MI; 2070 return TRI.regsOverlap(DstReg, Reg); 2071 }; 2072 2073 WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates - 2074 getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates); 2075 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2076 2077 int NumWaitStates = 2078 getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates); 2079 if (NumWaitStates == std::numeric_limits<int>::max()) 2080 continue; 2081 2082 int OpNo = MI->getOperandNo(&Use); 2083 unsigned Opc1 = MI1->getOpcode(); 2084 int NeedWaitStates = 0; 2085 if (OpNo == SrcCIdx) { 2086 if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) { 2087 NeedWaitStates = 0; 2088 } else if (FullReg) { 2089 if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 2090 Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) && 2091 (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || 2092 Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64)) 2093 NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates; 2094 else if (ST.hasGFX940Insts() && 2095 TSchedModel.computeInstrLatency(MI1) == 2) 2096 NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates; 2097 } else { 2098 switch (Opc1) { 2099 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 2100 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 2101 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 2102 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 2103 if (!isXDL(ST, *MI)) 2104 NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates; 2105 break; 2106 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 2107 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 2108 if (!isXDL(ST, *MI)) 2109 NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; 2110 break; 2111 default: 2112 if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1)) 2113 break; 2114 switch (TSchedModel.computeInstrLatency(MI1)) { 2115 case 2: 2116 NeedWaitStates = ST.hasGFX940Insts() 2117 ? isXDL(ST, *MI1) 2118 ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates 2119 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates 2120 : isDGEMM(Opc) 2121 ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates 2122 : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; 2123 break; 2124 case 4: 2125 assert(ST.hasGFX940Insts()); 2126 NeedWaitStates = isXDL(ST, *MI1) 2127 ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates 2128 : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates; 2129 break; 2130 case 8: 2131 NeedWaitStates = ST.hasGFX940Insts() 2132 ? isXDL(ST, *MI1) 2133 ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates 2134 : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates 2135 : isDGEMM(Opc) 2136 ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates 2137 : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; 2138 break; 2139 case 16: LLVM_FALLTHROUGH; 2140 default: 2141 NeedWaitStates = ST.hasGFX940Insts() 2142 ? isXDL(ST, *MI1) 2143 ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates 2144 : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates 2145 : isDGEMM(Opc) 2146 ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates 2147 : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; 2148 } 2149 } 2150 } 2151 } else { 2152 switch (Opc1) { 2153 case AMDGPU::V_MFMA_F64_16X16X4F64_e64: 2154 case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: 2155 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: 2156 case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: 2157 NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates; 2158 break; 2159 case AMDGPU::V_MFMA_F64_4X4X4F64_e64: 2160 case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: 2161 NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates; 2162 break; 2163 default: 2164 switch (TSchedModel.computeInstrLatency(MI1)) { 2165 case 2: 2166 NeedWaitStates = ST.hasGFX940Insts() 2167 ? isXDL(ST, *MI1) 2168 ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates 2169 : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates 2170 : SMFMA4x4WritesVGPROverlappedSrcABWaitStates; 2171 break; 2172 case 4: 2173 assert(ST.hasGFX940Insts()); 2174 NeedWaitStates = isXDL(ST, *MI1) 2175 ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates 2176 : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates; 2177 break; 2178 case 8: 2179 NeedWaitStates = ST.hasGFX940Insts() 2180 ? isXDL(ST, *MI1) 2181 ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates 2182 : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates 2183 : SMFMA16x16WritesVGPROverlappedSrcABWaitStates; 2184 break; 2185 case 16: LLVM_FALLTHROUGH; 2186 default: 2187 NeedWaitStates = ST.hasGFX940Insts() 2188 ? isXDL(ST, *MI1) 2189 ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates 2190 : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates 2191 : SMFMA32x32WritesVGPROverlappedSrcABWaitStates; 2192 } 2193 } 2194 } 2195 if (WaitStatesNeeded >= NeedWaitStates) 2196 continue; 2197 2198 WaitStatesNeededForUse = NeedWaitStates - NumWaitStates; 2199 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2200 2201 if (WaitStatesNeeded == MaxWaitStates) 2202 break; 2203 } 2204 2205 return WaitStatesNeeded; 2206 } 2207 2208 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { 2209 // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards() 2210 if (!ST.hasMAIInsts() || ST.hasGFX90AInsts()) 2211 return 0; 2212 2213 int WaitStatesNeeded = 0; 2214 2215 auto IsAccVgprReadFn = [](const MachineInstr &MI) { 2216 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; 2217 }; 2218 2219 for (const MachineOperand &Op : MI->explicit_uses()) { 2220 if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg())) 2221 continue; 2222 2223 Register Reg = Op.getReg(); 2224 2225 const int AccVgprReadLdStWaitStates = 2; 2226 const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1; 2227 const int MaxWaitStates = 2; 2228 2229 int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - 2230 getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates); 2231 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2232 2233 if (WaitStatesNeeded == MaxWaitStates) 2234 return WaitStatesNeeded; // Early exit. 2235 2236 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { 2237 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && 2238 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 2239 return false; 2240 auto IsVALUFn = [](const MachineInstr &MI) { 2241 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); 2242 }; 2243 return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) < 2244 std::numeric_limits<int>::max(); 2245 }; 2246 2247 WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates - 2248 getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates); 2249 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2250 } 2251 2252 return WaitStatesNeeded; 2253 } 2254 2255 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { 2256 if (!ST.hasGFX90AInsts()) 2257 return 0; 2258 2259 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { 2260 return isDGEMM(MI.getOpcode()); 2261 }; 2262 2263 // This is checked in checkMAIHazards90A() 2264 if (SIInstrInfo::isMFMA(*MI)) 2265 return 0; 2266 2267 int WaitStatesNeeded = 0; 2268 2269 bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) || 2270 SIInstrInfo::isFLAT(*MI) || 2271 SIInstrInfo::isDS(*MI) || 2272 SIInstrInfo::isEXP(*MI); 2273 bool IsVALU = SIInstrInfo::isVALU(*MI); 2274 2275 const MachineInstr *MFMA = nullptr; 2276 unsigned Reg; 2277 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { 2278 if (!SIInstrInfo::isMFMA(MI) || 2279 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 2280 return false; 2281 MFMA = &MI; 2282 return true; 2283 }; 2284 2285 const MachineInstr *DOT = nullptr; 2286 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { 2287 if (!SIInstrInfo::isDOT(MI) || 2288 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) 2289 return false; 2290 DOT = &MI; 2291 return true; 2292 }; 2293 2294 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 2295 AMDGPU::OpName::src2); 2296 2297 if (IsMemOrExport || IsVALU) { 2298 const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5; 2299 const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11; 2300 const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19; 2301 const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4; 2302 const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6; 2303 const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10; 2304 const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18; 2305 const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5; 2306 const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7; 2307 const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11; 2308 const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19; 2309 const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9; 2310 const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18; 2311 const int DMFMA4x4WriteVgprVALUReadWaitStates = 6; 2312 const int DMFMA16x16WriteVgprVALUReadWaitStates = 11; 2313 const int DotWriteSameDotReadSrcAB = 3; 2314 const int DotWriteDifferentVALURead = 3; 2315 const int MaxWaitStates = 19; 2316 2317 for (const MachineOperand &Use : MI->explicit_uses()) { 2318 if (!Use.isReg()) 2319 continue; 2320 Reg = Use.getReg(); 2321 2322 DOT = nullptr; 2323 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 2324 MaxWaitStates); 2325 if (DOT) { 2326 int NeedWaitStates = 0; 2327 if (DOT->getOpcode() == MI->getOpcode()) { 2328 if (&Use - &MI->getOperand(0) != SrcCIdx) 2329 NeedWaitStates = DotWriteSameDotReadSrcAB; 2330 } else { 2331 NeedWaitStates = DotWriteDifferentVALURead; 2332 } 2333 2334 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2335 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2336 } 2337 2338 MFMA = nullptr; 2339 WaitStatesSinceDef = 2340 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 2341 if (!MFMA) 2342 continue; 2343 2344 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 2345 int NeedWaitStates = MaxWaitStates; 2346 switch (HazardDefLatency) { 2347 case 2: 2348 NeedWaitStates = 2349 ST.hasGFX940Insts() 2350 ? isXDL(ST, *MFMA) 2351 ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates 2352 : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates 2353 : SMFMA4x4WriteVgprVALUMemExpReadWaitStates; 2354 break; 2355 case 4: 2356 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); 2357 NeedWaitStates = 2358 isDGEMM(MFMA->getOpcode()) 2359 ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates 2360 : DMFMA4x4WriteVgprVALUReadWaitStates 2361 : isXDL(ST, *MFMA) 2362 ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates 2363 : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates; 2364 break; 2365 case 8: 2366 NeedWaitStates = 2367 ST.hasGFX940Insts() 2368 ? isXDL(ST, *MFMA) 2369 ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates 2370 : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates 2371 : SMFMA16x16WriteVgprVALUMemExpReadWaitStates; 2372 break; 2373 case 16: LLVM_FALLTHROUGH; 2374 default: 2375 NeedWaitStates = 2376 isDGEMM(MFMA->getOpcode()) 2377 ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates 2378 : DMFMA16x16WriteVgprVALUReadWaitStates 2379 : ST.hasGFX940Insts() 2380 ? isXDL(ST, *MFMA) 2381 ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates 2382 : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates 2383 : SMFMA32x32WriteVgprVALUMemExpReadWaitStates; 2384 break; 2385 } 2386 2387 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2388 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2389 2390 if (WaitStatesNeeded == MaxWaitStates) 2391 break; 2392 } 2393 } 2394 2395 unsigned Opc = MI->getOpcode(); 2396 const int DMFMAToFMA64WaitStates = 2; 2397 if ((Opc == AMDGPU::V_FMA_F64_e64 || 2398 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 || 2399 Opc == AMDGPU::V_FMAC_F64_dpp) && 2400 WaitStatesNeeded < DMFMAToFMA64WaitStates) { 2401 int WaitStatesNeededForUse = DMFMAToFMA64WaitStates - 2402 getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates); 2403 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2404 } 2405 2406 if (!IsVALU && !IsMemOrExport) 2407 return WaitStatesNeeded; 2408 2409 for (const MachineOperand &Def : MI->defs()) { 2410 const int SMFMA4x4WriteVgprVALUWawWaitStates = 5; 2411 const int SMFMA16x16WriteVgprVALUWawWaitStates = 11; 2412 const int SMFMA32x32WriteVgprVALUWawWaitStates = 19; 2413 const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4; 2414 const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6; 2415 const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10; 2416 const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18; 2417 const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5; 2418 const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7; 2419 const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11; 2420 const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19; 2421 const int SMFMA4x4ReadVgprVALUWarWaitStates = 1; 2422 const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3; 2423 const int SMFMA16x16ReadVgprVALUWarWaitStates = 7; 2424 const int SMFMA32x32ReadVgprVALUWarWaitStates = 15; 2425 const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6; 2426 const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11; 2427 const int DotWriteDifferentVALUWrite = 3; 2428 const int MaxWaitStates = 19; 2429 const int MaxWarWaitStates = 15; 2430 2431 Reg = Def.getReg(); 2432 2433 DOT = nullptr; 2434 int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn, 2435 MaxWaitStates); 2436 if (DOT && DOT->getOpcode() != MI->getOpcode()) 2437 WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite - 2438 WaitStatesSinceDef); 2439 2440 MFMA = nullptr; 2441 WaitStatesSinceDef = 2442 getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates); 2443 if (MFMA) { 2444 int NeedWaitStates = MaxWaitStates; 2445 switch (TSchedModel.computeInstrLatency(MFMA)) { 2446 case 2: 2447 NeedWaitStates = ST.hasGFX940Insts() 2448 ? isXDL(ST, *MFMA) 2449 ? GFX940_XDL2PassWriteVgprVALUWawWaitStates 2450 : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates 2451 : SMFMA4x4WriteVgprVALUWawWaitStates; 2452 break; 2453 case 4: 2454 assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts()); 2455 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 2456 ? DMFMA4x4WriteVgprVALUWriteWaitStates 2457 : isXDL(ST, *MFMA) 2458 ? GFX940_XDL4PassWriteVgprVALUWawWaitStates 2459 : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates; 2460 break; 2461 case 8: 2462 NeedWaitStates = ST.hasGFX940Insts() 2463 ? isXDL(ST, *MFMA) 2464 ? GFX940_XDL8PassWriteVgprVALUWawWaitStates 2465 : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates 2466 : SMFMA16x16WriteVgprVALUWawWaitStates; 2467 break; 2468 case 16: LLVM_FALLTHROUGH; 2469 default: 2470 NeedWaitStates = isDGEMM(MFMA->getOpcode()) 2471 ? DMFMA16x16WriteVgprVALUWriteWaitStates 2472 : ST.hasGFX940Insts() 2473 ? isXDL(ST, *MFMA) 2474 ? GFX940_XDL16PassWriteVgprVALUWawWaitStates 2475 : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates 2476 : SMFMA32x32WriteVgprVALUWawWaitStates; 2477 break; 2478 } 2479 2480 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef; 2481 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2482 2483 if (WaitStatesNeeded == MaxWaitStates) 2484 break; 2485 } 2486 2487 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { 2488 if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || 2489 !MI.readsRegister(Reg, &TRI)) 2490 return false; 2491 2492 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) 2493 return false; 2494 2495 const MachineOperand *SrcC = 2496 TII.getNamedOperand(MI, AMDGPU::OpName::src2); 2497 assert(SrcC); 2498 if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg)) 2499 return false; 2500 2501 MFMA = &MI; 2502 return true; 2503 }; 2504 2505 MFMA = nullptr; 2506 int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn, 2507 MaxWarWaitStates); 2508 if (!MFMA) 2509 continue; 2510 2511 unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA); 2512 int NeedWaitStates = MaxWaitStates; 2513 switch (HazardDefLatency) { 2514 case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates; 2515 break; 2516 case 4: assert(ST.hasGFX940Insts()); 2517 NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates; 2518 break; 2519 case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates; 2520 break; 2521 case 16: LLVM_FALLTHROUGH; 2522 default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates; 2523 break; 2524 } 2525 2526 int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse; 2527 WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); 2528 } 2529 2530 return WaitStatesNeeded; 2531 } 2532 2533 bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { 2534 if (!SU->isInstr()) 2535 return false; 2536 2537 const MachineInstr *MAI = nullptr; 2538 2539 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { 2540 MAI = nullptr; 2541 if (SIInstrInfo::isMFMA(MI)) 2542 MAI = &MI; 2543 return MAI != nullptr; 2544 }; 2545 2546 MachineInstr *MI = SU->getInstr(); 2547 if (IsMFMAFn(*MI)) { 2548 int W = getWaitStatesSince(IsMFMAFn, 16); 2549 if (MAI) 2550 return W < (int)TSchedModel.computeInstrLatency(MAI); 2551 } 2552 2553 return false; 2554 } 2555