Lines Matching refs:MI
76 void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { in EmitInstruction() argument
77 CurrCycleInstr = MI; in EmitInstruction()
123 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { in isXDL() argument
124 unsigned Opcode = MI.getOpcode(); in isXDL()
126 if (!SIInstrInfo::isMAI(MI) || in isXDL()
139 const MachineInstr &MI) { in isSendMsgTraceDataOrGDS() argument
140 if (TII.isAlwaysGDS(MI.getOpcode())) in isSendMsgTraceDataOrGDS()
143 switch (MI.getOpcode()) { in isSendMsgTraceDataOrGDS()
154 if (TII.isDS(MI.getOpcode())) { in isSendMsgTraceDataOrGDS()
155 int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in isSendMsgTraceDataOrGDS()
157 if (MI.getOperand(GDS).getImm()) in isSendMsgTraceDataOrGDS()
164 static bool isPermlane(const MachineInstr &MI) { in isPermlane() argument
165 unsigned Opcode = MI.getOpcode(); in isPermlane()
173 static bool isLdsDma(const MachineInstr &MI) { in isLdsDma() argument
174 return SIInstrInfo::isVALU(MI) && in isLdsDma()
175 (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)); in isLdsDma()
186 MachineInstr *MI = SU->getInstr(); in getHazardType() local
191 if (MI->isBundle()) in getHazardType()
194 if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0) in getHazardType()
197 if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0) in getHazardType()
200 if (checkFPAtomicToDenormModeHazard(MI) > 0) in getHazardType()
207 if ((SIInstrInfo::isVMEM(*MI) || in getHazardType()
208 SIInstrInfo::isFLAT(*MI)) in getHazardType()
209 && checkVMEMHazards(MI) > 0) in getHazardType()
212 if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0) in getHazardType()
215 if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0) in getHazardType()
218 if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0) in getHazardType()
221 if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0) in getHazardType()
224 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || in getHazardType()
225 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || in getHazardType()
226 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) in getHazardType()
229 if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0) in getHazardType()
232 if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0) in getHazardType()
235 if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0) in getHazardType()
239 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || in getHazardType()
240 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || in getHazardType()
241 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || in getHazardType()
242 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in getHazardType()
243 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in getHazardType()
245 MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) && in getHazardType()
246 checkReadM0Hazards(MI) > 0) in getHazardType()
249 if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) in getHazardType()
252 if ((SIInstrInfo::isVMEM(*MI) || in getHazardType()
253 SIInstrInfo::isFLAT(*MI) || in getHazardType()
254 SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0) in getHazardType()
257 if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) in getHazardType()
263 static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, in insertNoopsInBundle() argument
268 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP)) in insertNoopsInBundle()
274 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { in getMFMAPipelineWaitStates()
275 const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI); in getMFMAPipelineWaitStates()
282 MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator()); in processBundle() local
285 for (; MI != E && MI->isInsideBundle(); ++MI) { in processBundle()
286 CurrCycleInstr = &*MI; in processBundle()
307 void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) { in runOnInstruction() argument
310 unsigned NumPreNoops = PreEmitNoops(MI); in runOnInstruction()
312 if (MI->isInsideBundle()) in runOnInstruction()
313 insertNoopsInBundle(MI, TII, NumPreNoops); in runOnInstruction()
315 TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI), in runOnInstruction()
317 EmitInstruction(MI); in runOnInstruction()
321 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { in PreEmitNoops() argument
323 CurrCycleInstr = MI; in PreEmitNoops()
324 unsigned W = PreEmitNoopsCommon(MI); in PreEmitNoops()
325 fixHazards(MI); in PreEmitNoops()
330 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { in PreEmitNoopsCommon() argument
331 if (MI->isBundle()) in PreEmitNoopsCommon()
336 if (SIInstrInfo::isSMRD(*MI)) in PreEmitNoopsCommon()
337 return std::max(WaitStates, checkSMRDHazards(MI)); in PreEmitNoopsCommon()
340 WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI)); in PreEmitNoopsCommon()
342 WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI)); in PreEmitNoopsCommon()
347 if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI)) in PreEmitNoopsCommon()
348 WaitStates = std::max(WaitStates, checkVMEMHazards(MI)); in PreEmitNoopsCommon()
350 if (SIInstrInfo::isVALU(*MI)) in PreEmitNoopsCommon()
351 WaitStates = std::max(WaitStates, checkVALUHazards(MI)); in PreEmitNoopsCommon()
353 if (SIInstrInfo::isDPP(*MI)) in PreEmitNoopsCommon()
354 WaitStates = std::max(WaitStates, checkDPPHazards(MI)); in PreEmitNoopsCommon()
356 if (isDivFMas(MI->getOpcode())) in PreEmitNoopsCommon()
357 WaitStates = std::max(WaitStates, checkDivFMasHazards(MI)); in PreEmitNoopsCommon()
359 if (isRWLane(MI->getOpcode())) in PreEmitNoopsCommon()
360 WaitStates = std::max(WaitStates, checkRWLaneHazards(MI)); in PreEmitNoopsCommon()
362 if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) || in PreEmitNoopsCommon()
363 SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) || in PreEmitNoopsCommon()
364 SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0) in PreEmitNoopsCommon()
365 WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI)); in PreEmitNoopsCommon()
367 if (MI->isInlineAsm()) in PreEmitNoopsCommon()
368 return std::max(WaitStates, checkInlineAsmHazards(MI)); in PreEmitNoopsCommon()
370 if (isSGetReg(MI->getOpcode())) in PreEmitNoopsCommon()
371 return std::max(WaitStates, checkGetRegHazards(MI)); in PreEmitNoopsCommon()
373 if (isSSetReg(MI->getOpcode())) in PreEmitNoopsCommon()
374 return std::max(WaitStates, checkSetRegHazards(MI)); in PreEmitNoopsCommon()
376 if (isRFE(MI->getOpcode())) in PreEmitNoopsCommon()
377 return std::max(WaitStates, checkRFEHazards(MI)); in PreEmitNoopsCommon()
380 (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) || in PreEmitNoopsCommon()
381 MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 || in PreEmitNoopsCommon()
382 MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) || in PreEmitNoopsCommon()
383 (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) || in PreEmitNoopsCommon()
384 (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) || in PreEmitNoopsCommon()
386 MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) in PreEmitNoopsCommon()
387 return std::max(WaitStates, checkReadM0Hazards(MI)); in PreEmitNoopsCommon()
389 if (SIInstrInfo::isMAI(*MI)) in PreEmitNoopsCommon()
390 return std::max(WaitStates, checkMAIHazards(MI)); in PreEmitNoopsCommon()
392 if (SIInstrInfo::isVMEM(*MI) || in PreEmitNoopsCommon()
393 SIInstrInfo::isFLAT(*MI) || in PreEmitNoopsCommon()
394 SIInstrInfo::isDS(*MI)) in PreEmitNoopsCommon()
395 return std::max(WaitStates, checkMAILdStHazards(MI)); in PreEmitNoopsCommon()
537 const MachineInstr *MI, IsExpiredFn IsExpired) { in getWaitStatesSince() argument
539 return getWaitStatesSince(IsHazard, MI->getParent(), in getWaitStatesSince()
540 std::next(MI->getReverseIterator()), in getWaitStatesSince()
553 for (MachineInstr *MI : EmittedInstrs) { in getWaitStatesSince()
554 if (MI) { in getWaitStatesSince()
555 if (IsHazard(*MI)) in getWaitStatesSince()
558 if (MI->isInlineAsm()) in getWaitStatesSince()
574 auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) { in getWaitStatesSinceDef() argument
575 return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI); in getWaitStatesSinceDef()
583 auto IsHazardFn = [IsHazard](const MachineInstr &MI) { in getWaitStatesSinceSetReg() argument
584 return isSSetReg(MI.getOpcode()) && IsHazard(MI); in getWaitStatesSinceSetReg()
609 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { in addClauseInst() argument
610 addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses); in addClauseInst()
613 static bool breaksSMEMSoftClause(MachineInstr *MI) { in breaksSMEMSoftClause() argument
614 return !SIInstrInfo::isSMRD(*MI); in breaksSMEMSoftClause()
617 static bool breaksVMEMSoftClause(MachineInstr *MI) { in breaksVMEMSoftClause() argument
618 return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI); in breaksVMEMSoftClause()
641 for (MachineInstr *MI : EmittedInstrs) { in checkSoftClauseHazards()
644 if (!MI) in checkSoftClauseHazards()
647 if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI)) in checkSoftClauseHazards()
650 addClauseInst(*MI); in checkSoftClauseHazards()
681 auto IsHazardDefFn = [this](const MachineInstr &MI) { in checkSMRDHazards() argument
682 return TII.isVALU(MI); in checkSMRDHazards()
684 auto IsBufferHazardDefFn = [this](const MachineInstr &MI) { in checkSMRDHazards() argument
685 return TII.isSALU(MI); in checkSMRDHazards()
726 auto IsHazardDefFn = [this](const MachineInstr &MI) { in checkVMEMHazards() argument
727 return TII.isVALU(MI); in checkVMEMHazards()
749 auto IsHazardDefFn = [TII](const MachineInstr &MI) { in checkDPPHazards() argument
750 return TII->isVALU(MI); in checkDPPHazards()
778 auto IsHazardDefFn = [TII](const MachineInstr &MI) { in checkDivFMasHazards() argument
779 return TII->isVALU(MI); in checkDivFMasHazards()
792 auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) { in checkGetRegHazards() argument
793 return GetRegHWReg == getHWReg(TII, MI); in checkGetRegHazards()
805 auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) { in checkSetRegHazards() argument
806 return HWReg == getHWReg(TII, MI); in checkSetRegHazards()
812 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { in createsVALUHazard() argument
813 if (!MI.mayStore()) in createsVALUHazard()
817 unsigned Opcode = MI.getOpcode(); in createsVALUHazard()
818 const MCInstrDesc &Desc = MI.getDesc(); in createsVALUHazard()
825 if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) { in createsVALUHazard()
833 TII->getNamedOperand(MI, AMDGPU::OpName::soffset); in createsVALUHazard()
845 if (TII->isMIMG(MI)) { in createsVALUHazard()
852 if (TII->isFLAT(MI)) { in createsVALUHazard()
874 auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) { in checkVALUHazardsHelper() argument
875 int DataIdx = createsVALUHazard(MI); in checkVALUHazardsHelper()
877 TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg); in checkVALUHazardsHelper()
892 auto IsTransDefFn = [this, VALU](const MachineInstr &MI) { in checkVALUHazards() argument
893 if (!SIInstrInfo::isTRANS(MI)) in checkVALUHazards()
897 Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg(); in checkVALUHazards()
916 auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) { in checkVALUHazards() argument
917 if (!SIInstrInfo::isVALU(MI)) in checkVALUHazards()
920 if (SIInstrInfo::isSDWA(MI)) { in checkVALUHazards()
921 if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel)) in checkVALUHazards()
925 if (!AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::op_sel) || in checkVALUHazards()
926 !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers) in checkVALUHazards()
932 if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in checkVALUHazards()
958 auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) { in checkVALUHazards() argument
959 if (!SIInstrInfo::isVALU(MI)) in checkVALUHazards()
961 return MI.modifiesRegister(UseReg, TRI); in checkVALUHazards()
1062 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); }; in checkRWLaneHazards() argument
1078 auto IsHazardFn = [TII](const MachineInstr &MI) { in checkRFEHazards() argument
1079 return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS; in checkRFEHazards()
1085 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { in checkReadM0Hazards() argument
1088 auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); }; in checkReadM0Hazards() argument
1093 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { in fixHazards() argument
1094 fixVMEMtoScalarWriteHazards(MI); in fixHazards()
1095 fixVcmpxPermlaneHazards(MI); in fixHazards()
1096 fixSMEMtoVectorWriteHazards(MI); in fixHazards()
1097 fixVcmpxExecWARHazard(MI); in fixHazards()
1098 fixLdsBranchVmemWARHazard(MI); in fixHazards()
1100 fixLdsDirectVALUHazard(MI); in fixHazards()
1101 fixLdsDirectVMEMHazard(MI); in fixHazards()
1103 fixVALUPartialForwardingHazard(MI); in fixHazards()
1104 fixVALUTransUseHazard(MI); in fixHazards()
1105 fixWMMAHazards(MI); in fixHazards()
1106 fixShift64HighRegBug(MI); in fixHazards()
1107 fixVALUMaskWriteHazard(MI); in fixHazards()
1108 fixRequiredExportPriority(MI); in fixHazards()
1111 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { in fixVcmpxPermlaneHazards() argument
1112 if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI)) in fixVcmpxPermlaneHazards()
1117 auto IsHazardFn = [TII, TRI](const MachineInstr &MI) { in fixVcmpxPermlaneHazards() argument
1118 return (TII->isVOPC(MI) || in fixVcmpxPermlaneHazards()
1119 ((TII->isVOP3(MI) || TII->isSDWA(MI)) && MI.isCompare())) && in fixVcmpxPermlaneHazards()
1120 MI.modifiesRegister(AMDGPU::EXEC, TRI); in fixVcmpxPermlaneHazards()
1123 auto IsExpiredFn = [](const MachineInstr &MI, int) { in fixVcmpxPermlaneHazards() argument
1124 unsigned Opc = MI.getOpcode(); in fixVcmpxPermlaneHazards()
1125 return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 && in fixVcmpxPermlaneHazards()
1129 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVcmpxPermlaneHazards()
1136 auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0); in fixVcmpxPermlaneHazards()
1139 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVcmpxPermlaneHazards()
1147 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { in fixVMEMtoScalarWriteHazards() argument
1152 if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI)) in fixVMEMtoScalarWriteHazards()
1155 if (MI->getNumDefs() == 0) in fixVMEMtoScalarWriteHazards()
1160 auto IsHazardFn = [TRI, MI](const MachineInstr &I) { in fixVMEMtoScalarWriteHazards()
1165 for (const MachineOperand &Def : MI->defs()) { in fixVMEMtoScalarWriteHazards()
1175 auto IsExpiredFn = [](const MachineInstr &MI, int) { in fixVMEMtoScalarWriteHazards() argument
1176 return SIInstrInfo::isVALU(MI) || in fixVMEMtoScalarWriteHazards()
1177 (MI.getOpcode() == AMDGPU::S_WAITCNT && in fixVMEMtoScalarWriteHazards()
1178 !MI.getOperand(0).getImm()) || in fixVMEMtoScalarWriteHazards()
1179 (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVMEMtoScalarWriteHazards()
1180 AMDGPU::DepCtr::decodeFieldVmVsrc(MI.getOperand(0).getImm()) == 0); in fixVMEMtoScalarWriteHazards()
1183 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVMEMtoScalarWriteHazards()
1188 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVMEMtoScalarWriteHazards()
1194 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { in fixSMEMtoVectorWriteHazards() argument
1199 if (!SIInstrInfo::isVALU(*MI)) in fixSMEMtoVectorWriteHazards()
1203 switch (MI->getOpcode()) { in fixSMEMtoVectorWriteHazards()
1216 const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName); in fixSMEMtoVectorWriteHazards()
1218 for (const auto &MO : MI->implicit_operands()) { in fixSMEMtoVectorWriteHazards()
1234 auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) { in fixSMEMtoVectorWriteHazards() argument
1235 if (TII->isSALU(MI)) { in fixSMEMtoVectorWriteHazards()
1236 switch (MI.getOpcode()) { in fixSMEMtoVectorWriteHazards()
1246 return (MI.getOperand(1).getImm() == 0) && in fixSMEMtoVectorWriteHazards()
1247 (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in fixSMEMtoVectorWriteHazards()
1249 const int64_t Imm = MI.getOperand(0).getImm(); in fixSMEMtoVectorWriteHazards()
1256 if (TII->isSOPP(MI)) in fixSMEMtoVectorWriteHazards()
1271 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixSMEMtoVectorWriteHazards()
1275 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixSMEMtoVectorWriteHazards()
1281 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { in fixVcmpxExecWARHazard() argument
1286 if (!SIInstrInfo::isVALU(*MI)) in fixVcmpxExecWARHazard()
1290 if (!MI->modifiesRegister(AMDGPU::EXEC, TRI)) in fixVcmpxExecWARHazard()
1300 auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) { in fixVcmpxExecWARHazard() argument
1301 if (SIInstrInfo::isVALU(MI)) { in fixVcmpxExecWARHazard()
1302 if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) in fixVcmpxExecWARHazard()
1304 for (auto MO : MI.implicit_operands()) in fixVcmpxExecWARHazard()
1308 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR && in fixVcmpxExecWARHazard()
1309 AMDGPU::DepCtr::decodeFieldSaSdst(MI.getOperand(0).getImm()) == 0) in fixVcmpxExecWARHazard()
1314 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVcmpxExecWARHazard()
1318 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVcmpxExecWARHazard()
1334 for (auto &MI : MBB) { in shouldRunLdsBranchVmemWARHazardFixup() local
1335 HasLds |= SIInstrInfo::isDS(MI); in shouldRunLdsBranchVmemWARHazardFixup()
1337 SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI); in shouldRunLdsBranchVmemWARHazardFixup()
1351 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { in fixLdsBranchVmemWARHazard() argument
1358 auto IsHazardInst = [](const MachineInstr &MI) { in fixLdsBranchVmemWARHazard() argument
1359 if (SIInstrInfo::isDS(MI)) in fixLdsBranchVmemWARHazard()
1361 if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) in fixLdsBranchVmemWARHazard()
1366 auto InstType = IsHazardInst(*MI); in fixLdsBranchVmemWARHazard()
1395 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixLdsBranchVmemWARHazard()
1400 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixLdsBranchVmemWARHazard()
1408 bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { in fixLdsDirectVALUHazard() argument
1409 if (!SIInstrInfo::isLDSDIR(*MI)) in fixLdsDirectVALUHazard()
1413 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVALUHazard()
1431 auto GetWaitStatesFn = [](const MachineInstr &MI) { in fixLdsDirectVALUHazard() argument
1432 return SIInstrInfo::isVALU(MI) ? 1 : 0; in fixLdsDirectVALUHazard()
1436 auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(), in fixLdsDirectVALUHazard()
1437 std::next(MI->getReverseIterator()), 0, in fixLdsDirectVALUHazard()
1446 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst); in fixLdsDirectVALUHazard()
1452 bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { in fixLdsDirectVMEMHazard() argument
1453 if (!SIInstrInfo::isLDSDIR(*MI)) in fixLdsDirectVMEMHazard()
1456 const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst); in fixLdsDirectVMEMHazard()
1477 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixLdsDirectVMEMHazard()
1482 TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0); in fixLdsDirectVMEMHazard()
1484 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixLdsDirectVMEMHazard()
1492 bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { in fixVALUPartialForwardingHazard() argument
1497 if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI)) in fixVALUPartialForwardingHazard()
1502 for (const MachineOperand &Use : MI->explicit_uses()) { in fixVALUPartialForwardingHazard()
1624 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { in fixVALUPartialForwardingHazard() argument
1625 if (SIInstrInfo::isVALU(MI)) in fixVALUPartialForwardingHazard()
1630 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), in fixVALUPartialForwardingHazard()
1631 std::next(MI->getReverseIterator()), Visited)) in fixVALUPartialForwardingHazard()
1634 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVALUPartialForwardingHazard()
1641 bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { in fixVALUTransUseHazard() argument
1646 if (!SIInstrInfo::isVALU(*MI)) in fixVALUTransUseHazard()
1651 for (const MachineOperand &Use : MI->explicit_uses()) { in fixVALUTransUseHazard()
1700 auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) { in fixVALUTransUseHazard() argument
1701 if (SIInstrInfo::isVALU(MI)) in fixVALUTransUseHazard()
1703 if (SIInstrInfo::isTRANS(MI)) in fixVALUTransUseHazard()
1708 if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(), in fixVALUTransUseHazard()
1709 std::next(MI->getReverseIterator()), Visited)) in fixVALUTransUseHazard()
1714 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), in fixVALUTransUseHazard()
1721 bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { in fixWMMAHazards() argument
1722 if (!SIInstrInfo::isWMMA(*MI) && !SIInstrInfo::isSWMMAC(*MI)) in fixWMMAHazards()
1728 auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) { in fixWMMAHazards()
1735 TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg(); in fixWMMAHazards()
1737 TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); in fixWMMAHazards()
1750 if (SIInstrInfo::isSWMMAC(*MI)) { in fixWMMAHazards()
1752 TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg(); in fixWMMAHazards()
1766 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixWMMAHazards()
1770 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32)); in fixWMMAHazards()
1775 bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) { in fixShift64HighRegBug() argument
1780 switch (MI->getOpcode()) { in fixShift64HighRegBug()
1789 MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0); in fixShift64HighRegBug()
1802 MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1); in fixShift64HighRegBug()
1804 bool OverlappedDst = MI->modifiesRegister(AmtReg, &TRI); in fixShift64HighRegBug()
1808 Src1->getReg() == MI->getOperand(0).getReg()); in fixShift64HighRegBug()
1815 if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) { in fixShift64HighRegBug()
1828 DebugLoc DL = MI->getDebugLoc(); in fixShift64HighRegBug()
1829 MachineBasicBlock *MBB = MI->getParent(); in fixShift64HighRegBug()
1831 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT)) in fixShift64HighRegBug()
1837 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo) in fixShift64HighRegBug()
1841 runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt) in fixShift64HighRegBug()
1848 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), in fixShift64HighRegBug()
1854 BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32), in fixShift64HighRegBug()
1868 MI->getOperand(0).setReg(NewReg); in fixShift64HighRegBug()
1878 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { in checkNSAtoVMEMHazard() argument
1884 if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI)) in checkNSAtoVMEMHazard()
1888 const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset); in checkNSAtoVMEMHazard()
1903 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { in checkFPAtomicToDenormModeHazard() argument
1910 if (MI->getOpcode() != AMDGPU::S_DENORM_MODE) in checkFPAtomicToDenormModeHazard()
1919 auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) { in checkFPAtomicToDenormModeHazard() argument
1920 if (WaitStates >= 3 || SIInstrInfo::isVALU(MI)) in checkFPAtomicToDenormModeHazard()
1923 switch (MI.getOpcode()) { in checkFPAtomicToDenormModeHazard()
1939 ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn); in checkFPAtomicToDenormModeHazard()
1942 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { in checkMAIHazards() argument
1943 assert(SIInstrInfo::isMAI(*MI)); in checkMAIHazards()
1945 return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI); in checkMAIHazards()
1948 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { in checkMFMAPadding() argument
1954 if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2) in checkMFMAPadding()
1959 this](const MachineInstr &MI) { in checkMFMAPadding() argument
1960 if (!SIInstrInfo::isMFMA(MI)) in checkMFMAPadding()
1963 NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI); in checkMFMAPadding()
1978 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { in checkMAIHazards908() argument
1980 unsigned Opc = MI->getOpcode(); in checkMAIHazards908()
1982 auto IsVALUFn = [](const MachineInstr &MI) { in checkMAIHazards908() argument
1983 return SIInstrInfo::isVALU(MI) || MI.isInlineAsm(); in checkMAIHazards908()
1996 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIHazards908()
2012 for (const MachineOperand &Op : MI->explicit_operands()) { in checkMAIHazards908()
2032 this](const MachineInstr &MI) { in checkMAIHazards908() argument
2033 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards908()
2035 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards908()
2039 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); in checkMAIHazards908()
2078 auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) { in checkMAIHazards908() argument
2079 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAIHazards908()
2081 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards908()
2107 Register DstReg = MI->getOperand(0).getReg(); in checkMAIHazards908()
2111 this](const MachineInstr &MI) { in checkMAIHazards908() argument
2112 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards908()
2114 Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg(); in checkMAIHazards908()
2116 std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI)); in checkMAIHazards908()
2137 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); in checkMAIHazards908()
2177 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { in checkMAIHazards90A() argument
2179 unsigned Opc = MI->getOpcode(); in checkMAIHazards90A()
2181 auto IsLegacyVALUFn = [](const MachineInstr &MI) { in checkMAIHazards90A() argument
2182 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI); in checkMAIHazards90A()
2185 auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) { in checkMAIHazards90A() argument
2186 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMFMA(MI) && in checkMAIHazards90A()
2187 !SIInstrInfo::isDOT(MI); in checkMAIHazards90A()
2190 if (!SIInstrInfo::isMFMA(*MI)) in checkMAIHazards90A()
2202 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIHazards90A()
2228 this](const MachineInstr &MI) { in checkMAIHazards90A() argument
2229 if (!SIInstrInfo::isMFMA(MI)) in checkMAIHazards90A()
2231 Register DstReg = MI.getOperand(0).getReg(); in checkMAIHazards90A()
2233 MI1 = &MI; in checkMAIHazards90A()
2267 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2272 if (!isXDL(ST, *MI)) in checkMAIHazards90A()
2278 if (isXDL(ST, *MI) && !isXDL(ST, *MI1)) in checkMAIHazards90A()
2364 WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI)); in checkMAIHazards90A()
2369 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { in checkMAILdStHazards() argument
2376 auto IsAccVgprReadFn = [](const MachineInstr &MI) { in checkMAILdStHazards() argument
2377 return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64; in checkMAILdStHazards()
2380 for (const MachineOperand &Op : MI->explicit_uses()) { in checkMAILdStHazards()
2397 auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) { in checkMAILdStHazards() argument
2398 if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 && in checkMAILdStHazards()
2399 MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) in checkMAILdStHazards()
2401 auto IsVALUFn = [](const MachineInstr &MI) { in checkMAILdStHazards() argument
2402 return SIInstrInfo::isVALU(MI) && !SIInstrInfo::isMAI(MI); in checkMAILdStHazards()
2448 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { in checkMAIVALUHazards() argument
2452 auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { in checkMAIVALUHazards() argument
2453 return isDGEMM(MI.getOpcode()); in checkMAIVALUHazards()
2457 if (SIInstrInfo::isMFMA(*MI)) in checkMAIVALUHazards()
2464 bool IsMem = SIInstrInfo::isVMEM(*MI) || in checkMAIVALUHazards()
2465 SIInstrInfo::isFLAT(*MI) || in checkMAIVALUHazards()
2466 SIInstrInfo::isDS(*MI); in checkMAIVALUHazards()
2467 bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI); in checkMAIVALUHazards()
2468 bool IsVALU = SIInstrInfo::isVALU(*MI); in checkMAIVALUHazards()
2472 auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2473 if (!SIInstrInfo::isMFMA(MI) || in checkMAIVALUHazards()
2474 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) in checkMAIVALUHazards()
2476 MFMA = &MI; in checkMAIVALUHazards()
2481 auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2482 if (!SIInstrInfo::isDOT(MI) || in checkMAIVALUHazards()
2483 !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg)) in checkMAIVALUHazards()
2485 DOT = &MI; in checkMAIVALUHazards()
2490 auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2492 if (isDGEMM(MI.getOpcode())) in checkMAIVALUHazards()
2497 if (!TII.isVALU(MI) || !DGEMMAfterVALUWrite) in checkMAIVALUHazards()
2503 int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), in checkMAIVALUHazards()
2519 for (const MachineOperand &Use : MI->explicit_uses()) { in checkMAIVALUHazards()
2529 if (DOT->getOpcode() == MI->getOpcode()) { in checkMAIVALUHazards()
2530 if (&Use - &MI->getOperand(0) != SrcCIdx) in checkMAIVALUHazards()
2611 unsigned Opc = MI->getOpcode(); in checkMAIVALUHazards()
2625 for (const MachineOperand &Def : MI->defs()) { in checkMAIVALUHazards()
2644 if (DOT && DOT->getOpcode() != MI->getOpcode()) in checkMAIVALUHazards()
2695 auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { in checkMAIVALUHazards() argument
2696 if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || in checkMAIVALUHazards()
2697 !MI.readsRegister(Reg, &TRI)) in checkMAIVALUHazards()
2700 if (ST.hasGFX940Insts() && !isXDL(ST, MI)) in checkMAIVALUHazards()
2704 TII.getNamedOperand(MI, AMDGPU::OpName::src2); in checkMAIVALUHazards()
2709 MFMA = &MI; in checkMAIVALUHazards()
2747 auto IsMFMAFn = [&MAI](const MachineInstr &MI) { in ShouldPreferAnother() argument
2749 if (SIInstrInfo::isMFMA(MI)) in ShouldPreferAnother()
2750 MAI = &MI; in ShouldPreferAnother()
2754 MachineInstr *MI = SU->getInstr(); in ShouldPreferAnother() local
2755 if (IsMFMAFn(*MI)) { in ShouldPreferAnother()
2764 bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { in fixVALUMaskWriteHazard() argument
2769 if (!ST.isWave64() || !SIInstrInfo::isSALU(*MI)) in fixVALUMaskWriteHazard()
2780 const MachineOperand *SDSTOp = TII.getNamedOperand(*MI, AMDGPU::OpName::sdst); in fixVALUMaskWriteHazard()
2874 if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) == in fixVALUMaskWriteHazard()
2878 auto NextMI = std::next(MI->getIterator()); in fixVALUMaskWriteHazard()
2881 BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(), in fixVALUMaskWriteHazard()
2886 if (MI->getOpcode() == AMDGPU::S_GETPC_B64) { in fixVALUMaskWriteHazard()
2888 while (NextMI != MI->getParent()->end() && in fixVALUMaskWriteHazard()
2916 bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) { in fixRequiredExportPriority() argument
2922 MachineBasicBlock *MBB = MI->getParent(); in fixRequiredExportPriority()
2939 auto It = MI->getIterator(); in fixRequiredExportPriority()
2940 switch (MI->getOpcode()) { in fixRequiredExportPriority()
2952 auto &PrioOp = MI->getOperand(0); in fixRequiredExportPriority()
2962 if (!TII.isEXP(*MI)) in fixRequiredExportPriority()
2986 const DebugLoc &DL = MI->getDebugLoc(); in fixRequiredExportPriority()