Lines Matching refs:AMDGPU
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
187 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode()); in getVmemType()
188 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo = in getVmemType()
189 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); in getVmemType()
198 unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) { in getCounterRef()
219 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { in addWait()
224 void setNoWait(AMDGPU::Waitcnt &Wait, InstCounterType T) { in setNoWait()
228 unsigned getWait(AMDGPU::Waitcnt &Wait, InstCounterType T) { in getWait()
310 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
312 void determineWait(InstCounterType T, int RegNo, AMDGPU::Waitcnt &Wait) const;
313 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
449 AMDGPU::IsaVersion IV;
457 IV(AMDGPU::getIsaVersion(ST->getCPU())), MaxCounter(MaxCounter), in WaitcntGenerator()
478 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
488 AMDGPU::Waitcnt Wait) = 0;
496 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
519 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
524 AMDGPU::Waitcnt Wait) override;
543 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
555 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
560 AMDGPU::Waitcnt Wait) override;
578 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
722 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
748 unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)) & in getRegInterval()
749 AMDGPU::HWEncoding::REG_IDX_MASK; in getRegInterval()
808 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr); in updateByEvent()
816 if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data0)) { in updateByEvent()
819 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0), in updateByEvent()
822 if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data1)) { in updateByEvent()
824 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), in updateByEvent()
825 AMDGPU::OpName::data1), in updateByEvent()
829 Inst.getOpcode() != AMDGPU::DS_APPEND && in updateByEvent()
830 Inst.getOpcode() != AMDGPU::DS_CONSUME && in updateByEvent()
831 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) { in updateByEvent()
844 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
849 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
858 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
871 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data), in updateByEvent()
878 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::vdst), in updateByEvent()
891 TRI->getEncodingValue(AMDGPU::getMCReg(DefMO.getReg(), *ST)), in updateByEvent()
1038 void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const { in simplifyWaitcnt()
1058 AMDGPU::Waitcnt &Wait) const { in determineWait()
1086 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) { in applyWaitcnt()
1136 int OpIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); in updateOperandIfDifferent()
1152 case AMDGPU::S_WAIT_LOADCNT: in counterTypeForInstr()
1154 case AMDGPU::S_WAIT_EXPCNT: in counterTypeForInstr()
1156 case AMDGPU::S_WAIT_STORECNT: in counterTypeForInstr()
1158 case AMDGPU::S_WAIT_SAMPLECNT: in counterTypeForInstr()
1160 case AMDGPU::S_WAIT_BVHCNT: in counterTypeForInstr()
1162 case AMDGPU::S_WAIT_DSCNT: in counterTypeForInstr()
1164 case AMDGPU::S_WAIT_KMCNT: in counterTypeForInstr()
1187 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const { in applyPreexistingWaitcnt()
1205 if (Opcode == AMDGPU::S_WAITCNT) { in applyPreexistingWaitcnt()
1207 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(IV, IEnc); in applyPreexistingWaitcnt()
1219 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT); in applyPreexistingWaitcnt()
1220 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL); in applyPreexistingWaitcnt()
1223 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1237 Modified |= updateOperandIfDifferent(*WaitcntInstr, AMDGPU::OpName::simm16, in applyPreexistingWaitcnt()
1238 AMDGPU::encodeWaitcnt(IV, Wait)); in applyPreexistingWaitcnt()
1259 AMDGPU::OpName::simm16, Wait.StoreCnt); in applyPreexistingWaitcnt()
1281 AMDGPU::Waitcnt Wait) { in createNewWaitcnt()
1291 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait); in createNewWaitcnt()
1293 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Enc); in createNewWaitcnt()
1305 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT)) in createNewWaitcnt()
1306 .addReg(AMDGPU::SGPR_NULL, RegState::Undef) in createNewWaitcnt()
1318 AMDGPU::Waitcnt
1320 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST->hasVscnt() ? 0 : ~0u); in getAllZeroWaitcnt()
1323 AMDGPU::Waitcnt
1325 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0); in getAllZeroWaitcnt()
1334 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const { in applyPreexistingWaitcnt()
1358 if (Opcode == AMDGPU::S_WAITCNT) in applyPreexistingWaitcnt()
1361 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) { in applyPreexistingWaitcnt()
1363 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1364 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(IV, OldEnc); in applyPreexistingWaitcnt()
1369 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) { in applyPreexistingWaitcnt()
1371 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1372 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(IV, OldEnc); in applyPreexistingWaitcnt()
1381 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm(); in applyPreexistingWaitcnt()
1406 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(IV, Wait); in applyPreexistingWaitcnt()
1408 AMDGPU::OpName::simm16, NewEnc); in applyPreexistingWaitcnt()
1431 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(IV, Wait); in applyPreexistingWaitcnt()
1433 AMDGPU::OpName::simm16, NewEnc); in applyPreexistingWaitcnt()
1493 AMDGPU::OpName::simm16, NewCnt); in applyPreexistingWaitcnt()
1518 AMDGPU::Waitcnt Wait) { in createNewWaitcnt()
1530 unsigned Enc = AMDGPU::encodeLoadcntDscnt(IV, Wait); in createNewWaitcnt()
1532 SWaitInst = BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT)) in createNewWaitcnt()
1538 unsigned Enc = AMDGPU::encodeStorecntDscnt(IV, Wait); in createNewWaitcnt()
1541 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAIT_STORECNT_DSCNT)) in createNewWaitcnt()
1581 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) && in readsVCCZ()
1621 AMDGPU::Waitcnt Wait; in generateWaitcntInstBefore()
1627 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 || in generateWaitcntInstBefore()
1628 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC || in generateWaitcntInstBefore()
1629 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL || in generateWaitcntInstBefore()
1630 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV || in generateWaitcntInstBefore()
1631 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) { in generateWaitcntInstBefore()
1638 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG || in generateWaitcntInstBefore()
1639 MI.getOpcode() == AMDGPU::SI_RETURN || in generateWaitcntInstBefore()
1640 MI.getOpcode() == AMDGPU::S_SETPC_B64_return || in generateWaitcntInstBefore()
1650 else if (MI.getOpcode() == AMDGPU::S_ENDPGM || in generateWaitcntInstBefore()
1651 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) { in generateWaitcntInstBefore()
1658 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG || in generateWaitcntInstBefore()
1659 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) && in generateWaitcntInstBefore()
1661 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) == in generateWaitcntInstBefore()
1662 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) { in generateWaitcntInstBefore()
1671 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) { in generateWaitcntInstBefore()
1686 Wait = AMDGPU::Waitcnt(); in generateWaitcntInstBefore()
1689 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); in generateWaitcntInstBefore()
1700 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst); in generateWaitcntInstBefore()
1854 bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait, in generateWaitcnt()
1875 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp); in generateWaitcnt()
1977 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB || in isCacheInvOrWBInst()
1978 Opc == AMDGPU::GLOBAL_WBINV; in isCacheInvOrWBInst()
1990 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) { in updateEventWaitcntAfter()
2025 !llvm::AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode())) { in updateEventWaitcntAfter()
2043 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt()); in updateEventWaitcntAfter()
2048 int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm(); in updateEventWaitcntAfter()
2051 unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm(); in updateEventWaitcntAfter()
2052 if (Imm >= AMDGPU::Exp::ET_PARAM0 && Imm <= AMDGPU::Exp::ET_PARAM31) in updateEventWaitcntAfter()
2054 else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST) in updateEventWaitcntAfter()
2060 case AMDGPU::S_SENDMSG: in updateEventWaitcntAfter()
2061 case AMDGPU::S_SENDMSG_RTN_B32: in updateEventWaitcntAfter()
2062 case AMDGPU::S_SENDMSG_RTN_B64: in updateEventWaitcntAfter()
2063 case AMDGPU::S_SENDMSGHALT: in updateEventWaitcntAfter()
2066 case AMDGPU::S_MEMTIME: in updateEventWaitcntAfter()
2067 case AMDGPU::S_MEMREALTIME: in updateEventWaitcntAfter()
2068 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0: in updateEventWaitcntAfter()
2069 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: in updateEventWaitcntAfter()
2070 case AMDGPU::S_BARRIER_LEAVE: in updateEventWaitcntAfter()
2071 case AMDGPU::S_GET_BARRIER_STATE_M0: in updateEventWaitcntAfter()
2072 case AMDGPU::S_GET_BARRIER_STATE_IMM: in updateEventWaitcntAfter()
2144 return Opcode == AMDGPU::S_WAITCNT || in isWaitInstr()
2145 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(0).isReg() && in isWaitInstr()
2146 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL) || in isWaitInstr()
2147 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT || in isWaitInstr()
2148 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT || in isWaitInstr()
2207 if (Inst.definesRegister(AMDGPU::VCC_LO, /*TRI=*/nullptr) || in insertWaitcntInBlock()
2208 Inst.definesRegister(AMDGPU::VCC_HI, /*TRI=*/nullptr)) { in insertWaitcntInBlock()
2212 } else if (Inst.definesRegister(AMDGPU::VCC, /*TRI=*/nullptr)) { in insertWaitcntInBlock()
2250 AMDGPU::Waitcnt Wait = WCG->getAllZeroWaitcnt( in insertWaitcntInBlock()
2269 TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), in insertWaitcntInBlock()
2281 AMDGPU::Waitcnt Wait; in insertWaitcntInBlock()
2407 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST->getCPU()); in runOnMachineFunction()
2429 Limits.LoadcntMax = AMDGPU::getLoadcntBitMask(IV); in runOnMachineFunction()
2430 Limits.DscntMax = AMDGPU::getDscntBitMask(IV); in runOnMachineFunction()
2432 Limits.LoadcntMax = AMDGPU::getVmcntBitMask(IV); in runOnMachineFunction()
2433 Limits.DscntMax = AMDGPU::getLgkmcntBitMask(IV); in runOnMachineFunction()
2435 Limits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); in runOnMachineFunction()
2436 Limits.StorecntMax = AMDGPU::getStorecntBitMask(IV); in runOnMachineFunction()
2437 Limits.SamplecntMax = AMDGPU::getSamplecntBitMask(IV); in runOnMachineFunction()
2438 Limits.BvhcntMax = AMDGPU::getBvhcntBitMask(IV); in runOnMachineFunction()
2439 Limits.KmcntMax = AMDGPU::getKmcntBitMask(IV); in runOnMachineFunction()
2448 TRI->getEncodingValue(AMDGPU::VGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK; in runOnMachineFunction()
2451 TRI->getEncodingValue(AMDGPU::SGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK; in runOnMachineFunction()
2472 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT)) in runOnMachineFunction()
2483 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0); in runOnMachineFunction()
2565 if (MI.getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
2566 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) in runOnMachineFunction()
2585 if (I->getOpcode() == AMDGPU::S_DCACHE_WB) in runOnMachineFunction()
2591 if ((I->getOpcode() == AMDGPU::S_ENDPGM || in runOnMachineFunction()
2592 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && in runOnMachineFunction()
2595 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); in runOnMachineFunction()
2606 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::S_NOP)) in runOnMachineFunction()
2610 TII->get(AMDGPU::S_SENDMSG)) in runOnMachineFunction()
2611 .addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus); in runOnMachineFunction()