Lines Matching refs:MRI
64 MRI = &MF.getRegInfo(); in setupMF()
78 const MachineRegisterInfo &MRI) const { in isVCC()
83 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); in isVCC()
87 const LLT Ty = MRI.getType(Reg); in isVCC()
91 return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC && in isVCC()
109 if (MRI->getType(Dst.getReg()) == LLT::scalar(1)) in constrainCopyLikeIntrin()
113 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in constrainCopyLikeIntrin()
115 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in constrainCopyLikeIntrin()
119 return RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI) && in constrainCopyLikeIntrin()
120 RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI); in constrainCopyLikeIntrin()
133 if (isVCC(DstReg, *MRI)) { in selectCOPY()
136 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectCOPY()
139 return RBI.constrainGenericRegister(DstReg, *RC, *MRI); in selectCOPY()
142 if (!isVCC(SrcReg, *MRI)) { in selectCOPY()
144 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI)) in selectCOPY()
148 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in selectCOPY()
151 getIConstantVRegValWithLookThrough(SrcReg, *MRI, true); in selectCOPY()
158 Register MaskedReg = MRI->createVirtualRegister(SrcRC); in selectCOPY()
194 if (!MRI->getRegClassOrNull(SrcReg)) in selectCOPY()
195 MRI->setRegClass(SrcReg, SrcRC); in selectCOPY()
201 TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectCOPY()
202 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) in selectCOPY()
213 TRI.getConstrainedRegClassForOperand(MO, *MRI); in selectCOPY()
216 RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI); in selectCOPY()
237 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectCOPY_SCC_VCC()
247 getIConstantVRegValWithLookThrough(I.getOperand(1).getReg(), *MRI); in selectCOPY_VCC_SCC()
259 return RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI); in selectCOPY_VCC_SCC()
291 const LLT DefTy = MRI->getType(DefReg); in selectPHI()
303 MRI->getRegClassOrRegBank(DefReg); in selectPHI()
326 const RegisterBank *RB = MRI->getRegBankOrNull(SrcReg); in selectPHI()
328 const LLT SrcTy = MRI->getType(SrcReg); in selectPHI()
331 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectPHI()
337 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI); in selectPHI()
347 Register DstReg = MRI->createVirtualRegister(&SubRC); in getSubOperand64()
390 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); in selectG_AND_OR_XOR()
392 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_AND_OR_XOR()
414 LLT Ty = MRI->getType(DstReg); in selectG_ADD_SUB()
419 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_ADD_SUB()
445 Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass()); in selectG_ADD_SUB()
468 Register DstLo = MRI->createVirtualRegister(&HalfRC); in selectG_ADD_SUB()
469 Register DstHi = MRI->createVirtualRegister(&HalfRC); in selectG_ADD_SUB()
481 Register CarryReg = MRI->createVirtualRegister(CarryRC); in selectG_ADD_SUB()
488 .addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead) in selectG_ADD_SUB()
505 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI)) in selectG_ADD_SUB()
524 if (isVCC(Dst1Reg, *MRI)) { in selectG_UADDO_USUBO_UADDE_USUBE()
549 if (MRI->use_nodbg_empty(Dst1Reg)) { in selectG_UADDO_USUBO_UADDE_USUBE()
554 if (!MRI->getRegClassOrNull(Dst1Reg)) in selectG_UADDO_USUBO_UADDE_USUBE()
555 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
558 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
559 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
560 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
565 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
595 LLT DstTy = MRI->getType(DstReg); in selectG_EXTRACT()
596 LLT SrcTy = MRI->getType(SrcReg); in selectG_EXTRACT()
611 TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI); in selectG_EXTRACT()
612 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectG_EXTRACT()
615 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_EXTRACT()
626 SrcReg = constrainOperandRegClass(*MF, TRI, *MRI, TII, RBI, I, in selectG_EXTRACT()
639 LLT DstTy = MRI->getType(DstReg); in selectG_MERGE_VALUES()
640 LLT SrcTy = MRI->getType(MI.getOperand(1).getReg()); in selectG_MERGE_VALUES()
647 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_MERGE_VALUES()
663 = TRI.getConstrainedRegClassForOperand(Src, *MRI); in selectG_MERGE_VALUES()
664 if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI)) in selectG_MERGE_VALUES()
668 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectG_MERGE_VALUES()
683 LLT DstTy = MRI->getType(DstReg0); in selectG_UNMERGE_VALUES()
684 LLT SrcTy = MRI->getType(SrcReg); in selectG_UNMERGE_VALUES()
689 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_UNMERGE_VALUES()
693 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectG_UNMERGE_VALUES()
707 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) in selectG_UNMERGE_VALUES()
711 TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectG_UNMERGE_VALUES()
712 if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI)) in selectG_UNMERGE_VALUES()
726 LLT SrcTy = MRI->getType(Src0); in selectG_BUILD_VECTOR()
737 if (MRI->getType(Dst) != LLT::fixed_vector(2, 16) || in selectG_BUILD_VECTOR()
742 const RegisterBank *DstBank = RBI.getRegBank(Dst, *MRI, TRI); in selectG_BUILD_VECTOR()
756 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true); in selectG_BUILD_VECTOR()
759 getAnyConstantVRegValWithLookThrough(Src0, *MRI, true, true); in selectG_BUILD_VECTOR()
771 return RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI); in selectG_BUILD_VECTOR()
777 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR()
787 MachineInstr *Src1Def = getDefIgnoringCopies(Src1, *MRI); in selectG_BUILD_VECTOR()
793 return RBI.constrainGenericRegister(Dst, RC, *MRI) && in selectG_BUILD_VECTOR()
794 RBI.constrainGenericRegister(Src0, RC, *MRI); in selectG_BUILD_VECTOR()
799 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_BUILD_VECTOR()
833 Src0, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc0), m_SpecificICst(16)))); in selectG_BUILD_VECTOR()
836 Src1, *MRI, m_OneUse(m_GLShr(m_Reg(ShiftSrc1), m_SpecificICst(16)))); in selectG_BUILD_VECTOR()
848 getAnyConstantVRegValWithLookThrough(Src1, *MRI, true, true); in selectG_BUILD_VECTOR()
874 const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, *MRI); in selectG_IMPLICIT_DEF()
875 if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) || in selectG_IMPLICIT_DEF()
876 (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI))) { in selectG_IMPLICIT_DEF()
890 LLT Src1Ty = MRI->getType(Src1Reg); in selectG_INSERT()
892 unsigned DstSize = MRI->getType(DstReg).getSizeInBits(); in selectG_INSERT()
909 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_INSERT()
915 const RegisterBank *Src0Bank = RBI.getRegBank(Src0Reg, *MRI, TRI); in selectG_INSERT()
916 const RegisterBank *Src1Bank = RBI.getRegBank(Src1Reg, *MRI, TRI); in selectG_INSERT()
928 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_INSERT()
929 !RBI.constrainGenericRegister(Src0Reg, *Src0RC, *MRI) || in selectG_INSERT()
930 !RBI.constrainGenericRegister(Src1Reg, *Src1RC, *MRI)) in selectG_INSERT()
949 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
951 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 && in selectG_SBFX_UBFX()
974 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
975 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
976 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
986 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
1032 getIConstantVRegValWithLookThrough(LaneSelect, *MRI); in selectWritelane()
1041 getIConstantVRegValWithLookThrough(Val, *MRI); in selectWritelane()
1055 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
1075 LLT Ty = MRI->getType(Dst0); in selectDivScale()
1130 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); in selectG_INTRINSIC()
1448 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); in selectG_ICMP_or_FCMP()
1453 if (!isVCC(CCReg, *MRI)) { in selectG_ICMP_or_FCMP()
1464 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP_or_FCMP()
1492 *TRI.getBoolRC(), *MRI); in selectG_ICMP_or_FCMP()
1500 if (isVCC(Dst, *MRI)) in selectIntrinsicCmp()
1503 LLT DstTy = MRI->getType(Dst); in selectIntrinsicCmp()
1510 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); in selectIntrinsicCmp()
1520 return RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI); in selectIntrinsicCmp()
1548 RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI); in selectIntrinsicCmp()
1560 static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI, in isLaneMaskFromSameBlock() argument
1562 MachineInstr *MI = MRI.getVRegDef(Reg); in isLaneMaskFromSameBlock()
1568 auto DstRB = MRI.getRegBankOrNull(MI->getOperand(0).getReg()); in isLaneMaskFromSameBlock()
1569 auto SrcRB = MRI.getRegBankOrNull(MI->getOperand(1).getReg()); in isLaneMaskFromSameBlock()
1581 if (mi_match(Reg, MRI, m_GAnd(m_Reg(LHS), m_Reg(RHS)))) in isLaneMaskFromSameBlock()
1582 return isLaneMaskFromSameBlock(LHS, MRI, MBB) || in isLaneMaskFromSameBlock()
1583 isLaneMaskFromSameBlock(RHS, MRI, MBB); in isLaneMaskFromSameBlock()
1593 const unsigned BallotSize = MRI->getType(DstReg).getSizeInBits(); in selectBallot()
1602 getIConstantVRegValWithLookThrough(SrcReg, *MRI); in selectBallot()
1607 Dst = MRI->createVirtualRegister(TRI.getBoolRC()); in selectBallot()
1621 if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI)) in selectBallot()
1624 if (isLaneMaskFromSameBlock(SrcReg, *MRI, BB)) { in selectBallot()
1627 if (!RBI.constrainGenericRegister(Dst, *TRI.getBoolRC(), *MRI)) in selectBallot()
1643 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectBallot()
1658 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectRelocConstant()
1660 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectRelocConstant()
1684 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectGroupStaticSize()
1717 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); in selectReturnAddress()
1719 !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) in selectReturnAddress()
1755 if (!MRI->getRegClassOrNull(Reg)) in selectEndCfIntrinsic()
1756 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); in selectEndCfIntrinsic()
1825 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1863 const RegisterBank *OffsetRB = RBI.getRegBank(BaseOffset, *MRI, TRI); in selectDSGWSIntrinsic()
1867 MachineInstr *OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); in selectDSGWSIntrinsic()
1881 OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); in selectDSGWSIntrinsic()
1895 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset, VT); in selectDSGWSIntrinsic()
1900 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1907 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1911 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1930 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1946 LLT PtrTy = MRI->getType(PtrBase); in selectDSAppendConsume()
1964 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
2078 LLT Ty = MRI->getType(VDataIn); in selectImageIntrinsic()
2100 VDataTy = MRI->getType(VDataIn); in selectImageIntrinsic()
2106 VDataTy = MRI->getType(VDataOut); in selectImageIntrinsic()
2145 NumVAddrDwords += (MRI->getType(Addr).getSizeInBits() + 31) / 32; in selectImageIntrinsic()
2204 const bool Is64 = MRI->getType(VDataOut).getSizeInBits() == 64; in selectImageIntrinsic()
2206 Register TmpReg = MRI->createVirtualRegister( in selectImageIntrinsic()
2211 if (!MRI->use_empty(VDataOut)) { in selectImageIntrinsic()
2382 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); in selectG_SELECT()
2386 if (!isVCC(CCReg, *MRI)) { in selectG_SELECT()
2395 if (!MRI->getRegClassOrNull(CCReg)) in selectG_SELECT()
2396 MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, *MRI)); in selectG_SELECT()
2428 const LLT DstTy = MRI->getType(DstReg); in selectG_TRUNC()
2429 const LLT SrcTy = MRI->getType(SrcReg); in selectG_TRUNC()
2432 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_TRUNC()
2439 DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_TRUNC()
2456 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_TRUNC()
2457 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) { in selectG_TRUNC()
2476 Register LoReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
2477 Register HiReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
2497 Register TmpReg0 = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
2498 Register TmpReg1 = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
2499 Register ImmReg = MRI->createVirtualRegister(DstRC); in selectG_TRUNC()
2552 if (!RBI.constrainGenericRegister(SrcReg, *SrcWithSubRC, *MRI)) in selectG_TRUNC()
2572 Register Reg, const MachineRegisterInfo &MRI, in getArtifactRegBank() argument
2574 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); in getArtifactRegBank()
2592 const LLT DstTy = MRI->getType(DstReg); in selectG_SZA_EXT()
2593 const LLT SrcTy = MRI->getType(SrcReg); in selectG_SZA_EXT()
2601 const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI); in selectG_SZA_EXT()
2610 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_SZA_EXT()
2614 Register UndefReg = MRI->createVirtualRegister(SrcRC); in selectG_SZA_EXT()
2623 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) && in selectG_SZA_EXT()
2624 RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI); in selectG_SZA_EXT()
2654 if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI)) in selectG_SZA_EXT()
2663 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2669 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2687 *MRI); in selectG_SZA_EXT()
2696 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2697 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2712 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2728 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2734 static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) { in stripCopy() argument
2735 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg; in stripCopy()
2738 static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) { in stripBitCast() argument
2740 if (mi_match(Reg, MRI, m_GBitcast(m_Reg(BitcastSrc)))) in stripBitCast()
2745 static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In, in isExtractHiElt() argument
2748 if (!mi_match(In, MRI, m_GTrunc(m_Reg(Trunc)))) in isExtractHiElt()
2753 if (mi_match(Trunc, MRI, m_GLShr(m_Reg(LShlSrc), m_Reg(Cst)))) { in isExtractHiElt()
2754 Cst = stripCopy(Cst, MRI); in isExtractHiElt()
2755 if (mi_match(Cst, MRI, m_SpecificICst(16))) { in isExtractHiElt()
2756 Out = stripBitCast(LShlSrc, MRI); in isExtractHiElt()
2761 MachineInstr *Shuffle = MRI.getVRegDef(Trunc); in isExtractHiElt()
2765 assert(MRI.getType(Shuffle->getOperand(0).getReg()) == in isExtractHiElt()
2784 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FPEXT()
2790 if (MRI->getType(Dst) == LLT::scalar(32) && in selectG_FPEXT()
2791 MRI->getType(Src) == LLT::scalar(16)) { in selectG_FPEXT()
2792 if (isExtractHiElt(*MRI, Src, Src)) { in selectG_FPEXT()
2797 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_FPEXT()
2817 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FNEG()
2819 MRI->getType(Dst) != LLT::scalar(64)) in selectG_FNEG()
2823 MachineInstr *Fabs = getOpcodeDef(TargetOpcode::G_FABS, Src, *MRI); in selectG_FNEG()
2827 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2828 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2833 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2834 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2835 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2836 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2863 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); in selectG_FABS()
2865 MRI->getType(Dst) != LLT::scalar(64)) in selectG_FABS()
2871 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2872 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2873 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2874 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2876 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2877 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2908 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const { in getAddrModeInfo() argument
2912 MRI.getUniqueVRegDef(Load.getOperand(OpNo).getReg()); in getAddrModeInfo()
2923 const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg()); in getAddrModeInfo()
2932 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI); in getAddrModeInfo()
2940 getAddrModeInfo(*PtrMI, MRI, AddrInfo); in getAddrModeInfo()
2944 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2965 return RBI.getRegBank(MI.getOperand(0).getReg(), *MRI, TRI)->getID() == in isInstrUniform()
2981 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg()); in initM0()
2999 static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) { in isVCmpResult() argument
3003 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg); in isVCmpResult()
3007 return isVCmpResult(MI.getOperand(1).getReg(), MRI); in isVCmpResult()
3011 return isVCmpResult(MI.getOperand(1).getReg(), MRI) && in isVCmpResult()
3012 isVCmpResult(MI.getOperand(2).getReg(), MRI); in isVCmpResult()
3035 if (!isVCC(CondReg, *MRI)) { in selectG_BRCOND()
3036 if (MRI->getType(CondReg) != LLT::scalar(32)) in selectG_BRCOND()
3047 if (!isVCmpResult(CondReg, *MRI)) { in selectG_BRCOND()
3052 Register TmpReg = MRI->createVirtualRegister(TRI.getBoolRC()); in selectG_BRCOND()
3065 if (!MRI->getRegClassOrNull(CondReg)) in selectG_BRCOND()
3066 MRI->setRegClass(CondReg, ConstrainRC); in selectG_BRCOND()
3080 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_GLOBAL_VALUE()
3087 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
3094 LLT Ty = MRI->getType(DstReg); in selectG_PTRMASK()
3095 LLT MaskTy = MRI->getType(MaskReg); in selectG_PTRMASK()
3099 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_PTRMASK()
3100 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_PTRMASK()
3101 const RegisterBank *MaskRB = RBI.getRegBank(MaskReg, *MRI, TRI); in selectG_PTRMASK()
3134 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_PTRMASK()
3135 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_PTRMASK()
3136 !RBI.constrainGenericRegister(MaskReg, *MaskRC, *MRI)) in selectG_PTRMASK()
3153 Register HiReg = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
3154 Register LoReg = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
3169 Register MaskLo = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
3170 MaskedLo = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
3183 Register MaskHi = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
3184 MaskedHi = MRI->createVirtualRegister(&RegRC); in selectG_PTRMASK()
3205 computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI, in computeIndirectRegIndex() argument
3212 AMDGPU::getBaseWithConstantOffset(MRI, IdxReg, &ValueTracking); in computeIndirectRegIndex()
3235 LLT DstTy = MRI->getType(DstReg); in selectG_EXTRACT_VECTOR_ELT()
3236 LLT SrcTy = MRI->getType(SrcReg); in selectG_EXTRACT_VECTOR_ELT()
3238 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
3239 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
3240 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); in selectG_EXTRACT_VECTOR_ELT()
3253 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || in selectG_EXTRACT_VECTOR_ELT()
3254 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || in selectG_EXTRACT_VECTOR_ELT()
3255 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
3264 *MRI, TRI, SrcRC, IdxReg, DstTy.getSizeInBits() / 8, *VT); in selectG_EXTRACT_VECTOR_ELT()
3313 LLT VecTy = MRI->getType(DstReg); in selectG_INSERT_VECTOR_ELT()
3314 LLT ValTy = MRI->getType(ValReg); in selectG_INSERT_VECTOR_ELT()
3318 const RegisterBank *VecRB = RBI.getRegBank(VecReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
3319 const RegisterBank *ValRB = RBI.getRegBank(ValReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
3320 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); in selectG_INSERT_VECTOR_ELT()
3334 if (!RBI.constrainGenericRegister(VecReg, *VecRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
3335 !RBI.constrainGenericRegister(DstReg, *VecRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
3336 !RBI.constrainGenericRegister(ValReg, *ValRC, *MRI) || in selectG_INSERT_VECTOR_ELT()
3337 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
3345 computeIndirectRegIndex(*MRI, TRI, VecRC, IdxReg, ValSize / 8, *VT); in selectG_INSERT_VECTOR_ELT()
3396 getIConstantVRegValWithLookThrough(VOffset, *MRI); in selectBufferLoadLds()
3448 Register IdxReg = MRI->createVirtualRegister(TRI.getVGPR64Class()); in selectBufferLoadLds()
3497 static Register matchZeroExtendFromS32(MachineRegisterInfo &MRI, Register Reg) { in matchZeroExtendFromS32() argument
3499 if (mi_match(Reg, MRI, m_GZExt(m_Reg(ZExtSrc)))) in matchZeroExtendFromS32()
3500 return MRI.getType(ZExtSrc) == LLT::scalar(32) ? ZExtSrc : Register(); in matchZeroExtendFromS32()
3503 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); in matchZeroExtendFromS32()
3508 MRI.getType(Def->getOperand(0).getReg()) == LLT::scalar(64)); in matchZeroExtendFromS32()
3509 if (mi_match(Def->getOperand(2).getReg(), MRI, m_ZeroInt())) { in matchZeroExtendFromS32()
3557 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectGlobalLoadLds()
3562 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI); in selectGlobalLoadLds()
3565 if (Register Off = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) { in selectGlobalLoadLds()
3576 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalLoadLds()
3750 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectWaveAddress()
3768 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI)) in selectWaveAddress()
3779 const MachineRegisterInfo &MRI) { in BitOp3_Op() argument
3783 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool { in BitOp3_Op()
3795 if (mi_match(Op, MRI, m_AllOnesInt())) { in BitOp3_Op()
3799 if (mi_match(Op, MRI, m_ZeroInt())) { in BitOp3_Op()
3823 if (mi_match(Op, MRI, m_Not(m_Reg(LHS)))) { in BitOp3_Op()
3824 LHS = getSrcRegIgnoringCopies(LHS, MRI); in BitOp3_Op()
3841 MachineInstr *MI = MRI.getVRegDef(R); in BitOp3_Op()
3846 Register LHS = getSrcRegIgnoringCopies(MI->getOperand(1).getReg(), MRI); in BitOp3_Op()
3847 Register RHS = getSrcRegIgnoringCopies(MI->getOperand(2).getReg(), MRI); in BitOp3_Op()
3857 auto Op = BitOp3_Op(LHS, Src, MRI); in BitOp3_Op()
3863 Op = BitOp3_Op(RHS, Src, MRI); in BitOp3_Op()
3897 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); in selectBITOP3()
3906 std::tie(NumOpcodes, TTbl) = BitOp3_Op(DstReg, Src, *MRI); in selectBITOP3()
3913 const bool IsB32 = MRI->getType(DstReg) == LLT::scalar(32); in selectBITOP3()
3918 if (mi_match(MI, *MRI, m_GXor(m_GXor(m_Reg(), m_Reg()), m_Reg())) || in selectBITOP3()
3919 mi_match(MI, *MRI, m_GOr(m_GOr(m_Reg(), m_Reg()), m_Reg())) || in selectBITOP3()
3920 mi_match(MI, *MRI, m_GOr(m_GAnd(m_Reg(), m_Reg()), m_Reg()))) in selectBITOP3()
3935 const RegisterBank *RB = RBI.getRegBank(Src[I], *MRI, TRI); in selectBITOP3()
3942 Register NewReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectBITOP3()
3978 if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, *MRI)) in selectStackRestore()
3981 MachineInstr *DefMI = MRI->getVRegDef(SrcReg); in selectStackRestore()
3989 WaveAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectStackRestore()
4104 if (MRI->getType(I.getOperand(1).getReg()) != LLT::scalar(1) && in select()
4173 MachineInstr *MI = getDefIgnoringCopies(Src, *MRI); in selectVOP3ModsImpl()
4178 MI = getDefIgnoringCopies(Src, *MRI); in selectVOP3ModsImpl()
4183 getConstantFPVRegVal(MI->getOperand(1).getReg(), *MRI); in selectVOP3ModsImpl()
4205 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in copyToVGPRIfSrcFolded()
4210 Register VGPRSrc = MRI->cloneVirtualRegister(Root.getReg()); in copyToVGPRIfSrcFolded()
4322 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI); in selectVOP3NoMods()
4350 const MachineRegisterInfo &MRI) { in isTruncHalf() argument
4354 unsigned DstSize = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits(); in isTruncHalf()
4355 unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits(); in isTruncHalf()
4361 static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) { in isLshrHalf() argument
4367 if (mi_match(MI->getOperand(0).getReg(), MRI, in isLshrHalf()
4369 unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits(); in isLshrHalf()
4378 static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) { in isShlHalf() argument
4384 if (mi_match(MI->getOperand(0).getReg(), MRI, in isShlHalf()
4386 unsigned SrcSize = MRI.getType(MI->getOperand(1).getReg()).getSizeInBits(); in isShlHalf()
4395 const MachineRegisterInfo &MRI) { in isUnmergeHalf() argument
4405 const MachineRegisterInfo &MRI) { in isVectorOfTwoOrScalar() argument
4406 LLT OpTy = MRI.getType(Reg); in isVectorOfTwoOrScalar()
4415 const MachineRegisterInfo &MRI) { in getNegStatus() argument
4416 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI); in getNegStatus()
4571 const MachineRegisterInfo &MRI) { in calcNextStatus() argument
4572 const MachineInstr *MI = MRI.getVRegDef(Curr.first); in calcNextStatus()
4587 SrcStatus Stat = getNegStatus(Curr.first, Curr.second, MRI); in calcNextStatus()
4600 if (isTruncHalf(MI, MRI)) in calcNextStatus()
4603 else if (isUnmergeHalf(MI, MRI)) { in calcNextStatus()
4612 if (isTruncHalf(MI, MRI)) { in calcNextStatus()
4622 if (isUnmergeHalf(MI, MRI)) { in calcNextStatus()
4631 if (isShlHalf(MI, MRI)) in calcNextStatus()
4636 if (isLshrHalf(MI, MRI)) in calcNextStatus()
4641 if (isShlHalf(MI, MRI)) in calcNextStatus()
4646 if (isLshrHalf(MI, MRI)) in calcNextStatus()
4668 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) { in SearchOptions() argument
4669 const MachineInstr *MI = MRI.getVRegDef(Reg); in SearchOptions()
4696 getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO, in getSrcStats() argument
4699 auto Curr = calcNextStatus({Reg, SrcStatus::IS_SAME}, MRI); in getSrcStats()
4706 Curr = calcNextStatus(Curr.value(), MRI); in getSrcStats()
4713 getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO, in getLastSameOrNeg() argument
4717 auto Curr = calcNextStatus(LastSameOrNeg, MRI); in getLastSameOrNeg()
4727 Curr = calcNextStatus(Curr.value(), MRI); in getLastSameOrNeg()
4734 const MachineRegisterInfo &MRI) { in isSameBitWidth() argument
4735 unsigned Width1 = MRI.getType(Reg1).getSizeInBits(); in isSameBitWidth()
4736 unsigned Width2 = MRI.getType(Reg2).getSizeInBits(); in isSameBitWidth()
4767 const MachineRegisterInfo &MRI) { in isValidToPack() argument
4772 return isSameBitWidth(NewReg, RootReg, MRI) && IsHalfState(LoStat) && in isValidToPack()
4777 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const { in selectVOP3PModsImpl() argument
4780 if (isVectorOfTwoOrScalar(RootReg, MRI) != TypeClass::VECTOR_OF_TWO) { in selectVOP3PModsImpl()
4785 SearchOptions SO(RootReg, MRI); in selectVOP3PModsImpl()
4787 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(RootReg, MRI, SO); in selectVOP3PModsImpl()
4796 MachineInstr *MI = MRI.getVRegDef(Stat.first); in selectVOP3PModsImpl()
4805 getSrcStats(MI->getOperand(2).getReg(), MRI, SO); in selectVOP3PModsImpl()
4813 getSrcStats(MI->getOperand(1).getReg(), MRI, SO); in selectVOP3PModsImpl()
4824 StatlistHi[I].first, RootReg, TII, MRI)) in selectVOP3PModsImpl()
4839 const MachineRegisterInfo &MRI, in checkRB() argument
4841 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI); in checkRB()
4854 MachineRegisterInfo &MRI, in getLegalRegBank() argument
4859 if (checkRB(RootReg, AMDGPU::SGPRRegBankID, RBI, MRI, TRI) || in getLegalRegBank()
4860 checkRB(NewReg, AMDGPU::VGPRRegBankID, RBI, MRI, TRI)) in getLegalRegBank()
4863 MachineInstr *MI = MRI.getVRegDef(RootReg); in getLegalRegBank()
4870 Register DstReg = MRI.cloneVirtualRegister(RootReg); in getLegalRegBank()
4883 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo(); in selectVOP3PRetHelper() local
4886 std::tie(Reg, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, IsDOT); in selectVOP3PRetHelper()
4888 Reg = getLegalRegBank(Reg, Root.getReg(), RBI, MRI, TRI, TII); in selectVOP3PRetHelper()
4938 MachineRegisterInfo &MRI) { in buildRegSequence() argument
4956 .addDef(MRI.createVirtualRegister(DstRegClass)); in buildRegSequence()
4967 MachineRegisterInfo &MRI) { in selectWMMAModsNegAbs() argument
4974 if (!mi_match(El, MRI, m_GFabs(m_Reg(FabsSrc)))) in selectWMMAModsNegAbs()
4980 Src = buildRegSequence(Elts, InsertPt, MRI); in selectWMMAModsNegAbs()
4984 Src = buildRegSequence(NegAbsElts, InsertPt, MRI); in selectWMMAModsNegAbs()
4990 Src = buildRegSequence(Elts, InsertPt, MRI); in selectWMMAModsNegAbs()
5000 if (GBuildVector *BV = dyn_cast<GBuildVector>(MRI->getVRegDef(Src))) { in selectWMMAModsF32NegAbs()
5003 MachineInstr *ElF32 = MRI->getVRegDef(BV->getSourceReg(0)); in selectWMMAModsF32NegAbs()
5008 ElF32 = MRI->getVRegDef(BV->getSourceReg(i)); in selectWMMAModsF32NegAbs()
5017 *MRI); in selectWMMAModsF32NegAbs()
5031 if (GConcatVectors *CV = dyn_cast<GConcatVectors>(MRI->getVRegDef(Src))) { in selectWMMAModsF16Neg()
5034 if (!mi_match(CV->getSourceReg(i), *MRI, m_GFNeg(m_Reg(FNegSrc)))) in selectWMMAModsF16Neg()
5043 Src = buildRegSequence(EltsV2F16, Root.getParent(), *MRI); in selectWMMAModsF16Neg()
5057 if (GConcatVectors *CV = dyn_cast<GConcatVectors>(MRI->getVRegDef(Src))) { in selectWMMAModsF16NegAbs()
5059 MachineInstr *ElV2F16 = MRI->getVRegDef(CV->getSourceReg(0)); in selectWMMAModsF16NegAbs()
5066 ElV2F16 = MRI->getVRegDef(CV->getSourceReg(i)); in selectWMMAModsF16NegAbs()
5076 *MRI); in selectWMMAModsF16NegAbs()
5087 if (mi_match(Root.getReg(), *MRI, m_GFCstOrSplat(FPValReg))) { in selectWMMAVISrc()
5099 if (mi_match(Root.getReg(), *MRI, m_ICstOrSplat(ICst))) { in selectWMMAVISrc()
5112 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg(); in selectSWMMACIndex8()
5117 if (mi_match(Src, *MRI, m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt))) && in selectSWMMACIndex8()
5118 MRI->getType(ShiftSrc).getSizeInBits() == 32 && in selectSWMMACIndex8()
5134 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg(); in selectSWMMACIndex16()
5139 if (mi_match(Src, *MRI, m_GLShr(m_Reg(ShiftSrc), m_GCst(ShiftAmt))) && in selectSWMMACIndex16()
5140 MRI->getType(ShiftSrc).getSizeInBits() == 32 && in selectSWMMACIndex16()
5212 getAddrModeInfo(*MI, *MRI, AddrInfo); in selectSmrdOffset()
5228 matchZeroExtendFromS32(*MRI, GEPI2.SgprParts[1])) { in selectSmrdOffset()
5266 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdOffset()
5273 if (Register OffsetReg = matchZeroExtendFromS32(*MRI, GEPI.SgprParts[1])) { in selectSmrdOffset()
5297 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); in selectSmrdImm32()
5350 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectFlatOffsetImpl()
5403 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectGlobalSAddr()
5411 auto PtrBaseDef = getDefSrcRegIgnoringCopies(PtrBase, *MRI); in selectGlobalSAddr()
5427 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
5458 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectGlobalSAddr()
5462 getSrcRegIgnoringCopies(AddrDef->MI->getOperand(1).getReg(), *MRI); in selectGlobalSAddr()
5469 if (Register VOffset = matchZeroExtendFromS32(*MRI, PtrBaseOffset)) { in selectGlobalSAddr()
5493 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
5514 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectScratchSAddr()
5523 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectScratchSAddr()
5537 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI); in selectScratchSAddr()
5538 auto RHSDef = getDefSrcRegIgnoringCopies(RHS, *MRI); in selectScratchSAddr()
5546 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
5590 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI); in selectScratchSVAddr()
5600 auto AddrDef = getDefSrcRegIgnoringCopies(Addr, *MRI); in selectScratchSVAddr()
5605 if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) in selectScratchSVAddr()
5609 auto LHSDef = getDefSrcRegIgnoringCopies(LHS, *MRI); in selectScratchSVAddr()
5649 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) && in selectMUBUFScratchOffen()
5651 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
5683 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectMUBUFScratchOffen()
5686 std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(VAddr, *MRI); in selectMUBUFScratchOffen()
5691 const MachineInstr *PtrBaseDef = MRI->getVRegDef(PtrBase); in selectMUBUFScratchOffen()
5761 MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI); in isFlatScratchBaseLegal()
5776 getIConstantVRegValWithLookThrough(RHS, *MRI); in isFlatScratchBaseLegal()
5792 MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI); in isFlatScratchBaseLegalSV()
5816 MachineInstr *AddrMI = getDefIgnoringCopies(Addr, *MRI); in isFlatScratchBaseLegalSVImm()
5819 getDefSrcRegIgnoringCopies(Base, *MRI); in isFlatScratchBaseLegalSVImm()
5821 getIConstantVRegValWithLookThrough(AddrMI->getOperand(2).getReg(), *MRI); in isFlatScratchBaseLegalSVImm()
5844 getIConstantVRegVal(MI.getOperand(2).getReg(), *MRI); in isUnneededShiftMask()
5862 getDefSrcRegIgnoringCopies(Reg, *MRI); in selectMUBUFScratchOffset()
5882 if (mi_match(Reg, *MRI, in selectMUBUFScratchOffset()
5887 MachineInstr *BasePtrDef = getDefIgnoringCopies(BasePtr, *MRI); in selectMUBUFScratchOffset()
5903 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) || in selectMUBUFScratchOffset()
5920 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDS1Addr1OffsetImpl()
5926 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDS1Addr1OffsetImpl()
5937 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDS1Addr1OffsetImpl()
5982 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDSReadWrite2Impl()
5988 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDSReadWrite2Impl()
6000 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDSReadWrite2Impl()
6014 Register Root, const MachineRegisterInfo &MRI) const { in getPtrBaseWithConstantOffset()
6015 MachineInstr *RootI = getDefIgnoringCopies(Root, MRI); in getPtrBaseWithConstantOffset()
6021 getIConstantVRegValWithLookThrough(RHS.getReg(), MRI); in getPtrBaseWithConstantOffset()
6033 static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildRSRC() argument
6036 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
6037 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
6038 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
6039 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
6060 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
6076 static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildAddr64RSrc() argument
6082 return buildRSRC(B, MRI, 0, Hi_32(DefaultFormat), BasePtr); in buildAddr64RSrc()
6085 static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, in buildOffsetSrc() argument
6091 return buildRSRC(B, MRI, -1, Hi_32(DefaultFormat), BasePtr); in buildOffsetSrc()
6102 std::tie(PtrBase, Offset) = getPtrBaseWithConstantOffset(Src, *MRI); in parseMUBUFAddress()
6109 = getOpcodeDef(TargetOpcode::G_PTR_ADD, Data.N0, *MRI)) { in parseMUBUFAddress()
6118 Data.N2 = getDefIgnoringCopies(Data.N2, *MRI)->getOperand(0).getReg(); in parseMUBUFAddress()
6119 Data.N3 = getDefIgnoringCopies(Data.N3, *MRI)->getOperand(0).getReg(); in parseMUBUFAddress()
6132 const RegisterBank *N0Bank = RBI.getRegBank(Addr.N0, *MRI, TRI); in shouldUseAddr64()
6145 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
6173 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
6175 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
6188 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
6198 RSrcReg = buildAddr64RSrc(B, *MRI, TII, SRDPtr); in selectMUBUFAddr64Impl()
6223 RSrcReg = buildOffsetSrc(B, *MRI, TII, SRDPtr); in selectMUBUFOffsetImpl()
6297 if (STI.hasRestrictedSOffset() && mi_match(SOffset, *MRI, m_ZeroInt())) in selectBUFSOffset()
6305 getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) { in getConstantZext32Val() argument
6307 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(Reg, MRI); in getConstantZext32Val()
6316 Root.isImm() ? Root.getImm() : getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm()
6332 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm32()
6351 *MRI, Root.getReg(), VT, /*CheckNUW*/ true); in selectSMRDBufferSgprImm()
6360 assert(MRI->getType(SOffset) == LLT::scalar(32)); in selectSMRDBufferSgprImm()
6374 if (mi_match(Src, *MRI, m_GFPExt(m_Reg(Src)))) { in selectVOP3PMadMixModsImpl()
6375 assert(MRI->getType(Src) == LLT::scalar(16)); in selectVOP3PMadMixModsImpl()
6380 Src = stripBitCast(Src, *MRI); in selectVOP3PMadMixModsImpl()
6406 if (isExtractHiElt(*MRI, Src, Src)) { in selectVOP3PMadMixModsImpl()
6462 *MRI); in selectSBarrierSignalIsfirst()
6471 getIConstantVRegSExtVal(BarOp.getReg(), *MRI); in selectSGetBarrierState()
6485 TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI); in selectSGetBarrierState()
6486 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectSGetBarrierState()
6522 Register TmpReg0 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInit()
6528 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInit()
6535 Register TmpReg2 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInit()
6541 Register TmpReg3 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInit()
6548 Register TmpReg4 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInit()
6573 getIConstantVRegSExtVal(BarOp.getReg(), *MRI); in selectNamedBarrierInst()
6577 Register TmpReg0 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInst()
6583 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInst()
6601 TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI); in selectNamedBarrierInst()
6602 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) in selectNamedBarrierInst()
6655 if (Op.isReg() && mi_match(Op.getReg(), *MRI, m_ICst(Imm))) in renderTruncTImm()