Lines Matching full:amdgpu
10 /// AMDGPU.
15 #include "AMDGPU.h"
32 #define DEBUG_TYPE "amdgpu-isel"
72 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS in getWaveAddress()
91 return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC && in isVCC()
96 return RB->getID() == AMDGPU::VCCRegBankID; in isVCC()
103 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in constrainCopyLikeIntrin()
134 if (SrcReg == AMDGPU::SCC) { in selectCOPY()
154 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectCOPY()
166 IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32; in selectCOPY()
173 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg) in selectCOPY()
254 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg) in getSubOperand64()
270 case AMDGPU::sub0: in getSubOperand64()
272 case AMDGPU::sub1: in getSubOperand64()
279 case AMDGPU::G_AND: in getLogicalBitOpcode()
280 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in getLogicalBitOpcode()
281 case AMDGPU::G_OR: in getLogicalBitOpcode()
282 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32; in getLogicalBitOpcode()
283 case AMDGPU::G_XOR: in getLogicalBitOpcode()
284 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32; in getLogicalBitOpcode()
295 if (DstRB->getID() != AMDGPU::SGPRRegBankID && in selectG_AND_OR_XOR()
296 DstRB->getID() != AMDGPU::VCCRegBankID) in selectG_AND_OR_XOR()
299 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID && in selectG_AND_OR_XOR()
304 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef in selectG_AND_OR_XOR()
322 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_ADD_SUB()
327 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32; in selectG_ADD_SUB()
338 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64; in selectG_ADD_SUB()
341 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_ADD_SUB()
345 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64; in selectG_ADD_SUB()
361 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass; in selectG_ADD_SUB()
363 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass; in selectG_ADD_SUB()
365 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
366 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0)); in selectG_ADD_SUB()
367 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
368 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1)); in selectG_ADD_SUB()
374 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo) in selectG_ADD_SUB()
377 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi) in selectG_ADD_SUB()
384 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_CO_U32_e64), DstLo) in selectG_ADD_SUB()
389 MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi) in selectG_ADD_SUB()
400 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_ADD_SUB()
402 .addImm(AMDGPU::sub0) in selectG_ADD_SUB()
404 .addImm(AMDGPU::sub1); in selectG_ADD_SUB()
421 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO || in selectG_UADDO_USUBO_UADDE_USUBE()
422 I.getOpcode() == AMDGPU::G_UADDE; in selectG_UADDO_USUBO_UADDE_USUBE()
423 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE || in selectG_UADDO_USUBO_UADDE_USUBE()
424 I.getOpcode() == AMDGPU::G_USUBE; in selectG_UADDO_USUBO_UADDE_USUBE()
428 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
429 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; in selectG_UADDO_USUBO_UADDE_USUBE()
431 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_UADDO_USUBO_UADDE_USUBE()
440 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_UADDO_USUBO_UADDE_USUBE()
444 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
445 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; in selectG_UADDO_USUBO_UADDE_USUBE()
454 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg) in selectG_UADDO_USUBO_UADDE_USUBE()
455 .addReg(AMDGPU::SCC); in selectG_UADDO_USUBO_UADDE_USUBE()
457 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); in selectG_UADDO_USUBO_UADDE_USUBE()
460 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
461 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || in selectG_UADDO_USUBO_UADDE_USUBE()
462 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
467 AMDGPU::SReg_32RegClass, *MRI)) in selectG_UADDO_USUBO_UADDE_USUBE()
478 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32; in selectG_AMDGPU_MAD_64_32()
482 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64 in selectG_AMDGPU_MAD_64_32()
483 : AMDGPU::V_MAD_I64_I32_gfx11_e64; in selectG_AMDGPU_MAD_64_32()
485 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64; in selectG_AMDGPU_MAD_64_32()
623 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC || in selectG_BUILD_VECTOR()
624 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR); in selectG_BUILD_VECTOR()
632 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) { in selectG_BUILD_VECTOR()
640 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC && in selectG_BUILD_VECTOR()
645 if (DstBank->getID() == AMDGPU::AGPRRegBankID) in selectG_BUILD_VECTOR()
648 assert(DstBank->getID() == AMDGPU::SGPRRegBankID || in selectG_BUILD_VECTOR()
649 DstBank->getID() == AMDGPU::VGPRRegBankID); in selectG_BUILD_VECTOR()
650 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID; in selectG_BUILD_VECTOR()
671 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), Dst).addImm(Imm); in selectG_BUILD_VECTOR()
673 return RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI); in selectG_BUILD_VECTOR()
677 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), Dst).addImm(Imm); in selectG_BUILD_VECTOR()
679 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_BUILD_VECTOR()
690 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) { in selectG_BUILD_VECTOR()
691 MI.setDesc(TII.get(AMDGPU::COPY)); in selectG_BUILD_VECTOR()
694 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_BUILD_VECTOR()
701 Register TmpReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectG_BUILD_VECTOR()
702 auto MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_AND_B32_e32), TmpReg) in selectG_BUILD_VECTOR()
708 MIB = BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_LSHL_OR_B32_e64), Dst) in selectG_BUILD_VECTOR()
740 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16; in selectG_BUILD_VECTOR()
742 Opc = AMDGPU::S_PACK_HH_B32_B16; in selectG_BUILD_VECTOR()
746 Opc = AMDGPU::S_PACK_LH_B32_B16; in selectG_BUILD_VECTOR()
753 auto MIB = BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst) in selectG_BUILD_VECTOR()
762 Opc = AMDGPU::S_PACK_HL_B32_B16; in selectG_BUILD_VECTOR()
808 if (SubReg == AMDGPU::NoSubRegister) in selectG_INSERT()
851 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID && in selectG_SBFX_UBFX()
860 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SBFX_UBFX()
876 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || in selectInterpP1F16()
877 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || in selectInterpP1F16()
878 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) in selectInterpP1F16()
888 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectInterpP1F16()
892 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectInterpP1F16()
894 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov) in selectInterpP1F16()
899 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst) in selectInterpP1F16()
921 if (STI.getConstantBusLimit(AMDGPU::V_WRITELANE_B32) > 1) in selectWritelane()
931 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_WRITELANE_B32), VDst); in selectWritelane()
947 if (ConstVal && AMDGPU::isInlinableLiteral32(ConstVal->Value.getSExtValue(), in selectWritelane()
957 RBI.constrainGenericRegister(LaneSelect, AMDGPU::SReg_32_XM0RegClass, *MRI); in selectWritelane()
959 BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectWritelane()
961 MIB.addReg(AMDGPU::M0); in selectWritelane()
980 Opc = AMDGPU::V_DIV_SCALE_F32_e64; in selectDivScale()
982 Opc = AMDGPU::V_DIV_SCALE_F64_e64; in selectDivScale()
1020 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK)) in selectG_INTRINSIC()
1039 return constrainCopyLikeIntrin(I, AMDGPU::WQM); in selectG_INTRINSIC()
1041 return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM); in selectG_INTRINSIC()
1044 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM); in selectG_INTRINSIC()
1046 return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM); in selectG_INTRINSIC()
1105 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64, in getV_CMPOpcode()
1106 AMDGPU::V_CMP_NE_U32_e64, AMDGPU::V_CMP_NE_U64_e64); in getV_CMPOpcode()
1108 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64, in getV_CMPOpcode()
1109 AMDGPU::V_CMP_EQ_U32_e64, AMDGPU::V_CMP_EQ_U64_e64); in getV_CMPOpcode()
1111 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64, in getV_CMPOpcode()
1112 AMDGPU::V_CMP_GT_I32_e64, AMDGPU::V_CMP_GT_I64_e64); in getV_CMPOpcode()
1114 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64, in getV_CMPOpcode()
1115 AMDGPU::V_CMP_GE_I32_e64, AMDGPU::V_CMP_GE_I64_e64); in getV_CMPOpcode()
1117 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64, in getV_CMPOpcode()
1118 AMDGPU::V_CMP_LT_I32_e64, AMDGPU::V_CMP_LT_I64_e64); in getV_CMPOpcode()
1120 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64, in getV_CMPOpcode()
1121 AMDGPU::V_CMP_LE_I32_e64, AMDGPU::V_CMP_LE_I64_e64); in getV_CMPOpcode()
1123 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64, in getV_CMPOpcode()
1124 AMDGPU::V_CMP_GT_U32_e64, AMDGPU::V_CMP_GT_U64_e64); in getV_CMPOpcode()
1126 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64, in getV_CMPOpcode()
1127 AMDGPU::V_CMP_GE_U32_e64, AMDGPU::V_CMP_GE_U64_e64); in getV_CMPOpcode()
1129 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64, in getV_CMPOpcode()
1130 AMDGPU::V_CMP_LT_U32_e64, AMDGPU::V_CMP_LT_U64_e64); in getV_CMPOpcode()
1132 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64, in getV_CMPOpcode()
1133 AMDGPU::V_CMP_LE_U32_e64, AMDGPU::V_CMP_LE_U64_e64); in getV_CMPOpcode()
1136 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64, in getV_CMPOpcode()
1137 AMDGPU::V_CMP_EQ_F32_e64, AMDGPU::V_CMP_EQ_F64_e64); in getV_CMPOpcode()
1139 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64, in getV_CMPOpcode()
1140 AMDGPU::V_CMP_GT_F32_e64, AMDGPU::V_CMP_GT_F64_e64); in getV_CMPOpcode()
1142 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64, in getV_CMPOpcode()
1143 AMDGPU::V_CMP_GE_F32_e64, AMDGPU::V_CMP_GE_F64_e64); in getV_CMPOpcode()
1145 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64, in getV_CMPOpcode()
1146 AMDGPU::V_CMP_LT_F32_e64, AMDGPU::V_CMP_LT_F64_e64); in getV_CMPOpcode()
1148 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64, in getV_CMPOpcode()
1149 AMDGPU::V_CMP_LE_F32_e64, AMDGPU::V_CMP_LE_F64_e64); in getV_CMPOpcode()
1151 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64, in getV_CMPOpcode()
1152 AMDGPU::V_CMP_NEQ_F32_e64, AMDGPU::V_CMP_NEQ_F64_e64); in getV_CMPOpcode()
1154 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64, in getV_CMPOpcode()
1155 AMDGPU::V_CMP_O_F32_e64, AMDGPU::V_CMP_O_F64_e64); in getV_CMPOpcode()
1157 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64, in getV_CMPOpcode()
1158 AMDGPU::V_CMP_U_F32_e64, AMDGPU::V_CMP_U_F64_e64); in getV_CMPOpcode()
1160 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64, in getV_CMPOpcode()
1161 AMDGPU::V_CMP_NLG_F32_e64, AMDGPU::V_CMP_NLG_F64_e64); in getV_CMPOpcode()
1163 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64, in getV_CMPOpcode()
1164 AMDGPU::V_CMP_NLE_F32_e64, AMDGPU::V_CMP_NLE_F64_e64); in getV_CMPOpcode()
1166 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64, in getV_CMPOpcode()
1167 AMDGPU::V_CMP_NLT_F32_e64, AMDGPU::V_CMP_NLT_F64_e64); in getV_CMPOpcode()
1169 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64, in getV_CMPOpcode()
1170 AMDGPU::V_CMP_NGE_F32_e64, AMDGPU::V_CMP_NGE_F64_e64); in getV_CMPOpcode()
1172 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64, in getV_CMPOpcode()
1173 AMDGPU::V_CMP_NGT_F32_e64, AMDGPU::V_CMP_NGT_F64_e64); in getV_CMPOpcode()
1175 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64, in getV_CMPOpcode()
1176 AMDGPU::V_CMP_NEQ_F32_e64, AMDGPU::V_CMP_NEQ_F64_e64); in getV_CMPOpcode()
1178 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64, in getV_CMPOpcode()
1179 AMDGPU::V_CMP_TRU_F32_e64, AMDGPU::V_CMP_TRU_F64_e64); in getV_CMPOpcode()
1181 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64, in getV_CMPOpcode()
1182 AMDGPU::V_CMP_F_F32_e64, AMDGPU::V_CMP_F_F64_e64); in getV_CMPOpcode()
1194 return AMDGPU::S_CMP_LG_U64; in getS_CMPOpcode()
1196 return AMDGPU::S_CMP_EQ_U64; in getS_CMPOpcode()
1205 return AMDGPU::S_CMP_LG_U32; in getS_CMPOpcode()
1207 return AMDGPU::S_CMP_EQ_U32; in getS_CMPOpcode()
1209 return AMDGPU::S_CMP_GT_I32; in getS_CMPOpcode()
1211 return AMDGPU::S_CMP_GE_I32; in getS_CMPOpcode()
1213 return AMDGPU::S_CMP_LT_I32; in getS_CMPOpcode()
1215 return AMDGPU::S_CMP_LE_I32; in getS_CMPOpcode()
1217 return AMDGPU::S_CMP_GT_U32; in getS_CMPOpcode()
1219 return AMDGPU::S_CMP_GE_U32; in getS_CMPOpcode()
1221 return AMDGPU::S_CMP_LT_U32; in getS_CMPOpcode()
1223 return AMDGPU::S_CMP_LE_U32; in getS_CMPOpcode()
1225 return AMDGPU::S_CMP_EQ_F32; in getS_CMPOpcode()
1227 return AMDGPU::S_CMP_GT_F32; in getS_CMPOpcode()
1229 return AMDGPU::S_CMP_GE_F32; in getS_CMPOpcode()
1231 return AMDGPU::S_CMP_LT_F32; in getS_CMPOpcode()
1233 return AMDGPU::S_CMP_LE_F32; in getS_CMPOpcode()
1235 return AMDGPU::S_CMP_LG_F32; in getS_CMPOpcode()
1237 return AMDGPU::S_CMP_O_F32; in getS_CMPOpcode()
1239 return AMDGPU::S_CMP_U_F32; in getS_CMPOpcode()
1241 return AMDGPU::S_CMP_NLG_F32; in getS_CMPOpcode()
1243 return AMDGPU::S_CMP_NLE_F32; in getS_CMPOpcode()
1245 return AMDGPU::S_CMP_NLT_F32; in getS_CMPOpcode()
1247 return AMDGPU::S_CMP_NGE_F32; in getS_CMPOpcode()
1249 return AMDGPU::S_CMP_NGT_F32; in getS_CMPOpcode()
1251 return AMDGPU::S_CMP_NEQ_F32; in getS_CMPOpcode()
1263 return AMDGPU::S_CMP_EQ_F16; in getS_CMPOpcode()
1265 return AMDGPU::S_CMP_GT_F16; in getS_CMPOpcode()
1267 return AMDGPU::S_CMP_GE_F16; in getS_CMPOpcode()
1269 return AMDGPU::S_CMP_LT_F16; in getS_CMPOpcode()
1271 return AMDGPU::S_CMP_LE_F16; in getS_CMPOpcode()
1273 return AMDGPU::S_CMP_LG_F16; in getS_CMPOpcode()
1275 return AMDGPU::S_CMP_O_F16; in getS_CMPOpcode()
1277 return AMDGPU::S_CMP_U_F16; in getS_CMPOpcode()
1279 return AMDGPU::S_CMP_NLG_F16; in getS_CMPOpcode()
1281 return AMDGPU::S_CMP_NLE_F16; in getS_CMPOpcode()
1283 return AMDGPU::S_CMP_NLT_F16; in getS_CMPOpcode()
1285 return AMDGPU::S_CMP_NGE_F16; in getS_CMPOpcode()
1287 return AMDGPU::S_CMP_NGT_F16; in getS_CMPOpcode()
1289 return AMDGPU::S_CMP_NEQ_F16; in getS_CMPOpcode()
1316 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg) in selectG_ICMP_or_FCMP()
1317 .addReg(AMDGPU::SCC); in selectG_ICMP_or_FCMP()
1320 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_ICMP_or_FCMP()
1325 if (I.getOpcode() == AMDGPU::G_FCMP) in selectG_ICMP_or_FCMP()
1363 BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Dst); in selectIntrinsicCmp()
1382 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers)) in selectIntrinsicCmp()
1385 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1_modifiers)) in selectIntrinsicCmp()
1388 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::clamp)) in selectIntrinsicCmp()
1390 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel)) in selectIntrinsicCmp()
1419 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), DstReg) in selectBallot()
1425 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectBallot()
1426 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg).addImm(0); in selectBallot()
1427 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectBallot()
1429 .addImm(AMDGPU::sub0) in selectBallot()
1431 .addImm(AMDGPU::sub1); in selectBallot()
1437 unsigned Opcode = Is64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; in selectBallot()
1440 BuildCopy(IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC); in selectBallot()
1457 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID; in selectRelocConstant()
1467 TII.get(IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DstReg) in selectRelocConstant()
1479 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ? in selectGroupStaticSize()
1480 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectGroupStaticSize()
1512 if (!RC->hasSubClassEq(&AMDGPU::SGPR_64RegClass) || in selectReturnAddress()
1519 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectReturnAddress()
1532 AMDGPU::SReg_64RegClass, DL); in selectReturnAddress()
1533 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), DstReg) in selectReturnAddress()
1543 BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF)) in selectEndCfIntrinsic()
1599 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSOrderedIntrinsic()
1605 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg) in selectDSOrderedIntrinsic()
1610 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) in selectDSOrderedIntrinsic()
1621 return AMDGPU::DS_GWS_INIT; in gwsIntrinToOpcode()
1623 return AMDGPU::DS_GWS_BARRIER; in gwsIntrinToOpcode()
1625 return AMDGPU::DS_GWS_SEMA_V; in gwsIntrinToOpcode()
1627 return AMDGPU::DS_GWS_SEMA_BR; in gwsIntrinToOpcode()
1629 return AMDGPU::DS_GWS_SEMA_P; in gwsIntrinToOpcode()
1631 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; in gwsIntrinToOpcode()
1649 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID) in selectDSGWSIntrinsic()
1663 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) { in selectDSGWSIntrinsic()
1669 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) { in selectDSGWSIntrinsic()
1676 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in selectDSGWSIntrinsic()
1680 AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset, KB); in selectDSGWSIntrinsic()
1685 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1692 AMDGPU::SReg_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1696 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectDSGWSIntrinsic()
1697 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base) in selectDSGWSIntrinsic()
1702 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSGWSIntrinsic()
1715 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) in selectDSGWSIntrinsic()
1722 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::data0); in selectDSGWSIntrinsic()
1745 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; in selectDSAppendConsume()
1747 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectDSAppendConsume()
1749 if (!RBI.constrainGenericRegister(PtrBase, AMDGPU::SReg_32RegClass, *MRI)) in selectDSAppendConsume()
1766 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::WAVE_BARRIER)); in selectSBarrier()
1776 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_IMM)) in selectSBarrier()
1777 .addImm(AMDGPU::Barrier::WORKGROUP); in selectSBarrier()
1778 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_WAIT)) in selectSBarrier()
1779 .addImm(AMDGPU::Barrier::WORKGROUP); in selectSBarrier()
1801 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const { in selectImageIntrinsic()
1805 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = in selectImageIntrinsic()
1806 AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1808 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); in selectImageIntrinsic()
1810 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI); in selectImageIntrinsic()
1811 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI); in selectImageIntrinsic()
1812 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI); in selectImageIntrinsic()
1819 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 || in selectImageIntrinsic()
1820 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16; in selectImageIntrinsic()
1857 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister); in selectImageIntrinsic()
1887 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo = in selectImageIntrinsic()
1888 AMDGPU::getMIMGG16MappingInfo(Intr->BaseOpcode); in selectImageIntrinsic()
1898 CPol |= AMDGPU::CPol::GLC; // TODO no-return optimization in selectImageIntrinsic()
1899 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) | in selectImageIntrinsic()
1900 AMDGPU::CPol::VOLATILE)) in selectImageIntrinsic()
1926 if (UseNSA && !STI.hasFeature(AMDGPU::FeatureNSAEncoding)) { in selectImageIntrinsic()
1936 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx12, in selectImageIntrinsic()
1939 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1940 UseNSA ? AMDGPU::MIMGEncGfx11NSA in selectImageIntrinsic()
1941 : AMDGPU::MIMGEncGfx11Default, in selectImageIntrinsic()
1944 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, in selectImageIntrinsic()
1945 UseNSA ? AMDGPU::MIMGEncGfx10NSA in selectImageIntrinsic()
1946 : AMDGPU::MIMGEncGfx10Default, in selectImageIntrinsic()
1950 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a, in selectImageIntrinsic()
1961 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx8, in selectImageIntrinsic()
1964 Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx6, in selectImageIntrinsic()
1978 Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass); in selectImageIntrinsic()
1979 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0; in selectImageIntrinsic()
1983 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), VDataOut) in selectImageIntrinsic()
2011 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::unorm)) in selectImageIntrinsic()
2016 STI.hasFeature(AMDGPU::FeatureR128A16) ? -1 : 0); in selectImageIntrinsic()
2027 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::lwe)) in selectImageIntrinsic()
2036 TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr); in selectImageIntrinsic()
2055 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_BVH_STACK_RTN_B32), Dst0) in selectDSBvhStackIntrinsic()
2134 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 : in selectG_SELECT()
2135 AMDGPU::S_CSELECT_B32; in selectG_SELECT()
2136 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) in selectG_SELECT()
2160 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg) in selectG_SELECT()
2175 return AMDGPU::sub0; in sizeToSubRegIndex()
2177 return AMDGPU::sub0_sub1; in sizeToSubRegIndex()
2179 return AMDGPU::sub0_sub1_sub2; in sizeToSubRegIndex()
2181 return AMDGPU::sub0_sub1_sub2_sub3; in sizeToSubRegIndex()
2183 return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7; in sizeToSubRegIndex()
2186 return AMDGPU::sub0; in sizeToSubRegIndex()
2212 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_TRUNC()
2236 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_TRUNC()
2237 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_TRUNC()
2238 BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_TRUNC()
2239 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_TRUNC()
2245 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_MOV_B32_sdwa), DstReg) in selectG_TRUNC()
2249 .addImm(AMDGPU::SDWA::WORD_1) // $dst_sel in selectG_TRUNC()
2250 .addImm(AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused in selectG_TRUNC()
2251 .addImm(AMDGPU::SDWA::WORD_0) // $src0_sel in selectG_TRUNC()
2259 BuildMI(*MBB, I, DL, TII.get(AMDGPU::V_LSHLREV_B32_e64), TmpReg0) in selectG_TRUNC()
2263 BuildMI(*MBB, I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0) in selectG_TRUNC()
2269 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; in selectG_TRUNC()
2270 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_TRUNC()
2271 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32; in selectG_TRUNC()
2341 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG; in selectG_SZA_EXT()
2342 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg; in selectG_SZA_EXT()
2350 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ? in selectG_SZA_EXT()
2360 if (I.getOpcode() == AMDGPU::G_ANYEXT) { in selectG_SZA_EXT()
2371 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2372 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_SZA_EXT()
2374 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2376 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2383 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) { in selectG_SZA_EXT()
2390 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg) in selectG_SZA_EXT()
2397 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64; in selectG_SZA_EXT()
2407 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) { in selectG_SZA_EXT()
2409 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass; in selectG_SZA_EXT()
2415 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16; in selectG_SZA_EXT()
2419 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2425 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2426 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister; in selectG_SZA_EXT()
2428 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_ASHR_I32), HiReg) in selectG_SZA_EXT()
2433 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg) in selectG_SZA_EXT()
2436 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_SZA_EXT()
2438 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2440 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2442 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, in selectG_SZA_EXT()
2446 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64; in selectG_SZA_EXT()
2447 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; in selectG_SZA_EXT()
2452 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); in selectG_SZA_EXT()
2453 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_SZA_EXT()
2454 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister; in selectG_SZA_EXT()
2456 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); in selectG_SZA_EXT()
2457 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg) in selectG_SZA_EXT()
2459 .addImm(AMDGPU::sub0) in selectG_SZA_EXT()
2461 .addImm(AMDGPU::sub1); in selectG_SZA_EXT()
2468 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); in selectG_SZA_EXT()
2473 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg) in selectG_SZA_EXT()
2484 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); in selectG_SZA_EXT()
2507 if (DstRB->getID() != AMDGPU::SGPRRegBankID) in selectG_FPEXT()
2516 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_CVT_HI_F32_F16), Dst) in selectG_FPEXT()
2519 return RBI.constrainGenericRegister(Dst, AMDGPU::SReg_32RegClass, *MRI); in selectG_FPEXT()
2533 // The AMDGPU backend only supports Imm operands and not CImm or FPImm. in selectG_CONSTANT()
2545 const bool IsSgpr = DstRB->getID() == AMDGPU::SGPRRegBankID; in selectG_CONSTANT()
2548 if (DstRB->getID() == AMDGPU::VCCRegBankID) { in selectG_CONSTANT()
2549 Opcode = STI.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in selectG_CONSTANT()
2551 AMDGPU::isValid32BitLiteral(I.getOperand(1).getImm(), IsFP)) { in selectG_CONSTANT()
2552 Opcode = IsSgpr ? AMDGPU::S_MOV_B64_IMM_PSEUDO : AMDGPU::V_MOV_B64_PSEUDO; in selectG_CONSTANT()
2557 Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; in selectG_CONSTANT()
2578 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) in selectG_CONSTANT()
2582 &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass; in selectG_CONSTANT()
2592 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_CONSTANT()
2594 .addImm(AMDGPU::sub0) in selectG_CONSTANT()
2596 .addImm(AMDGPU::sub1); in selectG_CONSTANT()
2623 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FNEG()
2632 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FNEG()
2633 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FNEG()
2638 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2639 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2640 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2641 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FNEG()
2643 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FNEG()
2644 .addReg(Src, 0, AMDGPU::sub0); in selectG_FNEG()
2645 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FNEG()
2646 .addReg(Src, 0, AMDGPU::sub1); in selectG_FNEG()
2647 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FNEG()
2651 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32; in selectG_FNEG()
2656 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FNEG()
2658 .addImm(AMDGPU::sub0) in selectG_FNEG()
2660 .addImm(AMDGPU::sub1); in selectG_FNEG()
2669 if (DstRB->getID() != AMDGPU::SGPRRegBankID || in selectG_FABS()
2676 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2677 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2678 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2679 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectG_FABS()
2681 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || in selectG_FABS()
2682 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) in selectG_FABS()
2685 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_FABS()
2686 .addReg(Src, 0, AMDGPU::sub0); in selectG_FABS()
2687 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_FABS()
2688 .addReg(Src, 0, AMDGPU::sub1); in selectG_FABS()
2689 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) in selectG_FABS()
2694 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_AND_B32), OpReg) in selectG_FABS()
2698 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) in selectG_FABS()
2700 .addImm(AMDGPU::sub0) in selectG_FABS()
2702 .addImm(AMDGPU::sub1); in selectG_FABS()
2715 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1; in getAddrModeInfo()
2738 if (OpBank->getID() == AMDGPU::SGPRRegBankID) in getAddrModeInfo()
2749 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID; in isSGPR()
2770 if (MI.getOpcode() == AMDGPU::G_PREFETCH) in isInstrUniform()
2772 AMDGPU::SGPRRegBankID; in isInstrUniform()
2775 return I && I->getMetadata("amdgpu.uniform"); in isInstrUniform()
2794 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) in initM0()
2812 if (Opcode == AMDGPU::COPY) in isVCmpResult()
2815 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR || in isVCmpResult()
2816 Opcode == AMDGPU::G_XOR) in isVCmpResult()
2823 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP; in isVCmpResult()
2845 CondPhysReg = AMDGPU::SCC; in selectG_BRCOND()
2846 BrOpcode = AMDGPU::S_CBRANCH_SCC1; in selectG_BRCOND()
2847 ConstrainRC = &AMDGPU::SReg_32RegClass; in selectG_BRCOND()
2855 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; in selectG_BRCOND()
2856 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO; in selectG_BRCOND()
2867 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ; in selectG_BRCOND()
2874 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg) in selectG_BRCOND()
2887 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_GLOBAL_VALUE()
2888 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32)); in selectG_GLOBAL_VALUE()
2890 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); in selectG_GLOBAL_VALUE()
2893 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); in selectG_GLOBAL_VALUE()
2908 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectG_PTRMASK()
2923 auto MIB = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_AND_B64), DstReg) in selectG_PTRMASK()
2931 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; in selectG_PTRMASK()
2933 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectG_PTRMASK()
2963 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg) in selectG_PTRMASK()
2964 .addReg(SrcReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
2965 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg) in selectG_PTRMASK()
2966 .addReg(SrcReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
2978 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskLo) in selectG_PTRMASK()
2979 .addReg(MaskReg, 0, AMDGPU::sub0); in selectG_PTRMASK()
2992 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), MaskHi) in selectG_PTRMASK()
2993 .addReg(MaskReg, 0, AMDGPU::sub1); in selectG_PTRMASK()
2999 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) in selectG_PTRMASK()
3001 .addImm(AMDGPU::sub0) in selectG_PTRMASK()
3003 .addImm(AMDGPU::sub1); in selectG_PTRMASK()
3018 AMDGPU::getBaseWithConstantOffset(MRI, IdxReg, &KnownBits); in computeIndirectRegIndex()
3019 if (IdxBaseReg == AMDGPU::NoRegister) { in computeIndirectRegIndex()
3050 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_EXTRACT_VECTOR_ELT()
3061 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_EXTRACT_VECTOR_ELT()
3072 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) { in selectG_EXTRACT_VECTOR_ELT()
3076 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
3079 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32; in selectG_EXTRACT_VECTOR_ELT()
3087 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32) in selectG_EXTRACT_VECTOR_ELT()
3091 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_EXTRACT_VECTOR_ELT()
3093 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg) in selectG_EXTRACT_VECTOR_ELT()
3132 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) in selectG_INSERT_VECTOR_ELT()
3143 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) in selectG_INSERT_VECTOR_ELT()
3146 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32) in selectG_INSERT_VECTOR_ELT()
3153 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID && in selectG_INSERT_VECTOR_ELT()
3160 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectG_INSERT_VECTOR_ELT()
3164 VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID); in selectG_INSERT_VECTOR_ELT()
3186 assert(!AMDGPU::isGFX12Plus(STI)); in selectBufferLoadLds()
3208 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN in selectBufferLoadLds()
3209 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN in selectBufferLoadLds()
3210 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN in selectBufferLoadLds()
3211 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET; in selectBufferLoadLds()
3214 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN in selectBufferLoadLds()
3215 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN in selectBufferLoadLds()
3216 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN in selectBufferLoadLds()
3217 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET; in selectBufferLoadLds()
3220 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN in selectBufferLoadLds()
3221 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN in selectBufferLoadLds()
3222 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN in selectBufferLoadLds()
3223 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET; in selectBufferLoadLds()
3229 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectBufferLoadLds()
3236 BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), IdxReg) in selectBufferLoadLds()
3238 .addImm(AMDGPU::sub0) in selectBufferLoadLds()
3240 .addImm(AMDGPU::sub1); in selectBufferLoadLds()
3253 MIB.addImm(Aux & AMDGPU::CPol::ALL); // cpol in selectBufferLoadLds()
3254 MIB.addImm(Aux & AMDGPU::CPol::SWZ_pregfx12 ? 1 : 0); // swz in selectBufferLoadLds()
3286 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES) in matchZeroExtendFromS32()
3306 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE; in selectGlobalLoadLds()
3309 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT; in selectGlobalLoadLds()
3312 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD; in selectGlobalLoadLds()
3318 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectGlobalLoadLds()
3329 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalLoadLds()
3343 Opc = AMDGPU::getGlobalSaddrOp(Opc); in selectGlobalLoadLds()
3345 VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalLoadLds()
3346 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalLoadLds()
3391 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64; in selectSMFMACIntrin()
3394 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64; in selectSMFMACIntrin()
3397 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64; in selectSMFMACIntrin()
3400 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64; in selectSMFMACIntrin()
3403 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64; in selectSMFMACIntrin()
3406 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64; in selectSMFMACIntrin()
3409 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64; in selectSMFMACIntrin()
3412 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64; in selectSMFMACIntrin()
3415 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64; in selectSMFMACIntrin()
3418 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64; in selectSMFMACIntrin()
3421 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64; in selectSMFMACIntrin()
3424 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64; in selectSMFMACIntrin()
3427 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64; in selectSMFMACIntrin()
3430 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64; in selectSMFMACIntrin()
3450 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID; in selectWaveAddress()
3455 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_LSHRREV_B32_e64), DstReg) in selectWaveAddress()
3459 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), DstReg) in selectWaveAddress()
3466 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; in selectWaveAddress()
3476 if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, *MRI)) in selectStackRestore()
3487 WaveAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectStackRestore()
3488 BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_LSHR_B32), WaveAddr) in selectStackRestore()
3494 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), SP) in selectStackRestore()
3527 case AMDGPU::G_AMDGPU_MAD_U64_U32: in select()
3528 case AMDGPU::G_AMDGPU_MAD_I64_I32: in select()
3619 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: in select()
3620 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16: in select()
3621 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET: in select()
3622 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: in select()
3623 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: { in select()
3624 const AMDGPU::ImageDimIntrinsicInfo *Intr = in select()
3625 AMDGPU::getImageDimIntrinsicInfo(AMDGPU::getIntrinsicID(I)); in select()
3629 case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY: in select()
3631 case AMDGPU::G_SBFX: in select()
3632 case AMDGPU::G_UBFX: in select()
3634 case AMDGPU::G_SI_CALL: in select()
3635 I.setDesc(TII.get(AMDGPU::SI_CALL)); in select()
3637 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: in select()
3639 case AMDGPU::G_STACKRESTORE: in select()
3641 case AMDGPU::G_PHI: in select()
3665 if (MI->getOpcode() == AMDGPU::G_FNEG) { in selectVOP3ModsImpl()
3669 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) { in selectVOP3ModsImpl()
3680 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) { in selectVOP3ModsImpl()
3695 RBI.getRegBank(Src, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) { in copyToVGPRIfSrcFolded()
3702 TII.get(AMDGPU::COPY), VGPRSrc) in copyToVGPRIfSrcFolded()
3811 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS) in selectVOP3NoMods()
3824 if (MI && MI->getOpcode() == AMDGPU::G_FNEG && in selectVOP3PModsImpl()
3909 DstRegClass = &AMDGPU::VReg_256RegClass; in buildRegSequence()
3912 DstRegClass = &AMDGPU::VReg_128RegClass; in buildRegSequence()
3915 DstRegClass = &AMDGPU::VReg_64RegClass; in buildRegSequence()
3922 auto MIB = B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRegSequence()
3971 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG) in selectWMMAModsF32NegAbs()
3972 ? AMDGPU::G_FNEG in selectWMMAModsF32NegAbs()
3973 : AMDGPU::G_FABS; in selectWMMAModsF32NegAbs()
4028 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG) in selectWMMAModsF16NegAbs()
4029 ? AMDGPU::G_FNEG in selectWMMAModsF16NegAbs()
4030 : AMDGPU::G_FABS; in selectWMMAModsF16NegAbs()
4187 EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, /*IsBuffer=*/false, in selectSmrdOffset()
4198 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(STI)) in selectSmrdOffset()
4216 EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPI.Imm, /*IsBuffer=*/false, in selectSmrdOffset()
4232 *SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectSmrdOffset()
4233 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), *SOffset) in selectSmrdOffset()
4271 AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm); in selectSmrdImm32()
4393 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4395 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectGlobalSAddr()
4417 if (STI.getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals) in selectGlobalSAddr()
4425 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectGlobalSAddr()
4451 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF || in selectGlobalSAddr()
4452 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(AddrDef->Reg)) in selectGlobalSAddr()
4459 Register VOffset = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectGlobalSAddr()
4461 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), VOffset) in selectGlobalSAddr()
4490 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSAddr()
4500 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) { in selectScratchSAddr()
4506 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX && in selectScratchSAddr()
4512 SAddr = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectScratchSAddr()
4514 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_I32), SAddr) in selectScratchSAddr()
4567 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD) in selectScratchSVAddr()
4571 if (RBI.getRegBank(RHS, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) in selectScratchSVAddr()
4588 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectScratchSVAddr()
4617 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in selectMUBUFScratchOffen()
4622 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), in selectMUBUFScratchOffen()
4657 if (PtrBaseDef->getOpcode() == AMDGPU::G_FRAME_INDEX) in selectMUBUFScratchOffen()
4663 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) { in selectMUBUFScratchOffen()
4902 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDS1Addr1OffsetImpl()
4969 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { in selectDSReadWrite2Impl()
5008 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
5009 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildRSRC()
5010 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
5011 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); in buildRSRC()
5013 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
5016 B.buildInstr(AMDGPU::S_MOV_B32) in buildRSRC()
5023 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
5026 .addImm(AMDGPU::sub0) in buildRSRC()
5028 .addImm(AMDGPU::sub1); in buildRSRC()
5032 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); in buildRSRC()
5033 B.buildInstr(AMDGPU::S_MOV_B64) in buildRSRC()
5038 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildRSRC()
5041 .addImm(AMDGPU::sub0_sub1) in buildRSRC()
5043 .addImm(AMDGPU::sub2_sub3); in buildRSRC()
5105 return N0Bank->getID() == AMDGPU::VGPRRegBankID; in shouldUseAddr64()
5117 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in splitIllegalMUBUFOffset()
5118 B.buildInstr(AMDGPU::S_MOV_B32) in splitIllegalMUBUFOffset()
5145 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
5147 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
5160 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { in selectMUBUFAddr64Impl()
5223 MIB.addReg(AMDGPU::SGPR_NULL); in selectMUBUFAddr64()
5253 MIB.addReg(AMDGPU::SGPR_NULL); in selectMUBUFOffset()
5270 SOffset = AMDGPU::SGPR_NULL; in selectBUFSOffset()
5292 AMDGPU::getSMRDEncodedOffset(STI, *OffsetVal, true); in selectSMRDBufferImm()
5308 AMDGPU::getSMRDEncodedLiteralOffset32(STI, *OffsetVal); in selectSMRDBufferImm32()
5321 std::tie(SOffset, Offset) = AMDGPU::getBaseWithConstantOffset( in selectSMRDBufferSgprImm()
5327 AMDGPU::getSMRDEncodedOffset(STI, Offset, /* IsBuffer */ true); in selectSMRDBufferSgprImm()
5339 if (MI->getOpcode() == AMDGPU::G_BITCAST) in stripBitCast()
5350 if (Inst->getOpcode() != AMDGPU::G_TRUNC) in isExtractHiElt()
5358 if (TruncOp->getOpcode() == AMDGPU::G_LSHR) { in isExtractHiElt()
5372 if (TruncOp->getOpcode() == AMDGPU::G_SHUFFLE_VECTOR) { in isExtractHiElt()
5399 if (MI->getOpcode() == AMDGPU::G_FPEXT) { in selectVOP3PMadMixModsImpl()
5408 if (MI->getOpcode() == AMDGPU::G_BITCAST) { in selectVOP3PMadMixModsImpl()
5492 auto CopyMIB = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) in selectSBarrierSignalIsfirst()
5494 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0)); in selectSBarrierSignalIsfirst()
5498 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM)) in selectSBarrierSignalIsfirst()
5502 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC); in selectSBarrierSignalIsfirst()
5505 return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass, in selectSBarrierSignalIsfirst()
5515 return AMDGPU::S_BARRIER_INIT_IMM; in getNamedBarrierOp()
5517 return AMDGPU::S_BARRIER_JOIN_IMM; in getNamedBarrierOp()
5519 return AMDGPU::S_WAKEUP_BARRIER_IMM; in getNamedBarrierOp()
5521 return AMDGPU::S_GET_BARRIER_STATE_IMM; in getNamedBarrierOp()
5528 return AMDGPU::S_BARRIER_INIT_M0; in getNamedBarrierOp()
5530 return AMDGPU::S_BARRIER_JOIN_M0; in getNamedBarrierOp()
5532 return AMDGPU::S_WAKEUP_BARRIER_M0; in getNamedBarrierOp()
5534 return AMDGPU::S_GET_BARRIER_STATE_M0; in getNamedBarrierOp()
5554 TmpReg0 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInst()
5557 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0) in selectNamedBarrierInst()
5569 Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); in selectNamedBarrierInst()
5570 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_OR_B32), TmpReg1) in selectNamedBarrierInst()
5582 BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0).addReg(M0Val); in selectNamedBarrierInst()
5605 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_BARRIER_LEAVE)); in selectSBarrierLeave()
5606 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC); in selectSBarrierLeave()
5609 return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass, in selectSBarrierLeave()
5671 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL in renderExtractCPol()
5672 : AMDGPU::CPol::ALL_pregfx12)); in renderExtractCPol()
5680 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ in renderExtractSWZ()
5681 : AMDGPU::CPol::SWZ_pregfx12); in renderExtractSWZ()
5689 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL in renderExtractCpolSetGLC()
5690 : AMDGPU::CPol::ALL_pregfx12); in renderExtractCpolSetGLC()
5691 MIB.addImm(Cpol | AMDGPU::CPol::GLC); in renderExtractCpolSetGLC()