Lines Matching refs:AMDGPU
116 using namespace AMDGPU::SDWA;
316 if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) { in getSrcMods()
317 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) { in getSrcMods()
320 } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) { in getSrcMods()
321 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) { in getSrcMods()
375 case AMDGPU::V_CVT_F32_FP8_sdwa: in convertToSDWA()
376 case AMDGPU::V_CVT_F32_BF8_sdwa: in convertToSDWA()
377 case AMDGPU::V_CVT_PK_F32_FP8_sdwa: in convertToSDWA()
378 case AMDGPU::V_CVT_PK_F32_BF8_sdwa: in convertToSDWA()
386 MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in convertToSDWA()
387 MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); in convertToSDWA()
389 TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); in convertToSDWA()
393 Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in convertToSDWA()
394 SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); in convertToSDWA()
395 SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); in convertToSDWA()
405 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
407 TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
410 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) { in convertToSDWA()
416 TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel)); in convertToSDWA()
417 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 && in convertToSDWA()
418 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) { in convertToSDWA()
420 auto DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), in convertToSDWA()
421 AMDGPU::OpName::vdst); in convertToSDWA()
434 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
435 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
436 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
437 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && in convertToSDWA()
480 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
481 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
482 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
483 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && in convertToSDWA()
484 getDstSel() != AMDGPU::SDWA::DWORD) { in convertToSDWA()
489 MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
494 MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); in convertToSDWA()
497 MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
529 MI.tieOperands(AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst), in convertToSDWA()
568 case AMDGPU::V_LSHRREV_B32_e32: in matchSDWAOperand()
569 case AMDGPU::V_ASHRREV_I32_e32: in matchSDWAOperand()
570 case AMDGPU::V_LSHLREV_B32_e32: in matchSDWAOperand()
571 case AMDGPU::V_LSHRREV_B32_e64: in matchSDWAOperand()
572 case AMDGPU::V_ASHRREV_I32_e64: in matchSDWAOperand()
573 case AMDGPU::V_LSHLREV_B32_e64: { in matchSDWAOperand()
582 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
590 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
591 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
596 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 || in matchSDWAOperand()
597 Opcode == AMDGPU::V_LSHLREV_B32_e64) { in matchSDWAOperand()
603 Opcode != AMDGPU::V_LSHRREV_B32_e32 && in matchSDWAOperand()
604 Opcode != AMDGPU::V_LSHRREV_B32_e64); in matchSDWAOperand()
608 case AMDGPU::V_LSHRREV_B16_e32: in matchSDWAOperand()
609 case AMDGPU::V_ASHRREV_I16_e32: in matchSDWAOperand()
610 case AMDGPU::V_LSHLREV_B16_e32: in matchSDWAOperand()
611 case AMDGPU::V_LSHRREV_B16_e64: in matchSDWAOperand()
612 case AMDGPU::V_ASHRREV_I16_e64: in matchSDWAOperand()
613 case AMDGPU::V_LSHLREV_B16_e64: { in matchSDWAOperand()
622 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
627 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
628 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
634 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 || in matchSDWAOperand()
635 Opcode == AMDGPU::V_LSHLREV_B16_e64) in matchSDWAOperand()
639 Opcode != AMDGPU::V_LSHRREV_B16_e32 && in matchSDWAOperand()
640 Opcode != AMDGPU::V_LSHRREV_B16_e64); in matchSDWAOperand()
644 case AMDGPU::V_BFE_I32_e64: in matchSDWAOperand()
645 case AMDGPU::V_BFE_U32_e64: { in matchSDWAOperand()
660 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
665 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); in matchSDWAOperand()
689 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
690 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
697 Src0, Dst, SrcSel, false, false, Opcode != AMDGPU::V_BFE_U32_e64); in matchSDWAOperand()
700 case AMDGPU::V_AND_B32_e32: in matchSDWAOperand()
701 case AMDGPU::V_AND_B32_e64: { in matchSDWAOperand()
706 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
707 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
719 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
729 case AMDGPU::V_OR_B32_e32: in matchSDWAOperand()
730 case AMDGPU::V_OR_B32_e64: { in matchSDWAOperand()
762 MachineOperand *OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
763 MachineOperand *OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
767 OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in matchSDWAOperand()
768 OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in matchSDWAOperand()
806 TII->getNamedImmOperand(*SDWAInst, AMDGPU::OpName::dst_sel)); in matchSDWAOperand()
808 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_sel)); in matchSDWAOperand()
848 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_unused)); in matchSDWAOperand()
853 MachineOperand *OrDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in matchSDWAOperand()
903 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) && in pseudoOpConvertToVOP2()
909 Opc = AMDGPU::getVOPe32(Opc); in pseudoOpConvertToVOP2()
911 const MachineOperand *Sdst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); in pseudoOpConvertToVOP2()
920 MachineOperand *CarryIn = TII->getNamedOperand(MISucc, AMDGPU::OpName::src2); in pseudoOpConvertToVOP2()
923 MachineOperand *CarryOut = TII->getNamedOperand(MISucc, AMDGPU::OpName::sdst); in pseudoOpConvertToVOP2()
930 auto Liveness = MBB.computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 25); in pseudoOpConvertToVOP2()
936 if (I->modifiesRegister(AMDGPU::VCC, TRI)) in pseudoOpConvertToVOP2()
942 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) in pseudoOpConvertToVOP2()
943 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src0)) in pseudoOpConvertToVOP2()
944 .add(*TII->getNamedOperand(MI, AMDGPU::OpName::src1)) in pseudoOpConvertToVOP2()
964 if (AMDGPU::getSDWAOp(Opc) == -1) in isConvertibleToSDWA()
965 Opc = AMDGPU::getVOPe32(Opc); in isConvertibleToSDWA()
967 if (AMDGPU::getSDWAOp(Opc) == -1) in isConvertibleToSDWA()
970 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) in isConvertibleToSDWA()
975 const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); in isConvertibleToSDWA()
976 if (SDst && (SDst->getReg() != AMDGPU::VCC && in isConvertibleToSDWA()
977 SDst->getReg() != AMDGPU::VCC_LO)) in isConvertibleToSDWA()
982 (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) || in isConvertibleToSDWA()
983 TII->hasModifiersSet(MI, AMDGPU::OpName::omod))) in isConvertibleToSDWA()
986 } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) || in isConvertibleToSDWA()
987 !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { in isConvertibleToSDWA()
991 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 || in isConvertibleToSDWA()
992 Opc == AMDGPU::V_FMAC_F32_e32 || in isConvertibleToSDWA()
993 Opc == AMDGPU::V_MAC_F16_e32 || in isConvertibleToSDWA()
994 Opc == AMDGPU::V_MAC_F32_e32)) in isConvertibleToSDWA()
1002 if (Opc == AMDGPU::V_CNDMASK_B32_e32) in isConvertibleToSDWA()
1005 if (MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0)) { in isConvertibleToSDWA()
1010 if (MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1)) { in isConvertibleToSDWA()
1030 SDWAOpcode = AMDGPU::getSDWAOp(Opcode); in convertToSDWA()
1032 SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(Opcode)); in convertToSDWA()
1044 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); in convertToSDWA()
1046 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::vdst)); in convertToSDWA()
1048 } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) { in convertToSDWA()
1049 assert(Dst && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst)); in convertToSDWA()
1052 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst)); in convertToSDWA()
1058 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); in convertToSDWA()
1059 assert(Src0 && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0) && in convertToSDWA()
1060 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_modifiers)); in convertToSDWA()
1061 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)) in convertToSDWA()
1068 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); in convertToSDWA()
1070 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1) && in convertToSDWA()
1071 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_modifiers)); in convertToSDWA()
1072 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)) in convertToSDWA()
1079 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa || in convertToSDWA()
1080 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa || in convertToSDWA()
1081 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa || in convertToSDWA()
1082 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) { in convertToSDWA()
1084 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); in convertToSDWA()
1090 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::clamp)); in convertToSDWA()
1091 MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp); in convertToSDWA()
1099 if (AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::omod)) { in convertToSDWA()
1100 MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod); in convertToSDWA()
1109 if (AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::dst_sel)) { in convertToSDWA()
1110 MachineOperand *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); in convertToSDWA()
1114 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1119 if (AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::dst_unused)) { in convertToSDWA()
1120 MachineOperand *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
1124 SDWAInst.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PAD); in convertToSDWA()
1129 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_sel)); in convertToSDWA()
1130 MachineOperand *Src0Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); in convertToSDWA()
1134 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1139 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_sel)); in convertToSDWA()
1140 MachineOperand *Src1Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); in convertToSDWA()
1144 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); in convertToSDWA()
1149 auto DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); in convertToSDWA()
1151 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) { in convertToSDWA()
1157 auto PreserveDstIdx = AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst); in convertToSDWA()
1226 Register VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); in legalizeScalarOperands()
1228 TII->get(AMDGPU::V_MOV_B32_e32), VGPR); in legalizeScalarOperands()
1262 (PotentialMI->getOpcode() == AMDGPU::V_ADD_CO_U32_e64 || in runOnMachineFunction()
1263 PotentialMI->getOpcode() == AMDGPU::V_SUB_CO_U32_e64)) in runOnMachineFunction()