Lines Matching full:amdgpu
10 /// AMDGPU.
14 /// AMDGPU has unique register bank constraints that require special high level
73 #include "AMDGPU.h"
124 if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT || in applyBank()
125 Opc == AMDGPU::G_SEXT) { in applyBank()
132 if (SrcBank == &AMDGPU::VCCRegBank) { in applyBank()
136 assert(NewBank == &AMDGPU::VGPRRegBank); in applyBank()
142 auto True = B.buildConstant(S32, Opc == AMDGPU::G_SEXT ? -1 : 1); in applyBank()
156 if (Opc == AMDGPU::G_TRUNC) { in applyBank()
159 assert(DstBank != &AMDGPU::VCCRegBank); in applyBank()
174 assert(NewBank == &AMDGPU::VGPRRegBank && in applyBank()
176 assert((MI.getOpcode() != AMDGPU::G_TRUNC && in applyBank()
177 MI.getOpcode() != AMDGPU::G_ANYEXT) && in applyBank()
179 RB = &AMDGPU::VCCRegBank; in applyBank()
212 assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank && in AMDGPURegisterBankInfo()
213 &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank && in AMDGPURegisterBankInfo()
214 &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank); in AMDGPURegisterBankInfo()
223 return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID; in isVectorRegisterBank()
227 return RB != &AMDGPU::SGPRRegBank; in isDivergentRegBank()
234 if (Dst.getID() == AMDGPU::SGPRRegBankID && in copyCost()
235 (isVectorRegisterBank(Src) || Src.getID() == AMDGPU::VCCRegBankID)) { in copyCost()
247 (Dst.getID() == AMDGPU::SGPRRegBankID) && in copyCost()
249 Src.getID() == AMDGPU::SGPRRegBankID || in copyCost()
250 Src.getID() == AMDGPU::VCCRegBankID)) in copyCost()
254 if (Dst.getID() == AMDGPU::AGPRRegBankID && in copyCost()
255 Src.getID() == AMDGPU::AGPRRegBankID) in copyCost()
289 if (&RC == &AMDGPU::SReg_1RegClass) in getRegBankFromRegClass()
290 return AMDGPU::VCCRegBank; in getRegBankFromRegClass()
299 return AMDGPU::SGPRRegBank; in getRegBankFromRegClass()
301 return Ty == LLT::scalar(1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank; in getRegBankFromRegClass()
304 return TRI->isAGPRClass(&RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank; in getRegBankFromRegClass()
326 Operands[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SizeI); in addMappingFromTable()
334 Operands[OpIdx] = AMDGPU::getValueMapping(Entry.RegBanks[I], Sizes[I]); in addMappingFromTable()
352 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 }, in getInstrAlternativeMappingsIntrinsic()
355 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 } in getInstrAlternativeMappingsIntrinsic()
364 …{ { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, … in getInstrAlternativeMappingsIntrinsic()
367 …{ { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, … in getInstrAlternativeMappingsIntrinsic()
370 …{ { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, … in getInstrAlternativeMappingsIntrinsic()
373 …{ { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, … in getInstrAlternativeMappingsIntrinsic()
393 { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 }, in getInstrAlternativeMappingsIntrinsicWSideEffects()
396 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 300 }, in getInstrAlternativeMappingsIntrinsicWSideEffects()
399 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1000 }, in getInstrAlternativeMappingsIntrinsicWSideEffects()
402 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1500 } in getInstrAlternativeMappingsIntrinsicWSideEffects()
414 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, in getInstrAlternativeMappingsIntrinsicWSideEffects()
417 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 } in getInstrAlternativeMappingsIntrinsicWSideEffects()
428 { { AMDGPU::SGPRRegBankID }, 1 }, in getInstrAlternativeMappingsIntrinsicWSideEffects()
431 { { AMDGPU::VGPRRegBankID }, 3 } in getInstrAlternativeMappingsIntrinsicWSideEffects()
484 { { AMDGPU::VGPRRegBankID }, 1 }, in getInstrAlternativeMappings()
485 { { AMDGPU::SGPRRegBankID }, 1 }, in getInstrAlternativeMappings()
486 { { AMDGPU::VCCRegBankID }, 1 } in getInstrAlternativeMappings()
498 { { AMDGPU::VGPRRegBankID }, 1 }, in getInstrAlternativeMappings()
499 { { AMDGPU::SGPRRegBankID }, 1 } in getInstrAlternativeMappings()
513 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32), in getInstrAlternativeMappings()
514 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32), in getInstrAlternativeMappings()
515 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32)}), in getInstrAlternativeMappings()
521 {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), in getInstrAlternativeMappings()
522 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), in getInstrAlternativeMappings()
523 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size)}), in getInstrAlternativeMappings()
534 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
535 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
536 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), in getInstrAlternativeMappings()
542 {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
543 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
544 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}), in getInstrAlternativeMappings()
562 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
563 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize)}), in getInstrAlternativeMappings()
571 {AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
572 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize)}), in getInstrAlternativeMappings()
588 getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
589 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), in getInstrAlternativeMappings()
590 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
591 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), in getInstrAlternativeMappings()
596 getOperandsMapping({AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
597 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), in getInstrAlternativeMappings()
598 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
599 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}), in getInstrAlternativeMappings()
612 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
613 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), in getInstrAlternativeMappings()
614 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
615 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), in getInstrAlternativeMappings()
616 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1)}), in getInstrAlternativeMappings()
621 getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
622 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), in getInstrAlternativeMappings()
623 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
624 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), in getInstrAlternativeMappings()
625 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1)}), in getInstrAlternativeMappings()
630 case AMDGPU::G_BRCOND: { in getInstrAlternativeMappings()
636 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), nullptr}), in getInstrAlternativeMappings()
642 {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), nullptr }), in getInstrAlternativeMappings()
647 case AMDGPU::G_INTRINSIC: in getInstrAlternativeMappings()
648 case AMDGPU::G_INTRINSIC_CONVERGENT: in getInstrAlternativeMappings()
650 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: in getInstrAlternativeMappings()
651 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: in getInstrAlternativeMappings()
675 B.buildInstr(AMDGPU::G_UNMERGE_VALUES) in split64BitValueForMapping()
709 if (Bank == &AMDGPU::SGPRRegBank) in buildReadFirstLane()
715 if (Bank != &AMDGPU::VGPRRegBank) { in buildReadFirstLane()
718 MRI.setRegBank(Src, AMDGPU::VGPRRegBank); in buildReadFirstLane()
736 Register DstPart = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); in buildReadFirstLane()
740 constrainGenericRegister(SrcPart, AMDGPU::VGPR_32RegClass, MRI); in buildReadFirstLane()
744 B.buildInstr(AMDGPU::V_READFIRSTLANE_B32, {DstPart}, {SrcPart}); in buildReadFirstLane()
753 MRI.setRegBank(Dst, AMDGPU::SGPRRegBank); in buildReadFirstLane()
787 Subtarget.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; in executeInWaterfallLoop()
789 Subtarget.isWave32() ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term; in executeInWaterfallLoop()
792 AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; in executeInWaterfallLoop()
794 AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; in executeInWaterfallLoop()
796 AMDGPU::EXEC_LO : AMDGPU::EXEC; in executeInWaterfallLoop()
883 if (OpBank != &AMDGPU::VGPRRegBank) { in executeInWaterfallLoop()
887 MRI.setRegBank(OpReg, AMDGPU::VGPRRegBank); in executeInWaterfallLoop()
911 MRI.setRegBank(OpParts[i], AMDGPU::VGPRRegBank); in executeInWaterfallLoop()
912 MRI.setRegBank(CurrentLaneParts[i], AMDGPU::SGPRRegBank); in executeInWaterfallLoop()
919 MRI.setRegBank(CmpReg, AMDGPU::VCCRegBank); in executeInWaterfallLoop()
925 MRI.setRegBank(CondReg, AMDGPU::VCCRegBank); in executeInWaterfallLoop()
962 B.buildInstr(AMDGPU::SI_WATERFALL_LOOP).addMBB(LoopBB); in executeInWaterfallLoop()
992 if (OpBank->getID() != AMDGPU::SGPRRegBankID) in collectWaterfallOperands()
1020 if (Bank == &AMDGPU::SGPRRegBank) in constrainOpWithReadfirstlane()
1066 if (DstBank == &AMDGPU::SGPRRegBank) { in applyMappingLoad()
1097 if (MI.getOpcode() == AMDGPU::G_SEXTLOAD) { in applyMappingLoad()
1101 } else if (MI.getOpcode() == AMDGPU::G_ZEXTLOAD) { in applyMappingLoad()
1154 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingLoad()
1165 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); in applyMappingLoad()
1190 if (SizeBank != &AMDGPU::SGPRRegBank) in applyMappingDynStackAlloc()
1198 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank); in applyMappingDynStackAlloc()
1260 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); in setBufferOffsets()
1261 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); in setBufferOffsets()
1270 AMDGPU::getBaseWithConstantOffset(*MRI, CombinedOffset); in setBufferOffsets()
1275 if (getRegBank(Base, *MRI, *TRI) == &AMDGPU::VGPRRegBank) { in setBufferOffsets()
1278 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); in setBufferOffsets()
1286 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); in setBufferOffsets()
1294 MachineInstr *Add = getOpcodeDef(AMDGPU::G_ADD, CombinedOffset, *MRI); in setBufferOffsets()
1302 if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) { in setBufferOffsets()
1308 if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) { in setBufferOffsets()
1317 if (getRegBank(CombinedOffset, *MRI, *TRI) == &AMDGPU::VGPRRegBank) { in setBufferOffsets()
1321 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); in setBufferOffsets()
1325 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); in setBufferOffsets()
1342 if (RSrcBank == &AMDGPU::SGPRRegBank && in applyMappingSBufferLoad()
1343 OffsetBank == &AMDGPU::SGPRRegBank) in applyMappingSBufferLoad()
1386 B.getMRI()->setRegBank(VIndex, AMDGPU::VGPRRegBank); in applyMappingSBufferLoad()
1398 MRI.setRegBank(LoadParts[i], AMDGPU::VGPRRegBank); in applyMappingSBufferLoad()
1405 B.buildInstr(AMDGPU::G_AMDGPU_BUFFER_LOAD) in applyMappingSBufferLoad()
1420 if (RSrcBank != &AMDGPU::SGPRRegBank) { in applyMappingSBufferLoad()
1441 if (RSrcBank == &AMDGPU::SGPRRegBank) in applyMappingSBufferLoad()
1468 if (DstBank == &AMDGPU::VGPRRegBank) { in applyMappingBFE()
1474 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingBFE()
1526 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank); in applyMappingBFE()
1542 unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) : in applyMappingBFE()
1543 (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64); in applyMappingBFE()
1567 if (MRI.getRegBankOrNull(Src0) == &AMDGPU::VGPRRegBank) in applyMappingMAD_64_32()
1570 bool IsUnsigned = MI.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32; in applyMappingMAD_64_32()
1574 bool DstOnValu = MRI.getRegBankOrNull(Src2) == &AMDGPU::VGPRRegBank; in applyMappingMAD_64_32()
1587 MRI.setRegBank(DstLo, AMDGPU::SGPRRegBank); in applyMappingMAD_64_32()
1592 MRI.setRegBank(DstHi, AMDGPU::SGPRRegBank); in applyMappingMAD_64_32()
1597 MRI.setRegBank(VSrc0, AMDGPU::VGPRRegBank); in applyMappingMAD_64_32()
1598 MRI.setRegBank(VSrc1, AMDGPU::VGPRRegBank); in applyMappingMAD_64_32()
1602 MRI.setRegBank(DstHi, AMDGPU::VGPRRegBank); in applyMappingMAD_64_32()
1620 DstOnValu ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank; in applyMappingMAD_64_32()
1622 DstOnValu ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank; in applyMappingMAD_64_32()
1629 MulHiInVgpr ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank); in applyMappingMAD_64_32()
1633 MRI.setRegBank(Carry, MulHiInVgpr ? AMDGPU::VCCRegBank in applyMappingMAD_64_32()
1634 : AMDGPU::SGPRRegBank); in applyMappingMAD_64_32()
1638 MRI.setRegBank(Carry, AMDGPU::VCCRegBank); in applyMappingMAD_64_32()
1646 MRI.setRegBank(DstLo, AMDGPU::VGPRRegBank); in applyMappingMAD_64_32()
1647 MRI.setRegBank(DstHi, AMDGPU::VGPRRegBank); in applyMappingMAD_64_32()
1803 // TODO: Use AMDGPU::getBaseWithConstantOffset() instead. in splitBufferOffsets()
1847 B.buildInstr(AMDGPU::V_MOV_B32_e32) in buildVCopy()
1850 return constrainGenericRegister(DstReg, AMDGPU::VGPR_32RegClass, MRI) && in buildVCopy()
1851 constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI); in buildVCopy()
1854 Register TmpReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in buildVCopy()
1855 Register TmpReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in buildVCopy()
1857 B.buildInstr(AMDGPU::V_MOV_B32_e32) in buildVCopy()
1859 .addUse(SrcReg, 0, AMDGPU::sub0); in buildVCopy()
1860 B.buildInstr(AMDGPU::V_MOV_B32_e32) in buildVCopy()
1862 .addUse(SrcReg, 0, AMDGPU::sub1); in buildVCopy()
1863 B.buildInstr(AMDGPU::REG_SEQUENCE) in buildVCopy()
1866 .addImm(AMDGPU::sub0) in buildVCopy()
1868 .addImm(AMDGPU::sub1); in buildVCopy()
1870 return constrainGenericRegister(SrcReg, AMDGPU::SReg_64RegClass, MRI) && in buildVCopy()
1871 constrainGenericRegister(DstReg, AMDGPU::VReg_64RegClass, MRI); in buildVCopy()
1888 MRI.setRegBank(MaterializedOffset.getReg(0), AMDGPU::SGPRRegBank); in reinsertVectorIndexAdd()
1889 MRI.setRegBank(Add.getReg(0), AMDGPU::SGPRRegBank); in reinsertVectorIndexAdd()
1902 if (ExtOpc == AMDGPU::G_ZEXT) { in extendLow32IntoHigh32()
1904 } else if (ExtOpc == AMDGPU::G_SEXT) { in extendLow32IntoHigh32()
1916 assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension"); in extendLow32IntoHigh32()
1932 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank; in foldExtractEltToCmpSelect()
1950 (DstBank == AMDGPU::SGPRRegBank && in foldExtractEltToCmpSelect()
1951 SrcBank == AMDGPU::SGPRRegBank && in foldExtractEltToCmpSelect()
1952 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank in foldExtractEltToCmpSelect()
1953 : AMDGPU::VCCRegBank; in foldExtractEltToCmpSelect()
1954 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1); in foldExtractEltToCmpSelect()
1956 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) { in foldExtractEltToCmpSelect()
1958 MRI.setRegBank(Idx, AMDGPU::VGPRRegBank); in foldExtractEltToCmpSelect()
1976 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank); in foldExtractEltToCmpSelect()
2030 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank; in foldInsertEltToCmpSelect()
2050 (DstBank == AMDGPU::SGPRRegBank && in foldInsertEltToCmpSelect()
2051 SrcBank == AMDGPU::SGPRRegBank && in foldInsertEltToCmpSelect()
2052 InsBank == AMDGPU::SGPRRegBank && in foldInsertEltToCmpSelect()
2053 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank in foldInsertEltToCmpSelect()
2054 : AMDGPU::VCCRegBank; in foldInsertEltToCmpSelect()
2055 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1); in foldInsertEltToCmpSelect()
2057 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) { in foldInsertEltToCmpSelect()
2059 MRI.setRegBank(Idx, AMDGPU::VGPRRegBank); in foldInsertEltToCmpSelect()
2077 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank); in foldInsertEltToCmpSelect()
2163 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingSMULU64()
2172 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); in applyMappingSMULU64()
2183 case AMDGPU::G_CONSTANT: in applyMappingImpl()
2184 case AMDGPU::G_IMPLICIT_DEF: { in applyMappingImpl()
2192 if (DstBank == &AMDGPU::VCCRegBank) in applyMappingImpl()
2204 if (Opc != AMDGPU::G_IMPLICIT_DEF) { in applyMappingImpl()
2214 case AMDGPU::G_PHI: { in applyMappingImpl()
2223 if (DstBank == &AMDGPU::VCCRegBank) { in applyMappingImpl()
2234 if (SrcBank != &AMDGPU::VCCRegBank) { in applyMappingImpl()
2239 MRI.setRegBank(Copy.getReg(0), AMDGPU::VCCRegBank); in applyMappingImpl()
2260 case AMDGPU::G_FCMP: in applyMappingImpl()
2264 case AMDGPU::G_ICMP: in applyMappingImpl()
2265 case AMDGPU::G_UADDO: in applyMappingImpl()
2266 case AMDGPU::G_USUBO: in applyMappingImpl()
2267 case AMDGPU::G_UADDE: in applyMappingImpl()
2268 case AMDGPU::G_SADDE: in applyMappingImpl()
2269 case AMDGPU::G_USUBE: in applyMappingImpl()
2270 case AMDGPU::G_SSUBE: { in applyMappingImpl()
2272 (Opc == AMDGPU::G_ICMP || Opc == AMDGPU::G_FCMP) ? 0 : 1; in applyMappingImpl()
2277 if (DstBank != &AMDGPU::SGPRRegBank) in applyMappingImpl()
2286 MRI.setRegBank(NewDstReg, AMDGPU::SGPRRegBank); in applyMappingImpl()
2291 MRI.setRegBank(NewSrcReg, AMDGPU::SGPRRegBank); in applyMappingImpl()
2307 case AMDGPU::G_SELECT: { in applyMappingImpl()
2319 if (CondBank == &AMDGPU::SGPRRegBank) { in applyMappingImpl()
2322 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank); in applyMappingImpl()
2359 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); in applyMappingImpl()
2363 case AMDGPU::G_BRCOND: { in applyMappingImpl()
2369 if (CondBank == &AMDGPU::SGPRRegBank) { in applyMappingImpl()
2372 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank); in applyMappingImpl()
2381 case AMDGPU::G_AND: in applyMappingImpl()
2382 case AMDGPU::G_OR: in applyMappingImpl()
2383 case AMDGPU::G_XOR: { in applyMappingImpl()
2392 if (DstBank == &AMDGPU::VCCRegBank) in applyMappingImpl()
2442 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); in applyMappingImpl()
2446 case AMDGPU::G_ABS: { in applyMappingImpl()
2452 if (SrcBank && SrcBank == &AMDGPU::VGPRRegBank) { in applyMappingImpl()
2454 ApplyRegBankMapping Apply(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingImpl()
2463 case AMDGPU::G_ADD: in applyMappingImpl()
2464 case AMDGPU::G_SUB: in applyMappingImpl()
2465 case AMDGPU::G_MUL: in applyMappingImpl()
2466 case AMDGPU::G_SHL: in applyMappingImpl()
2467 case AMDGPU::G_LSHR: in applyMappingImpl()
2468 case AMDGPU::G_ASHR: in applyMappingImpl()
2469 case AMDGPU::G_SMIN: in applyMappingImpl()
2470 case AMDGPU::G_SMAX: in applyMappingImpl()
2471 case AMDGPU::G_UMIN: in applyMappingImpl()
2472 case AMDGPU::G_UMAX: { in applyMappingImpl()
2479 if (Opc == AMDGPU::G_MUL && DstTy.getSizeInBits() == 64) { in applyMappingImpl()
2491 if (DstBank == &AMDGPU::VGPRRegBank) in applyMappingImpl()
2497 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank); in applyMappingImpl()
2499 if (DstTy.isVector() && Opc == AMDGPU::G_ABS) { in applyMappingImpl()
2504 auto Lo = B.buildInstr(AMDGPU::G_ABS, {S32}, {WideSrcLo}); in applyMappingImpl()
2505 auto Hi = B.buildInstr(AMDGPU::G_ABS, {S32}, {WideSrcHi}); in applyMappingImpl()
2531 if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR || in applyMappingImpl()
2532 Opc == AMDGPU::G_ASHR) { in applyMappingImpl()
2541 case AMDGPU::G_AMDGPU_S_MUL_I64_I32: in applyMappingImpl()
2542 case AMDGPU::G_AMDGPU_S_MUL_U64_U32: { in applyMappingImpl()
2567 if (DstBank == &AMDGPU::SGPRRegBank) { in applyMappingImpl()
2568 MI.setDesc(TII->get(AMDGPU::S_MUL_U64)); in applyMappingImpl()
2569 MRI.setRegClass(DstReg, &AMDGPU::SGPR_64RegClass); in applyMappingImpl()
2570 MRI.setRegClass(SrcReg0, &AMDGPU::SGPR_64RegClass); in applyMappingImpl()
2571 MRI.setRegClass(SrcReg1, &AMDGPU::SGPR_64RegClass); in applyMappingImpl()
2577 assert(MRI.getRegBankOrNull(DstReg) == &AMDGPU::VGPRRegBank && in applyMappingImpl()
2583 Register Op0L = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in applyMappingImpl()
2584 MRI.setRegClass(Op0L, &AMDGPU::VGPR_32RegClass); in applyMappingImpl()
2589 Register Op1L = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); in applyMappingImpl()
2590 MRI.setRegClass(Op1L, &AMDGPU::VGPR_32RegClass); in applyMappingImpl()
2594 unsigned NewOpc = Opc == AMDGPU::G_AMDGPU_S_MUL_U64_U32 in applyMappingImpl()
2595 ? AMDGPU::G_AMDGPU_MAD_U64_U32 in applyMappingImpl()
2596 : AMDGPU::G_AMDGPU_MAD_I64_I32; in applyMappingImpl()
2600 MRI.setRegClass(Zero64, &AMDGPU::VReg_64RegClass); in applyMappingImpl()
2601 Register CarryOut = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); in applyMappingImpl()
2602 MRI.setRegClass(CarryOut, &AMDGPU::VReg_64RegClass); in applyMappingImpl()
2607 case AMDGPU::G_SEXT_INREG: { in applyMappingImpl()
2613 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingImpl()
2642 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); in applyMappingImpl()
2646 case AMDGPU::G_CTPOP: in applyMappingImpl()
2647 case AMDGPU::G_BITREVERSE: { in applyMappingImpl()
2650 if (DstBank == &AMDGPU::SGPRRegBank) in applyMappingImpl()
2659 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingImpl()
2668 case AMDGPU::G_AMDGPU_FFBH_U32: in applyMappingImpl()
2669 case AMDGPU::G_AMDGPU_FFBL_B32: in applyMappingImpl()
2670 case AMDGPU::G_CTLZ_ZERO_UNDEF: in applyMappingImpl()
2671 case AMDGPU::G_CTTZ_ZERO_UNDEF: { in applyMappingImpl()
2674 if (DstBank == &AMDGPU::SGPRRegBank) in applyMappingImpl()
2689 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank); in applyMappingImpl()
2691 unsigned NewOpc = Opc == AMDGPU::G_CTLZ_ZERO_UNDEF in applyMappingImpl()
2692 ? (unsigned)AMDGPU::G_AMDGPU_FFBH_U32 in applyMappingImpl()
2693 : Opc == AMDGPU::G_CTTZ_ZERO_UNDEF in applyMappingImpl()
2694 ? (unsigned)AMDGPU::G_AMDGPU_FFBL_B32 in applyMappingImpl()
2696 unsigned Idx = NewOpc == AMDGPU::G_AMDGPU_FFBH_U32; in applyMappingImpl()
2700 Opc == AMDGPU::G_CTLZ_ZERO_UNDEF || Opc == AMDGPU::G_CTTZ_ZERO_UNDEF in applyMappingImpl()
2701 ? AMDGPU::G_ADD in applyMappingImpl()
2702 : AMDGPU::G_UADDSAT; in applyMappingImpl()
2709 case AMDGPU::G_SEXT: in applyMappingImpl()
2710 case AMDGPU::G_ZEXT: in applyMappingImpl()
2711 case AMDGPU::G_ANYEXT: { in applyMappingImpl()
2714 const bool Signed = Opc == AMDGPU::G_SEXT; in applyMappingImpl()
2724 SrcBank != &AMDGPU::SGPRRegBank && in applyMappingImpl()
2725 SrcBank != &AMDGPU::VCCRegBank && in applyMappingImpl()
2736 } else if (Opc == AMDGPU::G_ZEXT) { in applyMappingImpl()
2754 if (SrcBank == &AMDGPU::VCCRegBank) { in applyMappingImpl()
2757 const RegisterBank *DstBank = &AMDGPU::VGPRRegBank; in applyMappingImpl()
2762 SrcBank->getID() == AMDGPU::SGPRRegBankID; in applyMappingImpl()
2790 case AMDGPU::G_EXTRACT_VECTOR_ELT: { in applyMappingImpl()
2816 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(2).getReg()); in applyMappingImpl()
2822 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank && in applyMappingImpl()
2834 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank && in applyMappingImpl()
2835 SrcBank == &AMDGPU::SGPRRegBank; in applyMappingImpl()
2844 MRI.setRegBank(TmpReg, AMDGPU::SGPRRegBank); in applyMappingImpl()
2883 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank); in applyMappingImpl()
2884 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank); in applyMappingImpl()
2885 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank); in applyMappingImpl()
2904 MRI.setRegBank(TmpReg0, AMDGPU::SGPRRegBank); in applyMappingImpl()
2905 MRI.setRegBank(TmpReg1, AMDGPU::SGPRRegBank); in applyMappingImpl()
2921 case AMDGPU::G_INSERT_VECTOR_ELT: { in applyMappingImpl()
2947 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(3).getReg()); in applyMappingImpl()
2953 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank && in applyMappingImpl()
3005 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank); in applyMappingImpl()
3006 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank); in applyMappingImpl()
3007 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank); in applyMappingImpl()
3038 case AMDGPU::G_AMDGPU_BUFFER_LOAD: in applyMappingImpl()
3039 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: in applyMappingImpl()
3040 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: in applyMappingImpl()
3041 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: in applyMappingImpl()
3042 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: in applyMappingImpl()
3043 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE: in applyMappingImpl()
3044 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE: in applyMappingImpl()
3045 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE: in applyMappingImpl()
3046 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE: in applyMappingImpl()
3047 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE: in applyMappingImpl()
3048 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT: in applyMappingImpl()
3049 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE: in applyMappingImpl()
3050 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16: in applyMappingImpl()
3051 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT: in applyMappingImpl()
3052 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16: in applyMappingImpl()
3053 case AMDGPU::G_AMDGPU_BUFFER_STORE: in applyMappingImpl()
3054 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: in applyMappingImpl()
3055 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: in applyMappingImpl()
3056 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: in applyMappingImpl()
3057 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: in applyMappingImpl()
3058 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: in applyMappingImpl()
3059 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: { in applyMappingImpl()
3064 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP: in applyMappingImpl()
3065 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD: in applyMappingImpl()
3066 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB: in applyMappingImpl()
3067 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN: in applyMappingImpl()
3068 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN: in applyMappingImpl()
3069 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX: in applyMappingImpl()
3070 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX: in applyMappingImpl()
3071 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND: in applyMappingImpl()
3072 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR: in applyMappingImpl()
3073 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR: in applyMappingImpl()
3074 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC: in applyMappingImpl()
3075 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: { in applyMappingImpl()
3080 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: in applyMappingImpl()
3081 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN: in applyMappingImpl()
3082 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: { in applyMappingImpl()
3087 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: { in applyMappingImpl()
3092 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: in applyMappingImpl()
3093 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE: in applyMappingImpl()
3094 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE: in applyMappingImpl()
3095 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT: in applyMappingImpl()
3096 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: { in applyMappingImpl()
3100 case AMDGPU::G_INTRINSIC: in applyMappingImpl()
3101 case AMDGPU::G_INTRINSIC_CONVERGENT: { in applyMappingImpl()
3173 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: in applyMappingImpl()
3174 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16: in applyMappingImpl()
3175 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET: in applyMappingImpl()
3176 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: in applyMappingImpl()
3177 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: { in applyMappingImpl()
3178 const AMDGPU::RsrcIntrinsic *RSrcIntrin = in applyMappingImpl()
3179 AMDGPU::lookupRsrcIntrinsic(AMDGPU::getIntrinsicID(MI)); in applyMappingImpl()
3187 case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY: { in applyMappingImpl()
3193 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: in applyMappingImpl()
3194 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: { in applyMappingImpl()
3290 if (const AMDGPU::RsrcIntrinsic *RSrcIntrin = in applyMappingImpl()
3291 AMDGPU::lookupRsrcIntrinsic(IntrID)) { in applyMappingImpl()
3306 case AMDGPU::G_SI_CALL: { in applyMappingImpl()
3317 unsigned FrameSetupOpcode = AMDGPU::ADJCALLSTACKUP; in applyMappingImpl()
3318 unsigned FrameDestroyOpcode = AMDGPU::ADJCALLSTACKDOWN; in applyMappingImpl()
3333 if (Start->getOpcode() == AMDGPU::COPY) { in applyMappingImpl()
3373 if (End->getOpcode() == AMDGPU::COPY) { in applyMappingImpl()
3401 case AMDGPU::G_LOAD: in applyMappingImpl()
3402 case AMDGPU::G_ZEXTLOAD: in applyMappingImpl()
3403 case AMDGPU::G_SEXTLOAD: { in applyMappingImpl()
3408 case AMDGPU::G_DYN_STACKALLOC: in applyMappingImpl()
3411 case AMDGPU::G_STACKRESTORE: { in applyMappingImpl()
3416 case AMDGPU::G_SBFX: in applyMappingImpl()
3419 case AMDGPU::G_UBFX: in applyMappingImpl()
3422 case AMDGPU::G_AMDGPU_MAD_U64_U32: in applyMappingImpl()
3423 case AMDGPU::G_AMDGPU_MAD_I64_I32: in applyMappingImpl()
3426 case AMDGPU::G_PREFETCH: { in applyMappingImpl()
3432 unsigned PtrBank = getRegBankID(PtrReg, MRI, AMDGPU::SGPRRegBankID); in applyMappingImpl()
3433 if (PtrBank == AMDGPU::VGPRRegBankID) { in applyMappingImpl()
3438 if (!AMDGPU::isFlatGlobalAddrSpace(AS) && in applyMappingImpl()
3458 if (RB0 == AMDGPU::InvalidRegBankID) in regBankUnion()
3460 if (RB1 == AMDGPU::InvalidRegBankID) in regBankUnion()
3463 if (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID) in regBankUnion()
3464 return AMDGPU::SGPRRegBankID; in regBankUnion()
3466 if (RB0 == AMDGPU::AGPRRegBankID && RB1 == AMDGPU::AGPRRegBankID) in regBankUnion()
3467 return AMDGPU::AGPRRegBankID; in regBankUnion()
3469 return AMDGPU::VGPRRegBankID; in regBankUnion()
3473 if (RB0 == AMDGPU::InvalidRegBankID) in regBankBoolUnion()
3475 if (RB1 == AMDGPU::InvalidRegBankID) in regBankBoolUnion()
3481 if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID) in regBankBoolUnion()
3482 return AMDGPU::VCCRegBankID; in regBankBoolUnion()
3490 unsigned RegBank = AMDGPU::InvalidRegBankID; in getMappingType()
3498 if (RegBank == AMDGPU::VGPRRegBankID) in getMappingType()
3514 if (Bank->getID() != AMDGPU::SGPRRegBankID) in isSALUMapping()
3533 OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getDefaultMappingSOP()
3555 unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID; in getDefaultMappingVOP()
3556 OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size); in getDefaultMappingVOP()
3575 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getDefaultMappingAllVGPR()
3614 unsigned NewBank = getRegBankID(OpReg, MRI, AMDGPU::SGPRRegBankID); in getImageMapping()
3615 OpdsMapping[I] = AMDGPU::getValueMapping(NewBank, Size); in getImageMapping()
3618 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getImageMapping()
3632 !AMDGPU::isFlatGlobalAddrSpace(PtrTy.getAddressSpace())) in getValueMappingForPtr()
3633 return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getValueMappingForPtr()
3638 return AMDGPU::getValueMapping(PtrBank->getID(), Size); in getValueMappingForPtr()
3658 if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) { in getInstrMappingForLoad()
3661 ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMappingForLoad()
3662 PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize); in getInstrMappingForLoad()
3664 ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMappingForLoad()
3669 AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID; in getInstrMappingForLoad()
3671 PtrMapping = AMDGPU::getValueMapping(PtrBankID, PtrSize); in getInstrMappingForLoad()
3674 ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMappingForLoad()
3675 PtrMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize); in getInstrMappingForLoad()
3702 unsigned Bank = getRegBankID(Reg, MRI, AMDGPU::SGPRRegBankID); in getSGPROpMapping()
3704 return AMDGPU::getValueMapping(Bank, Size); in getSGPROpMapping()
3712 return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getVGPROpMapping()
3720 return AMDGPU::getValueMapping(AMDGPU::AGPRRegBankID, Size); in getAGPROpMapping()
3737 if (MI.isCopy() || MI.getOpcode() == AMDGPU::G_FREEZE) { in getInstrMapping()
3749 if (MI.getOpcode() != AMDGPU::G_FREEZE && in getInstrMapping()
3757 if (MI.getOpcode() == AMDGPU::G_FREEZE) in getInstrMapping()
3768 unsigned BankID = AMDGPU::SGPRRegBankID; in getInstrMapping()
3774 if (OpBank != AMDGPU::SGPRRegBankID) { in getInstrMapping()
3775 BankID = AMDGPU::VGPRRegBankID; in getInstrMapping()
3792 unsigned ResultBank = AMDGPU::InvalidRegBankID; in getInstrMapping()
3804 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) { in getInstrMapping()
3805 ResultBank = AMDGPU::VGPRRegBankID; in getInstrMapping()
3814 assert(ResultBank != AMDGPU::InvalidRegBankID); in getInstrMapping()
3835 case AMDGPU::G_AND: in getInstrMapping()
3836 case AMDGPU::G_OR: in getInstrMapping()
3837 case AMDGPU::G_XOR: in getInstrMapping()
3838 case AMDGPU::G_MUL: { in getInstrMapping()
3844 unsigned TargetBankID = AMDGPU::InvalidRegBankID; in getInstrMapping()
3845 unsigned BankLHS = AMDGPU::InvalidRegBankID; in getInstrMapping()
3846 unsigned BankRHS = AMDGPU::InvalidRegBankID; in getInstrMapping()
3849 if (DstBank == &AMDGPU::VCCRegBank) { in getInstrMapping()
3850 TargetBankID = AMDGPU::VCCRegBankID; in getInstrMapping()
3851 BankLHS = AMDGPU::VCCRegBankID; in getInstrMapping()
3852 BankRHS = AMDGPU::VCCRegBankID; in getInstrMapping()
3855 AMDGPU::SGPRRegBankID); in getInstrMapping()
3857 AMDGPU::SGPRRegBankID); in getInstrMapping()
3861 AMDGPU::VCCRegBankID); in getInstrMapping()
3863 AMDGPU::VCCRegBankID); in getInstrMapping()
3866 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) { in getInstrMapping()
3867 TargetBankID = AMDGPU::VGPRRegBankID; in getInstrMapping()
3868 } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) { in getInstrMapping()
3869 TargetBankID = AMDGPU::VCCRegBankID; in getInstrMapping()
3870 BankLHS = AMDGPU::VCCRegBankID; in getInstrMapping()
3871 BankRHS = AMDGPU::VCCRegBankID; in getInstrMapping()
3872 } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) { in getInstrMapping()
3873 TargetBankID = AMDGPU::SGPRRegBankID; in getInstrMapping()
3877 OpdsMapping[0] = AMDGPU::getValueMapping(TargetBankID, Size); in getInstrMapping()
3878 OpdsMapping[1] = AMDGPU::getValueMapping(BankLHS, Size); in getInstrMapping()
3879 OpdsMapping[2] = AMDGPU::getValueMapping(BankRHS, Size); in getInstrMapping()
3886 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
3889 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
3891 OpdsMapping[1] = AMDGPU::getValueMapping(Bank1, Size); in getInstrMapping()
3894 OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size); in getInstrMapping()
3902 case AMDGPU::G_PTR_ADD: in getInstrMapping()
3903 case AMDGPU::G_PTRMASK: in getInstrMapping()
3904 case AMDGPU::G_ADD: in getInstrMapping()
3905 case AMDGPU::G_SUB: in getInstrMapping()
3906 case AMDGPU::G_SHL: in getInstrMapping()
3907 case AMDGPU::G_LSHR: in getInstrMapping()
3908 case AMDGPU::G_ASHR: in getInstrMapping()
3909 case AMDGPU::G_UADDO: in getInstrMapping()
3910 case AMDGPU::G_USUBO: in getInstrMapping()
3911 case AMDGPU::G_UADDE: in getInstrMapping()
3912 case AMDGPU::G_SADDE: in getInstrMapping()
3913 case AMDGPU::G_USUBE: in getInstrMapping()
3914 case AMDGPU::G_SSUBE: in getInstrMapping()
3915 case AMDGPU::G_SMIN: in getInstrMapping()
3916 case AMDGPU::G_SMAX: in getInstrMapping()
3917 case AMDGPU::G_UMIN: in getInstrMapping()
3918 case AMDGPU::G_UMAX: in getInstrMapping()
3919 case AMDGPU::G_ABS: in getInstrMapping()
3920 case AMDGPU::G_SHUFFLE_VECTOR: in getInstrMapping()
3921 case AMDGPU::G_SBFX: in getInstrMapping()
3922 case AMDGPU::G_UBFX: in getInstrMapping()
3923 case AMDGPU::G_AMDGPU_S_MUL_I64_I32: in getInstrMapping()
3924 case AMDGPU::G_AMDGPU_S_MUL_U64_U32: in getInstrMapping()
3928 case AMDGPU::G_FADD: in getInstrMapping()
3929 case AMDGPU::G_FSUB: in getInstrMapping()
3930 case AMDGPU::G_FMUL: in getInstrMapping()
3931 case AMDGPU::G_FMA: in getInstrMapping()
3932 case AMDGPU::G_FFLOOR: in getInstrMapping()
3933 case AMDGPU::G_FCEIL: in getInstrMapping()
3934 case AMDGPU::G_INTRINSIC_ROUNDEVEN: in getInstrMapping()
3935 case AMDGPU::G_FMINNUM: in getInstrMapping()
3936 case AMDGPU::G_FMAXNUM: in getInstrMapping()
3937 case AMDGPU::G_FMINIMUM: in getInstrMapping()
3938 case AMDGPU::G_FMAXIMUM: in getInstrMapping()
3939 case AMDGPU::G_INTRINSIC_TRUNC: in getInstrMapping()
3940 case AMDGPU::G_STRICT_FADD: in getInstrMapping()
3941 case AMDGPU::G_STRICT_FSUB: in getInstrMapping()
3942 case AMDGPU::G_STRICT_FMUL: in getInstrMapping()
3943 case AMDGPU::G_STRICT_FMA: { in getInstrMapping()
3951 case AMDGPU::G_FPTOSI: in getInstrMapping()
3952 case AMDGPU::G_FPTOUI: in getInstrMapping()
3953 case AMDGPU::G_SITOFP: in getInstrMapping()
3954 case AMDGPU::G_UITOFP: { in getInstrMapping()
3962 case AMDGPU::G_FPTRUNC: in getInstrMapping()
3963 case AMDGPU::G_FPEXT: { in getInstrMapping()
3971 case AMDGPU::G_FSQRT: in getInstrMapping()
3972 case AMDGPU::G_FEXP2: in getInstrMapping()
3973 case AMDGPU::G_FLOG2: { in getInstrMapping()
3980 case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU in getInstrMapping()
3981 case AMDGPU::G_SSUBSAT: in getInstrMapping()
3982 case AMDGPU::G_UADDSAT: in getInstrMapping()
3983 case AMDGPU::G_USUBSAT: in getInstrMapping()
3984 case AMDGPU::G_FMAD: in getInstrMapping()
3985 case AMDGPU::G_FLDEXP: in getInstrMapping()
3986 case AMDGPU::G_FMINNUM_IEEE: in getInstrMapping()
3987 case AMDGPU::G_FMAXNUM_IEEE: in getInstrMapping()
3988 case AMDGPU::G_FCANONICALIZE: in getInstrMapping()
3989 case AMDGPU::G_STRICT_FLDEXP: in getInstrMapping()
3990 case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar? in getInstrMapping()
3991 case AMDGPU::G_FSHR: // TODO: Expand for scalar in getInstrMapping()
3992 case AMDGPU::G_AMDGPU_FMIN_LEGACY: in getInstrMapping()
3993 case AMDGPU::G_AMDGPU_FMAX_LEGACY: in getInstrMapping()
3994 case AMDGPU::G_AMDGPU_RCP_IFLAG: in getInstrMapping()
3995 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0: in getInstrMapping()
3996 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1: in getInstrMapping()
3997 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2: in getInstrMapping()
3998 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3: in getInstrMapping()
3999 case AMDGPU::G_AMDGPU_CVT_PK_I16_I32: in getInstrMapping()
4000 case AMDGPU::G_AMDGPU_SMED3: in getInstrMapping()
4001 case AMDGPU::G_AMDGPU_FMED3: in getInstrMapping()
4003 case AMDGPU::G_UMULH: in getInstrMapping()
4004 case AMDGPU::G_SMULH: { in getInstrMapping()
4009 case AMDGPU::G_AMDGPU_MAD_U64_U32: in getInstrMapping()
4010 case AMDGPU::G_AMDGPU_MAD_I64_I32: { in getInstrMapping()
4024 if (Bank->getID() != AMDGPU::SGPRRegBankID) { in getInstrMapping()
4044 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 64); in getInstrMapping()
4045 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
4046 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); in getInstrMapping()
4047 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); in getInstrMapping()
4048 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 64); in getInstrMapping()
4051 case AMDGPU::G_IMPLICIT_DEF: { in getInstrMapping()
4053 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4056 case AMDGPU::G_FCONSTANT: in getInstrMapping()
4057 case AMDGPU::G_CONSTANT: in getInstrMapping()
4058 case AMDGPU::G_GLOBAL_VALUE: in getInstrMapping()
4059 case AMDGPU::G_BLOCK_ADDR: in getInstrMapping()
4060 case AMDGPU::G_READSTEADYCOUNTER: in getInstrMapping()
4061 case AMDGPU::G_READCYCLECOUNTER: { in getInstrMapping()
4063 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4066 case AMDGPU::G_FRAME_INDEX: { in getInstrMapping()
4070 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4073 case AMDGPU::G_DYN_STACKALLOC: { in getInstrMapping()
4075 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); in getInstrMapping()
4077 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32); in getInstrMapping()
4080 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: { in getInstrMapping()
4085 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); in getInstrMapping()
4086 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); in getInstrMapping()
4089 case AMDGPU::G_INSERT: { in getInstrMapping()
4094 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize); in getInstrMapping()
4095 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize); in getInstrMapping()
4096 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, EltSize); in getInstrMapping()
4100 case AMDGPU::G_EXTRACT: { in getInstrMapping()
4104 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize); in getInstrMapping()
4105 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize); in getInstrMapping()
4109 case AMDGPU::G_BUILD_VECTOR: in getInstrMapping()
4110 case AMDGPU::G_BUILD_VECTOR_TRUNC: { in getInstrMapping()
4119 OpdsMapping[0] = AMDGPU::getValueMapping(DstBankID, DstSize); in getInstrMapping()
4120 OpdsMapping[1] = AMDGPU::getValueMapping(Src0BankID, SrcSize); in getInstrMapping()
4121 OpdsMapping[2] = AMDGPU::getValueMapping(Src1BankID, SrcSize); in getInstrMapping()
4127 case AMDGPU::G_MERGE_VALUES: in getInstrMapping()
4128 case AMDGPU::G_CONCAT_VECTORS: { in getInstrMapping()
4133 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize); in getInstrMapping()
4136 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, SrcSize); in getInstrMapping()
4139 case AMDGPU::G_BITREVERSE: in getInstrMapping()
4140 case AMDGPU::G_BITCAST: in getInstrMapping()
4141 case AMDGPU::G_INTTOPTR: in getInstrMapping()
4142 case AMDGPU::G_PTRTOINT: in getInstrMapping()
4143 case AMDGPU::G_FABS: in getInstrMapping()
4144 case AMDGPU::G_FNEG: { in getInstrMapping()
4147 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size); in getInstrMapping()
4150 case AMDGPU::G_AMDGPU_FFBH_U32: in getInstrMapping()
4151 case AMDGPU::G_AMDGPU_FFBL_B32: in getInstrMapping()
4152 case AMDGPU::G_CTLZ_ZERO_UNDEF: in getInstrMapping()
4153 case AMDGPU::G_CTTZ_ZERO_UNDEF: { in getInstrMapping()
4156 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32); in getInstrMapping()
4157 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID, Size); in getInstrMapping()
4160 case AMDGPU::G_CTPOP: { in getInstrMapping()
4163 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32); in getInstrMapping()
4168 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size); in getInstrMapping()
4171 case AMDGPU::G_TRUNC: { in getInstrMapping()
4177 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize); in getInstrMapping()
4178 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize); in getInstrMapping()
4181 case AMDGPU::G_ZEXT: in getInstrMapping()
4182 case AMDGPU::G_SEXT: in getInstrMapping()
4183 case AMDGPU::G_ANYEXT: in getInstrMapping()
4184 case AMDGPU::G_SEXT_INREG: { in getInstrMapping()
4194 case AMDGPU::SGPRRegBankID: in getInstrMapping()
4195 DstBank = AMDGPU::SGPRRegBankID; in getInstrMapping()
4198 DstBank = AMDGPU::VGPRRegBankID; in getInstrMapping()
4204 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize); in getInstrMapping()
4205 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(), in getInstrMapping()
4209 case AMDGPU::G_IS_FPCLASS: { in getInstrMapping()
4213 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize); in getInstrMapping()
4214 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); in getInstrMapping()
4217 case AMDGPU::G_STORE: { in getInstrMapping()
4224 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4229 case AMDGPU::G_ICMP: in getInstrMapping()
4230 case AMDGPU::G_FCMP: { in getInstrMapping()
4236 AMDGPU::SGPRRegBankID); in getInstrMapping()
4252 bool isICMP = MI.getOpcode() == AMDGPU::G_ICMP; in getInstrMapping()
4253 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID && in getInstrMapping()
4254 Op2Bank == AMDGPU::SGPRRegBankID && in getInstrMapping()
4255 Op3Bank == AMDGPU::SGPRRegBankID && in getInstrMapping()
4258 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; in getInstrMapping()
4259 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; in getInstrMapping()
4265 OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, ResultSize); in getInstrMapping()
4267 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, Size); in getInstrMapping()
4268 OpdsMapping[3] = AMDGPU::getValueMapping(SrcBank, Size); in getInstrMapping()
4271 case AMDGPU::G_EXTRACT_VECTOR_ELT: { in getInstrMapping()
4280 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(OutputBankID, DstSize); in getInstrMapping()
4281 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, SrcSize); in getInstrMapping()
4284 OpdsMapping[2] = AMDGPU::getValueMapping(IdxBank, IdxSize); in getInstrMapping()
4287 case AMDGPU::G_INSERT_VECTOR_ELT: { in getInstrMapping()
4289 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; in getInstrMapping()
4297 OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize); in getInstrMapping()
4298 OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize); in getInstrMapping()
4302 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) { in getInstrMapping()
4303 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID, in getInstrMapping()
4307 OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize); in getInstrMapping()
4311 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBankID, IdxSize); in getInstrMapping()
4314 case AMDGPU::G_UNMERGE_VALUES: { in getInstrMapping()
4321 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, Size); in getInstrMapping()
4325 case AMDGPU::G_AMDGPU_BUFFER_LOAD: in getInstrMapping()
4326 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: in getInstrMapping()
4327 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: in getInstrMapping()
4328 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: in getInstrMapping()
4329 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: in getInstrMapping()
4330 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE: in getInstrMapping()
4331 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE: in getInstrMapping()
4332 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE: in getInstrMapping()
4333 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE: in getInstrMapping()
4334 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE: in getInstrMapping()
4335 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT: in getInstrMapping()
4336 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE: in getInstrMapping()
4337 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16: in getInstrMapping()
4338 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT: in getInstrMapping()
4339 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16: in getInstrMapping()
4340 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: in getInstrMapping()
4341 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: in getInstrMapping()
4342 case AMDGPU::G_AMDGPU_BUFFER_STORE: in getInstrMapping()
4343 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: in getInstrMapping()
4344 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: in getInstrMapping()
4345 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: in getInstrMapping()
4346 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: { in getInstrMapping()
4365 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP: in getInstrMapping()
4366 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD: in getInstrMapping()
4367 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB: in getInstrMapping()
4368 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN: in getInstrMapping()
4369 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN: in getInstrMapping()
4370 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX: in getInstrMapping()
4371 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX: in getInstrMapping()
4372 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND: in getInstrMapping()
4373 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR: in getInstrMapping()
4374 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR: in getInstrMapping()
4375 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC: in getInstrMapping()
4376 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: in getInstrMapping()
4377 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: in getInstrMapping()
4378 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN: in getInstrMapping()
4379 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: { in getInstrMapping()
4402 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: { in getInstrMapping()
4428 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: in getInstrMapping()
4429 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE: in getInstrMapping()
4430 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE: in getInstrMapping()
4431 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT: in getInstrMapping()
4432 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: { in getInstrMapping()
4445 OpdsMapping[0] = AMDGPU::getValueMapping(ResultBank, Size0); in getInstrMapping()
4448 case AMDGPU::G_INTRINSIC: in getInstrMapping()
4449 case AMDGPU::G_INTRINSIC_CONVERGENT: { in getInstrMapping()
4585 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4591 = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size); in getInstrMapping()
4595 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
4601 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Dst0Size); in getInstrMapping()
4602 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Dst1Size); in getInstrMapping()
4605 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); in getInstrMapping()
4606 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); in getInstrMapping()
4615 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize); in getInstrMapping()
4616 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size); in getInstrMapping()
4617 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src1Size); in getInstrMapping()
4624 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); in getInstrMapping()
4626 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize); in getInstrMapping()
4627 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize); in getInstrMapping()
4634 unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4635 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize); in getInstrMapping()
4641 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); in getInstrMapping()
4642 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); in getInstrMapping()
4649 unsigned SrcBank = getRegBankID(SrcReg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4652 unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4653 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); in getInstrMapping()
4657 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, SrcSize); in getInstrMapping()
4658 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize); in getInstrMapping()
4659 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); in getInstrMapping()
4664 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4665 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
4666 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4672 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4673 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4674 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4682 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4683 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4684 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4685 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4776 unsigned M0Bank = getRegBankID(M0Reg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4779 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); in getInstrMapping()
4781 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4785 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32); in getInstrMapping()
4795 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); in getInstrMapping()
4796 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4797 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4798 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4804 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); in getInstrMapping()
4805 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize); in getInstrMapping()
4812 unsigned MaskBank = getRegBankID(MaskReg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4813 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
4814 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, MaskSize); in getInstrMapping()
4821 unsigned MaskBank = getRegBankID(MaskReg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4822 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, MaskSize); in getInstrMapping()
4823 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, MaskSize); in getInstrMapping()
4829 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); in getInstrMapping()
4832 isSALUMapping(MI) ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; in getInstrMapping()
4833 OpdsMapping[2] = AMDGPU::getValueMapping(regBankID, OpSize); in getInstrMapping()
4838 unsigned MaskBank = getRegBankID(MaskReg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4839 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 64); in getInstrMapping()
4840 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, 32); in getInstrMapping()
4844 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: in getInstrMapping()
4845 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16: in getInstrMapping()
4846 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET: in getInstrMapping()
4847 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: in getInstrMapping()
4848 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: { in getInstrMapping()
4849 auto IntrID = AMDGPU::getIntrinsicID(MI); in getInstrMapping()
4850 const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(IntrID); in getInstrMapping()
4858 case AMDGPU::G_AMDGPU_INTRIN_BVH_INTERSECT_RAY: { in getInstrMapping()
4860 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 128); in getInstrMapping()
4867 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4872 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); in getInstrMapping()
4877 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: in getInstrMapping()
4878 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: { in getInstrMapping()
4887 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4911 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); in getInstrMapping()
4913 AMDGPU::SGPRRegBankID); in getInstrMapping()
4914 OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32); in getInstrMapping()
4915 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4921 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); in getInstrMapping()
4926 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4927 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4931 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4932 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4933 OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4934 OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4937 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4938 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4939 OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4940 OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
4947 AMDGPU::SGPRRegBankID); in getInstrMapping()
4948 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); in getInstrMapping()
4954 AMDGPU::SGPRRegBankID); in getInstrMapping()
4955 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); in getInstrMapping()
4961 getRegBankID(MI.getOperand(1).getReg(), MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
4962 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32); in getInstrMapping()
4967 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
4972 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
4973 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize); in getInstrMapping()
4974 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize); in getInstrMapping()
4978 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
4983 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); in getInstrMapping()
5053 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); in getInstrMapping()
5059 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
5063 AMDGPU::SGPRRegBankID); in getInstrMapping()
5064 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); in getInstrMapping()
5072 AMDGPU::SGPRRegBankID); in getInstrMapping()
5073 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32); in getInstrMapping()
5084 unsigned M0Bank = getRegBankID(M0Reg, MRI, AMDGPU::SGPRRegBankID); in getInstrMapping()
5087 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); in getInstrMapping()
5089 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); in getInstrMapping()
5093 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32); in getInstrMapping()
5129 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, ResultSize); in getInstrMapping()
5137 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, ResultSize); in getInstrMapping()
5152 case AMDGPU::G_SELECT: { in getInstrMapping()
5155 AMDGPU::SGPRRegBankID); in getInstrMapping()
5157 AMDGPU::SGPRRegBankID); in getInstrMapping()
5158 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID && in getInstrMapping()
5159 Op3Bank == AMDGPU::SGPRRegBankID; in getInstrMapping()
5162 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; in getInstrMapping()
5165 if (CondBank == AMDGPU::SGPRRegBankID) in getInstrMapping()
5166 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; in getInstrMapping()
5167 else if (CondBank == AMDGPU::VGPRRegBankID) in getInstrMapping()
5168 CondBank = AMDGPU::VCCRegBankID; in getInstrMapping()
5170 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ? in getInstrMapping()
5171 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; in getInstrMapping()
5173 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID); in getInstrMapping()
5177 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); in getInstrMapping()
5178 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1); in getInstrMapping()
5179 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); in getInstrMapping()
5180 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); in getInstrMapping()
5182 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, Size); in getInstrMapping()
5183 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1); in getInstrMapping()
5184 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, Size); in getInstrMapping()
5185 OpdsMapping[3] = AMDGPU::getValueMapping(Bank, Size); in getInstrMapping()
5191 case AMDGPU::G_SI_CALL: { in getInstrMapping()
5192 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 64); in getInstrMapping()
5203 OpdsMapping[I] = AMDGPU::getValueMapping(OpBank, Size); in getInstrMapping()
5208 case AMDGPU::G_LOAD: in getInstrMapping()
5209 case AMDGPU::G_ZEXTLOAD: in getInstrMapping()
5210 case AMDGPU::G_SEXTLOAD: in getInstrMapping()
5213 case AMDGPU::G_ATOMICRMW_XCHG: in getInstrMapping()
5214 case AMDGPU::G_ATOMICRMW_ADD: in getInstrMapping()
5215 case AMDGPU::G_ATOMICRMW_SUB: in getInstrMapping()
5216 case AMDGPU::G_ATOMICRMW_AND: in getInstrMapping()
5217 case AMDGPU::G_ATOMICRMW_OR: in getInstrMapping()
5218 case AMDGPU::G_ATOMICRMW_XOR: in getInstrMapping()
5219 case AMDGPU::G_ATOMICRMW_MAX: in getInstrMapping()
5220 case AMDGPU::G_ATOMICRMW_MIN: in getInstrMapping()
5221 case AMDGPU::G_ATOMICRMW_UMAX: in getInstrMapping()
5222 case AMDGPU::G_ATOMICRMW_UMIN: in getInstrMapping()
5223 case AMDGPU::G_ATOMICRMW_FADD: in getInstrMapping()
5224 case AMDGPU::G_ATOMICRMW_FMIN: in getInstrMapping()
5225 case AMDGPU::G_ATOMICRMW_FMAX: in getInstrMapping()
5226 case AMDGPU::G_ATOMICRMW_UINC_WRAP: in getInstrMapping()
5227 case AMDGPU::G_ATOMICRMW_UDEC_WRAP: in getInstrMapping()
5228 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: { in getInstrMapping()
5234 case AMDGPU::G_ATOMIC_CMPXCHG: { in getInstrMapping()
5241 case AMDGPU::G_BRCOND: { in getInstrMapping()
5243 AMDGPU::SGPRRegBankID); in getInstrMapping()
5245 if (Bank != AMDGPU::SGPRRegBankID) in getInstrMapping()
5246 Bank = AMDGPU::VCCRegBankID; in getInstrMapping()
5248 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1); in getInstrMapping()
5251 case AMDGPU::G_FPTRUNC_ROUND_UPWARD: in getInstrMapping()
5252 case AMDGPU::G_FPTRUNC_ROUND_DOWNWARD: in getInstrMapping()
5254 case AMDGPU::G_PREFETCH: in getInstrMapping()