Lines Matching refs:Root
3650 AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const { in selectVCSRC()
3652 [=](MachineInstrBuilder &MIB) { MIB.add(Root); } in selectVCSRC()
3658 AMDGPUInstructionSelector::selectVOP3ModsImpl(MachineOperand &Root, in selectVOP3ModsImpl() argument
3661 Register Src = Root.getReg(); in selectVOP3ModsImpl()
3692 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt, in copyToVGPRIfSrcFolded() argument
3700 Register VGPRSrc = MRI->cloneVirtualRegister(Root.getReg()); in copyToVGPRIfSrcFolded()
3714 AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const { in selectVSRC0()
3716 [=](MachineInstrBuilder &MIB) { MIB.add(Root); } in selectVSRC0()
3721 AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const { in selectVOP3Mods0()
3724 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3Mods0()
3728 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3Mods0()
3737 AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const { in selectVOP3BMods0()
3740 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, in selectVOP3BMods0()
3746 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3BMods0()
3755 AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const { in selectVOP3OMods()
3757 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }, in selectVOP3OMods()
3764 AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const { in selectVOP3Mods()
3767 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3Mods()
3771 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3Mods()
3779 MachineOperand &Root) const { in selectVOP3ModsNonCanonicalizing()
3782 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, /*IsCanonicalizing=*/false); in selectVOP3ModsNonCanonicalizing()
3786 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3ModsNonCanonicalizing()
3793 AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const { in selectVOP3BMods()
3796 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, /*IsCanonicalizing=*/true, in selectVOP3BMods()
3801 MIB.addReg(copyToVGPRIfSrcFolded(Src, Mods, Root, MIB)); in selectVOP3BMods()
3808 AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const { in selectVOP3NoMods()
3809 Register Reg = Root.getReg(); in selectVOP3NoMods()
3845 AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const { in selectVOP3PMods()
3847 = Root.getParent()->getParent()->getParent()->getRegInfo(); in selectVOP3PMods()
3851 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI); in selectVOP3PMods()
3860 AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const { in selectVOP3PModsDOT()
3862 = Root.getParent()->getParent()->getParent()->getRegInfo(); in selectVOP3PModsDOT()
3866 std::tie(Src, Mods) = selectVOP3PModsImpl(Root.getReg(), MRI, true); in selectVOP3PModsDOT()
3875 AMDGPUInstructionSelector::selectVOP3PModsNeg(MachineOperand &Root) const { in selectVOP3PModsNeg()
3879 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) && in selectVOP3PModsNeg()
3882 if (Root.getImm() == -1) in selectVOP3PModsNeg()
3891 MachineOperand &Root) const { in selectWMMAOpSelVOP3PMods()
3892 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) && in selectWMMAOpSelVOP3PMods()
3895 if (Root.getImm() != 0) in selectWMMAOpSelVOP3PMods()
3962 AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const { in selectWMMAModsF32NegAbs()
3963 Register Src = Root.getReg(); in selectWMMAModsF32NegAbs()
3983 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, Root.getParent(), in selectWMMAModsF32NegAbs()
3993 AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const { in selectWMMAModsF16Neg()
3994 Register Src = Root.getReg(); in selectWMMAModsF16Neg()
4010 Src = buildRegSequence(EltsV2F16, Root.getParent(), *MRI); in selectWMMAModsF16Neg()
4019 AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const { in selectWMMAModsF16NegAbs()
4020 Register Src = Root.getReg(); in selectWMMAModsF16NegAbs()
4041 MachineIRBuilder B(*Root.getParent()); in selectWMMAModsF16NegAbs()
4042 selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, Root.getParent(), in selectWMMAModsF16NegAbs()
4052 AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const { in selectWMMAVISrc()
4054 if (mi_match(Root.getReg(), *MRI, m_GFCstOrSplat(FPValReg))) { in selectWMMAVISrc()
4066 if (mi_match(Root.getReg(), *MRI, m_ICstOrSplat(ICst))) { in selectWMMAVISrc()
4077 AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const { in selectSWMMACIndex8()
4079 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg(); in selectSWMMACIndex8()
4098 AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const { in selectSWMMACIndex16()
4101 getDefIgnoringCopies(Root.getReg(), *MRI)->getOperand(0).getReg(); in selectSWMMACIndex16()
4120 AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const { in selectVOP3OpSelMods()
4123 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3OpSelMods()
4133 AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const { in selectVINTERPMods()
4136 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, in selectVINTERPMods()
4144 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true)); in selectVINTERPMods()
4151 AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const { in selectVINTERPModsHi()
4154 std::tie(Src, Mods) = selectVOP3ModsImpl(Root, in selectVINTERPModsHi()
4162 copyToVGPRIfSrcFolded(Src, Mods, Root, MIB, /* ForceVGPR */ true)); in selectVINTERPModsHi()
4168 bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root, in selectSmrdOffset() argument
4172 MachineInstr *MI = Root.getParent(); in selectSmrdOffset()
4250 AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const { in selectSmrdImm()
4253 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, &Offset)) in selectSmrdImm()
4261 AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const { in selectSmrdImm32()
4263 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); in selectSmrdImm32()
4282 AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const { in selectSmrdSgpr()
4284 if (!selectSmrdOffset(Root, Base, &SOffset, /* Offset= */ nullptr)) in selectSmrdSgpr()
4292 AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const { in selectSmrdSgprImm()
4295 if (!selectSmrdOffset(Root, Base, &SOffset, &Offset)) in selectSmrdSgprImm()
4304 AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root, in selectFlatOffsetImpl() argument
4306 MachineInstr *MI = Root.getParent(); in selectFlatOffsetImpl()
4308 auto Default = std::pair(Root.getReg(), 0); in selectFlatOffsetImpl()
4316 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectFlatOffsetImpl()
4319 !isFlatScratchBaseLegal(Root.getReg()))) in selectFlatOffsetImpl()
4330 AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const { in selectFlatOffset()
4331 auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FLAT); in selectFlatOffset()
4340 AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const { in selectGlobalOffset()
4341 auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FlatGlobal); in selectGlobalOffset()
4350 AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const { in selectScratchOffset()
4351 auto PtrWithOffset = selectFlatOffsetImpl(Root, SIInstrFlags::FlatScratch); in selectScratchOffset()
4361 AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const { in selectGlobalSAddr()
4362 Register Addr = Root.getReg(); in selectGlobalSAddr()
4390 MachineInstr *MI = Root.getParent(); in selectGlobalSAddr()
4457 MachineInstr *MI = Root.getParent(); in selectGlobalSAddr()
4472 AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const { in selectScratchSAddr()
4473 Register Addr = Root.getReg(); in selectScratchSAddr()
4509 MachineInstr &I = *Root.getParent(); in selectScratchSAddr()
4549 AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const { in selectScratchSVAddr()
4550 Register Addr = Root.getReg(); in selectScratchSVAddr()
4608 AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const { in selectMUBUFScratchOffen()
4609 MachineInstr *MI = Root.getParent(); in selectMUBUFScratchOffen()
4615 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset)) && in selectMUBUFScratchOffen()
4647 Register VAddr = Root.getReg(); in selectMUBUFScratchOffen()
4648 if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) { in selectMUBUFScratchOffen()
4823 MachineOperand &Root) const { in selectMUBUFScratchOffset()
4824 Register Reg = Root.getReg(); in selectMUBUFScratchOffset()
4869 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) || in selectMUBUFScratchOffset()
4885 AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const { in selectDS1Addr1OffsetImpl()
4886 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDS1Addr1OffsetImpl()
4888 return std::pair(Root.getReg(), 0); in selectDS1Addr1OffsetImpl()
4895 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDS1Addr1OffsetImpl()
4906 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDS1Addr1OffsetImpl()
4911 return std::pair(Root.getReg(), 0); in selectDS1Addr1OffsetImpl()
4915 AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const { in selectDS1Addr1Offset()
4918 std::tie(Reg, Offset) = selectDS1Addr1OffsetImpl(Root); in selectDS1Addr1Offset()
4926 AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const { in selectDS64Bit4ByteAligned()
4927 return selectDSReadWrite2(Root, 4); in selectDS64Bit4ByteAligned()
4931 AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const { in selectDS128Bit8ByteAligned()
4932 return selectDSReadWrite2(Root, 8); in selectDS128Bit8ByteAligned()
4936 AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root, in selectDSReadWrite2() argument
4940 std::tie(Reg, Offset) = selectDSReadWrite2Impl(Root, Size); in selectDSReadWrite2()
4949 AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root, in selectDSReadWrite2Impl() argument
4951 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); in selectDSReadWrite2Impl()
4953 return std::pair(Root.getReg(), 0); in selectDSReadWrite2Impl()
4960 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); in selectDSReadWrite2Impl()
4972 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { in selectDSReadWrite2Impl()
4977 return std::pair(Root.getReg(), 0); in selectDSReadWrite2Impl()
4986 Register Root, const MachineRegisterInfo &MRI) const { in getPtrBaseWithConstantOffset() argument
4987 MachineInstr *RootI = getDefIgnoringCopies(Root, MRI); in getPtrBaseWithConstantOffset()
4989 return {Root, 0}; in getPtrBaseWithConstantOffset()
4995 return {Root, 0}; in getPtrBaseWithConstantOffset()
5125 MachineOperand &Root, Register &VAddr, Register &RSrcReg, in selectMUBUFAddr64Impl() argument
5132 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg()); in selectMUBUFAddr64Impl()
5169 MachineIRBuilder B(*Root.getParent()); in selectMUBUFAddr64Impl()
5176 MachineOperand &Root, Register &RSrcReg, Register &SOffset, in selectMUBUFOffsetImpl() argument
5183 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg()); in selectMUBUFOffsetImpl()
5193 MachineIRBuilder B(*Root.getParent()); in selectMUBUFOffsetImpl()
5201 AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const { in selectMUBUFAddr64()
5207 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset)) in selectMUBUFAddr64()
5237 AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const { in selectMUBUFOffset()
5242 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset)) in selectMUBUFOffset()
5265 AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const { in selectBUFSOffset()
5267 Register SOffset = Root.getReg(); in selectBUFSOffset()
5286 AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const { in selectSMRDBufferImm()
5287 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm()
5300 AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const { in selectSMRDBufferImm32()
5303 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Root.getReg(), *MRI); in selectSMRDBufferImm32()
5316 AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const { in selectSMRDBufferSgprImm()
5322 *MRI, Root.getReg(), KB, /*CheckNUW*/ true); in selectSMRDBufferSgprImm()
5390 AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root, in selectVOP3PMadMixModsImpl() argument
5396 std::tie(Src, Mods) = selectVOP3ModsImpl(Root); in selectVOP3PMadMixModsImpl()
5456 MachineOperand &Root) const { in selectVOP3PMadMixModsExt()
5460 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched); in selectVOP3PMadMixModsExt()
5471 AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const { in selectVOP3PMadMixMods()
5475 std::tie(Src, Mods) = selectVOP3PMadMixModsImpl(Root, Matched); in selectVOP3PMadMixMods()