Lines Matching refs:Paired
214 const CombineInfo &Paired);
216 CombineInfo &Paired, bool Modify = false);
218 const CombineInfo &Paired);
219 unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
221 const CombineInfo &Paired);
224 const CombineInfo &Paired) const;
227 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
229 void copyToDestRegs(CombineInfo &CI, CombineInfo &Paired,
232 Register copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
239 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
245 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
248 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
251 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
254 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
257 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
260 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
263 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
266 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
269 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
293 const CombineInfo &Paired);
296 const CombineInfo &Paired);
646 const CombineInfo &Paired) { in getCommonInstClass() argument
647 assert(CI.InstClass == Paired.InstClass); in getCommonInstClass()
650 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I)) in getCommonInstClass()
901 const CombineInfo &Paired) { in combineKnownAdjacentMMOs() argument
903 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); in combineKnownAdjacentMMOs()
909 if (Paired < CI) in combineKnownAdjacentMMOs()
923 const CombineInfo &Paired) { in dmasksCanBeCombined() argument
940 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx) in dmasksCanBeCombined()
943 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
948 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
949 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
998 CombineInfo &Paired, in offsetsCanBeCombined() argument
1004 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
1008 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
1018 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI); in offsetsCanBeCombined()
1031 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
1036 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
1043 EltOffset1 + Paired.Width != EltOffset0) in offsetsCanBeCombined()
1045 if (CI.CPol != Paired.CPol) in offsetsCanBeCombined()
1054 if (CI.Width != Paired.Width && in offsetsCanBeCombined()
1055 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset)) in offsetsCanBeCombined()
1067 Paired.Offset = EltOffset1 / 64; in offsetsCanBeCombined()
1077 Paired.Offset = EltOffset1; in offsetsCanBeCombined()
1098 Paired.Offset = (EltOffset1 - BaseOff) / 64; in offsetsCanBeCombined()
1112 Paired.Offset = EltOffset1 - BaseOff; in offsetsCanBeCombined()
1122 const CombineInfo &Paired) { in widthsFit() argument
1123 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
1167 CombineInfo &Paired) { in checkAndPrepareMerge() argument
1170 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN) in checkAndPrepareMerge()
1172 assert(CI.InstClass == Paired.InstClass); in checkAndPrepareMerge()
1175 getInstSubclass(Paired.I->getOpcode(), *TII)) in checkAndPrepareMerge()
1181 if (!dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
1184 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired)) in checkAndPrepareMerge()
1193 addDefsUsesToList(*Paired.I, RegDefs, RegUses); in checkAndPrepareMerge()
1194 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) { in checkAndPrepareMerge()
1195 if (!canSwapInstructions(RegDefs, RegUses, *Paired.I, *MBBI)) in checkAndPrepareMerge()
1202 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) { in checkAndPrepareMerge()
1206 Where = &Paired; in checkAndPrepareMerge()
1214 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1221 CombineInfo &CI, CombineInfo &Paired, in copyToDestRegs() argument
1227 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired); in copyToDestRegs()
1232 auto *Dest1 = TII->getNamedOperand(*Paired.I, OpName); in copyToDestRegs()
1251 SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired, in copyFromSrcRegs() argument
1257 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired); in copyFromSrcRegs()
1260 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in copyFromSrcRegs()
1264 const auto *Src1 = TII->getNamedOperand(*Paired.I, OpName); in copyFromSrcRegs()
1290 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1298 unsigned NewOffset0 = std::min(CI.Offset, Paired.Offset); in mergeRead2Pair()
1299 unsigned NewOffset1 = std::max(CI.Offset, Paired.Offset); in mergeRead2Pair()
1308 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1337 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1339 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdst, DestReg); in mergeRead2Pair()
1342 Paired.I->eraseFromParent(); in mergeRead2Pair()
1365 CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1376 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1379 unsigned NewOffset1 = Paired.Offset; in mergeWrite2Pair()
1421 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1424 Paired.I->eraseFromParent(); in mergeWrite2Pair()
1431 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1435 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1437 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1440 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1455 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1457 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeImagePair()
1459 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata, DestReg); in mergeImagePair()
1462 Paired.I->eraseFromParent(); in mergeImagePair()
1467 CombineInfo &CI, CombineInfo &Paired, in mergeSMemLoadImmPair() argument
1471 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSMemLoadImmPair()
1473 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSMemLoadImmPair()
1476 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSMemLoadImmPair()
1481 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSMemLoadImmPair()
1489 New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeSMemLoadImmPair()
1491 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::sdst, DestReg); in mergeSMemLoadImmPair()
1494 Paired.I->eraseFromParent(); in mergeSMemLoadImmPair()
1499 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1504 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1506 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1510 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1522 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1530 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferLoadPair()
1532 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata, DestReg); in mergeBufferLoadPair()
1535 Paired.I->eraseFromParent(); in mergeBufferLoadPair()
1540 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1545 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1547 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1551 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1561 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1566 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1575 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferLoadPair()
1577 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata, DestReg); in mergeTBufferLoadPair()
1580 Paired.I->eraseFromParent(); in mergeTBufferLoadPair()
1585 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1590 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1593 copyFromSrcRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1604 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1609 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1614 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1618 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferStorePair()
1621 Paired.I->eraseFromParent(); in mergeTBufferStorePair()
1626 CombineInfo &CI, CombineInfo &Paired, in mergeFlatLoadPair() argument
1631 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatLoadPair()
1633 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatLoadPair()
1643 .addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatLoadPair()
1645 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatLoadPair()
1647 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdst, DestReg); in mergeFlatLoadPair()
1650 Paired.I->eraseFromParent(); in mergeFlatLoadPair()
1655 CombineInfo &CI, CombineInfo &Paired, in mergeFlatStorePair() argument
1660 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatStorePair()
1663 copyFromSrcRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata); in mergeFlatStorePair()
1673 MIB.addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatStorePair()
1675 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatStorePair()
1678 Paired.I->eraseFromParent(); in mergeFlatStorePair()
1683 const CombineInfo &Paired) { in getNewOpcode() argument
1684 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1686 switch (getCommonInstClass(CI, Paired)) { in getNewOpcode()
1815 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) && in getNewOpcode()
1823 const CombineInfo &Paired) { in getSubRegIdxs() argument
1825 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == in getSubRegIdxs()
1826 CI.Width + Paired.Width)) && in getSubRegIdxs()
1841 assert(Paired.Width >= 1 && Paired.Width <= 4); in getSubRegIdxs()
1843 if (Paired < CI) { in getSubRegIdxs()
1844 Idx1 = Idxs[0][Paired.Width - 1]; in getSubRegIdxs()
1845 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1848 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1856 const CombineInfo &Paired) const { in getTargetRegisterClass()
1859 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1875 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1882 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1887 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1890 copyFromSrcRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1904 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1909 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1912 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferStorePair()
1915 Paired.I->eraseFromParent(); in mergeBufferStorePair()
2389 CombineInfo &Paired = *Second; in optimizeInstsWithSameBaseAddr() local
2391 CombineInfo *Where = checkAndPrepareMerge(CI, Paired); in optimizeInstsWithSameBaseAddr()
2399 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2407 NewMI = mergeRead2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2410 NewMI = mergeWrite2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2415 NewMI = mergeSMemLoadImmPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2416 OptimizeListAgain |= CI.Width + Paired.Width < 8; in optimizeInstsWithSameBaseAddr()
2419 NewMI = mergeBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2420 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2423 NewMI = mergeBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2424 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2427 NewMI = mergeImagePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2428 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2431 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2432 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2435 NewMI = mergeTBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2436 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2441 NewMI = mergeFlatLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2442 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2447 NewMI = mergeFlatStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2448 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()