Lines Matching full:ci
125 bool hasSameBaseAddress(const CombineInfo &CI) { in hasSameBaseAddress()
126 if (NumAddresses != CI.NumAddresses) in hasSameBaseAddress()
129 const MachineInstr &MI = *CI.I; in hasSameBaseAddress()
212 static bool dmasksCanBeCombined(const CombineInfo &CI,
215 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
217 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
219 unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
220 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
223 getTargetRegisterClass(const CombineInfo &CI,
227 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
229 void copyToDestRegs(CombineInfo &CI, CombineInfo &Paired,
232 Register copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
239 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
245 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
248 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
251 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
254 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
257 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
260 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
263 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
266 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
269 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
281 bool promoteConstantOffsetToImm(MachineInstr &CI,
284 void addInstToMergeableList(const CombineInfo &CI,
292 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
295 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
645 SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI, in getCommonInstClass() argument
647 assert(CI.InstClass == Paired.InstClass); in getCommonInstClass()
649 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) && in getCommonInstClass()
650 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I)) in getCommonInstClass()
651 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD; in getCommonInstClass()
653 return CI.InstClass; in getCommonInstClass()
897 // Given that \p CI and \p Paired are adjacent memory operations produce a new
900 SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI, in combineKnownAdjacentMMOs() argument
902 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); in combineKnownAdjacentMMOs()
909 if (Paired < CI) in combineKnownAdjacentMMOs()
917 MachineFunction *MF = CI.I->getMF(); in combineKnownAdjacentMMOs()
921 bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, in dmasksCanBeCombined() argument
924 assert(CI.InstClass == MIMG); in dmasksCanBeCombined()
927 const auto *TFEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::tfe); in dmasksCanBeCombined()
928 const auto *LWEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::lwe); in dmasksCanBeCombined()
939 int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op); in dmasksCanBeCombined()
943 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) in dmasksCanBeCombined()
948 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
949 unsigned MinMask = std::min(CI.DMask, Paired.DMask); in dmasksCanBeCombined()
996 bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI, in offsetsCanBeCombined() argument
1000 assert(CI.InstClass != MIMG); in offsetsCanBeCombined()
1004 if (CI.Offset == Paired.Offset) in offsetsCanBeCombined()
1008 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) in offsetsCanBeCombined()
1011 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) { in offsetsCanBeCombined()
1014 llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI); in offsetsCanBeCombined()
1031 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) in offsetsCanBeCombined()
1035 uint32_t EltOffset0 = CI.Offset / CI.EltSize; in offsetsCanBeCombined()
1036 uint32_t EltOffset1 = Paired.Offset / CI.EltSize; in offsetsCanBeCombined()
1037 CI.UseST64 = false; in offsetsCanBeCombined()
1038 CI.BaseOff = 0; in offsetsCanBeCombined()
1041 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) { in offsetsCanBeCombined()
1042 if (EltOffset0 + CI.Width != EltOffset1 && in offsetsCanBeCombined()
1045 if (CI.CPol != Paired.CPol) in offsetsCanBeCombined()
1047 if (CI.InstClass == S_LOAD_IMM || CI.InstClass == S_BUFFER_LOAD_IMM || in offsetsCanBeCombined()
1048 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) { in offsetsCanBeCombined()
1054 if (CI.Width != Paired.Width && in offsetsCanBeCombined()
1055 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset)) in offsetsCanBeCombined()
1066 CI.Offset = EltOffset0 / 64; in offsetsCanBeCombined()
1068 CI.UseST64 = true; in offsetsCanBeCombined()
1076 CI.Offset = EltOffset0; in offsetsCanBeCombined()
1096 CI.BaseOff = BaseOff * CI.EltSize; in offsetsCanBeCombined()
1097 CI.Offset = (EltOffset0 - BaseOff) / 64; in offsetsCanBeCombined()
1099 CI.UseST64 = true; in offsetsCanBeCombined()
1110 CI.BaseOff = BaseOff * CI.EltSize; in offsetsCanBeCombined()
1111 CI.Offset = EltOffset0 - BaseOff; in offsetsCanBeCombined()
1121 const CombineInfo &CI, in widthsFit() argument
1123 const unsigned Width = (CI.Width + Paired.Width); in widthsFit()
1124 switch (CI.InstClass) { in widthsFit()
1163 /// This function assumes that CI comes before Paired in a basic block. Return
1166 SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI, in checkAndPrepareMerge() argument
1168 // If another instruction has already been merged into CI, it may now be a in checkAndPrepareMerge()
1170 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN) in checkAndPrepareMerge()
1172 assert(CI.InstClass == Paired.InstClass); in checkAndPrepareMerge()
1174 if (getInstSubclass(CI.I->getOpcode(), *TII) != in checkAndPrepareMerge()
1180 if (CI.InstClass == MIMG) { in checkAndPrepareMerge()
1181 if (!dmasksCanBeCombined(CI, *TII, Paired)) in checkAndPrepareMerge()
1184 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired)) in checkAndPrepareMerge()
1191 if (CI.I->mayLoad()) { in checkAndPrepareMerge()
1192 // Try to hoist Paired up to CI. in checkAndPrepareMerge()
1194 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) { in checkAndPrepareMerge()
1198 Where = &CI; in checkAndPrepareMerge()
1200 // Try to sink CI down to Paired. in checkAndPrepareMerge()
1201 addDefsUsesToList(*CI.I, RegDefs, RegUses); in checkAndPrepareMerge()
1202 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) { in checkAndPrepareMerge()
1203 if (!canSwapInstructions(RegDefs, RegUses, *CI.I, *MBBI)) in checkAndPrepareMerge()
1213 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) in checkAndPrepareMerge()
1214 offsetsCanBeCombined(CI, *STM, Paired, true); in checkAndPrepareMerge()
1218 // Copy the merged load result from DestReg to the original dest regs of CI and
1221 CombineInfo &CI, CombineInfo &Paired, in copyToDestRegs() argument
1224 MachineBasicBlock *MBB = CI.I->getParent(); in copyToDestRegs()
1225 DebugLoc DL = CI.I->getDebugLoc(); in copyToDestRegs()
1227 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired); in copyToDestRegs()
1231 auto *Dest0 = TII->getNamedOperand(*CI.I, OpName); in copyToDestRegs()
1249 // original source regs of CI and Paired into it.
1251 SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired, in copyFromSrcRegs() argument
1254 MachineBasicBlock *MBB = CI.I->getParent(); in copyFromSrcRegs()
1255 DebugLoc DL = CI.I->getDebugLoc(); in copyFromSrcRegs()
1257 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired); in copyFromSrcRegs()
1260 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in copyFromSrcRegs()
1263 const auto *Src0 = TII->getNamedOperand(*CI.I, OpName); in copyFromSrcRegs()
1290 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, in mergeRead2Pair() argument
1292 MachineBasicBlock *MBB = CI.I->getParent(); in mergeRead2Pair()
1296 const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); in mergeRead2Pair()
1298 unsigned NewOffset0 = std::min(CI.Offset, Paired.Offset); in mergeRead2Pair()
1299 unsigned NewOffset1 = std::max(CI.Offset, Paired.Offset); in mergeRead2Pair()
1301 CI.UseST64 ? read2ST64Opcode(CI.EltSize) : read2Opcode(CI.EltSize); in mergeRead2Pair()
1308 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeRead2Pair()
1311 DebugLoc DL = CI.I->getDebugLoc(); in mergeRead2Pair()
1316 if (CI.BaseOff) { in mergeRead2Pair()
1319 .addImm(CI.BaseOff); in mergeRead2Pair()
1337 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeRead2Pair()
1339 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdst, DestReg); in mergeRead2Pair()
1341 CI.I->eraseFromParent(); in mergeRead2Pair()
1365 CombineInfo &CI, CombineInfo &Paired, in mergeWrite2Pair() argument
1367 MachineBasicBlock *MBB = CI.I->getParent(); in mergeWrite2Pair()
1372 TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); in mergeWrite2Pair()
1374 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0); in mergeWrite2Pair()
1378 unsigned NewOffset0 = CI.Offset; in mergeWrite2Pair()
1381 CI.UseST64 ? write2ST64Opcode(CI.EltSize) : write2Opcode(CI.EltSize); in mergeWrite2Pair()
1393 DebugLoc DL = CI.I->getDebugLoc(); in mergeWrite2Pair()
1398 if (CI.BaseOff) { in mergeWrite2Pair()
1401 .addImm(CI.BaseOff); in mergeWrite2Pair()
1421 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); in mergeWrite2Pair()
1423 CI.I->eraseFromParent(); in mergeWrite2Pair()
1431 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, in mergeImagePair() argument
1433 MachineBasicBlock *MBB = CI.I->getParent(); in mergeImagePair()
1434 DebugLoc DL = CI.I->getDebugLoc(); in mergeImagePair()
1435 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeImagePair()
1437 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeImagePair()
1440 unsigned MergedDMask = CI.DMask | Paired.DMask; in mergeImagePair()
1442 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask); in mergeImagePair()
1445 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) { in mergeImagePair()
1449 MIB.add((*CI.I).getOperand(I)); in mergeImagePair()
1455 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeImagePair()
1457 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeImagePair()
1459 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata, DestReg); in mergeImagePair()
1461 CI.I->eraseFromParent(); in mergeImagePair()
1467 CombineInfo &CI, CombineInfo &Paired, in mergeSMemLoadImmPair() argument
1469 MachineBasicBlock *MBB = CI.I->getParent(); in mergeSMemLoadImmPair()
1470 DebugLoc DL = CI.I->getDebugLoc(); in mergeSMemLoadImmPair()
1471 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeSMemLoadImmPair()
1473 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeSMemLoadImmPair()
1476 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeSMemLoadImmPair()
1481 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeSMemLoadImmPair()
1485 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase)); in mergeSMemLoadImmPair()
1486 if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) in mergeSMemLoadImmPair()
1487 New.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)); in mergeSMemLoadImmPair()
1489 New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeSMemLoadImmPair()
1491 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::sdst, DestReg); in mergeSMemLoadImmPair()
1493 CI.I->eraseFromParent(); in mergeSMemLoadImmPair()
1499 CombineInfo &CI, CombineInfo &Paired, in mergeBufferLoadPair() argument
1501 MachineBasicBlock *MBB = CI.I->getParent(); in mergeBufferLoadPair()
1502 DebugLoc DL = CI.I->getDebugLoc(); in mergeBufferLoadPair()
1504 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferLoadPair()
1506 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeBufferLoadPair()
1510 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeBufferLoadPair()
1517 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeBufferLoadPair()
1522 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferLoadPair()
1525 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeBufferLoadPair()
1526 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeBufferLoadPair()
1528 .addImm(CI.CPol) // cpol in mergeBufferLoadPair()
1530 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferLoadPair()
1532 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata, DestReg); in mergeBufferLoadPair()
1534 CI.I->eraseFromParent(); in mergeBufferLoadPair()
1540 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferLoadPair() argument
1542 MachineBasicBlock *MBB = CI.I->getParent(); in mergeTBufferLoadPair()
1543 DebugLoc DL = CI.I->getDebugLoc(); in mergeTBufferLoadPair()
1545 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferLoadPair()
1547 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeTBufferLoadPair()
1551 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); in mergeTBufferLoadPair()
1558 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeTBufferLoadPair()
1561 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferLoadPair()
1566 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferLoadPair()
1569 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeTBufferLoadPair()
1570 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeTBufferLoadPair()
1573 .addImm(CI.CPol) // cpol in mergeTBufferLoadPair()
1575 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferLoadPair()
1577 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata, DestReg); in mergeTBufferLoadPair()
1579 CI.I->eraseFromParent(); in mergeTBufferLoadPair()
1585 CombineInfo &CI, CombineInfo &Paired, in mergeTBufferStorePair() argument
1587 MachineBasicBlock *MBB = CI.I->getParent(); in mergeTBufferStorePair()
1588 DebugLoc DL = CI.I->getDebugLoc(); in mergeTBufferStorePair()
1590 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeTBufferStorePair()
1593 copyFromSrcRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata); in mergeTBufferStorePair()
1601 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeTBufferStorePair()
1604 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); in mergeTBufferStorePair()
1609 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeTBufferStorePair()
1612 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeTBufferStorePair()
1613 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeTBufferStorePair()
1614 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeTBufferStorePair()
1616 .addImm(CI.CPol) // cpol in mergeTBufferStorePair()
1618 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeTBufferStorePair()
1620 CI.I->eraseFromParent(); in mergeTBufferStorePair()
1626 CombineInfo &CI, CombineInfo &Paired, in mergeFlatLoadPair() argument
1628 MachineBasicBlock *MBB = CI.I->getParent(); in mergeFlatLoadPair()
1629 DebugLoc DL = CI.I->getDebugLoc(); in mergeFlatLoadPair()
1631 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatLoadPair()
1633 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); in mergeFlatLoadPair()
1638 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr)) in mergeFlatLoadPair()
1642 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)) in mergeFlatLoadPair()
1643 .addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatLoadPair()
1644 .addImm(CI.CPol) in mergeFlatLoadPair()
1645 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatLoadPair()
1647 copyToDestRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdst, DestReg); in mergeFlatLoadPair()
1649 CI.I->eraseFromParent(); in mergeFlatLoadPair()
1655 CombineInfo &CI, CombineInfo &Paired, in mergeFlatStorePair() argument
1657 MachineBasicBlock *MBB = CI.I->getParent(); in mergeFlatStorePair()
1658 DebugLoc DL = CI.I->getDebugLoc(); in mergeFlatStorePair()
1660 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeFlatStorePair()
1663 copyFromSrcRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata); in mergeFlatStorePair()
1666 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)) in mergeFlatStorePair()
1669 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr)) in mergeFlatStorePair()
1673 MIB.addImm(std::min(CI.Offset, Paired.Offset)) in mergeFlatStorePair()
1674 .addImm(CI.CPol) in mergeFlatStorePair()
1675 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeFlatStorePair()
1677 CI.I->eraseFromParent(); in mergeFlatStorePair()
1682 unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI, in getNewOpcode() argument
1684 const unsigned Width = CI.Width + Paired.Width; in getNewOpcode()
1686 switch (getCommonInstClass(CI, Paired)) { in getNewOpcode()
1688 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE); in getNewOpcode()
1690 return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()), in getNewOpcode()
1694 return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()), in getNewOpcode()
1728 const MachineMemOperand *MMO = *CI.I->memoperands_begin(); in getNewOpcode()
1815 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) && in getNewOpcode()
1817 return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width); in getNewOpcode()
1822 SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI, in getSubRegIdxs() argument
1824 assert((CI.InstClass != MIMG || in getSubRegIdxs()
1825 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == in getSubRegIdxs()
1826 CI.Width + Paired.Width)) && in getSubRegIdxs()
1840 assert(CI.Width >= 1 && CI.Width <= 4); in getSubRegIdxs()
1843 if (Paired < CI) { in getSubRegIdxs()
1845 Idx0 = Idxs[Paired.Width][CI.Width - 1]; in getSubRegIdxs()
1847 Idx0 = Idxs[0][CI.Width - 1]; in getSubRegIdxs()
1848 Idx1 = Idxs[CI.Width][Paired.Width - 1]; in getSubRegIdxs()
1855 SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI, in getTargetRegisterClass() argument
1857 if (CI.InstClass == S_BUFFER_LOAD_IMM || in getTargetRegisterClass()
1858 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM || CI.InstClass == S_LOAD_IMM) { in getTargetRegisterClass()
1859 switch (CI.Width + Paired.Width) { in getTargetRegisterClass()
1875 unsigned BitWidth = 32 * (CI.Width + Paired.Width); in getTargetRegisterClass()
1876 return TRI->isAGPRClass(getDataRegClass(*CI.I)) in getTargetRegisterClass()
1882 CombineInfo &CI, CombineInfo &Paired, in mergeBufferStorePair() argument
1884 MachineBasicBlock *MBB = CI.I->getParent(); in mergeBufferStorePair()
1885 DebugLoc DL = CI.I->getDebugLoc(); in mergeBufferStorePair()
1887 const unsigned Opcode = getNewOpcode(CI, Paired); in mergeBufferStorePair()
1890 copyFromSrcRegs(CI, Paired, InsertBefore, AMDGPU::OpName::vdata); in mergeBufferStorePair()
1898 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); in mergeBufferStorePair()
1904 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); in mergeBufferStorePair()
1907 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) in mergeBufferStorePair()
1908 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) in mergeBufferStorePair()
1909 .addImm(std::min(CI.Offset, Paired.Offset)) // offset in mergeBufferStorePair()
1910 .addImm(CI.CPol) // cpol in mergeBufferStorePair()
1912 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired)); in mergeBufferStorePair()
1914 CI.I->eraseFromParent(); in mergeBufferStorePair()
2231 void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI, in addInstToMergeableList() argument
2234 if (AddrList.front().InstClass == CI.InstClass && in addInstToMergeableList()
2235 AddrList.front().IsAGPR == CI.IsAGPR && in addInstToMergeableList()
2236 AddrList.front().hasSameBaseAddress(CI)) { in addInstToMergeableList()
2237 AddrList.emplace_back(CI); in addInstToMergeableList()
2243 MergeableInsts.emplace_back(1, CI); in addInstToMergeableList()
2284 CombineInfo CI; in collectMergeableInsts() local
2285 CI.setMI(MI, *this); in collectMergeableInsts()
2286 CI.Order = Order++; in collectMergeableInsts()
2288 if (!CI.hasMergeableAddress(*MRI)) in collectMergeableInsts()
2291 if (CI.InstClass == DS_WRITE && CI.IsAGPR) { in collectMergeableInsts()
2303 addInstToMergeableList(CI, MergeableInsts); in collectMergeableInsts()
2388 CombineInfo &CI = *First; in optimizeInstsWithSameBaseAddr() local
2391 CombineInfo *Where = checkAndPrepareMerge(CI, Paired); in optimizeInstsWithSameBaseAddr()
2399 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); in optimizeInstsWithSameBaseAddr()
2402 switch (CI.InstClass) { in optimizeInstsWithSameBaseAddr()
2407 NewMI = mergeRead2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2410 NewMI = mergeWrite2Pair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2415 NewMI = mergeSMemLoadImmPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2416 OptimizeListAgain |= CI.Width + Paired.Width < 8; in optimizeInstsWithSameBaseAddr()
2419 NewMI = mergeBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2420 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2423 NewMI = mergeBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2424 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2427 NewMI = mergeImagePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2428 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2431 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2432 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2435 NewMI = mergeTBufferStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2436 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2441 NewMI = mergeFlatLoadPair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2442 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2447 NewMI = mergeFlatStorePair(CI, Paired, Where->I); in optimizeInstsWithSameBaseAddr()
2448 OptimizeListAgain |= CI.Width + Paired.Width < 4; in optimizeInstsWithSameBaseAddr()
2451 CI.setMI(NewMI, *this); in optimizeInstsWithSameBaseAddr()
2452 CI.Order = Where->Order; in optimizeInstsWithSameBaseAddr()