1 //===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
17 //===----------------------------------------------------------------------===//
46 #include "llvm/Support/CommandLine.h"
47 #include "llvm/Support/Debug.h"
48 #include "llvm/Support/ErrorHandling.h"
49 #include "llvm/Support/raw_ostream.h"
59     DisablePacketizer("disable-packetizer", cl::Hidden,
62 static cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden,
67     "hexagon-packetize-volatiles", cl::Hidden, cl::init(true),
68     cl::desc("Allow non-solo packetization of volatile memory references"));
71     EnableGenAllInsnClass("enable-gen-insn", cl::Hidden,
75     DisableVecDblNVStores("disable-vecdbl-nv-stores", cl::Hidden,
76                           cl::desc("Disable vector double new-value-stores"));
125 INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer",
131 INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer",  in INITIALIZE_PASS_DEPENDENCY()
189   for (++I; I != E && I->isBundledWithPred(); ++I)  in moveInstrOut()
199   MachineInstr &SingleI = *BundleIt->getNextNode();  in moveInstrOut()
202   BundleIt->eraseFromParent();  in moveInstrOut()
220     HII->genAllInsnTimingClasses(MF);  in runOnMachineFunction()
230   // Loop over all basic blocks and remove KILL pseudo-instructions  in runOnMachineFunction()
244   // TinyCore with Duplexes: Translate to big-instructions.  in runOnMachineFunction()
246     HII->translateInstrsForDup(MF, true);  in runOnMachineFunction()
252       // Find the first non-boundary starting from the end of the last  in runOnMachineFunction()
255       while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))  in runOnMachineFunction()
260       while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))  in runOnMachineFunction()
273   // TinyCore with Duplexes: Translate to tiny-instructions.  in runOnMachineFunction()
275     HII->translateInstrsForDup(MF, false);  in runOnMachineFunction()
295   auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc());  in tryAllocateResourcesForConstExt()
296   bool Avail = ResourceTracker->canReserveResources(*ExtMI);  in tryAllocateResourcesForConstExt()
298     ResourceTracker->reserveResources(*ExtMI);  in tryAllocateResourcesForConstExt()
306   if (DepReg == HRI->getRARegister())  in isCallDependent()
309   if (HII->isDeallocRet(MI))  in isCallDependent()
310     if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())  in isCallDependent()
313   // Call-like instructions can be packetized with preceding instructions  in isCallDependent()
348 /// Returns true if the instruction modifies a callee-saved register.
351   const MachineFunction &MF = *MI.getParent()->getParent();  in doesModifyCalleeSavedReg()
352   for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)  in doesModifyCalleeSavedReg()
359 // new-value store.
362   // Vector stores can be predicated, and can be new-value stores, but  in isNewifiable()
365     if (HII->isHVXVec(MI) && MI.mayStore())  in isNewifiable()
367     return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0;  in isNewifiable()
369   // If the class is not PredRegs, it could only apply to new-value stores.  in isNewifiable()
370   return HII->mayBeNewStore(MI);  in isNewifiable()
380   int CurOpcode = HII->getDotCurOp(MI);  in promoteToDotCur()
381   MI.setDesc(HII->get(CurOpcode));  in promoteToDotCur()
388     LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););  in cleanUpDotCur()
389     if (HII->isDotCurInst(*BI)) {  in cleanUpDotCur()
394       for (auto &MO : BI->operands())  in cleanUpDotCur()
395         if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())  in cleanUpDotCur()
401   // We did not find a use of the CUR, so de-cur it.  in cleanUpDotCur()
402   MI->setDesc(HII->get(HII->getNonDotCurOp(*MI)));  in cleanUpDotCur()
403   LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump(););  in cleanUpDotCur()
410   if (!HII->isHVXVec(MI))  in canPromoteToDotCur()
412   if (!HII->isHVXVec(*MII))  in canPromoteToDotCur()
416   if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))  in canPromoteToDotCur()
419   if (!HII->mayBeCurLoad(MI))  in canPromoteToDotCur()
423   if (PacketSU->getInstr()->isInlineAsm())  in canPromoteToDotCur()
445     LLVM_DEBUG(dbgs() << "packet has "; BI->dump(););  in canPromoteToDotCur()
446     if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))  in canPromoteToDotCur()
464     NewOpcode = HII->getDotNewPredOp(MI, MBPI);  in promoteToDotNew()
466     NewOpcode = HII->getDotNewOp(MI);  in promoteToDotNew()
467   MI.setDesc(HII->get(NewOpcode));  in promoteToDotNew()
472   int NewOpcode = HII->getDotOldOp(MI);  in demoteToDotOld()
473   MI.setDesc(HII->get(NewOpcode));  in demoteToDotOld()
490   int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);  in useCallersSP()
491   if (HII->isValidOffset(Opc, NewOff, HRI)) {  in useCallersSP()
517   assert(SUI->getInstr() && SUJ->getInstr());  in updateOffset()
518   MachineInstr &MI = *SUI->getInstr();  in updateOffset()
519   MachineInstr &MJ = *SUJ->getInstr();  in updateOffset()
522   if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI))  in updateOffset()
525   if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ))  in updateOffset()
533   for (const auto &PI : SUI->Preds)  in updateOffset()
538   if (!HII->getIncrementValue(MJ, Incr))  in updateOffset()
542   if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI))  in updateOffset()
554   if (!HII->getBaseAndOffsetPosition(MI, BP, OP))  in undoChangedOffset()
569   if (!HII->isPredicated(MI))  in getPredicateSense()
571   if (HII->isPredicatedTrue(MI))  in getPredicateSense()
578   assert(HII->isPostIncrement(MI) && "Not a post increment operation.");  in getPostIncrementOperand()
612   return MI.getOperand(MI.getNumOperands()-1);  in getStoreValueOperand()
637 // 1. If an instruction uses auto-increment, its address register cannot
638 //    be a new-value register. Arch Spec 5.4.2.1
639 // 2. If an instruction uses absolute-set addressing mode, its address
640 //    register cannot be a new-value register. Arch Spec 5.4.2.1.
641 // 3. If an instruction produces a 64-bit result, its registers cannot be used
642 //    as new-value registers. Arch Spec 5.4.2.2.
643 // 4. If the instruction that sets the new-value register is conditional, then
644 //    the instruction that uses the new-value register must also be conditional,
654   if (!HII->mayBeNewStore(MI))  in canPromoteToNewValueStore()
665   const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF);  in canPromoteToNewValueStore()
670   // New-value stores are of class NV (slot 0), dual stores require class ST  in canPromoteToNewValueStore()
673     SUnit *PacketSU = MIToSUnit.find(I)->second;  in canPromoteToNewValueStore()
674     if (PacketSU->getInstr()->mayStore())  in canPromoteToNewValueStore()
680   if (HII->isPostIncrement(MI) &&  in canPromoteToNewValueStore()
685   if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() &&  in canPromoteToNewValueStore()
687     // If source is post_inc, or absolute-set addressing, it can not feed  in canPromoteToNewValueStore()
690     //   memw(r30 + #-1404) = r2.new -> can not be new value store  in canPromoteToNewValueStore()
700   if (HII->isPredicated(PacketMI)) {  in canPromoteToNewValueStore()
701     if (!HII->isPredicated(MI))  in canPromoteToNewValueStore()
715       predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);  in canPromoteToNewValueStore()
722     // Get predicate register used in new-value store instruction.  in canPromoteToNewValueStore()
727       predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);  in canPromoteToNewValueStore()
734     // New-value register producer and user (store) need to satisfy these  in canPromoteToNewValueStore()
737     // 2) If producer of the new-value register is .new predicated then store  in canPromoteToNewValueStore()
740     // 3) Both new-value register producer and user should have same predicate  in canPromoteToNewValueStore()
741     // sense, i.e, either both should be negated or both should be non-negated.  in canPromoteToNewValueStore()
743         HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||  in canPromoteToNewValueStore()
748   // Make sure that other than the new-value register no other store instruction  in canPromoteToNewValueStore()
759     SUnit *TempSU = MIToSUnit.find(I)->second;  in canPromoteToNewValueStore()
760     MachineInstr &TempMI = *TempSU->getInstr();  in canPromoteToNewValueStore()
773       if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI))  in canPromoteToNewValueStore()
777   // Make sure that for non-POST_INC stores:  in canPromoteToNewValueStore()
783   if (!HII->isPostIncrement(MI)) {  in canPromoteToNewValueStore()
784     for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {  in canPromoteToNewValueStore()
793   // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12  in canPromoteToNewValueStore()
801     if (R == DepReg || HRI->isSuperRegister(DepReg, R))  in canPromoteToNewValueStore()
805   // Handle imp-use of super reg case. There is a target independent side  in canPromoteToNewValueStore()
807   // just-in-case. For example, we cannot newify R2 in the following case:  in canPromoteToNewValueStore()
823   if (!HII->mayBeNewStore(MI))  in canPromoteToNewValue()
827   MachineInstr &PacketMI = *PacketSU->getInstr();  in canPromoteToNewValue()
854   if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))  in canPromoteToDotNew()
860   const MachineInstr &PI = *PacketSU->getInstr();  in canPromoteToDotNew()
878   const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF);  in canPromoteToDotNew()
884     return HII->predCanBeUsedAsDotNew(PI, DepReg);  in canPromoteToDotNew()
886   if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))  in canPromoteToDotNew()
891   int NewOpcode = (RC != &Hexagon::PredRegsRegClass) ? HII->getDotNewOp(MI) :  in canPromoteToDotNew()
892     HII->getDotNewPredOp(MI, MBPI);  in canPromoteToDotNew()
893   const MCInstrDesc &D = HII->get(NewOpcode);  in canPromoteToDotNew()
895   bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);  in canPromoteToDotNew()
919 // Anti-dep between c) and b) is irrelevant for this case
922   SUnit *PacketSUDep = MIToSUnit.find(&MI)->second;  in restrictingDepExistInPacket()
926     if (!HII->isPredicated(*I))  in restrictingDepExistInPacket()
930     SUnit *PacketSU = MIToSUnit.find(I)->second;  in restrictingDepExistInPacket()
935     if (PacketSU->isSucc(PacketSUDep)) {  in restrictingDepExistInPacket()
936       for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {  in restrictingDepExistInPacket()
937         auto &Dep = PacketSU->Succs[i];  in restrictingDepExistInPacket()
953   assert(QII->isPredicated(MI) && "Must be predicated instruction");  in getPredicatedRegister()
996     SUnit *PacketSU = MIToSUnit.find(I)->second;  in arePredicatesComplements()
999     if (PacketSU->isSucc(SU)) {  in arePredicatesComplements()
1000       for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {  in arePredicatesComplements()
1001         auto Dep = PacketSU->Succs[i];  in arePredicatesComplements()
1009           // data dep to candidate on the register we care about - c) in the  in arePredicatesComplements()
1030          HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2);  in arePredicatesComplements()
1062   auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass());  in ignorePseudoInstruction()
1063   return !IS->getUnits();  in ignorePseudoInstruction()
1085   if (HII->isSolo(MI))  in isSoloInstruction()
1099 // Quick check if instructions MI and MJ cannot coexist in the same packet.
1100 // Limit the tests to be "one-way", e.g.  "if MI->isBranch and MJ->isInlineAsm",
1101 // but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
1108   const MachineFunction *MF = MI.getParent()->getParent();  in cannotCoexistAsymm()
1109   if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() &&  in cannotCoexistAsymm()
1126   // New-value stores cannot coexist with any other stores.  in cannotCoexistAsymm()
1141     // These instructions can only be grouped with ALU32 or non-floating-point  in cannotCoexistAsymm()
1156   // I and J cannot coexist.  in cannotCoexistAsymm()
1214   if (HII->isPredicated(I) || HII->isPredicated(J))  in hasDeadDependence()
1236   // A save callee-save register function call can only be in a packet  in hasControlDependence()
1237   // with instructions that don't write to the callee-save registers.  in hasControlDependence()
1238   if ((HII->isSaveCalleeSavedRegsCall(I) &&  in hasControlDependence()
1240       (HII->isSaveCalleeSavedRegsCall(J) &&  in hasControlDependence()
1248   // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot  in hasControlDependence()
1250   // a new-value compare jump or a dealloc_return.  in hasControlDependence()
1251   auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {  in hasControlDependence()
1252     if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))  in hasControlDependence()
1254     if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))  in hasControlDependence()
1259   if (HII->isLoopN(I) && isBadForLoopN(J))  in hasControlDependence()
1261   if (HII->isLoopN(J) && isBadForLoopN(I))  in hasControlDependence()
1266   return HII->isDeallocRet(I) &&  in hasControlDependence()
1286     assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");  in hasRegMaskDependence()
1308     if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))  in hasDualStoreDependence()
1313     bool MopStI = HII->isMemOp(I) || StoreI;  in hasDualStoreDependence()
1314     bool MopStJ = HII->isMemOp(J) || StoreJ;  in hasDualStoreDependence()
1319   return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));  in hasDualStoreDependence()
1326   assert(SUI->getInstr() && SUJ->getInstr());  in isLegalToPacketizeTogether()
1327   MachineInstr &I = *SUI->getInstr();  in isLegalToPacketizeTogether()
1328   MachineInstr &J = *SUJ->getInstr();  in isLegalToPacketizeTogether()
1354   // Dual-store does not allow second store, if the first store is not  in isLegalToPacketizeTogether()
1364   if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) {  in isLegalToPacketizeTogether()
1375       // NVJ can not be part of the dual jump - Arch Spec: section 7.8.  in isLegalToPacketizeTogether()
1376       if (PI->isCall()) {  in isLegalToPacketizeTogether()
1388       if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||  in isLegalToPacketizeTogether()
1389           HII->isLoopN(*PI)) {  in isLegalToPacketizeTogether()
1395       if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) {  in isLegalToPacketizeTogether()
1407   if (!SUJ->isSucc(SUI))  in isLegalToPacketizeTogether()
1410   for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {  in isLegalToPacketizeTogether()
1414     if (SUJ->Succs[i].getSUnit() != SUI)  in isLegalToPacketizeTogether()
1417     SDep::Kind DepType = SUJ->Succs[i].getKind();  in isLegalToPacketizeTogether()
1436       DepReg = SUJ->Succs[i].getReg();  in isLegalToPacketizeTogether()
1437       RC = HRI->getMinimalPhysRegClass(DepReg);  in isLegalToPacketizeTogether()
1440     if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) {  in isLegalToPacketizeTogether()
1443       if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg()))  in isLegalToPacketizeTogether()
1454     if (DepType == SDep::Data && HII->isDotCurInst(J)) {  in isLegalToPacketizeTogether()
1455       if (HII->isHVXVec(I))  in isLegalToPacketizeTogether()
1459     // For instructions that can be promoted to dot-new, try to promote.  in isLegalToPacketizeTogether()
1469       if (HII->isNewValueJump(I))  in isLegalToPacketizeTogether()
1475     if (HII->isPredicated(I) && HII->isPredicated(J) &&  in isLegalToPacketizeTogether()
1485       // There will be an output dependence between (1)->(2) and (2)->(3).  in isLegalToPacketizeTogether()
1486       // However, there is no dependence edge between (1)->(3). This results  in isLegalToPacketizeTogether()
1499     // and non-control-flow instructions.  in isLegalToPacketizeTogether()
1532       bool NVStoreJ = HII->isNewValueStore(J);  in isLegalToPacketizeTogether()
1533       bool NVStoreI = HII->isNewValueStore(I);  in isLegalToPacketizeTogether()
1534       bool IsVecJ = HII->isHVXVec(J);  in isLegalToPacketizeTogether()
1535       bool IsVecI = HII->isHVXVec(I);  in isLegalToPacketizeTogether()
1539       if (LoadJ && LoadI && HII->isPureSlot0(J)) {  in isLegalToPacketizeTogether()
1551           (!HII->isMemOp(J) && !HII->isMemOp(I)) && (!IsVecJ && !IsVecI))  in isLegalToPacketizeTogether()
1584           if (I.getOperand(0).getReg() == HRI->getStackRegister()) {  in isLegalToPacketizeTogether()
1599     // There are certain anti-dependencies that cannot be ignored.  in isLegalToPacketizeTogether()
1601     //   J2_call ... implicit-def %r0   ; SUJ  in isLegalToPacketizeTogether()
1623     // Skip over remaining anti-dependences. Two instructions that are  in isLegalToPacketizeTogether()
1624     // anti-dependent can share a packet, since in most such cases all  in isLegalToPacketizeTogether()
1644   assert(SUI->getInstr() && SUJ->getInstr());  in isLegalToPruneDependencies()
1645   MachineInstr &I = *SUI->getInstr();  in isLegalToPruneDependencies()
1646   MachineInstr &J = *SUJ->getInstr();  in isLegalToPruneDependencies()
1648   bool Coexist = !cannotCoexist(I, J);  in isLegalToPruneDependencies()  local
1650   if (Coexist && !Dependence)  in isLegalToPruneDependencies()
1653   // Check if the instruction was promoted to a dot-new. If so, demote it  in isLegalToPruneDependencies()
1654   // back into a dot-old.  in isLegalToPruneDependencies()
1671     // Putting I and J together would prevent the new-value jump from being  in isLegalToPruneDependencies()
1677   if (!Coexist)  in isLegalToPruneDependencies()
1695     unsigned Opc = MJ->getOpcode();  in foundLSInPacket()
1698     if (HII->isMemOp(*MJ))  in foundLSInPacket()
1700     if (MJ->mayLoad())  in foundLSInPacket()
1702     if (MJ->mayStore() && !HII->isNewValueStore(*MJ))  in foundLSInPacket()
1727   assert(ResourceTracker->canReserveResources(MI));  in addToPacket()
1729   bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);  in addToPacket()
1738     ResourceTracker->reserveResources(MI);  in addToPacket()
1742     bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI);  in addToPacket()
1744       if (ResourceTracker->canReserveResources(NvjMI))  in addToPacket()
1745         ResourceTracker->reserveResources(NvjMI);  in addToPacket()
1754       assert(ResourceTracker->canReserveResources(MI));  in addToPacket()
1755       ResourceTracker->reserveResources(MI);  in addToPacket()
1760       assert(ResourceTracker->canReserveResources(NvjMI));  in addToPacket()
1761       ResourceTracker->reserveResources(NvjMI);  in addToPacket()
1772   ResourceTracker->reserveResources(MI);  in addToPacket()
1781     ResourceTracker->reserveResources(MI);  in addToPacket()
1797         unsigned R = ResourceTracker->getUsedResources(Idx++);  in endPacket()
1812     MachineBasicBlock::instr_iterator NextMI = std::next(MI->getIterator());  in endPacket()
1813     for (auto &I : make_range(HII->expandVGatherPseudo(*MI), NextMI))  in endPacket()
1824       HII->setBundleNoShuf(BundleMII);  in endPacket()
1831   ResourceTracker->clearResources();  in endPacket()
1844   auto &HST = MI.getParent()->getParent()->getSubtarget<HexagonSubtarget>();  in shouldAddToPacket()
1854     // Check for SLOT0 only non-duplexable instruction in packet.  in shouldAddToPacket()
1856       PacketHasSLOT0OnlyInsn |= HII->isPureSlot0(*MJ);  in shouldAddToPacket()
1858     int Opcode = HII->getDuplexOpcode(MI, false);  in shouldAddToPacket()
1862         if (HII->isDuplexPair(MI, *MJ) && !PacketHasSLOT0OnlyInsn) {  in shouldAddToPacket()
1870       MIRef.setDesc(HII->get(Opcode));  in shouldAddToPacket()
1871       return ResourceTracker->canReserveResources(MIRef);  in shouldAddToPacket()
1888     auto *OldBB = OldPacketMIs.front()->getParent();  in calcStall()
1890     if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB))  in calcStall()
1907   // non-zero even though we can have  both .cur and .new in the same packet.  in calcStall()
1921     for (auto &Pred : SUI->Preds)  in calcStall()
1924             HII->isNewValueJump(I) || HII->isToBeScheduledASAP(*J, I))  in calcStall()
1932     for (auto &Pred : SUI->Preds)  in calcStall()
1950 //===----------------------------------------------------------------------===//
1952 //===----------------------------------------------------------------------===//