1 //===- RegisterPressure.cpp - Dynamic Register Pressure -------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
30 #include "llvm/Config/llvm-config.h"
75     CurrSetPressure[*PSetI] -= Weight;  in decreaseSetPressure()
86       dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << '\n';  in dumpRegSetPressure()
161   PSetIterator PSetI = MRI->getPressureSets(RegUnit);  in increaseRegPressure()
192 /// If the current top is not less than or equal to the next index, open it.
201 /// If the current top is the previous instruction (before receding), open it.
209 /// If the current bottom is not greater than the previous index, open it.
217 /// If the current bottom is the previous instr (before advancing), open it.
230   this->NumRegUnits = NumRegUnits;  in init()
272   TRI = MF->getSubtarget().getRegisterInfo();  in init()
274   MRI = &MF->getRegInfo();  in init()
276   this->TrackUntiedDefs = TrackUntiedDefs;  in init()
277   this->TrackLaneMasks = TrackLaneMasks;  in init()
285   CurrSetPressure.assign(TRI->getNumRegPressureSets(), 0);  in init()
291     UntiedDefs.setUniverse(MRI->getNumVirtRegs());  in init()
312     skipDebugInstructionsForward(CurrPos, MBB->end());  in getCurrSlot()
313   if (IdxPos == MBB->end())  in getCurrSlot()
314     return LIS->getMBBEndIdx(MBB);  in getCurrSlot()
315   return LIS->getInstructionIndex(*IdxPos).getRegSlot();  in getCurrSlot()
356 /// live-thru ranges. However, two-address or coalesced chains can also lead
360   LiveThruPressure.assign(TRI->getNumRegPressureSets(), 0);  in initLiveThru()
361   assert(isBottomClosed() && "need bottom-up tracking to intialize.");  in initLiveThru()
377   return I->LaneMask;  in getRegLanes()
390     I->LaneMask |= Pair.LaneMask;  in addRegLanes()
402     I->LaneMask = LaneBitmask::getNone();  in setRegZero()
414     I->LaneMask &= ~Pair.LaneMask;  in removeRegLanes()
415     if (I->LaneMask.none())  in removeRegLanes()
539       // Treat read-undef subreg defs as definitions of the whole register.  in collectOperandLanes()
582     Register Reg = RI->RegUnit;  in detectDeadDefs()
585       LiveQueryResult LRQ = LR->Query(SlotIdx);  in detectDeadDefs()
603     LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit,  in adjustLaneLiveness()
606     // of a subregister def we need a read-undef flag.  in adjustLaneLiveness()
607     Register RegUnit = I->RegUnit;  in adjustLaneLiveness()
609         (LiveAfter & ~I->LaneMask).none())  in adjustLaneLiveness()
610       AddFlagsMI->setRegisterDefReadUndef(RegUnit);  in adjustLaneLiveness()
612     LaneBitmask ActualDef = I->LaneMask & LiveAfter;  in adjustLaneLiveness()
616       I->LaneMask = ActualDef;  in adjustLaneLiveness()
633         AddFlagsMI->setRegisterDefReadUndef(RegUnit);  in adjustLaneLiveness()
654   assert(!PDiff.begin()->isValid() && "stale PDiff");  in addInstruction()
665   PSetIterator PSetI = MRI->getPressureSets(RegUnit);  in addPressureChange()
666   int Weight = IsDec ? -PSetI.getWeight() : PSetI.getWeight();  in addPressureChange()
670     for (; I != E && I->isValid(); ++I) {  in addPressureChange()
671       if (I->getPSet() >= *PSetI)  in addPressureChange()
678     if (!I->isValid() || I->getPSet() != *PSetI) {  in addPressureChange()
684     unsigned NewUnitInc = I->getUnitInc() + Weight;  in addPressureChange()
686       I->setUnitInc(NewUnitInc);  in addPressureChange()
690       for (J = std::next(I); J != E && J->isValid(); ++J, ++I)  in addPressureChange()
721     PrevMask = I->LaneMask;  in discoverLiveInOrOut()
723     I->LaneMask = NewMask;  in discoverLiveInOrOut()
752 /// RegUnits that are made live by the current instruction's uses. This includes
758   assert(!CurrPos->isDebugOrPseudoInstr());  in recede()
760   // Boost pressure for all dead defs together.  in recede()
792     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();  in recede()
813           bool IsRedef = I != LiveUses->end();  in recede()
815             // ignore re-defs here...  in recede()
816             assert(I->LaneMask.none());  in recede()
845   assert(CurrPos != MBB->begin());  in recedeSkipDebugValues()
854   CurrPos = prev_nodbg(CurrPos, MBB->begin());  in recedeSkipDebugValues()
857   if (RequireIntervals && !CurrPos->isDebugOrPseudoInstr())  in recedeSkipDebugValues()
858     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();  in recedeSkipDebugValues()
867   if (CurrPos->isDebugInstr() || CurrPos->isPseudoProbe()) {  in recede()
870     assert(CurrPos == MBB->begin());  in recede()
878     SlotIndex SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();  in recede()
887 /// Advance across the current instruction.
890   assert(CurrPos != MBB->end());  in advance()
932   // Boost pressure for all dead defs together.  in advance()
936   CurrPos = next_nodbg(CurrPos, MBB->end());  in advance()
960     int PDiff = (int)PNew - (int)POld;  in computeExcessPressureDelta()
963     // Only consider change beyond the limit.  in computeExcessPressureDelta()
964     unsigned Limit = RCI->getRegPressureSetLimit(i);  in computeExcessPressureDelta()  local
966       Limit += LiveThruPressureVec[i];  in computeExcessPressureDelta()
968     if (Limit > POld) {  in computeExcessPressureDelta()
969       if (Limit > PNew)  in computeExcessPressureDelta()
970         PDiff = 0;            // Under the limit  in computeExcessPressureDelta()
972         PDiff = PNew - Limit; // Just exceeded limit.  in computeExcessPressureDelta()
973     } else if (Limit > PNew)  in computeExcessPressureDelta()
974       PDiff = Limit - POld;   // Just obeyed limit.  in computeExcessPressureDelta()
985 /// limit or exceeds the current MaxPressureLimit.
1010         int PDiff = (int)PNew - (int)CriticalPSets[CritIdx].getUnitInc();  in computeMaxPressureDelta()
1021       Delta.CurrentMax.setUnitInc(PNew - POld);  in computeMaxPressureDelta()
1028 /// Record the upward impact of a single instruction on current register
1033 /// with the current position, so must be restored by the caller.
1035   assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");  in bumpUpwardPressure()
1039     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();  in bumpUpwardPressure()
1050   // Boost max pressure for all dead defs together.  in bumpUpwardPressure()
1082 /// bottom-up. Find the pressure set with the most change beyond its pressure
1083 /// limit based on the tracker's current pressure, and return the change in
1087 /// This assumes that the current LiveOut set is sufficient.
1089 /// This is expensive for an on-the-fly query because it calls
1090 /// bumpUpwardPressure to recompute the pressure sets based on current
1092 /// -verify-misched. getUpwardPressureDelta is the fast version of this query
1093 /// that uses the per-SUnit cache of the PressureDiff.
1127     PDiff->dump(*TRI);  in getMaxUpwardPressureDelta()
1130       dbgs() << "Excess1 " << TRI->getRegPressureSetName(Delta.Excess.getPSet())  in getMaxUpwardPressureDelta()
1133       dbgs() << "Critic1 " << TRI->getRegPressureSetName(Delta.CriticalMax.getPSet())  in getMaxUpwardPressureDelta()
1136       dbgs() << "CurrMx1 " << TRI->getRegPressureSetName(Delta.CurrentMax.getPSet())  in getMaxUpwardPressureDelta()
1139       dbgs() << "Excess2 " << TRI->getRegPressureSetName(Delta2.Excess.getPSet())  in getMaxUpwardPressureDelta()
1142       dbgs() << "Critic2 " << TRI->getRegPressureSetName(Delta2.CriticalMax.getPSet())  in getMaxUpwardPressureDelta()
1145       dbgs() << "CurrMx2 " << TRI->getRegPressureSetName(Delta2.CurrentMax.getPSet())  in getMaxUpwardPressureDelta()
1153 /// directly depend on current liveness.
1158 /// limit within the region, not necessarily at the current position.
1161 /// necessarily at the current position.
1170        PDiffI != PDiffE && PDiffI->isValid(); ++PDiffI) {  in getUpwardPressureDelta()
1172     unsigned PSetID = PDiffI->getPSet();  in getUpwardPressureDelta()
1173     unsigned Limit = RCI->getRegPressureSetLimit(PSetID);  in getUpwardPressureDelta()  local
1175       Limit += LiveThruPressure[PSetID];  in getUpwardPressureDelta()
1181     unsigned PNew = POld + PDiffI->getUnitInc();  in getUpwardPressureDelta()
1182     assert((PDiffI->getUnitInc() >= 0) == (PNew >= POld)  in getUpwardPressureDelta()
1186     // Check if current pressure has exceeded the limit.  in getUpwardPressureDelta()
1189       if (PNew > Limit)  in getUpwardPressureDelta()
1190         ExcessInc = POld > Limit ? PNew - POld : PNew - Limit;  in getUpwardPressureDelta()
1191       else if (POld > Limit)  in getUpwardPressureDelta()
1192         ExcessInc = Limit - POld;  in getUpwardPressureDelta()
1206         int CritInc = (int)MNew - (int)CriticalPSets[CritIdx].getUnitInc();  in getUpwardPressureDelta()
1213     // Check if max pressure has exceeded the current max.  in getUpwardPressureDelta()
1216       Delta.CurrentMax.setUnitInc(MNew - MOld);  in getUpwardPressureDelta()
1233     SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();  in findUseBetween()
1262         return S != nullptr && S->end == Pos.getRegSlot();  in getLastUsedLanes()
1273         return S != nullptr && S->start < Pos.getRegSlot(true) &&  in getLiveThroughAt()
1274                S->end != Pos.getDeadSlot();  in getLiveThroughAt()
1278 /// Record the downward impact of a single instruction on current register
1283 /// with the current position, so must be restored by the caller.
1285   assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");  in bumpDownwardPressure()
1289     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();  in bumpDownwardPressure()
1305       // last uses for the current position.  in bumpDownwardPressure()
1307       // to be bottom-scheduled to avoid searching uses at each query.  in bumpDownwardPressure()
1328   // Boost pressure for all dead defs together.  in bumpDownwardPressure()
1333 /// top-down. Find the register class with the most change in its pressure limit
1334 /// based on the tracker's current pressure, and return the number of excess
1337 /// This assumes that the current LiveIn set is sufficient.
1339 /// This is expensive for an on-the-fly query because it calls
1340 /// bumpDownwardPressure to recompute the pressure sets based on current
1365 /// Get the pressure of each PSet after traversing this instruction bottom-up.
1376   // Current pressure becomes the result. Restore current pressure.  in getUpwardPressure()
1381 /// Get the pressure of each PSet after traversing this instruction top-down.
1392   // Current pressure becomes the result. Restore current pressure.  in getDownwardPressure()