Lines Matching refs:DAG
67 void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) { in initialize() argument
68 GenericScheduler::initialize(DAG); in initialize()
70 MF = &DAG->MF; in initialize()
173 if (!DAG->isTrackingPressure()) in initCandidate()
198 for (const auto &Diff : DAG->getPressureDiff(SU)) { in initCandidate()
296 if (DAG->isTrackingPressure()) { in pickNodeFromQueue()
312 TryCand.initResourceDelta(Zone.DAG, SchedModel); in pickNodeFromQueue()
346 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand, in pickNodeBidirectional()
355 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand, in pickNodeBidirectional()
368 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand, in pickNodeBidirectional()
377 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand, in pickNodeBidirectional()
403 if (DAG->top() == DAG->bottom()) { in pickNode()
415 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand, in pickNode()
426 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand, in pickNode()
496 if (DAG->isTrackingPressure() && in tryCandidate()
498 RegExcess, TRI, DAG->MF)) in tryCandidate()
514 TryCand.initResourceDelta(DAG, SchedModel); in tryCandidate()
540 Cand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred(); in tryCandidate()
542 TryCand.AtTop ? DAG->getNextClusterSucc() : DAG->getNextClusterPred(); in tryCandidate()
548 if (DAG->isTrackingPressure() && in tryCandidate()
550 TryCand, Cand, RegCritical, TRI, DAG->MF)) in tryCandidate()
554 if (DAG->isTrackingPressure() && in tryCandidate()
556 Cand, RegMax, TRI, DAG->MF)) in tryCandidate()
780 GCNSchedStage::GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG) in GCNSchedStage() argument
781 : DAG(DAG), S(static_cast<GCNSchedStrategy &>(*DAG.SchedImpl)), MF(DAG.MF), in GCNSchedStage()
782 MFI(DAG.MFI), ST(DAG.ST), StageID(StageID) {} in GCNSchedStage()
785 if (!DAG.LIS) in initGCNSchedStage()
799 if (DAG.RegionsWithHighRP.none() && DAG.RegionsWithExcessRP.none()) in initGCNSchedStage()
802 SavedMutations.swap(DAG.Mutations); in initGCNSchedStage()
803 DAG.addMutation( in initGCNSchedStage()
806 InitialOccupancy = DAG.MinOccupancy; in initGCNSchedStage()
811 if (MFI.getMaxWavesPerEU() > DAG.MinOccupancy) in initGCNSchedStage()
812 MFI.increaseOccupancy(MF, ++DAG.MinOccupancy); in initGCNSchedStage()
818 << DAG.MinOccupancy << ".\n"); in initGCNSchedStage()
833 if (DAG.StartingOccupancy <= DAG.MinOccupancy) in initGCNSchedStage()
838 << DAG.MinOccupancy << ".\n"); in initGCNSchedStage()
846 if (DAG.RegionsWithMinOcc.none() || DAG.Regions.size() == 1) in initGCNSchedStage()
852 DAG.MinOccupancy) in initGCNSchedStage()
867 << DAG.MinOccupancy << " from rematerializing\n"); in initGCNSchedStage()
872 DAG.finishBlock(); in finalizeGCNSchedStage()
877 SavedMutations.swap(DAG.Mutations); in finalizeGCNSchedStage()
879 if (DAG.MinOccupancy > InitialOccupancy) { in finalizeGCNSchedStage()
880 for (unsigned IDX = 0; IDX < DAG.Pressure.size(); ++IDX) in finalizeGCNSchedStage()
881 DAG.RegionsWithMinOcc[IDX] = in finalizeGCNSchedStage()
882 DAG.Pressure[IDX].getOccupancy(DAG.ST) == DAG.MinOccupancy; in finalizeGCNSchedStage()
886 << DAG.MinOccupancy << '\n'); in finalizeGCNSchedStage()
894 if (DAG.RegionBegin->getParent() != CurrentMBB) in initGCNRegion()
897 unsigned NumRegionInstrs = std::distance(DAG.begin(), DAG.end()); in initGCNRegion()
898 DAG.enterRegion(CurrentMBB, DAG.begin(), DAG.end(), NumRegionInstrs); in initGCNRegion()
901 if (DAG.begin() == DAG.end() || DAG.begin() == std::prev(DAG.end())) in initGCNRegion()
907 << "\n From: " << *DAG.begin() << " To: "; in initGCNRegion()
908 if (DAG.RegionEnd != CurrentMBB->end()) dbgs() << *DAG.RegionEnd; in initGCNRegion()
914 Unsched.reserve(DAG.NumRegionInstrs); in initGCNRegion()
917 for (auto &I : DAG) { in initGCNRegion()
921 DAG.RegionsWithIGLPInstrs[RegionIdx] = true; in initGCNRegion()
924 for (auto &I : DAG) in initGCNRegion()
928 PressureBefore = DAG.Pressure[RegionIdx]; in initGCNRegion()
932 << print(DAG.LiveIns[RegionIdx], DAG.MRI) in initGCNRegion()
934 << print(llvm::getRegPressure(DAG.MRI, DAG.LiveIns[RegionIdx])) in initGCNRegion()
940 if (DAG.RegionsWithIGLPInstrs[RegionIdx] && in initGCNRegion()
943 SavedMutations.swap(DAG.Mutations); in initGCNRegion()
946 DAG.addMutation(createIGroupLPDAGMutation( in initGCNRegion()
957 if ((!DAG.RegionsWithMinOcc[RegionIdx] || in initGCNRegion()
958 DAG.MinOccupancy <= InitialOccupancy) && in initGCNRegion()
959 !DAG.RegionsWithExcessRP[RegionIdx]) in initGCNRegion()
971 if (!DAG.RegionsWithHighRP[RegionIdx]) in initGCNRegion()
978 if (!DAG.RescheduleRegions[RegionIdx]) in initGCNRegion()
986 DAG.finishBlock(); in setupNewBlock()
988 CurrentMBB = DAG.RegionBegin->getParent(); in setupNewBlock()
989 DAG.startBlock(CurrentMBB); in setupNewBlock()
994 DAG.computeBlockPressure(RegionIdx, CurrentMBB); in setupNewBlock()
998 DAG.Regions[RegionIdx] = std::pair(DAG.RegionBegin, DAG.RegionEnd); in finalizeGCNRegion()
999 DAG.RescheduleRegions[RegionIdx] = false; in finalizeGCNRegion()
1001 DAG.RegionsWithHighRP[RegionIdx] = true; in finalizeGCNRegion()
1007 if (DAG.RegionsWithIGLPInstrs[RegionIdx] && in finalizeGCNRegion()
1009 SavedMutations.swap(DAG.Mutations); in finalizeGCNRegion()
1011 DAG.exitRegion(); in finalizeGCNRegion()
1017 PressureAfter = DAG.getRealRegPressure(RegionIdx); in checkScheduling()
1023 DAG.Pressure[RegionIdx] = PressureAfter; in checkScheduling()
1024 DAG.RegionsWithMinOcc[RegionIdx] = in checkScheduling()
1025 PressureAfter.getOccupancy(ST) == DAG.MinOccupancy; in checkScheduling()
1049 if (WavesAfter < WavesBefore && WavesAfter < DAG.MinOccupancy && in checkScheduling()
1056 if (NewOccupancy < DAG.MinOccupancy) { in checkScheduling()
1057 DAG.MinOccupancy = NewOccupancy; in checkScheduling()
1058 MFI.limitOccupancy(DAG.MinOccupancy); in checkScheduling()
1059 DAG.RegionsWithMinOcc.reset(); in checkScheduling()
1061 << DAG.MinOccupancy << ".\n"); in checkScheduling()
1075 DAG.RescheduleRegions[RegionIdx] = true; in checkScheduling()
1076 DAG.RegionsWithHighRP[RegionIdx] = true; in checkScheduling()
1077 DAG.RegionsWithExcessRP[RegionIdx] = true; in checkScheduling()
1085 DAG.Pressure[RegionIdx] = PressureAfter; in checkScheduling()
1086 DAG.RegionsWithMinOcc[RegionIdx] = in checkScheduling()
1087 PressureAfter.getOccupancy(ST) == DAG.MinOccupancy; in checkScheduling()
1100 unsigned DefReady = ReadyCycles[DAG.getSUnit(DefMI)->NodeNum]; in computeSUnitReadyCycle()
1170 GCNSchedStage::getScheduleMetrics(const GCNScheduleDAGMILive &DAG) { in getScheduleMetrics() argument
1179 for (auto &MI : DAG) { in getScheduleMetrics()
1180 SUnit *SU = DAG.getSUnit(&MI); in getScheduleMetrics()
1206 if (WavesAfter < DAG.MinOccupancy) in shouldRevertScheduling()
1244 getScheduleMetrics(DAG.SUnits); in shouldRevertScheduling()
1248 ScheduleMetrics MAfter = getScheduleMetrics(DAG); in shouldRevertScheduling()
1304 DAG.RegionsWithMinOcc[RegionIdx] = in revertScheduling()
1305 PressureBefore.getOccupancy(ST) == DAG.MinOccupancy; in revertScheduling()
1307 DAG.RescheduleRegions[RegionIdx] = in revertScheduling()
1310 DAG.RegionEnd = DAG.RegionBegin; in revertScheduling()
1318 if (MI->getIterator() != DAG.RegionEnd) { in revertScheduling()
1319 DAG.BB->remove(MI); in revertScheduling()
1320 DAG.BB->insert(DAG.RegionEnd, MI); in revertScheduling()
1322 DAG.LIS->handleMove(*MI, true); in revertScheduling()
1329 RegOpers.collect(*MI, *DAG.TRI, DAG.MRI, DAG.ShouldTrackLaneMasks, false); in revertScheduling()
1331 if (DAG.ShouldTrackLaneMasks) { in revertScheduling()
1333 SlotIndex SlotIdx = DAG.LIS->getInstructionIndex(*MI).getRegSlot(); in revertScheduling()
1334 RegOpers.adjustLaneLiveness(*DAG.LIS, DAG.MRI, SlotIdx, MI); in revertScheduling()
1337 RegOpers.detectDeadDefs(*MI, *DAG.LIS); in revertScheduling()
1340 DAG.RegionEnd = MI->getIterator(); in revertScheduling()
1341 ++DAG.RegionEnd; in revertScheduling()
1349 ++DAG.RegionEnd; in revertScheduling()
1354 DAG.RegionBegin = Unsched.front()->getIterator(); in revertScheduling()
1355 if (DAG.RegionBegin->isDebugInstr()) { in revertScheduling()
1359 DAG.RegionBegin = MI->getIterator(); in revertScheduling()
1366 DAG.placeDebugValues(); in revertScheduling()
1368 DAG.Regions[RegionIdx] = std::pair(DAG.RegionBegin, DAG.RegionEnd); in revertScheduling()
1372 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(DAG.TRI); in collectRematerializableInstructions()
1373 for (unsigned I = 0, E = DAG.MRI.getNumVirtRegs(); I != E; ++I) { in collectRematerializableInstructions()
1375 if (!DAG.LIS->hasInterval(Reg)) in collectRematerializableInstructions()
1379 if (!SRI->isVGPRClass(DAG.MRI.getRegClass(Reg)) || in collectRematerializableInstructions()
1380 !DAG.MRI.hasOneDef(Reg) || !DAG.MRI.hasOneNonDBGUse(Reg)) in collectRematerializableInstructions()
1383 MachineOperand *Op = DAG.MRI.getOneDef(Reg); in collectRematerializableInstructions()
1388 MachineInstr *UseI = &*DAG.MRI.use_instr_nodbg_begin(Reg); in collectRematerializableInstructions()
1396 for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) { in collectRematerializableInstructions()
1397 auto It = DAG.LiveIns[I].find(Reg); in collectRematerializableInstructions()
1398 if (It != DAG.LiveIns[I].end() && !It->second.none()) { in collectRematerializableInstructions()
1399 if (DAG.RegionsWithMinOcc[I]) { in collectRematerializableInstructions()
1424 LiveIntervals *LIS = DAG.LIS; in sinkTriviallyRematInsts()
1426 NewRegions.resize(DAG.Regions.size()); in sinkTriviallyRematInsts()
1427 NewRescheduleRegions.resize(DAG.Regions.size()); in sinkTriviallyRematInsts()
1437 NewPressure[Idx] = DAG.Pressure[Idx]; in sinkTriviallyRematInsts()
1438 NewLiveIns[Idx] = DAG.LiveIns[Idx]; in sinkTriviallyRematInsts()
1440 NewRegions = DAG.Regions; in sinkTriviallyRematInsts()
1446 if (!DAG.RegionsWithMinOcc[I]) in sinkTriviallyRematInsts()
1455 if (ST.getOccupancyWithNumSGPRs(SGPRUsage) == DAG.MinOccupancy) in sinkTriviallyRematInsts()
1460 if (NewPressure[I].getOccupancy(ST) > DAG.MinOccupancy) { in sinkTriviallyRematInsts()
1480 if (OptimisticOccupancy <= DAG.MinOccupancy) in sinkTriviallyRematInsts()
1495 Def->getOperand(0).getSubReg(), *Def, *DAG.TRI); in sinkTriviallyRematInsts()
1504 DAG.updateRegionBoundaries(NewRegions, Def, /*NewMI =*/nullptr, in sinkTriviallyRematInsts()
1508 DAG.updateRegionBoundaries(NewRegions, InsertPos, NewMI); in sinkTriviallyRematInsts()
1516 if (InsertPos->getParent() != DAG.Regions[Idx].first->getParent()) { in sinkTriviallyRematInsts()
1518 NewPressure[Idx].inc(Reg, PrevMask, LaneBitmask::getNone(), DAG.MRI); in sinkTriviallyRematInsts()
1532 if (ImproveOccupancy > DAG.MinOccupancy) in sinkTriviallyRematInsts()
1541 if (ImproveOccupancy <= DAG.MinOccupancy) in sinkTriviallyRematInsts()
1570 DAG.BBLiveInMap.erase(OldMI); in sinkTriviallyRematInsts()
1582 DAG.LiveIns[Idx] = NewLiveIns[Idx]; in sinkTriviallyRematInsts()
1583 DAG.Pressure[Idx] = NewPressure[Idx]; in sinkTriviallyRematInsts()
1584 DAG.MBBLiveIns.erase(DAG.Regions[Idx].first->getParent()); in sinkTriviallyRematInsts()
1586 DAG.Regions = NewRegions; in sinkTriviallyRematInsts()
1587 DAG.RescheduleRegions = NewRescheduleRegions; in sinkTriviallyRematInsts()
1590 MFI.increaseOccupancy(MF, ++DAG.MinOccupancy); in sinkTriviallyRematInsts()
1597 if (!DAG.TII->isTriviallyReMaterializable(MI)) in isTriviallyReMaterializable()
1649 static bool hasIGLPInstrs(ScheduleDAGInstrs *DAG) { in hasIGLPInstrs() argument
1651 DAG->begin(), DAG->end(), [](MachineBasicBlock::iterator MI) { in hasIGLPInstrs()