1 //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This contains a MachineSchedStrategy implementation for maximizing wave 11 /// occupancy on GCN hardware. 12 //===----------------------------------------------------------------------===// 13 14 #include "GCNSchedStrategy.h" 15 #include "SIMachineFunctionInfo.h" 16 17 #define DEBUG_TYPE "machine-scheduler" 18 19 using namespace llvm; 20 21 GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy( 22 const MachineSchedContext *C) : 23 GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { } 24 25 void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) { 26 GenericScheduler::initialize(DAG); 27 28 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI); 29 30 MF = &DAG->MF; 31 32 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 33 34 // FIXME: This is also necessary, because some passes that run after 35 // scheduling and before regalloc increase register pressure. 36 const int ErrorMargin = 3; 37 38 SGPRExcessLimit = Context->RegClassInfo 39 ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin; 40 VGPRExcessLimit = Context->RegClassInfo 41 ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin; 42 if (TargetOccupancy) { 43 SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true); 44 VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy); 45 } else { 46 SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF, 47 AMDGPU::RegisterPressureSets::SReg_32); 48 VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF, 49 AMDGPU::RegisterPressureSets::VGPR_32); 50 } 51 52 SGPRCriticalLimit -= ErrorMargin; 53 VGPRCriticalLimit -= ErrorMargin; 54 } 55 56 void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, 57 bool AtTop, const RegPressureTracker &RPTracker, 58 const SIRegisterInfo *SRI, 59 unsigned SGPRPressure, 60 unsigned VGPRPressure) { 61 62 Cand.SU = SU; 63 Cand.AtTop = AtTop; 64 65 // getDownwardPressure() and getUpwardPressure() make temporary changes to 66 // the tracker, so we need to pass those function a non-const copy. 67 RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker); 68 69 Pressure.clear(); 70 MaxPressure.clear(); 71 72 if (AtTop) 73 TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure); 74 else { 75 // FIXME: I think for bottom up scheduling, the register pressure is cached 76 // and can be retrieved by DAG->getPressureDif(SU). 77 TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure); 78 } 79 80 unsigned NewSGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32]; 81 unsigned NewVGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32]; 82 83 // If two instructions increase the pressure of different register sets 84 // by the same amount, the generic scheduler will prefer to schedule the 85 // instruction that increases the set with the least amount of registers, 86 // which in our case would be SGPRs. This is rarely what we want, so 87 // when we report excess/critical register pressure, we do it either 88 // only for VGPRs or only for SGPRs. 89 90 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs. 91 const unsigned MaxVGPRPressureInc = 16; 92 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit; 93 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit; 94 95 96 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold 97 // to increase the likelihood we don't go over the limits. We should improve 98 // the analysis to look through dependencies to find the path with the least 99 // register pressure. 100 101 // We only need to update the RPDelta for instructions that increase register 102 // pressure. Instructions that decrease or keep reg pressure the same will be 103 // marked as RegExcess in tryCandidate() when they are compared with 104 // instructions that increase the register pressure. 105 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) { 106 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::VGPR_32); 107 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit); 108 } 109 110 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) { 111 Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::SReg_32); 112 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit); 113 } 114 115 // Register pressure is considered 'CRITICAL' if it is approaching a value 116 // that would reduce the wave occupancy for the execution unit. When 117 // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both 118 // has the same cost, so we don't need to prefer one over the other. 119 120 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit; 121 int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit; 122 123 if (SGPRDelta >= 0 || VGPRDelta >= 0) { 124 if (SGPRDelta > VGPRDelta) { 125 Cand.RPDelta.CriticalMax = 126 PressureChange(AMDGPU::RegisterPressureSets::SReg_32); 127 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta); 128 } else { 129 Cand.RPDelta.CriticalMax = 130 PressureChange(AMDGPU::RegisterPressureSets::VGPR_32); 131 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta); 132 } 133 } 134 } 135 136 // This function is mostly cut and pasted from 137 // GenericScheduler::pickNodeFromQueue() 138 void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone, 139 const CandPolicy &ZonePolicy, 140 const RegPressureTracker &RPTracker, 141 SchedCandidate &Cand) { 142 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI); 143 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos(); 144 unsigned SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32]; 145 unsigned VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32]; 146 ReadyQueue &Q = Zone.Available; 147 for (SUnit *SU : Q) { 148 149 SchedCandidate TryCand(ZonePolicy); 150 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, 151 SGPRPressure, VGPRPressure); 152 // Pass SchedBoundary only when comparing nodes from the same boundary. 153 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr; 154 GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg); 155 if (TryCand.Reason != NoCand) { 156 // Initialize resource delta if needed in case future heuristics query it. 157 if (TryCand.ResDelta == SchedResourceDelta()) 158 TryCand.initResourceDelta(Zone.DAG, SchedModel); 159 Cand.setBest(TryCand); 160 LLVM_DEBUG(traceCandidate(Cand)); 161 } 162 } 163 } 164 165 // This function is mostly cut and pasted from 166 // GenericScheduler::pickNodeBidirectional() 167 SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) { 168 // Schedule as far as possible in the direction of no choice. This is most 169 // efficient, but also provides the best heuristics for CriticalPSets. 170 if (SUnit *SU = Bot.pickOnlyChoice()) { 171 IsTopNode = false; 172 return SU; 173 } 174 if (SUnit *SU = Top.pickOnlyChoice()) { 175 IsTopNode = true; 176 return SU; 177 } 178 // Set the bottom-up policy based on the state of the current bottom zone and 179 // the instructions outside the zone, including the top zone. 180 CandPolicy BotPolicy; 181 setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top); 182 // Set the top-down policy based on the state of the current top zone and 183 // the instructions outside the zone, including the bottom zone. 184 CandPolicy TopPolicy; 185 setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot); 186 187 // See if BotCand is still valid (because we previously scheduled from Top). 188 LLVM_DEBUG(dbgs() << "Picking from Bot:\n"); 189 if (!BotCand.isValid() || BotCand.SU->isScheduled || 190 BotCand.Policy != BotPolicy) { 191 BotCand.reset(CandPolicy()); 192 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand); 193 assert(BotCand.Reason != NoCand && "failed to find the first candidate"); 194 } else { 195 LLVM_DEBUG(traceCandidate(BotCand)); 196 #ifndef NDEBUG 197 if (VerifyScheduling) { 198 SchedCandidate TCand; 199 TCand.reset(CandPolicy()); 200 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand); 201 assert(TCand.SU == BotCand.SU && 202 "Last pick result should correspond to re-picking right now"); 203 } 204 #endif 205 } 206 207 // Check if the top Q has a better candidate. 208 LLVM_DEBUG(dbgs() << "Picking from Top:\n"); 209 if (!TopCand.isValid() || TopCand.SU->isScheduled || 210 TopCand.Policy != TopPolicy) { 211 TopCand.reset(CandPolicy()); 212 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand); 213 assert(TopCand.Reason != NoCand && "failed to find the first candidate"); 214 } else { 215 LLVM_DEBUG(traceCandidate(TopCand)); 216 #ifndef NDEBUG 217 if (VerifyScheduling) { 218 SchedCandidate TCand; 219 TCand.reset(CandPolicy()); 220 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand); 221 assert(TCand.SU == TopCand.SU && 222 "Last pick result should correspond to re-picking right now"); 223 } 224 #endif 225 } 226 227 // Pick best from BotCand and TopCand. 228 LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand); 229 dbgs() << "Bot Cand: "; traceCandidate(BotCand);); 230 SchedCandidate Cand = BotCand; 231 TopCand.Reason = NoCand; 232 GenericScheduler::tryCandidate(Cand, TopCand, nullptr); 233 if (TopCand.Reason != NoCand) { 234 Cand.setBest(TopCand); 235 } 236 LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand);); 237 238 IsTopNode = Cand.AtTop; 239 return Cand.SU; 240 } 241 242 // This function is mostly cut and pasted from 243 // GenericScheduler::pickNode() 244 SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) { 245 if (DAG->top() == DAG->bottom()) { 246 assert(Top.Available.empty() && Top.Pending.empty() && 247 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage"); 248 return nullptr; 249 } 250 SUnit *SU; 251 do { 252 if (RegionPolicy.OnlyTopDown) { 253 SU = Top.pickOnlyChoice(); 254 if (!SU) { 255 CandPolicy NoPolicy; 256 TopCand.reset(NoPolicy); 257 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand); 258 assert(TopCand.Reason != NoCand && "failed to find a candidate"); 259 SU = TopCand.SU; 260 } 261 IsTopNode = true; 262 } else if (RegionPolicy.OnlyBottomUp) { 263 SU = Bot.pickOnlyChoice(); 264 if (!SU) { 265 CandPolicy NoPolicy; 266 BotCand.reset(NoPolicy); 267 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand); 268 assert(BotCand.Reason != NoCand && "failed to find a candidate"); 269 SU = BotCand.SU; 270 } 271 IsTopNode = false; 272 } else { 273 SU = pickNodeBidirectional(IsTopNode); 274 } 275 } while (SU->isScheduled); 276 277 if (SU->isTopReady()) 278 Top.removeReady(SU); 279 if (SU->isBottomReady()) 280 Bot.removeReady(SU); 281 282 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " 283 << *SU->getInstr()); 284 return SU; 285 } 286 287 GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C, 288 std::unique_ptr<MachineSchedStrategy> S) : 289 ScheduleDAGMILive(C, std::move(S)), 290 ST(MF.getSubtarget<GCNSubtarget>()), 291 MFI(*MF.getInfo<SIMachineFunctionInfo>()), 292 StartingOccupancy(MFI.getOccupancy()), 293 MinOccupancy(StartingOccupancy), Stage(Collect), RegionIdx(0) { 294 295 LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n"); 296 } 297 298 void GCNScheduleDAGMILive::schedule() { 299 if (Stage == Collect) { 300 // Just record regions at the first pass. 301 Regions.push_back(std::make_pair(RegionBegin, RegionEnd)); 302 return; 303 } 304 305 std::vector<MachineInstr*> Unsched; 306 Unsched.reserve(NumRegionInstrs); 307 for (auto &I : *this) { 308 Unsched.push_back(&I); 309 } 310 311 GCNRegPressure PressureBefore; 312 if (LIS) { 313 PressureBefore = Pressure[RegionIdx]; 314 315 LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:"; 316 GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI); 317 dbgs() << "Region live-in pressure: "; 318 llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs()); 319 dbgs() << "Region register pressure: "; 320 PressureBefore.print(dbgs())); 321 } 322 323 ScheduleDAGMILive::schedule(); 324 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd); 325 RescheduleRegions[RegionIdx] = false; 326 327 if (!LIS) 328 return; 329 330 // Check the results of scheduling. 331 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl; 332 auto PressureAfter = getRealRegPressure(); 333 334 LLVM_DEBUG(dbgs() << "Pressure after scheduling: "; 335 PressureAfter.print(dbgs())); 336 337 if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit && 338 PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) { 339 Pressure[RegionIdx] = PressureAfter; 340 LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n"); 341 return; 342 } 343 unsigned Occ = MFI.getOccupancy(); 344 unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST)); 345 unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST)); 346 LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore 347 << ", after " << WavesAfter << ".\n"); 348 349 // We could not keep current target occupancy because of the just scheduled 350 // region. Record new occupancy for next scheduling cycle. 351 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore); 352 // Allow memory bound functions to drop to 4 waves if not limited by an 353 // attribute. 354 if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy && 355 WavesAfter >= MFI.getMinAllowedOccupancy()) { 356 LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to " 357 << MFI.getMinAllowedOccupancy() << " waves\n"); 358 NewOccupancy = WavesAfter; 359 } 360 if (NewOccupancy < MinOccupancy) { 361 MinOccupancy = NewOccupancy; 362 MFI.limitOccupancy(MinOccupancy); 363 LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to " 364 << MinOccupancy << ".\n"); 365 } 366 367 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF); 368 unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF); 369 if (PressureAfter.getVGPRNum() > MaxVGPRs || 370 PressureAfter.getSGPRNum() > MaxSGPRs) 371 RescheduleRegions[RegionIdx] = true; 372 373 if (WavesAfter >= MinOccupancy) { 374 if (Stage == UnclusteredReschedule && 375 !PressureAfter.less(ST, PressureBefore)) { 376 LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n"); 377 } else if (WavesAfter > MFI.getMinWavesPerEU() || 378 PressureAfter.less(ST, PressureBefore) || 379 !RescheduleRegions[RegionIdx]) { 380 Pressure[RegionIdx] = PressureAfter; 381 return; 382 } else { 383 LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n"); 384 } 385 } 386 387 LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n"); 388 RescheduleRegions[RegionIdx] = true; 389 RegionEnd = RegionBegin; 390 for (MachineInstr *MI : Unsched) { 391 if (MI->isDebugInstr()) 392 continue; 393 394 if (MI->getIterator() != RegionEnd) { 395 BB->remove(MI); 396 BB->insert(RegionEnd, MI); 397 if (!MI->isDebugInstr()) 398 LIS->handleMove(*MI, true); 399 } 400 // Reset read-undef flags and update them later. 401 for (auto &Op : MI->operands()) 402 if (Op.isReg() && Op.isDef()) 403 Op.setIsUndef(false); 404 RegisterOperands RegOpers; 405 RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false); 406 if (!MI->isDebugInstr()) { 407 if (ShouldTrackLaneMasks) { 408 // Adjust liveness and add missing dead+read-undef flags. 409 SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot(); 410 RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI); 411 } else { 412 // Adjust for missing dead-def flags. 413 RegOpers.detectDeadDefs(*MI, *LIS); 414 } 415 } 416 RegionEnd = MI->getIterator(); 417 ++RegionEnd; 418 LLVM_DEBUG(dbgs() << "Scheduling " << *MI); 419 } 420 RegionBegin = Unsched.front()->getIterator(); 421 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd); 422 423 placeDebugValues(); 424 } 425 426 GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const { 427 GCNDownwardRPTracker RPTracker(*LIS); 428 RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]); 429 return RPTracker.moveMaxPressure(); 430 } 431 432 void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) { 433 GCNDownwardRPTracker RPTracker(*LIS); 434 435 // If the block has the only successor then live-ins of that successor are 436 // live-outs of the current block. We can reuse calculated live set if the 437 // successor will be sent to scheduling past current block. 438 const MachineBasicBlock *OnlySucc = nullptr; 439 if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) { 440 SlotIndexes *Ind = LIS->getSlotIndexes(); 441 if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin())) 442 OnlySucc = *MBB->succ_begin(); 443 } 444 445 // Scheduler sends regions from the end of the block upwards. 446 size_t CurRegion = RegionIdx; 447 for (size_t E = Regions.size(); CurRegion != E; ++CurRegion) 448 if (Regions[CurRegion].first->getParent() != MBB) 449 break; 450 --CurRegion; 451 452 auto I = MBB->begin(); 453 auto LiveInIt = MBBLiveIns.find(MBB); 454 if (LiveInIt != MBBLiveIns.end()) { 455 auto LiveIn = std::move(LiveInIt->second); 456 RPTracker.reset(*MBB->begin(), &LiveIn); 457 MBBLiveIns.erase(LiveInIt); 458 } else { 459 auto &Rgn = Regions[CurRegion]; 460 I = Rgn.first; 461 auto *NonDbgMI = &*skipDebugInstructionsForward(Rgn.first, Rgn.second); 462 auto LRS = BBLiveInMap.lookup(NonDbgMI); 463 assert(isEqual(getLiveRegsBefore(*NonDbgMI, *LIS), LRS)); 464 RPTracker.reset(*I, &LRS); 465 } 466 467 for ( ; ; ) { 468 I = RPTracker.getNext(); 469 470 if (Regions[CurRegion].first == I) { 471 LiveIns[CurRegion] = RPTracker.getLiveRegs(); 472 RPTracker.clearMaxPressure(); 473 } 474 475 if (Regions[CurRegion].second == I) { 476 Pressure[CurRegion] = RPTracker.moveMaxPressure(); 477 if (CurRegion-- == RegionIdx) 478 break; 479 } 480 RPTracker.advanceToNext(); 481 RPTracker.advanceBeforeNext(); 482 } 483 484 if (OnlySucc) { 485 if (I != MBB->end()) { 486 RPTracker.advanceToNext(); 487 RPTracker.advance(MBB->end()); 488 } 489 RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs()); 490 RPTracker.advanceBeforeNext(); 491 MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs(); 492 } 493 } 494 495 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> 496 GCNScheduleDAGMILive::getBBLiveInMap() const { 497 assert(!Regions.empty()); 498 std::vector<MachineInstr *> BBStarters; 499 BBStarters.reserve(Regions.size()); 500 auto I = Regions.rbegin(), E = Regions.rend(); 501 auto *BB = I->first->getParent(); 502 do { 503 auto *MI = &*skipDebugInstructionsForward(I->first, I->second); 504 BBStarters.push_back(MI); 505 do { 506 ++I; 507 } while (I != E && I->first->getParent() == BB); 508 } while (I != E); 509 return getLiveRegMap(BBStarters, false /*After*/, *LIS); 510 } 511 512 void GCNScheduleDAGMILive::finalizeSchedule() { 513 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl; 514 LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n"); 515 516 LiveIns.resize(Regions.size()); 517 Pressure.resize(Regions.size()); 518 RescheduleRegions.resize(Regions.size()); 519 RescheduleRegions.set(); 520 521 if (!Regions.empty()) 522 BBLiveInMap = getBBLiveInMap(); 523 524 std::vector<std::unique_ptr<ScheduleDAGMutation>> SavedMutations; 525 526 do { 527 Stage++; 528 RegionIdx = 0; 529 MachineBasicBlock *MBB = nullptr; 530 531 if (Stage > InitialSchedule) { 532 if (!LIS) 533 break; 534 535 // Retry function scheduling if we found resulting occupancy and it is 536 // lower than used for first pass scheduling. This will give more freedom 537 // to schedule low register pressure blocks. 538 // Code is partially copied from MachineSchedulerBase::scheduleRegions(). 539 540 if (Stage == UnclusteredReschedule) { 541 if (RescheduleRegions.none()) 542 continue; 543 LLVM_DEBUG(dbgs() << 544 "Retrying function scheduling without clustering.\n"); 545 } 546 547 if (Stage == ClusteredLowOccupancyReschedule) { 548 if (StartingOccupancy <= MinOccupancy) 549 break; 550 551 LLVM_DEBUG( 552 dbgs() 553 << "Retrying function scheduling with lowest recorded occupancy " 554 << MinOccupancy << ".\n"); 555 556 S.setTargetOccupancy(MinOccupancy); 557 } 558 } 559 560 if (Stage == UnclusteredReschedule) 561 SavedMutations.swap(Mutations); 562 563 for (auto Region : Regions) { 564 if (Stage == UnclusteredReschedule && !RescheduleRegions[RegionIdx]) { 565 ++RegionIdx; 566 continue; 567 } 568 569 RegionBegin = Region.first; 570 RegionEnd = Region.second; 571 572 if (RegionBegin->getParent() != MBB) { 573 if (MBB) finishBlock(); 574 MBB = RegionBegin->getParent(); 575 startBlock(MBB); 576 if (Stage == InitialSchedule) 577 computeBlockPressure(MBB); 578 } 579 580 unsigned NumRegionInstrs = std::distance(begin(), end()); 581 enterRegion(MBB, begin(), end(), NumRegionInstrs); 582 583 // Skip empty scheduling regions (0 or 1 schedulable instructions). 584 if (begin() == end() || begin() == std::prev(end())) { 585 exitRegion(); 586 continue; 587 } 588 589 LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n"); 590 LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " " 591 << MBB->getName() << "\n From: " << *begin() 592 << " To: "; 593 if (RegionEnd != MBB->end()) dbgs() << *RegionEnd; 594 else dbgs() << "End"; 595 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n'); 596 597 schedule(); 598 599 exitRegion(); 600 ++RegionIdx; 601 } 602 finishBlock(); 603 604 if (Stage == UnclusteredReschedule) 605 SavedMutations.swap(Mutations); 606 } while (Stage != LastStage); 607 } 608