xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/GCNIterativeScheduler.cpp (revision 3e8eb5c7f4909209c042403ddee340b2ee7003a5)
1 //===- GCNIterativeScheduler.cpp ------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements the class GCNIterativeScheduler.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "GCNIterativeScheduler.h"
15 #include "GCNSchedStrategy.h"
16 #include "SIMachineFunctionInfo.h"
17 
18 using namespace llvm;
19 
20 #define DEBUG_TYPE "machine-scheduler"
21 
22 namespace llvm {
23 
24 std::vector<const SUnit *> makeMinRegSchedule(ArrayRef<const SUnit *> TopRoots,
25                                               const ScheduleDAG &DAG);
26 
27   std::vector<const SUnit*> makeGCNILPScheduler(ArrayRef<const SUnit*> BotRoots,
28     const ScheduleDAG &DAG);
29 }
30 
31 // shim accessors for different order containers
32 static inline MachineInstr *getMachineInstr(MachineInstr *MI) {
33   return MI;
34 }
35 static inline MachineInstr *getMachineInstr(const SUnit *SU) {
36   return SU->getInstr();
37 }
38 static inline MachineInstr *getMachineInstr(const SUnit &SU) {
39   return SU.getInstr();
40 }
41 
42 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
43 LLVM_DUMP_METHOD
44 static void printRegion(raw_ostream &OS,
45                         MachineBasicBlock::iterator Begin,
46                         MachineBasicBlock::iterator End,
47                         const LiveIntervals *LIS,
48                         unsigned MaxInstNum =
49                           std::numeric_limits<unsigned>::max()) {
50   auto BB = Begin->getParent();
51   OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
52      << BB->getName() << ":\n";
53   auto I = Begin;
54   MaxInstNum = std::max(MaxInstNum, 1u);
55   for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
56     if (!I->isDebugInstr() && LIS)
57       OS << LIS->getInstructionIndex(*I);
58     OS << '\t' << *I;
59   }
60   if (I != End) {
61     OS << "\t...\n";
62     I = std::prev(End);
63     if (!I->isDebugInstr() && LIS)
64       OS << LIS->getInstructionIndex(*I);
65     OS << '\t' << *I;
66   }
67   if (End != BB->end()) { // print boundary inst if present
68     OS << "----\n";
69     if (LIS) OS << LIS->getInstructionIndex(*End) << '\t';
70     OS << *End;
71   }
72 }
73 
74 LLVM_DUMP_METHOD
75 static void printLivenessInfo(raw_ostream &OS,
76                               MachineBasicBlock::iterator Begin,
77                               MachineBasicBlock::iterator End,
78                               const LiveIntervals *LIS) {
79   const auto BB = Begin->getParent();
80   const auto &MRI = BB->getParent()->getRegInfo();
81 
82   const auto LiveIns = getLiveRegsBefore(*Begin, *LIS);
83   OS << "LIn RP: ";
84   getRegPressure(MRI, LiveIns).print(OS);
85 
86   const auto BottomMI = End == BB->end() ? std::prev(End) : End;
87   const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
88   OS << "LOt RP: ";
89   getRegPressure(MRI, LiveOuts).print(OS);
90 }
91 
92 LLVM_DUMP_METHOD
93 void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
94   const auto &ST = MF.getSubtarget<GCNSubtarget>();
95   for (const auto R : Regions) {
96     OS << "Region to schedule ";
97     printRegion(OS, R->Begin, R->End, LIS, 1);
98     printLivenessInfo(OS, R->Begin, R->End, LIS);
99     OS << "Max RP: ";
100     R->MaxPressure.print(OS, &ST);
101   }
102 }
103 
104 LLVM_DUMP_METHOD
105 void GCNIterativeScheduler::printSchedResult(raw_ostream &OS,
106                                              const Region *R,
107                                              const GCNRegPressure &RP) const {
108   OS << "\nAfter scheduling ";
109   printRegion(OS, R->Begin, R->End, LIS);
110   printSchedRP(OS, R->MaxPressure, RP);
111   OS << '\n';
112 }
113 
114 LLVM_DUMP_METHOD
115 void GCNIterativeScheduler::printSchedRP(raw_ostream &OS,
116                                          const GCNRegPressure &Before,
117                                          const GCNRegPressure &After) const {
118   const auto &ST = MF.getSubtarget<GCNSubtarget>();
119   OS << "RP before: ";
120   Before.print(OS, &ST);
121   OS << "RP after:  ";
122   After.print(OS, &ST);
123 }
124 #endif
125 
126 // DAG builder helper
127 class GCNIterativeScheduler::BuildDAG {
128   GCNIterativeScheduler &Sch;
129   SmallVector<SUnit *, 8> TopRoots;
130 
131   SmallVector<SUnit*, 8> BotRoots;
132 public:
133   BuildDAG(const Region &R, GCNIterativeScheduler &_Sch)
134     : Sch(_Sch) {
135     auto BB = R.Begin->getParent();
136     Sch.BaseClass::startBlock(BB);
137     Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
138 
139     Sch.buildSchedGraph(Sch.AA, nullptr, nullptr, nullptr,
140                         /*TrackLaneMask*/true);
141     Sch.Topo.InitDAGTopologicalSorting();
142     Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
143   }
144 
145   ~BuildDAG() {
146     Sch.BaseClass::exitRegion();
147     Sch.BaseClass::finishBlock();
148   }
149 
150   ArrayRef<const SUnit *> getTopRoots() const {
151     return TopRoots;
152   }
153   ArrayRef<SUnit*> getBottomRoots() const {
154     return BotRoots;
155   }
156 };
157 
158 class GCNIterativeScheduler::OverrideLegacyStrategy {
159   GCNIterativeScheduler &Sch;
160   Region &Rgn;
161   std::unique_ptr<MachineSchedStrategy> SaveSchedImpl;
162   GCNRegPressure SaveMaxRP;
163 
164 public:
165   OverrideLegacyStrategy(Region &R,
166                          MachineSchedStrategy &OverrideStrategy,
167                          GCNIterativeScheduler &_Sch)
168     : Sch(_Sch)
169     , Rgn(R)
170     , SaveSchedImpl(std::move(_Sch.SchedImpl))
171     , SaveMaxRP(R.MaxPressure) {
172     Sch.SchedImpl.reset(&OverrideStrategy);
173     auto BB = R.Begin->getParent();
174     Sch.BaseClass::startBlock(BB);
175     Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
176   }
177 
178   ~OverrideLegacyStrategy() {
179     Sch.BaseClass::exitRegion();
180     Sch.BaseClass::finishBlock();
181     Sch.SchedImpl.release();
182     Sch.SchedImpl = std::move(SaveSchedImpl);
183   }
184 
185   void schedule() {
186     assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
187     LLVM_DEBUG(dbgs() << "\nScheduling ";
188                printRegion(dbgs(), Rgn.Begin, Rgn.End, Sch.LIS, 2));
189     Sch.BaseClass::schedule();
190 
191     // Unfortunately placeDebugValues incorrectly modifies RegionEnd, restore
192     Sch.RegionEnd = Rgn.End;
193     //assert(Rgn.End == Sch.RegionEnd);
194     Rgn.Begin = Sch.RegionBegin;
195     Rgn.MaxPressure.clear();
196   }
197 
198   void restoreOrder() {
199     assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
200     // DAG SUnits are stored using original region's order
201     // so just use SUnits as the restoring schedule
202     Sch.scheduleRegion(Rgn, Sch.SUnits, SaveMaxRP);
203   }
204 };
205 
206 namespace {
207 
208 // just a stub to make base class happy
209 class SchedStrategyStub : public MachineSchedStrategy {
210 public:
211   bool shouldTrackPressure() const override { return false; }
212   bool shouldTrackLaneMasks() const override { return false; }
213   void initialize(ScheduleDAGMI *DAG) override {}
214   SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
215   void schedNode(SUnit *SU, bool IsTopNode) override {}
216   void releaseTopNode(SUnit *SU) override {}
217   void releaseBottomNode(SUnit *SU) override {}
218 };
219 
220 } // end anonymous namespace
221 
222 GCNIterativeScheduler::GCNIterativeScheduler(MachineSchedContext *C,
223                                              StrategyKind S)
224   : BaseClass(C, std::make_unique<SchedStrategyStub>())
225   , Context(C)
226   , Strategy(S)
227   , UPTracker(*LIS) {
228 }
229 
230 // returns max pressure for a region
231 GCNRegPressure
232 GCNIterativeScheduler::getRegionPressure(MachineBasicBlock::iterator Begin,
233                                          MachineBasicBlock::iterator End)
234   const {
235   // For the purpose of pressure tracking bottom inst of the region should
236   // be also processed. End is either BB end, BB terminator inst or sched
237   // boundary inst.
238   auto const BBEnd = Begin->getParent()->end();
239   auto const BottomMI = End == BBEnd ? std::prev(End) : End;
240 
241   // scheduleRegions walks bottom to top, so its likely we just get next
242   // instruction to track
243   auto AfterBottomMI = std::next(BottomMI);
244   if (AfterBottomMI == BBEnd ||
245       &*AfterBottomMI != UPTracker.getLastTrackedMI()) {
246     UPTracker.reset(*BottomMI);
247   } else {
248     assert(UPTracker.isValid());
249   }
250 
251   for (auto I = BottomMI; I != Begin; --I)
252     UPTracker.recede(*I);
253 
254   UPTracker.recede(*Begin);
255 
256   assert(UPTracker.isValid() ||
257          (dbgs() << "Tracked region ",
258           printRegion(dbgs(), Begin, End, LIS), false));
259   return UPTracker.moveMaxPressure();
260 }
261 
262 // returns max pressure for a tentative schedule
263 template <typename Range> GCNRegPressure
264 GCNIterativeScheduler::getSchedulePressure(const Region &R,
265                                            Range &&Schedule) const {
266   auto const BBEnd = R.Begin->getParent()->end();
267   GCNUpwardRPTracker RPTracker(*LIS);
268   if (R.End != BBEnd) {
269     // R.End points to the boundary instruction but the
270     // schedule doesn't include it
271     RPTracker.reset(*R.End);
272     RPTracker.recede(*R.End);
273   } else {
274     // R.End doesn't point to the boundary instruction
275     RPTracker.reset(*std::prev(BBEnd));
276   }
277   for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
278     RPTracker.recede(*getMachineInstr(*--I));
279   }
280   return RPTracker.moveMaxPressure();
281 }
282 
283 void GCNIterativeScheduler::enterRegion(MachineBasicBlock *BB, // overridden
284                                         MachineBasicBlock::iterator Begin,
285                                         MachineBasicBlock::iterator End,
286                                         unsigned NumRegionInstrs) {
287   BaseClass::enterRegion(BB, Begin, End, NumRegionInstrs);
288   if (NumRegionInstrs > 2) {
289     Regions.push_back(
290       new (Alloc.Allocate())
291       Region { Begin, End, NumRegionInstrs,
292                getRegionPressure(Begin, End), nullptr });
293   }
294 }
295 
296 void GCNIterativeScheduler::schedule() { // overridden
297   // do nothing
298   LLVM_DEBUG(printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS);
299              if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
300                dbgs() << "Max RP: ";
301                Regions.back()->MaxPressure.print(
302                    dbgs(), &MF.getSubtarget<GCNSubtarget>());
303              } dbgs()
304              << '\n';);
305 }
306 
307 void GCNIterativeScheduler::finalizeSchedule() { // overridden
308   if (Regions.empty())
309     return;
310   switch (Strategy) {
311   case SCHEDULE_MINREGONLY: scheduleMinReg(); break;
312   case SCHEDULE_MINREGFORCED: scheduleMinReg(true); break;
313   case SCHEDULE_LEGACYMAXOCCUPANCY: scheduleLegacyMaxOccupancy(); break;
314   case SCHEDULE_ILP: scheduleILP(false); break;
315   }
316 }
317 
318 // Detach schedule from SUnits and interleave it with debug values.
319 // Returned schedule becomes independent of DAG state.
320 std::vector<MachineInstr*>
321 GCNIterativeScheduler::detachSchedule(ScheduleRef Schedule) const {
322   std::vector<MachineInstr*> Res;
323   Res.reserve(Schedule.size() * 2);
324 
325   if (FirstDbgValue)
326     Res.push_back(FirstDbgValue);
327 
328   const auto DbgB = DbgValues.begin(), DbgE = DbgValues.end();
329   for (auto SU : Schedule) {
330     Res.push_back(SU->getInstr());
331     const auto &D = std::find_if(DbgB, DbgE, [SU](decltype(*DbgB) &P) {
332       return P.second == SU->getInstr();
333     });
334     if (D != DbgE)
335       Res.push_back(D->first);
336   }
337   return Res;
338 }
339 
340 void GCNIterativeScheduler::setBestSchedule(Region &R,
341                                             ScheduleRef Schedule,
342                                             const GCNRegPressure &MaxRP) {
343   R.BestSchedule.reset(
344     new TentativeSchedule{ detachSchedule(Schedule), MaxRP });
345 }
346 
347 void GCNIterativeScheduler::scheduleBest(Region &R) {
348   assert(R.BestSchedule.get() && "No schedule specified");
349   scheduleRegion(R, R.BestSchedule->Schedule, R.BestSchedule->MaxPressure);
350   R.BestSchedule.reset();
351 }
352 
353 // minimal required region scheduler, works for ranges of SUnits*,
354 // SUnits or MachineIntrs*
355 template <typename Range>
356 void GCNIterativeScheduler::scheduleRegion(Region &R, Range &&Schedule,
357                                            const GCNRegPressure &MaxRP) {
358   assert(RegionBegin == R.Begin && RegionEnd == R.End);
359   assert(LIS != nullptr);
360 #ifndef NDEBUG
361   const auto SchedMaxRP = getSchedulePressure(R, Schedule);
362 #endif
363   auto BB = R.Begin->getParent();
364   auto Top = R.Begin;
365   for (const auto &I : Schedule) {
366     auto MI = getMachineInstr(I);
367     if (MI != &*Top) {
368       BB->remove(MI);
369       BB->insert(Top, MI);
370       if (!MI->isDebugInstr())
371         LIS->handleMove(*MI, true);
372     }
373     if (!MI->isDebugInstr()) {
374       // Reset read - undef flags and update them later.
375       for (auto &Op : MI->operands())
376         if (Op.isReg() && Op.isDef())
377           Op.setIsUndef(false);
378 
379       RegisterOperands RegOpers;
380       RegOpers.collect(*MI, *TRI, MRI, /*ShouldTrackLaneMasks*/true,
381                                        /*IgnoreDead*/false);
382       // Adjust liveness and add missing dead+read-undef flags.
383       auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
384       RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
385     }
386     Top = std::next(MI->getIterator());
387   }
388   RegionBegin = getMachineInstr(Schedule.front());
389 
390   // Schedule consisting of MachineInstr* is considered 'detached'
391   // and already interleaved with debug values
392   if (!std::is_same<decltype(*Schedule.begin()), MachineInstr*>::value) {
393     placeDebugValues();
394     // Unfortunately placeDebugValues incorrectly modifies RegionEnd, restore
395     // assert(R.End == RegionEnd);
396     RegionEnd = R.End;
397   }
398 
399   R.Begin = RegionBegin;
400   R.MaxPressure = MaxRP;
401 
402 #ifndef NDEBUG
403   const auto RegionMaxRP = getRegionPressure(R);
404   const auto &ST = MF.getSubtarget<GCNSubtarget>();
405 #endif
406   assert((SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP))
407   || (dbgs() << "Max RP mismatch!!!\n"
408                 "RP for schedule (calculated): ",
409       SchedMaxRP.print(dbgs(), &ST),
410       dbgs() << "RP for schedule (reported): ",
411       MaxRP.print(dbgs(), &ST),
412       dbgs() << "RP after scheduling: ",
413       RegionMaxRP.print(dbgs(), &ST),
414       false));
415 }
416 
417 // Sort recorded regions by pressure - highest at the front
418 void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) {
419   const auto &ST = MF.getSubtarget<GCNSubtarget>();
420   llvm::sort(Regions, [&ST, TargetOcc](const Region *R1, const Region *R2) {
421     return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc);
422   });
423 }
424 
425 ///////////////////////////////////////////////////////////////////////////////
426 // Legacy MaxOccupancy Strategy
427 
428 // Tries to increase occupancy applying minreg scheduler for a sequence of
429 // most demanding regions. Obtained schedules are saved as BestSchedule for a
430 // region.
431 // TargetOcc is the best achievable occupancy for a kernel.
432 // Returns better occupancy on success or current occupancy on fail.
433 // BestSchedules aren't deleted on fail.
434 unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
435   // TODO: assert Regions are sorted descending by pressure
436   const auto &ST = MF.getSubtarget<GCNSubtarget>();
437   const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
438   LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
439                     << ", current = " << Occ << '\n');
440 
441   auto NewOcc = TargetOcc;
442   for (auto R : Regions) {
443     if (R->MaxPressure.getOccupancy(ST) >= NewOcc)
444       break;
445 
446     LLVM_DEBUG(printRegion(dbgs(), R->Begin, R->End, LIS, 3);
447                printLivenessInfo(dbgs(), R->Begin, R->End, LIS));
448 
449     BuildDAG DAG(*R, *this);
450     const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
451     const auto MaxRP = getSchedulePressure(*R, MinSchedule);
452     LLVM_DEBUG(dbgs() << "Occupancy improvement attempt:\n";
453                printSchedRP(dbgs(), R->MaxPressure, MaxRP));
454 
455     NewOcc = std::min(NewOcc, MaxRP.getOccupancy(ST));
456     if (NewOcc <= Occ)
457       break;
458 
459     setBestSchedule(*R, MinSchedule, MaxRP);
460   }
461   LLVM_DEBUG(dbgs() << "New occupancy = " << NewOcc
462                     << ", prev occupancy = " << Occ << '\n');
463   if (NewOcc > Occ) {
464     SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
465     MFI->increaseOccupancy(MF, NewOcc);
466   }
467 
468   return std::max(NewOcc, Occ);
469 }
470 
471 void GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
472   bool TryMaximizeOccupancy) {
473   const auto &ST = MF.getSubtarget<GCNSubtarget>();
474   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
475   auto TgtOcc = MFI->getMinAllowedOccupancy();
476 
477   sortRegionsByPressure(TgtOcc);
478   auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
479 
480   if (TryMaximizeOccupancy && Occ < TgtOcc)
481     Occ = tryMaximizeOccupancy(TgtOcc);
482 
483   // This is really weird but for some magic scheduling regions twice
484   // gives performance improvement
485   const int NumPasses = Occ < TgtOcc ? 2 : 1;
486 
487   TgtOcc = std::min(Occ, TgtOcc);
488   LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
489                        "target occupancy = "
490                     << TgtOcc << '\n');
491   GCNMaxOccupancySchedStrategy LStrgy(Context);
492   unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
493 
494   for (int I = 0; I < NumPasses; ++I) {
495     // running first pass with TargetOccupancy = 0 mimics previous scheduling
496     // approach and is a performance magic
497     LStrgy.setTargetOccupancy(I == 0 ? 0 : TgtOcc);
498     for (auto R : Regions) {
499       OverrideLegacyStrategy Ovr(*R, LStrgy, *this);
500 
501       Ovr.schedule();
502       const auto RP = getRegionPressure(*R);
503       LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
504 
505       if (RP.getOccupancy(ST) < TgtOcc) {
506         LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
507         if (R->BestSchedule.get() &&
508             R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
509           LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
510           scheduleBest(*R);
511         } else {
512           LLVM_DEBUG(dbgs() << ", restoring\n");
513           Ovr.restoreOrder();
514           assert(R->MaxPressure.getOccupancy(ST) >= TgtOcc);
515         }
516       }
517       FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
518     }
519   }
520   MFI->limitOccupancy(FinalOccupancy);
521 }
522 
523 ///////////////////////////////////////////////////////////////////////////////
524 // Minimal Register Strategy
525 
526 void GCNIterativeScheduler::scheduleMinReg(bool force) {
527   const auto &ST = MF.getSubtarget<GCNSubtarget>();
528   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
529   const auto TgtOcc = MFI->getOccupancy();
530   sortRegionsByPressure(TgtOcc);
531 
532   auto MaxPressure = Regions.front()->MaxPressure;
533   for (auto R : Regions) {
534     if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc))
535       break;
536 
537     BuildDAG DAG(*R, *this);
538     const auto MinSchedule = makeMinRegSchedule(DAG.getTopRoots(), *this);
539 
540     const auto RP = getSchedulePressure(*R, MinSchedule);
541     LLVM_DEBUG(if (R->MaxPressure.less(ST, RP, TgtOcc)) {
542       dbgs() << "\nWarning: Pressure becomes worse after minreg!";
543       printSchedRP(dbgs(), R->MaxPressure, RP);
544     });
545 
546     if (!force && MaxPressure.less(ST, RP, TgtOcc))
547       break;
548 
549     scheduleRegion(*R, MinSchedule, RP);
550     LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
551 
552     MaxPressure = RP;
553   }
554 }
555 
556 ///////////////////////////////////////////////////////////////////////////////
557 // ILP scheduler port
558 
559 void GCNIterativeScheduler::scheduleILP(
560   bool TryMaximizeOccupancy) {
561   const auto &ST = MF.getSubtarget<GCNSubtarget>();
562   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
563   auto TgtOcc = MFI->getMinAllowedOccupancy();
564 
565   sortRegionsByPressure(TgtOcc);
566   auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
567 
568   if (TryMaximizeOccupancy && Occ < TgtOcc)
569     Occ = tryMaximizeOccupancy(TgtOcc);
570 
571   TgtOcc = std::min(Occ, TgtOcc);
572   LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
573                        "target occupancy = "
574                     << TgtOcc << '\n');
575 
576   unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
577   for (auto R : Regions) {
578     BuildDAG DAG(*R, *this);
579     const auto ILPSchedule = makeGCNILPScheduler(DAG.getBottomRoots(), *this);
580 
581     const auto RP = getSchedulePressure(*R, ILPSchedule);
582     LLVM_DEBUG(printSchedRP(dbgs(), R->MaxPressure, RP));
583 
584     if (RP.getOccupancy(ST) < TgtOcc) {
585       LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
586       if (R->BestSchedule.get() &&
587         R->BestSchedule->MaxPressure.getOccupancy(ST) >= TgtOcc) {
588         LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
589         scheduleBest(*R);
590       }
591     } else {
592       scheduleRegion(*R, ILPSchedule, RP);
593       LLVM_DEBUG(printSchedResult(dbgs(), R, RP));
594       FinalOccupancy = std::min(FinalOccupancy, RP.getOccupancy(ST));
595     }
596   }
597   MFI->limitOccupancy(FinalOccupancy);
598 }
599