xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/TargetPassConfig.cpp (revision 95eb4b873b6a8b527c5bd78d7191975dfca38998)
1 //===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines interfaces to access the target independent code
10 // generation passes provided by the LLVM backend.
11 //
12 //===---------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/TargetPassConfig.h"
15 #include "llvm/ADT/DenseMap.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/Analysis/BasicAliasAnalysis.h"
19 #include "llvm/Analysis/CallGraphSCCPass.h"
20 #include "llvm/Analysis/ScopedNoAliasAA.h"
21 #include "llvm/Analysis/TargetTransformInfo.h"
22 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
23 #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
24 #include "llvm/CodeGen/CSEConfigBase.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachinePassRegistry.h"
27 #include "llvm/CodeGen/Passes.h"
28 #include "llvm/CodeGen/RegAllocRegistry.h"
29 #include "llvm/IR/IRPrintingPasses.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #include "llvm/IR/PassInstrumentation.h"
32 #include "llvm/IR/Verifier.h"
33 #include "llvm/InitializePasses.h"
34 #include "llvm/MC/MCAsmInfo.h"
35 #include "llvm/MC/MCTargetOptions.h"
36 #include "llvm/Pass.h"
37 #include "llvm/Support/CodeGen.h"
38 #include "llvm/Support/CommandLine.h"
39 #include "llvm/Support/Compiler.h"
40 #include "llvm/Support/Debug.h"
41 #include "llvm/Support/Discriminator.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/SaveAndRestore.h"
44 #include "llvm/Support/Threading.h"
45 #include "llvm/Support/VirtualFileSystem.h"
46 #include "llvm/Support/WithColor.h"
47 #include "llvm/Target/CGPassBuilderOption.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Transforms/Scalar.h"
50 #include "llvm/Transforms/Utils.h"
51 #include <cassert>
52 #include <optional>
53 #include <string>
54 
55 using namespace llvm;
56 
57 static cl::opt<bool>
58     EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,
59                cl::desc("Enable interprocedural register allocation "
60                         "to reduce load/store at procedure calls."));
61 static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
62     cl::desc("Disable Post Regalloc Scheduler"));
63 static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
64     cl::desc("Disable branch folding"));
65 static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
66     cl::desc("Disable tail duplication"));
67 static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
68     cl::desc("Disable pre-register allocation tail duplication"));
69 static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
70     cl::Hidden, cl::desc("Disable probability-driven block placement"));
71 static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
72     cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
73 static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
74     cl::desc("Disable Stack Slot Coloring"));
75 static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
76     cl::desc("Disable Machine Dead Code Elimination"));
77 static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
78     cl::desc("Disable Early If-conversion"));
79 static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
80     cl::desc("Disable Machine LICM"));
81 static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
82     cl::desc("Disable Machine Common Subexpression Elimination"));
83 static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
84     "optimize-regalloc", cl::Hidden,
85     cl::desc("Enable optimized register allocation compilation path."));
86 static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
87     cl::Hidden,
88     cl::desc("Disable Machine LICM"));
89 static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
90     cl::desc("Disable Machine Sinking"));
91 static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
92     cl::Hidden,
93     cl::desc("Disable PostRA Machine Sinking"));
94 static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
95     cl::desc("Disable Loop Strength Reduction Pass"));
96 static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
97     cl::Hidden, cl::desc("Disable ConstantHoisting"));
98 static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
99     cl::desc("Disable Codegen Prepare"));
100 static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
101     cl::desc("Disable Copy Propagation pass"));
102 static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
103     cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
104 static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(
105     "disable-atexit-based-global-dtor-lowering", cl::Hidden,
106     cl::desc("For MachO, disable atexit()-based global destructor lowering"));
107 static cl::opt<bool> EnableImplicitNullChecks(
108     "enable-implicit-null-checks",
109     cl::desc("Fold null checks into faulting memory operations"),
110     cl::init(false), cl::Hidden);
111 static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",
112     cl::desc("Disable MergeICmps Pass"),
113     cl::init(false), cl::Hidden);
114 static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
115     cl::desc("Print LLVM IR produced by the loop-reduce pass"));
116 static cl::opt<bool>
117     PrintISelInput("print-isel-input", cl::Hidden,
118                    cl::desc("Print LLVM IR input to isel pass"));
119 static cl::opt<cl::boolOrDefault>
120     VerifyMachineCode("verify-machineinstrs", cl::Hidden,
121                       cl::desc("Verify generated machine code"));
122 static cl::opt<cl::boolOrDefault>
123     DebugifyAndStripAll("debugify-and-strip-all-safe", cl::Hidden,
124                         cl::desc("Debugify MIR before and Strip debug after "
125                                  "each pass except those known to be unsafe "
126                                  "when debug info is present"));
127 static cl::opt<cl::boolOrDefault> DebugifyCheckAndStripAll(
128     "debugify-check-and-strip-all-safe", cl::Hidden,
129     cl::desc(
130         "Debugify MIR before, by checking and stripping the debug info after, "
131         "each pass except those known to be unsafe when debug info is "
132         "present"));
133 // Enable or disable the MachineOutliner.
134 static cl::opt<RunOutliner> EnableMachineOutliner(
135     "enable-machine-outliner", cl::desc("Enable the machine outliner"),
136     cl::Hidden, cl::ValueOptional, cl::init(RunOutliner::TargetDefault),
137     cl::values(clEnumValN(RunOutliner::AlwaysOutline, "always",
138                           "Run on all functions guaranteed to be beneficial"),
139                clEnumValN(RunOutliner::NeverOutline, "never",
140                           "Disable all outlining"),
141                // Sentinel value for unspecified option.
142                clEnumValN(RunOutliner::AlwaysOutline, "", "")));
143 // Disable the pass to fix unwind information. Whether the pass is included in
144 // the pipeline is controlled via the target options, this option serves as
145 // manual override.
146 static cl::opt<bool> DisableCFIFixup("disable-cfi-fixup", cl::Hidden,
147                                      cl::desc("Disable the CFI fixup pass"));
148 // Enable or disable FastISel. Both options are needed, because
149 // FastISel is enabled by default with -fast, and we wish to be
150 // able to enable or disable fast-isel independently from -O0.
151 static cl::opt<cl::boolOrDefault>
152 EnableFastISelOption("fast-isel", cl::Hidden,
153   cl::desc("Enable the \"fast\" instruction selector"));
154 
155 static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
156     "global-isel", cl::Hidden,
157     cl::desc("Enable the \"global\" instruction selector"));
158 
159 // FIXME: remove this after switching to NPM or GlobalISel, whichever gets there
160 //        first...
161 static cl::opt<bool>
162     PrintAfterISel("print-after-isel", cl::init(false), cl::Hidden,
163                    cl::desc("Print machine instrs after ISel"));
164 
165 static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
166     "global-isel-abort", cl::Hidden,
167     cl::desc("Enable abort calls when \"global\" instruction selection "
168              "fails to lower/select an instruction"),
169     cl::values(
170         clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
171         clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
172         clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
173                    "Disable the abort but emit a diagnostic on failure")));
174 
175 // Disable MIRProfileLoader before RegAlloc. This is for for debugging and
176 // tuning purpose.
177 static cl::opt<bool> DisableRAFSProfileLoader(
178     "disable-ra-fsprofile-loader", cl::init(false), cl::Hidden,
179     cl::desc("Disable MIRProfileLoader before RegAlloc"));
180 // Disable MIRProfileLoader before BloackPlacement. This is for for debugging
181 // and tuning purpose.
182 static cl::opt<bool> DisableLayoutFSProfileLoader(
183     "disable-layout-fsprofile-loader", cl::init(false), cl::Hidden,
184     cl::desc("Disable MIRProfileLoader before BlockPlacement"));
185 // Specify FSProfile file name.
186 static cl::opt<std::string>
187     FSProfileFile("fs-profile-file", cl::init(""), cl::value_desc("filename"),
188                   cl::desc("Flow Sensitive profile file name."), cl::Hidden);
189 // Specify Remapping file for FSProfile.
190 static cl::opt<std::string> FSRemappingFile(
191     "fs-remapping-file", cl::init(""), cl::value_desc("filename"),
192     cl::desc("Flow Sensitive profile remapping file name."), cl::Hidden);
193 
194 // Temporary option to allow experimenting with MachineScheduler as a post-RA
195 // scheduler. Targets can "properly" enable this with
196 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
197 // Targets can return true in targetSchedulesPostRAScheduling() and
198 // insert a PostRA scheduling pass wherever it wants.
199 static cl::opt<bool> MISchedPostRA(
200     "misched-postra", cl::Hidden,
201     cl::desc(
202         "Run MachineScheduler post regalloc (independent of preRA sched)"));
203 
204 // Experimental option to run live interval analysis early.
205 static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
206     cl::desc("Run live interval analysis earlier in the pipeline"));
207 
208 /// Option names for limiting the codegen pipeline.
209 /// Those are used in error reporting and we didn't want
210 /// to duplicate their names all over the place.
211 static const char StartAfterOptName[] = "start-after";
212 static const char StartBeforeOptName[] = "start-before";
213 static const char StopAfterOptName[] = "stop-after";
214 static const char StopBeforeOptName[] = "stop-before";
215 
216 static cl::opt<std::string>
217     StartAfterOpt(StringRef(StartAfterOptName),
218                   cl::desc("Resume compilation after a specific pass"),
219                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
220 
221 static cl::opt<std::string>
222     StartBeforeOpt(StringRef(StartBeforeOptName),
223                    cl::desc("Resume compilation before a specific pass"),
224                    cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
225 
226 static cl::opt<std::string>
227     StopAfterOpt(StringRef(StopAfterOptName),
228                  cl::desc("Stop compilation after a specific pass"),
229                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
230 
231 static cl::opt<std::string>
232     StopBeforeOpt(StringRef(StopBeforeOptName),
233                   cl::desc("Stop compilation before a specific pass"),
234                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
235 
236 /// Enable the machine function splitter pass.
237 static cl::opt<bool> EnableMachineFunctionSplitter(
238     "enable-split-machine-functions", cl::Hidden,
239     cl::desc("Split out cold blocks from machine functions based on profile "
240              "information."));
241 
242 /// Disable the expand reductions pass for testing.
243 static cl::opt<bool> DisableExpandReductions(
244     "disable-expand-reductions", cl::init(false), cl::Hidden,
245     cl::desc("Disable the expand reduction intrinsics pass from running"));
246 
247 /// Disable the select optimization pass.
248 static cl::opt<bool> DisableSelectOptimize(
249     "disable-select-optimize", cl::init(true), cl::Hidden,
250     cl::desc("Disable the select-optimization pass from running"));
251 
252 /// Enable garbage-collecting empty basic blocks.
253 static cl::opt<bool>
254     GCEmptyBlocks("gc-empty-basic-blocks", cl::init(false), cl::Hidden,
255                   cl::desc("Enable garbage-collecting empty basic blocks"));
256 
257 /// Allow standard passes to be disabled by command line options. This supports
258 /// simple binary flags that either suppress the pass or do nothing.
259 /// i.e. -disable-mypass=false has no effect.
260 /// These should be converted to boolOrDefault in order to use applyOverride.
261 static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
262                                        bool Override) {
263   if (Override)
264     return IdentifyingPassPtr();
265   return PassID;
266 }
267 
268 /// Allow standard passes to be disabled by the command line, regardless of who
269 /// is adding the pass.
270 ///
271 /// StandardID is the pass identified in the standard pass pipeline and provided
272 /// to addPass(). It may be a target-specific ID in the case that the target
273 /// directly adds its own pass, but in that case we harmlessly fall through.
274 ///
275 /// TargetID is the pass that the target has configured to override StandardID.
276 ///
277 /// StandardID may be a pseudo ID. In that case TargetID is the name of the real
278 /// pass to run. This allows multiple options to control a single pass depending
279 /// on where in the pipeline that pass is added.
280 static IdentifyingPassPtr overridePass(AnalysisID StandardID,
281                                        IdentifyingPassPtr TargetID) {
282   if (StandardID == &PostRASchedulerID)
283     return applyDisable(TargetID, DisablePostRASched);
284 
285   if (StandardID == &BranchFolderPassID)
286     return applyDisable(TargetID, DisableBranchFold);
287 
288   if (StandardID == &TailDuplicateID)
289     return applyDisable(TargetID, DisableTailDuplicate);
290 
291   if (StandardID == &EarlyTailDuplicateID)
292     return applyDisable(TargetID, DisableEarlyTailDup);
293 
294   if (StandardID == &MachineBlockPlacementID)
295     return applyDisable(TargetID, DisableBlockPlacement);
296 
297   if (StandardID == &StackSlotColoringID)
298     return applyDisable(TargetID, DisableSSC);
299 
300   if (StandardID == &DeadMachineInstructionElimID)
301     return applyDisable(TargetID, DisableMachineDCE);
302 
303   if (StandardID == &EarlyIfConverterID)
304     return applyDisable(TargetID, DisableEarlyIfConversion);
305 
306   if (StandardID == &EarlyMachineLICMID)
307     return applyDisable(TargetID, DisableMachineLICM);
308 
309   if (StandardID == &MachineCSEID)
310     return applyDisable(TargetID, DisableMachineCSE);
311 
312   if (StandardID == &MachineLICMID)
313     return applyDisable(TargetID, DisablePostRAMachineLICM);
314 
315   if (StandardID == &MachineSinkingID)
316     return applyDisable(TargetID, DisableMachineSink);
317 
318   if (StandardID == &PostRAMachineSinkingID)
319     return applyDisable(TargetID, DisablePostRAMachineSink);
320 
321   if (StandardID == &MachineCopyPropagationID)
322     return applyDisable(TargetID, DisableCopyProp);
323 
324   return TargetID;
325 }
326 
327 // Find the FSProfile file name. The internal option takes the precedence
328 // before getting from TargetMachine.
329 static std::string getFSProfileFile(const TargetMachine *TM) {
330   if (!FSProfileFile.empty())
331     return FSProfileFile.getValue();
332   const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
333   if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
334     return std::string();
335   return PGOOpt->ProfileFile;
336 }
337 
338 // Find the Profile remapping file name. The internal option takes the
339 // precedence before getting from TargetMachine.
340 static std::string getFSRemappingFile(const TargetMachine *TM) {
341   if (!FSRemappingFile.empty())
342     return FSRemappingFile.getValue();
343   const std::optional<PGOOptions> &PGOOpt = TM->getPGOOption();
344   if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse)
345     return std::string();
346   return PGOOpt->ProfileRemappingFile;
347 }
348 
349 //===---------------------------------------------------------------------===//
350 /// TargetPassConfig
351 //===---------------------------------------------------------------------===//
352 
353 INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
354                 "Target Pass Configuration", false, false)
355 char TargetPassConfig::ID = 0;
356 
357 namespace {
358 
359 struct InsertedPass {
360   AnalysisID TargetPassID;
361   IdentifyingPassPtr InsertedPassID;
362 
363   InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID)
364       : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID) {}
365 
366   Pass *getInsertedPass() const {
367     assert(InsertedPassID.isValid() && "Illegal Pass ID!");
368     if (InsertedPassID.isInstance())
369       return InsertedPassID.getInstance();
370     Pass *NP = Pass::createPass(InsertedPassID.getID());
371     assert(NP && "Pass ID not registered");
372     return NP;
373   }
374 };
375 
376 } // end anonymous namespace
377 
378 namespace llvm {
379 
380 extern cl::opt<bool> EnableFSDiscriminator;
381 
382 class PassConfigImpl {
383 public:
384   // List of passes explicitly substituted by this target. Normally this is
385   // empty, but it is a convenient way to suppress or replace specific passes
386   // that are part of a standard pass pipeline without overridding the entire
387   // pipeline. This mechanism allows target options to inherit a standard pass's
388   // user interface. For example, a target may disable a standard pass by
389   // default by substituting a pass ID of zero, and the user may still enable
390   // that standard pass with an explicit command line option.
391   DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
392 
393   /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
394   /// is inserted after each instance of the first one.
395   SmallVector<InsertedPass, 4> InsertedPasses;
396 };
397 
398 } // end namespace llvm
399 
400 // Out of line virtual method.
401 TargetPassConfig::~TargetPassConfig() {
402   delete Impl;
403 }
404 
405 static const PassInfo *getPassInfo(StringRef PassName) {
406   if (PassName.empty())
407     return nullptr;
408 
409   const PassRegistry &PR = *PassRegistry::getPassRegistry();
410   const PassInfo *PI = PR.getPassInfo(PassName);
411   if (!PI)
412     report_fatal_error(Twine('\"') + Twine(PassName) +
413                        Twine("\" pass is not registered."));
414   return PI;
415 }
416 
417 static AnalysisID getPassIDFromName(StringRef PassName) {
418   const PassInfo *PI = getPassInfo(PassName);
419   return PI ? PI->getTypeInfo() : nullptr;
420 }
421 
422 static std::pair<StringRef, unsigned>
423 getPassNameAndInstanceNum(StringRef PassName) {
424   StringRef Name, InstanceNumStr;
425   std::tie(Name, InstanceNumStr) = PassName.split(',');
426 
427   unsigned InstanceNum = 0;
428   if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum))
429     report_fatal_error("invalid pass instance specifier " + PassName);
430 
431   return std::make_pair(Name, InstanceNum);
432 }
433 
434 void TargetPassConfig::setStartStopPasses() {
435   StringRef StartBeforeName;
436   std::tie(StartBeforeName, StartBeforeInstanceNum) =
437     getPassNameAndInstanceNum(StartBeforeOpt);
438 
439   StringRef StartAfterName;
440   std::tie(StartAfterName, StartAfterInstanceNum) =
441     getPassNameAndInstanceNum(StartAfterOpt);
442 
443   StringRef StopBeforeName;
444   std::tie(StopBeforeName, StopBeforeInstanceNum)
445     = getPassNameAndInstanceNum(StopBeforeOpt);
446 
447   StringRef StopAfterName;
448   std::tie(StopAfterName, StopAfterInstanceNum)
449     = getPassNameAndInstanceNum(StopAfterOpt);
450 
451   StartBefore = getPassIDFromName(StartBeforeName);
452   StartAfter = getPassIDFromName(StartAfterName);
453   StopBefore = getPassIDFromName(StopBeforeName);
454   StopAfter = getPassIDFromName(StopAfterName);
455   if (StartBefore && StartAfter)
456     report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
457                        Twine(StartAfterOptName) + Twine(" specified!"));
458   if (StopBefore && StopAfter)
459     report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
460                        Twine(StopAfterOptName) + Twine(" specified!"));
461   Started = (StartAfter == nullptr) && (StartBefore == nullptr);
462 }
463 
464 CGPassBuilderOption llvm::getCGPassBuilderOption() {
465   CGPassBuilderOption Opt;
466 
467 #define SET_OPTION(Option)                                                     \
468   if (Option.getNumOccurrences())                                              \
469     Opt.Option = Option;
470 
471   SET_OPTION(EnableFastISelOption)
472   SET_OPTION(EnableGlobalISelAbort)
473   SET_OPTION(EnableGlobalISelOption)
474   SET_OPTION(EnableIPRA)
475   SET_OPTION(OptimizeRegAlloc)
476   SET_OPTION(VerifyMachineCode)
477   SET_OPTION(DisableAtExitBasedGlobalDtorLowering)
478   SET_OPTION(DisableExpandReductions)
479   SET_OPTION(PrintAfterISel)
480   SET_OPTION(FSProfileFile)
481   SET_OPTION(GCEmptyBlocks)
482 
483 #define SET_BOOLEAN_OPTION(Option) Opt.Option = Option;
484 
485   SET_BOOLEAN_OPTION(EarlyLiveIntervals)
486   SET_BOOLEAN_OPTION(EnableBlockPlacementStats)
487   SET_BOOLEAN_OPTION(EnableImplicitNullChecks)
488   SET_BOOLEAN_OPTION(EnableMachineOutliner)
489   SET_BOOLEAN_OPTION(MISchedPostRA)
490   SET_BOOLEAN_OPTION(DisableMergeICmps)
491   SET_BOOLEAN_OPTION(DisableLSR)
492   SET_BOOLEAN_OPTION(DisableConstantHoisting)
493   SET_BOOLEAN_OPTION(DisableCGP)
494   SET_BOOLEAN_OPTION(DisablePartialLibcallInlining)
495   SET_BOOLEAN_OPTION(DisableSelectOptimize)
496   SET_BOOLEAN_OPTION(PrintLSR)
497   SET_BOOLEAN_OPTION(PrintISelInput)
498   SET_BOOLEAN_OPTION(DebugifyAndStripAll)
499   SET_BOOLEAN_OPTION(DebugifyCheckAndStripAll)
500   SET_BOOLEAN_OPTION(DisableRAFSProfileLoader)
501   SET_BOOLEAN_OPTION(DisableCFIFixup)
502   SET_BOOLEAN_OPTION(EnableMachineFunctionSplitter)
503 
504   return Opt;
505 }
506 
507 void llvm::registerCodeGenCallback(PassInstrumentationCallbacks &PIC,
508                                    LLVMTargetMachine &LLVMTM) {
509 
510   // Register a callback for disabling passes.
511   PIC.registerShouldRunOptionalPassCallback([](StringRef P, Any) {
512 
513 #define DISABLE_PASS(Option, Name)                                             \
514   if (Option && P.contains(#Name))                                             \
515     return false;
516     DISABLE_PASS(DisableBlockPlacement, MachineBlockPlacementPass)
517     DISABLE_PASS(DisableBranchFold, BranchFolderPass)
518     DISABLE_PASS(DisableCopyProp, MachineCopyPropagationPass)
519     DISABLE_PASS(DisableEarlyIfConversion, EarlyIfConverterPass)
520     DISABLE_PASS(DisableEarlyTailDup, EarlyTailDuplicatePass)
521     DISABLE_PASS(DisableMachineCSE, MachineCSEPass)
522     DISABLE_PASS(DisableMachineDCE, DeadMachineInstructionElimPass)
523     DISABLE_PASS(DisableMachineLICM, EarlyMachineLICMPass)
524     DISABLE_PASS(DisableMachineSink, MachineSinkingPass)
525     DISABLE_PASS(DisablePostRAMachineLICM, MachineLICMPass)
526     DISABLE_PASS(DisablePostRAMachineSink, PostRAMachineSinkingPass)
527     DISABLE_PASS(DisablePostRASched, PostRASchedulerPass)
528     DISABLE_PASS(DisableSSC, StackSlotColoringPass)
529     DISABLE_PASS(DisableTailDuplicate, TailDuplicatePass)
530 
531     return true;
532   });
533 }
534 
535 Expected<TargetPassConfig::StartStopInfo>
536 TargetPassConfig::getStartStopInfo(PassInstrumentationCallbacks &PIC) {
537   auto [StartBefore, StartBeforeInstanceNum] =
538       getPassNameAndInstanceNum(StartBeforeOpt);
539   auto [StartAfter, StartAfterInstanceNum] =
540       getPassNameAndInstanceNum(StartAfterOpt);
541   auto [StopBefore, StopBeforeInstanceNum] =
542       getPassNameAndInstanceNum(StopBeforeOpt);
543   auto [StopAfter, StopAfterInstanceNum] =
544       getPassNameAndInstanceNum(StopAfterOpt);
545 
546   if (!StartBefore.empty() && !StartAfter.empty())
547     return make_error<StringError>(
548         Twine(StartBeforeOptName) + " and " + StartAfterOptName + " specified!",
549         std::make_error_code(std::errc::invalid_argument));
550   if (!StopBefore.empty() && !StopAfter.empty())
551     return make_error<StringError>(
552         Twine(StopBeforeOptName) + " and " + StopAfterOptName + " specified!",
553         std::make_error_code(std::errc::invalid_argument));
554 
555   StartStopInfo Result;
556   Result.StartPass = StartBefore.empty() ? StartAfter : StartBefore;
557   Result.StopPass = StopBefore.empty() ? StopAfter : StopBefore;
558   Result.StartInstanceNum =
559       StartBefore.empty() ? StartAfterInstanceNum : StartBeforeInstanceNum;
560   Result.StopInstanceNum =
561       StopBefore.empty() ? StopAfterInstanceNum : StopBeforeInstanceNum;
562   Result.StartAfter = !StartAfter.empty();
563   Result.StopAfter = !StopAfter.empty();
564   Result.StartInstanceNum += Result.StartInstanceNum == 0;
565   Result.StopInstanceNum += Result.StopInstanceNum == 0;
566   return Result;
567 }
568 
569 // Out of line constructor provides default values for pass options and
570 // registers all common codegen passes.
571 TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
572     : ImmutablePass(ID), PM(&pm), TM(&TM) {
573   Impl = new PassConfigImpl();
574 
575   // Register all target independent codegen passes to activate their PassIDs,
576   // including this pass itself.
577   initializeCodeGen(*PassRegistry::getPassRegistry());
578 
579   // Also register alias analysis passes required by codegen passes.
580   initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
581   initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
582 
583   if (EnableIPRA.getNumOccurrences())
584     TM.Options.EnableIPRA = EnableIPRA;
585   else {
586     // If not explicitly specified, use target default.
587     TM.Options.EnableIPRA |= TM.useIPRA();
588   }
589 
590   if (TM.Options.EnableIPRA)
591     setRequiresCodeGenSCCOrder();
592 
593   if (EnableGlobalISelAbort.getNumOccurrences())
594     TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
595 
596   setStartStopPasses();
597 }
598 
599 CodeGenOptLevel TargetPassConfig::getOptLevel() const {
600   return TM->getOptLevel();
601 }
602 
603 /// Insert InsertedPassID pass after TargetPassID.
604 void TargetPassConfig::insertPass(AnalysisID TargetPassID,
605                                   IdentifyingPassPtr InsertedPassID) {
606   assert(((!InsertedPassID.isInstance() &&
607            TargetPassID != InsertedPassID.getID()) ||
608           (InsertedPassID.isInstance() &&
609            TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
610          "Insert a pass after itself!");
611   Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID);
612 }
613 
614 /// createPassConfig - Create a pass configuration object to be used by
615 /// addPassToEmitX methods for generating a pipeline of CodeGen passes.
616 ///
617 /// Targets may override this to extend TargetPassConfig.
618 TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) {
619   return new TargetPassConfig(*this, PM);
620 }
621 
622 TargetPassConfig::TargetPassConfig()
623   : ImmutablePass(ID) {
624   report_fatal_error("Trying to construct TargetPassConfig without a target "
625                      "machine. Scheduling a CodeGen pass without a target "
626                      "triple set?");
627 }
628 
629 bool TargetPassConfig::willCompleteCodeGenPipeline() {
630   return StopBeforeOpt.empty() && StopAfterOpt.empty();
631 }
632 
633 bool TargetPassConfig::hasLimitedCodeGenPipeline() {
634   return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
635          !willCompleteCodeGenPipeline();
636 }
637 
638 std::string TargetPassConfig::getLimitedCodeGenPipelineReason() {
639   if (!hasLimitedCodeGenPipeline())
640     return std::string();
641   std::string Res;
642   static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
643                                               &StopAfterOpt, &StopBeforeOpt};
644   static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
645                                    StopAfterOptName, StopBeforeOptName};
646   bool IsFirst = true;
647   for (int Idx = 0; Idx < 4; ++Idx)
648     if (!PassNames[Idx]->empty()) {
649       if (!IsFirst)
650         Res += " and ";
651       IsFirst = false;
652       Res += OptNames[Idx];
653     }
654   return Res;
655 }
656 
657 // Helper to verify the analysis is really immutable.
658 void TargetPassConfig::setOpt(bool &Opt, bool Val) {
659   assert(!Initialized && "PassConfig is immutable");
660   Opt = Val;
661 }
662 
663 void TargetPassConfig::substitutePass(AnalysisID StandardID,
664                                       IdentifyingPassPtr TargetID) {
665   Impl->TargetPasses[StandardID] = TargetID;
666 }
667 
668 IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
669   DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
670     I = Impl->TargetPasses.find(ID);
671   if (I == Impl->TargetPasses.end())
672     return ID;
673   return I->second;
674 }
675 
676 bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
677   IdentifyingPassPtr TargetID = getPassSubstitution(ID);
678   IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);
679   return !FinalPtr.isValid() || FinalPtr.isInstance() ||
680       FinalPtr.getID() != ID;
681 }
682 
683 /// Add a pass to the PassManager if that pass is supposed to be run.  If the
684 /// Started/Stopped flags indicate either that the compilation should start at
685 /// a later pass or that it should stop after an earlier pass, then do not add
686 /// the pass.  Finally, compare the current pass against the StartAfter
687 /// and StopAfter options and change the Started/Stopped flags accordingly.
688 void TargetPassConfig::addPass(Pass *P) {
689   assert(!Initialized && "PassConfig is immutable");
690 
691   // Cache the Pass ID here in case the pass manager finds this pass is
692   // redundant with ones already scheduled / available, and deletes it.
693   // Fundamentally, once we add the pass to the manager, we no longer own it
694   // and shouldn't reference it.
695   AnalysisID PassID = P->getPassID();
696 
697   if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
698     Started = true;
699   if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
700     Stopped = true;
701   if (Started && !Stopped) {
702     if (AddingMachinePasses) {
703       // Construct banner message before PM->add() as that may delete the pass.
704       std::string Banner =
705           std::string("After ") + std::string(P->getPassName());
706       addMachinePrePasses();
707       PM->add(P);
708       addMachinePostPasses(Banner);
709     } else {
710       PM->add(P);
711     }
712 
713     // Add the passes after the pass P if there is any.
714     for (const auto &IP : Impl->InsertedPasses)
715       if (IP.TargetPassID == PassID)
716         addPass(IP.getInsertedPass());
717   } else {
718     delete P;
719   }
720 
721   if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
722     Stopped = true;
723 
724   if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
725     Started = true;
726   if (Stopped && !Started)
727     report_fatal_error("Cannot stop compilation after pass that is not run");
728 }
729 
730 /// Add a CodeGen pass at this point in the pipeline after checking for target
731 /// and command line overrides.
732 ///
733 /// addPass cannot return a pointer to the pass instance because is internal the
734 /// PassManager and the instance we create here may already be freed.
735 AnalysisID TargetPassConfig::addPass(AnalysisID PassID) {
736   IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
737   IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
738   if (!FinalPtr.isValid())
739     return nullptr;
740 
741   Pass *P;
742   if (FinalPtr.isInstance())
743     P = FinalPtr.getInstance();
744   else {
745     P = Pass::createPass(FinalPtr.getID());
746     if (!P)
747       llvm_unreachable("Pass ID not registered");
748   }
749   AnalysisID FinalID = P->getPassID();
750   addPass(P); // Ends the lifetime of P.
751 
752   return FinalID;
753 }
754 
755 void TargetPassConfig::printAndVerify(const std::string &Banner) {
756   addPrintPass(Banner);
757   addVerifyPass(Banner);
758 }
759 
760 void TargetPassConfig::addPrintPass(const std::string &Banner) {
761   if (PrintAfterISel)
762     PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
763 }
764 
765 void TargetPassConfig::addVerifyPass(const std::string &Banner) {
766   bool Verify = VerifyMachineCode == cl::BOU_TRUE;
767 #ifdef EXPENSIVE_CHECKS
768   if (VerifyMachineCode == cl::BOU_UNSET)
769     Verify = TM->isMachineVerifierClean();
770 #endif
771   if (Verify)
772     PM->add(createMachineVerifierPass(Banner));
773 }
774 
775 void TargetPassConfig::addDebugifyPass() {
776   PM->add(createDebugifyMachineModulePass());
777 }
778 
779 void TargetPassConfig::addStripDebugPass() {
780   PM->add(createStripDebugMachineModulePass(/*OnlyDebugified=*/true));
781 }
782 
783 void TargetPassConfig::addCheckDebugPass() {
784   PM->add(createCheckDebugMachineModulePass());
785 }
786 
787 void TargetPassConfig::addMachinePrePasses(bool AllowDebugify) {
788   if (AllowDebugify && DebugifyIsSafe &&
789       (DebugifyAndStripAll == cl::BOU_TRUE ||
790        DebugifyCheckAndStripAll == cl::BOU_TRUE))
791     addDebugifyPass();
792 }
793 
794 void TargetPassConfig::addMachinePostPasses(const std::string &Banner) {
795   if (DebugifyIsSafe) {
796     if (DebugifyCheckAndStripAll == cl::BOU_TRUE) {
797       addCheckDebugPass();
798       addStripDebugPass();
799     } else if (DebugifyAndStripAll == cl::BOU_TRUE)
800       addStripDebugPass();
801   }
802   addVerifyPass(Banner);
803 }
804 
805 /// Add common target configurable passes that perform LLVM IR to IR transforms
806 /// following machine independent optimization.
807 void TargetPassConfig::addIRPasses() {
808   // Before running any passes, run the verifier to determine if the input
809   // coming from the front-end and/or optimizer is valid.
810   if (!DisableVerify)
811     addPass(createVerifierPass());
812 
813   if (getOptLevel() != CodeGenOptLevel::None) {
814     // Basic AliasAnalysis support.
815     // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
816     // BasicAliasAnalysis wins if they disagree. This is intended to help
817     // support "obvious" type-punning idioms.
818     addPass(createTypeBasedAAWrapperPass());
819     addPass(createScopedNoAliasAAWrapperPass());
820     addPass(createBasicAAWrapperPass());
821 
822     // Run loop strength reduction before anything else.
823     if (!DisableLSR) {
824       addPass(createCanonicalizeFreezeInLoopsPass());
825       addPass(createLoopStrengthReducePass());
826       if (PrintLSR)
827         addPass(createPrintFunctionPass(dbgs(),
828                                         "\n\n*** Code after LSR ***\n"));
829     }
830 
831     // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
832     // loads and compares. ExpandMemCmpPass then tries to expand those calls
833     // into optimally-sized loads and compares. The transforms are enabled by a
834     // target lowering hook.
835     if (!DisableMergeICmps)
836       addPass(createMergeICmpsLegacyPass());
837     addPass(createExpandMemCmpLegacyPass());
838   }
839 
840   // Run GC lowering passes for builtin collectors
841   // TODO: add a pass insertion point here
842   addPass(&GCLoweringID);
843   addPass(&ShadowStackGCLoweringID);
844   addPass(createLowerConstantIntrinsicsPass());
845 
846   // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
847   // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
848   if (TM->getTargetTriple().isOSBinFormatMachO() &&
849       !DisableAtExitBasedGlobalDtorLowering)
850     addPass(createLowerGlobalDtorsLegacyPass());
851 
852   // Make sure that no unreachable blocks are instruction selected.
853   addPass(createUnreachableBlockEliminationPass());
854 
855   // Prepare expensive constants for SelectionDAG.
856   if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting)
857     addPass(createConstantHoistingPass());
858 
859   if (getOptLevel() != CodeGenOptLevel::None)
860     addPass(createReplaceWithVeclibLegacyPass());
861 
862   if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining)
863     addPass(createPartiallyInlineLibCallsPass());
864 
865   // Expand vector predication intrinsics into standard IR instructions.
866   // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction
867   // passes since it emits those kinds of intrinsics.
868   addPass(createExpandVectorPredicationPass());
869 
870   // Add scalarization of target's unsupported masked memory intrinsics pass.
871   // the unsupported intrinsic will be replaced with a chain of basic blocks,
872   // that stores/loads element one-by-one if the appropriate mask bit is set.
873   addPass(createScalarizeMaskedMemIntrinLegacyPass());
874 
875   // Expand reduction intrinsics into shuffle sequences if the target wants to.
876   // Allow disabling it for testing purposes.
877   if (!DisableExpandReductions)
878     addPass(createExpandReductionsPass());
879 
880   if (getOptLevel() != CodeGenOptLevel::None)
881     addPass(createTLSVariableHoistPass());
882 
883   // Convert conditional moves to conditional jumps when profitable.
884   if (getOptLevel() != CodeGenOptLevel::None && !DisableSelectOptimize)
885     addPass(createSelectOptimizePass());
886 }
887 
888 /// Turn exception handling constructs into something the code generators can
889 /// handle.
890 void TargetPassConfig::addPassesToHandleExceptions() {
891   const MCAsmInfo *MCAI = TM->getMCAsmInfo();
892   assert(MCAI && "No MCAsmInfo");
893   switch (MCAI->getExceptionHandlingType()) {
894   case ExceptionHandling::SjLj:
895     // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
896     // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
897     // catch info can get misplaced when a selector ends up more than one block
898     // removed from the parent invoke(s). This could happen when a landing
899     // pad is shared by multiple invokes and is also a target of a normal
900     // edge from elsewhere.
901     addPass(createSjLjEHPreparePass(TM));
902     [[fallthrough]];
903   case ExceptionHandling::DwarfCFI:
904   case ExceptionHandling::ARM:
905   case ExceptionHandling::AIX:
906   case ExceptionHandling::ZOS:
907     addPass(createDwarfEHPass(getOptLevel()));
908     break;
909   case ExceptionHandling::WinEH:
910     // We support using both GCC-style and MSVC-style exceptions on Windows, so
911     // add both preparation passes. Each pass will only actually run if it
912     // recognizes the personality function.
913     addPass(createWinEHPass());
914     addPass(createDwarfEHPass(getOptLevel()));
915     break;
916   case ExceptionHandling::Wasm:
917     // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
918     // on catchpads and cleanuppads because it does not outline them into
919     // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
920     // should remove PHIs there.
921     addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
922     addPass(createWasmEHPass());
923     break;
924   case ExceptionHandling::None:
925     addPass(createLowerInvokePass());
926 
927     // The lower invoke pass may create unreachable code. Remove it.
928     addPass(createUnreachableBlockEliminationPass());
929     break;
930   }
931 }
932 
933 /// Add pass to prepare the LLVM IR for code generation. This should be done
934 /// before exception handling preparation passes.
935 void TargetPassConfig::addCodeGenPrepare() {
936   if (getOptLevel() != CodeGenOptLevel::None && !DisableCGP)
937     addPass(createCodeGenPrepareLegacyPass());
938 }
939 
940 /// Add common passes that perform LLVM IR to IR transforms in preparation for
941 /// instruction selection.
942 void TargetPassConfig::addISelPrepare() {
943   addPreISel();
944 
945   // Force codegen to run according to the callgraph.
946   if (requiresCodeGenSCCOrder())
947     addPass(new DummyCGSCCPass);
948 
949   addPass(createCallBrPass());
950 
951   // Add both the safe stack and the stack protection passes: each of them will
952   // only protect functions that have corresponding attributes.
953   addPass(createSafeStackPass());
954   addPass(createStackProtectorPass());
955 
956   if (PrintISelInput)
957     addPass(createPrintFunctionPass(
958         dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));
959 
960   // All passes which modify the LLVM IR are now complete; run the verifier
961   // to ensure that the IR is valid.
962   if (!DisableVerify)
963     addPass(createVerifierPass());
964 }
965 
966 bool TargetPassConfig::addCoreISelPasses() {
967   // Enable FastISel with -fast-isel, but allow that to be overridden.
968   TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
969 
970   // Determine an instruction selector.
971   enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
972   SelectorType Selector;
973 
974   if (EnableFastISelOption == cl::BOU_TRUE)
975     Selector = SelectorType::FastISel;
976   else if (EnableGlobalISelOption == cl::BOU_TRUE ||
977            (TM->Options.EnableGlobalISel &&
978             EnableGlobalISelOption != cl::BOU_FALSE))
979     Selector = SelectorType::GlobalISel;
980   else if (TM->getOptLevel() == CodeGenOptLevel::None &&
981            TM->getO0WantsFastISel())
982     Selector = SelectorType::FastISel;
983   else
984     Selector = SelectorType::SelectionDAG;
985 
986   // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
987   if (Selector == SelectorType::FastISel) {
988     TM->setFastISel(true);
989     TM->setGlobalISel(false);
990   } else if (Selector == SelectorType::GlobalISel) {
991     TM->setFastISel(false);
992     TM->setGlobalISel(true);
993   }
994 
995   // FIXME: Injecting into the DAGISel pipeline seems to cause issues with
996   //        analyses needing to be re-run. This can result in being unable to
997   //        schedule passes (particularly with 'Function Alias Analysis
998   //        Results'). It's not entirely clear why but AFAICT this seems to be
999   //        due to one FunctionPassManager not being able to use analyses from a
1000   //        previous one. As we're injecting a ModulePass we break the usual
1001   //        pass manager into two. GlobalISel with the fallback path disabled
1002   //        and -run-pass seem to be unaffected. The majority of GlobalISel
1003   //        testing uses -run-pass so this probably isn't too bad.
1004   SaveAndRestore SavedDebugifyIsSafe(DebugifyIsSafe);
1005   if (Selector != SelectorType::GlobalISel || !isGlobalISelAbortEnabled())
1006     DebugifyIsSafe = false;
1007 
1008   // Add instruction selector passes.
1009   if (Selector == SelectorType::GlobalISel) {
1010     SaveAndRestore SavedAddingMachinePasses(AddingMachinePasses, true);
1011     if (addIRTranslator())
1012       return true;
1013 
1014     addPreLegalizeMachineIR();
1015 
1016     if (addLegalizeMachineIR())
1017       return true;
1018 
1019     // Before running the register bank selector, ask the target if it
1020     // wants to run some passes.
1021     addPreRegBankSelect();
1022 
1023     if (addRegBankSelect())
1024       return true;
1025 
1026     addPreGlobalInstructionSelect();
1027 
1028     if (addGlobalInstructionSelect())
1029       return true;
1030 
1031     // Pass to reset the MachineFunction if the ISel failed.
1032     addPass(createResetMachineFunctionPass(
1033         reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
1034 
1035     // Provide a fallback path when we do not want to abort on
1036     // not-yet-supported input.
1037     if (!isGlobalISelAbortEnabled() && addInstSelector())
1038       return true;
1039 
1040   } else if (addInstSelector())
1041     return true;
1042 
1043   // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
1044   // FinalizeISel.
1045   addPass(&FinalizeISelID);
1046 
1047   // Print the instruction selected machine code...
1048   printAndVerify("After Instruction Selection");
1049 
1050   return false;
1051 }
1052 
1053 bool TargetPassConfig::addISelPasses() {
1054   if (TM->useEmulatedTLS())
1055     addPass(createLowerEmuTLSPass());
1056 
1057   PM->add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
1058   addPass(createPreISelIntrinsicLoweringPass());
1059   addPass(createExpandLargeDivRemPass());
1060   addPass(createExpandLargeFpConvertPass());
1061   addIRPasses();
1062   addCodeGenPrepare();
1063   addPassesToHandleExceptions();
1064   addISelPrepare();
1065 
1066   return addCoreISelPasses();
1067 }
1068 
1069 /// -regalloc=... command line option.
1070 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
1071 static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
1072                RegisterPassParser<RegisterRegAlloc>>
1073     RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
1074              cl::desc("Register allocator to use"));
1075 
1076 /// Add the complete set of target-independent postISel code generator passes.
1077 ///
1078 /// This can be read as the standard order of major LLVM CodeGen stages. Stages
1079 /// with nontrivial configuration or multiple passes are broken out below in
1080 /// add%Stage routines.
1081 ///
1082 /// Any TargetPassConfig::addXX routine may be overriden by the Target. The
1083 /// addPre/Post methods with empty header implementations allow injecting
1084 /// target-specific fixups just before or after major stages. Additionally,
1085 /// targets have the flexibility to change pass order within a stage by
1086 /// overriding default implementation of add%Stage routines below. Each
1087 /// technique has maintainability tradeoffs because alternate pass orders are
1088 /// not well supported. addPre/Post works better if the target pass is easily
1089 /// tied to a common pass. But if it has subtle dependencies on multiple passes,
1090 /// the target should override the stage instead.
1091 ///
1092 /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
1093 /// before/after any target-independent pass. But it's currently overkill.
1094 void TargetPassConfig::addMachinePasses() {
1095   AddingMachinePasses = true;
1096 
1097   // Add passes that optimize machine instructions in SSA form.
1098   if (getOptLevel() != CodeGenOptLevel::None) {
1099     addMachineSSAOptimization();
1100   } else {
1101     // If the target requests it, assign local variables to stack slots relative
1102     // to one another and simplify frame index references where possible.
1103     addPass(&LocalStackSlotAllocationID);
1104   }
1105 
1106   if (TM->Options.EnableIPRA)
1107     addPass(createRegUsageInfoPropPass());
1108 
1109   // Run pre-ra passes.
1110   addPreRegAlloc();
1111 
1112   // Debugifying the register allocator passes seems to provoke some
1113   // non-determinism that affects CodeGen and there doesn't seem to be a point
1114   // where it becomes safe again so stop debugifying here.
1115   DebugifyIsSafe = false;
1116 
1117   // Add a FSDiscriminator pass right before RA, so that we could get
1118   // more precise SampleFDO profile for RA.
1119   if (EnableFSDiscriminator) {
1120     addPass(createMIRAddFSDiscriminatorsPass(
1121         sampleprof::FSDiscriminatorPass::Pass1));
1122     const std::string ProfileFile = getFSProfileFile(TM);
1123     if (!ProfileFile.empty() && !DisableRAFSProfileLoader)
1124       addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
1125                                          sampleprof::FSDiscriminatorPass::Pass1,
1126                                          nullptr));
1127   }
1128 
1129   // Run register allocation and passes that are tightly coupled with it,
1130   // including phi elimination and scheduling.
1131   if (getOptimizeRegAlloc())
1132     addOptimizedRegAlloc();
1133   else
1134     addFastRegAlloc();
1135 
1136   // Run post-ra passes.
1137   addPostRegAlloc();
1138 
1139   addPass(&RemoveRedundantDebugValuesID);
1140 
1141   addPass(&FixupStatepointCallerSavedID);
1142 
1143   // Insert prolog/epilog code.  Eliminate abstract frame index references...
1144   if (getOptLevel() != CodeGenOptLevel::None) {
1145     addPass(&PostRAMachineSinkingID);
1146     addPass(&ShrinkWrapID);
1147   }
1148 
1149   // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
1150   // do so if it hasn't been disabled, substituted, or overridden.
1151   if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))
1152       addPass(createPrologEpilogInserterPass());
1153 
1154   /// Add passes that optimize machine instructions after register allocation.
1155   if (getOptLevel() != CodeGenOptLevel::None)
1156       addMachineLateOptimization();
1157 
1158   // Expand pseudo instructions before second scheduling pass.
1159   addPass(&ExpandPostRAPseudosID);
1160 
1161   // Run pre-sched2 passes.
1162   addPreSched2();
1163 
1164   if (EnableImplicitNullChecks)
1165     addPass(&ImplicitNullChecksID);
1166 
1167   // Second pass scheduler.
1168   // Let Target optionally insert this pass by itself at some other
1169   // point.
1170   if (getOptLevel() != CodeGenOptLevel::None &&
1171       !TM->targetSchedulesPostRAScheduling()) {
1172     if (MISchedPostRA)
1173       addPass(&PostMachineSchedulerID);
1174     else
1175       addPass(&PostRASchedulerID);
1176   }
1177 
1178   // GC
1179   addGCPasses();
1180 
1181   // Basic block placement.
1182   if (getOptLevel() != CodeGenOptLevel::None)
1183     addBlockPlacement();
1184 
1185   // Insert before XRay Instrumentation.
1186   addPass(&FEntryInserterID);
1187 
1188   addPass(&XRayInstrumentationID);
1189   addPass(&PatchableFunctionID);
1190 
1191   addPreEmitPass();
1192 
1193   if (TM->Options.EnableIPRA)
1194     // Collect register usage information and produce a register mask of
1195     // clobbered registers, to be used to optimize call sites.
1196     addPass(createRegUsageInfoCollector());
1197 
1198   // FIXME: Some backends are incompatible with running the verifier after
1199   // addPreEmitPass.  Maybe only pass "false" here for those targets?
1200   addPass(&FuncletLayoutID);
1201 
1202   addPass(&StackMapLivenessID);
1203   addPass(&LiveDebugValuesID);
1204   addPass(&MachineSanitizerBinaryMetadataID);
1205 
1206   if (TM->Options.EnableMachineOutliner &&
1207       getOptLevel() != CodeGenOptLevel::None &&
1208       EnableMachineOutliner != RunOutliner::NeverOutline) {
1209     bool RunOnAllFunctions =
1210         (EnableMachineOutliner == RunOutliner::AlwaysOutline);
1211     bool AddOutliner =
1212         RunOnAllFunctions || TM->Options.SupportsDefaultOutlining;
1213     if (AddOutliner)
1214       addPass(createMachineOutlinerPass(RunOnAllFunctions));
1215   }
1216 
1217   if (GCEmptyBlocks)
1218     addPass(llvm::createGCEmptyBasicBlocksPass());
1219 
1220   if (EnableFSDiscriminator)
1221     addPass(createMIRAddFSDiscriminatorsPass(
1222         sampleprof::FSDiscriminatorPass::PassLast));
1223 
1224   // Machine function splitter uses the basic block sections feature. Both
1225   // cannot be enabled at the same time. Basic block sections takes precedence.
1226   // FIXME: In principle, BasicBlockSection::Labels and splitting can used
1227   // together. Update this check once we have addressed any issues.
1228   if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) {
1229     if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
1230       addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
1231           TM->getBBSectionsFuncListBuf()));
1232       addPass(llvm::createBasicBlockPathCloningPass());
1233     }
1234     addPass(llvm::createBasicBlockSectionsPass());
1235   } else if (TM->Options.EnableMachineFunctionSplitter ||
1236              EnableMachineFunctionSplitter) {
1237     const std::string ProfileFile = getFSProfileFile(TM);
1238     if (!ProfileFile.empty()) {
1239       if (EnableFSDiscriminator) {
1240         addPass(createMIRProfileLoaderPass(
1241             ProfileFile, getFSRemappingFile(TM),
1242             sampleprof::FSDiscriminatorPass::PassLast, nullptr));
1243       } else {
1244         // Sample profile is given, but FSDiscriminator is not
1245         // enabled, this may result in performance regression.
1246         WithColor::warning()
1247             << "Using AutoFDO without FSDiscriminator for MFS may regress "
1248                "performance.\n";
1249       }
1250     }
1251     addPass(createMachineFunctionSplitterPass());
1252   }
1253 
1254   addPostBBSections();
1255 
1256   if (!DisableCFIFixup && TM->Options.EnableCFIFixup)
1257     addPass(createCFIFixup());
1258 
1259   PM->add(createStackFrameLayoutAnalysisPass());
1260 
1261   // Add passes that directly emit MI after all other MI passes.
1262   addPreEmitPass2();
1263 
1264   AddingMachinePasses = false;
1265 }
1266 
1267 /// Add passes that optimize machine instructions in SSA form.
1268 void TargetPassConfig::addMachineSSAOptimization() {
1269   // Pre-ra tail duplication.
1270   addPass(&EarlyTailDuplicateID);
1271 
1272   // Optimize PHIs before DCE: removing dead PHI cycles may make more
1273   // instructions dead.
1274   addPass(&OptimizePHIsID);
1275 
1276   // This pass merges large allocas. StackSlotColoring is a different pass
1277   // which merges spill slots.
1278   addPass(&StackColoringID);
1279 
1280   // If the target requests it, assign local variables to stack slots relative
1281   // to one another and simplify frame index references where possible.
1282   addPass(&LocalStackSlotAllocationID);
1283 
1284   // With optimization, dead code should already be eliminated. However
1285   // there is one known exception: lowered code for arguments that are only
1286   // used by tail calls, where the tail calls reuse the incoming stack
1287   // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1288   addPass(&DeadMachineInstructionElimID);
1289 
1290   // Allow targets to insert passes that improve instruction level parallelism,
1291   // like if-conversion. Such passes will typically need dominator trees and
1292   // loop info, just like LICM and CSE below.
1293   addILPOpts();
1294 
1295   addPass(&EarlyMachineLICMID);
1296   addPass(&MachineCSEID);
1297 
1298   addPass(&MachineSinkingID);
1299 
1300   addPass(&PeepholeOptimizerID);
1301   // Clean-up the dead code that may have been generated by peephole
1302   // rewriting.
1303   addPass(&DeadMachineInstructionElimID);
1304 }
1305 
1306 //===---------------------------------------------------------------------===//
1307 /// Register Allocation Pass Configuration
1308 //===---------------------------------------------------------------------===//
1309 
1310 bool TargetPassConfig::getOptimizeRegAlloc() const {
1311   switch (OptimizeRegAlloc) {
1312   case cl::BOU_UNSET:
1313     return getOptLevel() != CodeGenOptLevel::None;
1314   case cl::BOU_TRUE:  return true;
1315   case cl::BOU_FALSE: return false;
1316   }
1317   llvm_unreachable("Invalid optimize-regalloc state");
1318 }
1319 
1320 /// A dummy default pass factory indicates whether the register allocator is
1321 /// overridden on the command line.
1322 static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1323 
1324 static RegisterRegAlloc
1325 defaultRegAlloc("default",
1326                 "pick register allocator based on -O option",
1327                 useDefaultRegisterAllocator);
1328 
1329 static void initializeDefaultRegisterAllocatorOnce() {
1330   if (!RegisterRegAlloc::getDefault())
1331     RegisterRegAlloc::setDefault(RegAlloc);
1332 }
1333 
1334 /// Instantiate the default register allocator pass for this target for either
1335 /// the optimized or unoptimized allocation path. This will be added to the pass
1336 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1337 /// in the optimized case.
1338 ///
1339 /// A target that uses the standard regalloc pass order for fast or optimized
1340 /// allocation may still override this for per-target regalloc
1341 /// selection. But -regalloc=... always takes precedence.
1342 FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1343   if (Optimized)
1344     return createGreedyRegisterAllocator();
1345   else
1346     return createFastRegisterAllocator();
1347 }
1348 
1349 /// Find and instantiate the register allocation pass requested by this target
1350 /// at the current optimization level.  Different register allocators are
1351 /// defined as separate passes because they may require different analysis.
1352 ///
1353 /// This helper ensures that the regalloc= option is always available,
1354 /// even for targets that override the default allocator.
1355 ///
1356 /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1357 /// this can be folded into addPass.
1358 FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1359   // Initialize the global default.
1360   llvm::call_once(InitializeDefaultRegisterAllocatorFlag,
1361                   initializeDefaultRegisterAllocatorOnce);
1362 
1363   RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1364   if (Ctor != useDefaultRegisterAllocator)
1365     return Ctor();
1366 
1367   // With no -regalloc= override, ask the target for a regalloc pass.
1368   return createTargetRegisterAllocator(Optimized);
1369 }
1370 
1371 bool TargetPassConfig::isCustomizedRegAlloc() {
1372   return RegAlloc !=
1373          (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator;
1374 }
1375 
1376 bool TargetPassConfig::addRegAssignAndRewriteFast() {
1377   if (RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&useDefaultRegisterAllocator &&
1378       RegAlloc != (RegisterRegAlloc::FunctionPassCtor)&createFastRegisterAllocator)
1379     report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc.");
1380 
1381   addPass(createRegAllocPass(false));
1382 
1383   // Allow targets to change the register assignments after
1384   // fast register allocation.
1385   addPostFastRegAllocRewrite();
1386   return true;
1387 }
1388 
1389 bool TargetPassConfig::addRegAssignAndRewriteOptimized() {
1390   // Add the selected register allocation pass.
1391   addPass(createRegAllocPass(true));
1392 
1393   // Allow targets to change the register assignments before rewriting.
1394   addPreRewrite();
1395 
1396   // Finally rewrite virtual registers.
1397   addPass(&VirtRegRewriterID);
1398 
1399   // Regalloc scoring for ML-driven eviction - noop except when learning a new
1400   // eviction policy.
1401   addPass(createRegAllocScoringPass());
1402   return true;
1403 }
1404 
1405 /// Return true if the default global register allocator is in use and
1406 /// has not be overriden on the command line with '-regalloc=...'
1407 bool TargetPassConfig::usingDefaultRegAlloc() const {
1408   return RegAlloc.getNumOccurrences() == 0;
1409 }
1410 
1411 /// Add the minimum set of target-independent passes that are required for
1412 /// register allocation. No coalescing or scheduling.
1413 void TargetPassConfig::addFastRegAlloc() {
1414   addPass(&PHIEliminationID);
1415   addPass(&TwoAddressInstructionPassID);
1416 
1417   addRegAssignAndRewriteFast();
1418 }
1419 
1420 /// Add standard target-independent passes that are tightly coupled with
1421 /// optimized register allocation, including coalescing, machine instruction
1422 /// scheduling, and register allocation itself.
1423 void TargetPassConfig::addOptimizedRegAlloc() {
1424   addPass(&DetectDeadLanesID);
1425 
1426   addPass(&ProcessImplicitDefsID);
1427 
1428   // LiveVariables currently requires pure SSA form.
1429   //
1430   // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1431   // LiveVariables can be removed completely, and LiveIntervals can be directly
1432   // computed. (We still either need to regenerate kill flags after regalloc, or
1433   // preferably fix the scavenger to not depend on them).
1434   // FIXME: UnreachableMachineBlockElim is a dependant pass of LiveVariables.
1435   // When LiveVariables is removed this has to be removed/moved either.
1436   // Explicit addition of UnreachableMachineBlockElim allows stopping before or
1437   // after it with -stop-before/-stop-after.
1438   addPass(&UnreachableMachineBlockElimID);
1439   addPass(&LiveVariablesID);
1440 
1441   // Edge splitting is smarter with machine loop info.
1442   addPass(&MachineLoopInfoID);
1443   addPass(&PHIEliminationID);
1444 
1445   // Eventually, we want to run LiveIntervals before PHI elimination.
1446   if (EarlyLiveIntervals)
1447     addPass(&LiveIntervalsID);
1448 
1449   addPass(&TwoAddressInstructionPassID);
1450   addPass(&RegisterCoalescerID);
1451 
1452   // The machine scheduler may accidentally create disconnected components
1453   // when moving subregister definitions around, avoid this by splitting them to
1454   // separate vregs before. Splitting can also improve reg. allocation quality.
1455   addPass(&RenameIndependentSubregsID);
1456 
1457   // PreRA instruction scheduling.
1458   addPass(&MachineSchedulerID);
1459 
1460   if (addRegAssignAndRewriteOptimized()) {
1461     // Perform stack slot coloring and post-ra machine LICM.
1462     addPass(&StackSlotColoringID);
1463 
1464     // Allow targets to expand pseudo instructions depending on the choice of
1465     // registers before MachineCopyPropagation.
1466     addPostRewrite();
1467 
1468     // Copy propagate to forward register uses and try to eliminate COPYs that
1469     // were not coalesced.
1470     addPass(&MachineCopyPropagationID);
1471 
1472     // Run post-ra machine LICM to hoist reloads / remats.
1473     //
1474     // FIXME: can this move into MachineLateOptimization?
1475     addPass(&MachineLICMID);
1476   }
1477 }
1478 
1479 //===---------------------------------------------------------------------===//
1480 /// Post RegAlloc Pass Configuration
1481 //===---------------------------------------------------------------------===//
1482 
1483 /// Add passes that optimize machine instructions after register allocation.
1484 void TargetPassConfig::addMachineLateOptimization() {
1485   // Cleanup of redundant immediate/address loads.
1486   addPass(&MachineLateInstrsCleanupID);
1487 
1488   // Branch folding must be run after regalloc and prolog/epilog insertion.
1489   addPass(&BranchFolderPassID);
1490 
1491   // Tail duplication.
1492   // Note that duplicating tail just increases code size and degrades
1493   // performance for targets that require Structured Control Flow.
1494   // In addition it can also make CFG irreducible. Thus we disable it.
1495   if (!TM->requiresStructuredCFG())
1496     addPass(&TailDuplicateID);
1497 
1498   // Copy propagation.
1499   addPass(&MachineCopyPropagationID);
1500 }
1501 
1502 /// Add standard GC passes.
1503 bool TargetPassConfig::addGCPasses() {
1504   addPass(&GCMachineCodeAnalysisID);
1505   return true;
1506 }
1507 
1508 /// Add standard basic block placement passes.
1509 void TargetPassConfig::addBlockPlacement() {
1510   if (EnableFSDiscriminator) {
1511     addPass(createMIRAddFSDiscriminatorsPass(
1512         sampleprof::FSDiscriminatorPass::Pass2));
1513     const std::string ProfileFile = getFSProfileFile(TM);
1514     if (!ProfileFile.empty() && !DisableLayoutFSProfileLoader)
1515       addPass(createMIRProfileLoaderPass(ProfileFile, getFSRemappingFile(TM),
1516                                          sampleprof::FSDiscriminatorPass::Pass2,
1517                                          nullptr));
1518   }
1519   if (addPass(&MachineBlockPlacementID)) {
1520     // Run a separate pass to collect block placement statistics.
1521     if (EnableBlockPlacementStats)
1522       addPass(&MachineBlockPlacementStatsID);
1523   }
1524 }
1525 
1526 //===---------------------------------------------------------------------===//
1527 /// GlobalISel Configuration
1528 //===---------------------------------------------------------------------===//
1529 bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1530   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1531 }
1532 
1533 bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1534   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1535 }
1536 
1537 bool TargetPassConfig::isGISelCSEEnabled() const {
1538   return true;
1539 }
1540 
1541 std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1542   return std::make_unique<CSEConfigBase>();
1543 }
1544