xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp (revision 770cf0a5f02dc8983a89c6568d741fbc25baa999) 
1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implements the info about RISC-V target spec.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVTargetMachine.h"
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVTargetObjectFile.h"
18 #include "RISCVTargetTransformInfo.h"
19 #include "TargetInfo/RISCVTargetInfo.h"
20 #include "llvm/Analysis/TargetTransformInfo.h"
21 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
22 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
25 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26 #include "llvm/CodeGen/MIRParser/MIParser.h"
27 #include "llvm/CodeGen/MIRYamlMapping.h"
28 #include "llvm/CodeGen/MachineScheduler.h"
29 #include "llvm/CodeGen/MacroFusion.h"
30 #include "llvm/CodeGen/Passes.h"
31 #include "llvm/CodeGen/RegAllocRegistry.h"
32 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
33 #include "llvm/CodeGen/TargetPassConfig.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/MC/TargetRegistry.h"
36 #include "llvm/Passes/PassBuilder.h"
37 #include "llvm/Support/Compiler.h"
38 #include "llvm/Target/TargetOptions.h"
39 #include "llvm/Transforms/IPO.h"
40 #include "llvm/Transforms/Scalar.h"
41 #include "llvm/Transforms/Vectorize/EVLIndVarSimplify.h"
42 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
43 #include <optional>
44 using namespace llvm;
45 
46 static cl::opt<bool> EnableRedundantCopyElimination(
47     "riscv-enable-copyelim",
48     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
49     cl::Hidden);
50 
51 // FIXME: Unify control over GlobalMerge.
52 static cl::opt<cl::boolOrDefault>
53     EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
54                       cl::desc("Enable the global merge pass"));
55 
56 static cl::opt<bool>
57     EnableMachineCombiner("riscv-enable-machine-combiner",
58                           cl::desc("Enable the machine combiner pass"),
59                           cl::init(true), cl::Hidden);
60 
61 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
62     "riscv-v-vector-bits-max",
63     cl::desc("Assume V extension vector registers are at most this big, "
64              "with zero meaning no maximum size is assumed."),
65     cl::init(0), cl::Hidden);
66 
67 static cl::opt<int> RVVVectorBitsMinOpt(
68     "riscv-v-vector-bits-min",
69     cl::desc("Assume V extension vector registers are at least this big, "
70              "with zero meaning no minimum size is assumed. A value of -1 "
71              "means use Zvl*b extension. This is primarily used to enable "
72              "autovectorization with fixed width vectors."),
73     cl::init(-1), cl::Hidden);
74 
75 static cl::opt<bool> EnableRISCVCopyPropagation(
76     "riscv-enable-copy-propagation",
77     cl::desc("Enable the copy propagation with RISC-V copy instr"),
78     cl::init(true), cl::Hidden);
79 
80 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
81     "riscv-enable-dead-defs", cl::Hidden,
82     cl::desc("Enable the pass that removes dead"
83              " definitions and replaces stores to"
84              " them with stores to x0"),
85     cl::init(true));
86 
87 static cl::opt<bool>
88     EnableSinkFold("riscv-enable-sink-fold",
89                    cl::desc("Enable sinking and folding of instruction copies"),
90                    cl::init(true), cl::Hidden);
91 
92 static cl::opt<bool>
93     EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
94                            cl::desc("Enable the loop data prefetch pass"),
95                            cl::init(true));
96 
97 static cl::opt<bool> EnableMISchedLoadStoreClustering(
98     "riscv-misched-load-store-clustering", cl::Hidden,
99     cl::desc("Enable load and store clustering in the machine scheduler"),
100     cl::init(true));
101 
102 static cl::opt<bool> EnablePostMISchedLoadStoreClustering(
103     "riscv-postmisched-load-store-clustering", cl::Hidden,
104     cl::desc("Enable PostRA load and store clustering in the machine scheduler"),
105     cl::init(true));
106 
107 static cl::opt<bool>
108     EnableVLOptimizer("riscv-enable-vl-optimizer",
109                       cl::desc("Enable the RISC-V VL Optimizer pass"),
110                       cl::init(true), cl::Hidden);
111 
112 static cl::opt<bool> DisableVectorMaskMutation(
113     "riscv-disable-vector-mask-mutation",
114     cl::desc("Disable the vector mask scheduling mutation"), cl::init(false),
115     cl::Hidden);
116 
117 static cl::opt<bool>
118     EnableMachinePipeliner("riscv-enable-pipeliner",
119                            cl::desc("Enable Machine Pipeliner for RISC-V"),
120                            cl::init(false), cl::Hidden);
121 
122 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
123   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
124   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
125   auto *PR = PassRegistry::getPassRegistry();
126   initializeGlobalISel(*PR);
127   initializeRISCVO0PreLegalizerCombinerPass(*PR);
128   initializeRISCVPreLegalizerCombinerPass(*PR);
129   initializeRISCVPostLegalizerCombinerPass(*PR);
130   initializeKCFIPass(*PR);
131   initializeRISCVDeadRegisterDefinitionsPass(*PR);
132   initializeRISCVLateBranchOptPass(*PR);
133   initializeRISCVMakeCompressibleOptPass(*PR);
134   initializeRISCVGatherScatterLoweringPass(*PR);
135   initializeRISCVCodeGenPreparePass(*PR);
136   initializeRISCVPostRAExpandPseudoPass(*PR);
137   initializeRISCVMergeBaseOffsetOptPass(*PR);
138   initializeRISCVOptWInstrsPass(*PR);
139   initializeRISCVFoldMemOffsetPass(*PR);
140   initializeRISCVPreRAExpandPseudoPass(*PR);
141   initializeRISCVExpandPseudoPass(*PR);
142   initializeRISCVVectorPeepholePass(*PR);
143   initializeRISCVVLOptimizerPass(*PR);
144   initializeRISCVVMV0EliminationPass(*PR);
145   initializeRISCVInsertVSETVLIPass(*PR);
146   initializeRISCVInsertReadWriteCSRPass(*PR);
147   initializeRISCVInsertWriteVXRMPass(*PR);
148   initializeRISCVDAGToDAGISelLegacyPass(*PR);
149   initializeRISCVMoveMergePass(*PR);
150   initializeRISCVPushPopOptPass(*PR);
151   initializeRISCVIndirectBranchTrackingPass(*PR);
152   initializeRISCVLoadStoreOptPass(*PR);
153   initializeRISCVExpandAtomicPseudoPass(*PR);
154   initializeRISCVRedundantCopyEliminationPass(*PR);
155   initializeRISCVAsmPrinterPass(*PR);
156 }
157 
158 static StringRef computeDataLayout(const Triple &TT,
159                                    const TargetOptions &Options) {
160   StringRef ABIName = Options.MCOptions.getABIName();
161   if (TT.isArch64Bit()) {
162     if (ABIName == "lp64e")
163       return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
164 
165     return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
166   }
167   assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
168 
169   if (ABIName == "ilp32e")
170     return "e-m:e-p:32:32-i64:64-n32-S32";
171 
172   return "e-m:e-p:32:32-i64:64-n32-S128";
173 }
174 
175 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
176                                            std::optional<Reloc::Model> RM) {
177   return RM.value_or(Reloc::Static);
178 }
179 
180 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
181                                        StringRef CPU, StringRef FS,
182                                        const TargetOptions &Options,
183                                        std::optional<Reloc::Model> RM,
184                                        std::optional<CodeModel::Model> CM,
185                                        CodeGenOptLevel OL, bool JIT)
186     : CodeGenTargetMachineImpl(T, computeDataLayout(TT, Options), TT, CPU, FS,
187                                Options, getEffectiveRelocModel(TT, RM),
188                                getEffectiveCodeModel(CM, CodeModel::Small), OL),
189       TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
190   initAsmInfo();
191 
192   // RISC-V supports the MachineOutliner.
193   setMachineOutliner(true);
194   setSupportsDefaultOutlining(true);
195 
196   if (TT.isOSFuchsia() && !TT.isArch64Bit())
197     report_fatal_error("Fuchsia is only supported for 64-bit");
198 
199   setCFIFixup(true);
200 }
201 
202 const RISCVSubtarget *
203 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
204   Attribute CPUAttr = F.getFnAttribute("target-cpu");
205   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
206   Attribute FSAttr = F.getFnAttribute("target-features");
207 
208   std::string CPU =
209       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
210   std::string TuneCPU =
211       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
212   std::string FS =
213       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
214 
215   unsigned RVVBitsMin = RVVVectorBitsMinOpt;
216   unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
217 
218   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
219   if (VScaleRangeAttr.isValid()) {
220     if (!RVVVectorBitsMinOpt.getNumOccurrences())
221       RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
222     std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
223     if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
224       RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
225   }
226 
227   if (RVVBitsMin != -1U) {
228     // FIXME: Change to >= 32 when VLEN = 32 is supported.
229     assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
230                                 isPowerOf2_32(RVVBitsMin))) &&
231            "V or Zve* extension requires vector length to be in the range of "
232            "64 to 65536 and a power 2!");
233     assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
234            "Minimum V extension vector length should not be larger than its "
235            "maximum!");
236   }
237   assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
238                               isPowerOf2_32(RVVBitsMax))) &&
239          "V or Zve* extension requires vector length to be in the range of "
240          "64 to 65536 and a power 2!");
241 
242   if (RVVBitsMin != -1U) {
243     if (RVVBitsMax != 0) {
244       RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
245       RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
246     }
247 
248     RVVBitsMin = llvm::bit_floor(
249         (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
250   }
251   RVVBitsMax =
252       llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
253 
254   SmallString<512> Key;
255   raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
256                            << CPU << TuneCPU << FS;
257   auto &I = SubtargetMap[Key];
258   if (!I) {
259     // This needs to be done before we create a new subtarget since any
260     // creation will depend on the TM and the code generation flags on the
261     // function that reside in TargetOptions.
262     resetTargetOptions(F);
263     auto ABIName = Options.MCOptions.getABIName();
264     if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
265             F.getParent()->getModuleFlag("target-abi"))) {
266       auto TargetABI = RISCVABI::getTargetABI(ABIName);
267       if (TargetABI != RISCVABI::ABI_Unknown &&
268           ModuleTargetABI->getString() != ABIName) {
269         report_fatal_error("-target-abi option != target-abi module flag");
270       }
271       ABIName = ModuleTargetABI->getString();
272     }
273     I = std::make_unique<RISCVSubtarget>(
274         TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
275   }
276   return I.get();
277 }
278 
279 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
280     BumpPtrAllocator &Allocator, const Function &F,
281     const TargetSubtargetInfo *STI) const {
282   return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
283       Allocator, F, static_cast<const RISCVSubtarget *>(STI));
284 }
285 
286 TargetTransformInfo
287 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
288   return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(this, F));
289 }
290 
291 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
292 // for all memory accesses, so it is reasonable to assume that an
293 // implementation has no-op address space casts. If an implementation makes a
294 // change to this, they can override it here.
295 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
296                                              unsigned DstAS) const {
297   return true;
298 }
299 
300 ScheduleDAGInstrs *
301 RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
302   ScheduleDAGMILive *DAG = createSchedLive(C);
303   if (EnableMISchedLoadStoreClustering) {
304     DAG->addMutation(createLoadClusterDAGMutation(
305         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
306     DAG->addMutation(createStoreClusterDAGMutation(
307         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
308   }
309 
310   const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
311   if (!DisableVectorMaskMutation && ST.hasVInstructions())
312     DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
313 
314   return DAG;
315 }
316 
317 ScheduleDAGInstrs *
318 RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
319   ScheduleDAGMI *DAG = createSchedPostRA(C);
320   if (EnablePostMISchedLoadStoreClustering) {
321     DAG->addMutation(createLoadClusterDAGMutation(
322         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
323     DAG->addMutation(createStoreClusterDAGMutation(
324         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
325   }
326 
327   return DAG;
328 }
329 
330 namespace {
331 
332 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
333 public:
334   RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
335       : RegisterRegAllocBase(N, D, C) {}
336 };
337 
338 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
339                                const MachineRegisterInfo &MRI,
340                                const Register Reg) {
341   const TargetRegisterClass *RC = MRI.getRegClass(Reg);
342   return RISCVRegisterInfo::isRVVRegClass(RC);
343 }
344 
345 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
346 
347 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
348 
349 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
350 /// This option could designate the rvv register allocator only.
351 /// For example: -riscv-rvv-regalloc=basic
352 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
353                RegisterPassParser<RVVRegisterRegAlloc>>
354     RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
355                 cl::init(&useDefaultRegisterAllocator),
356                 cl::desc("Register allocator to use for RVV register."));
357 
358 static void initializeDefaultRVVRegisterAllocatorOnce() {
359   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
360 
361   if (!Ctor) {
362     Ctor = RVVRegAlloc;
363     RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
364   }
365 }
366 
367 static FunctionPass *createBasicRVVRegisterAllocator() {
368   return createBasicRegisterAllocator(onlyAllocateRVVReg);
369 }
370 
371 static FunctionPass *createGreedyRVVRegisterAllocator() {
372   return createGreedyRegisterAllocator(onlyAllocateRVVReg);
373 }
374 
375 static FunctionPass *createFastRVVRegisterAllocator() {
376   return createFastRegisterAllocator(onlyAllocateRVVReg, false);
377 }
378 
379 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
380                                                "basic register allocator",
381                                                createBasicRVVRegisterAllocator);
382 static RVVRegisterRegAlloc
383     greedyRegAllocRVVReg("greedy", "greedy register allocator",
384                          createGreedyRVVRegisterAllocator);
385 
386 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
387                                               createFastRVVRegisterAllocator);
388 
389 class RISCVPassConfig : public TargetPassConfig {
390 public:
391   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
392       : TargetPassConfig(TM, PM) {
393     if (TM.getOptLevel() != CodeGenOptLevel::None)
394       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
395     setEnableSinkAndFold(EnableSinkFold);
396     EnableLoopTermFold = true;
397   }
398 
399   RISCVTargetMachine &getRISCVTargetMachine() const {
400     return getTM<RISCVTargetMachine>();
401   }
402 
403   void addIRPasses() override;
404   bool addPreISel() override;
405   void addCodeGenPrepare() override;
406   bool addInstSelector() override;
407   bool addIRTranslator() override;
408   void addPreLegalizeMachineIR() override;
409   bool addLegalizeMachineIR() override;
410   void addPreRegBankSelect() override;
411   bool addRegBankSelect() override;
412   bool addGlobalInstructionSelect() override;
413   void addPreEmitPass() override;
414   void addPreEmitPass2() override;
415   void addPreSched2() override;
416   void addMachineSSAOptimization() override;
417   FunctionPass *createRVVRegAllocPass(bool Optimized);
418   bool addRegAssignAndRewriteFast() override;
419   bool addRegAssignAndRewriteOptimized() override;
420   void addPreRegAlloc() override;
421   void addPostRegAlloc() override;
422   void addFastRegAlloc() override;
423 
424   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
425 };
426 } // namespace
427 
428 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
429   return new RISCVPassConfig(*this, PM);
430 }
431 
432 std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
433   return getStandardCSEConfigForOpt(TM->getOptLevel());
434 }
435 
436 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
437   // Initialize the global default.
438   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
439                   initializeDefaultRVVRegisterAllocatorOnce);
440 
441   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
442   if (Ctor != useDefaultRegisterAllocator)
443     return Ctor();
444 
445   if (Optimized)
446     return createGreedyRVVRegisterAllocator();
447 
448   return createFastRVVRegisterAllocator();
449 }
450 
451 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
452   addPass(createRVVRegAllocPass(false));
453   addPass(createRISCVInsertVSETVLIPass());
454   if (TM->getOptLevel() != CodeGenOptLevel::None &&
455       EnableRISCVDeadRegisterElimination)
456     addPass(createRISCVDeadRegisterDefinitionsPass());
457   return TargetPassConfig::addRegAssignAndRewriteFast();
458 }
459 
460 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
461   addPass(createRVVRegAllocPass(true));
462   addPass(createVirtRegRewriter(false));
463   addPass(createRISCVInsertVSETVLIPass());
464   if (TM->getOptLevel() != CodeGenOptLevel::None &&
465       EnableRISCVDeadRegisterElimination)
466     addPass(createRISCVDeadRegisterDefinitionsPass());
467   return TargetPassConfig::addRegAssignAndRewriteOptimized();
468 }
469 
470 void RISCVPassConfig::addIRPasses() {
471   addPass(createAtomicExpandLegacyPass());
472   addPass(createRISCVZacasABIFixPass());
473 
474   if (getOptLevel() != CodeGenOptLevel::None) {
475     if (EnableLoopDataPrefetch)
476       addPass(createLoopDataPrefetchPass());
477 
478     addPass(createRISCVGatherScatterLoweringPass());
479     addPass(createInterleavedAccessPass());
480     addPass(createRISCVCodeGenPreparePass());
481   }
482 
483   TargetPassConfig::addIRPasses();
484 }
485 
486 bool RISCVPassConfig::addPreISel() {
487   if (TM->getOptLevel() != CodeGenOptLevel::None) {
488     // Add a barrier before instruction selection so that we will not get
489     // deleted block address after enabling default outlining. See D99707 for
490     // more details.
491     addPass(createBarrierNoopPass());
492   }
493 
494   if ((TM->getOptLevel() != CodeGenOptLevel::None &&
495        EnableGlobalMerge == cl::BOU_UNSET) ||
496       EnableGlobalMerge == cl::BOU_TRUE) {
497     // FIXME: Like AArch64, we disable extern global merging by default due to
498     // concerns it might regress some workloads. Unlike AArch64, we don't
499     // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
500     // Investigating and addressing both items are TODO.
501     addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
502                                   /* OnlyOptimizeForSize */ false,
503                                   /* MergeExternalByDefault */ true));
504   }
505 
506   return false;
507 }
508 
509 void RISCVPassConfig::addCodeGenPrepare() {
510   if (getOptLevel() != CodeGenOptLevel::None)
511     addPass(createTypePromotionLegacyPass());
512   TargetPassConfig::addCodeGenPrepare();
513 }
514 
515 bool RISCVPassConfig::addInstSelector() {
516   addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
517 
518   return false;
519 }
520 
521 bool RISCVPassConfig::addIRTranslator() {
522   addPass(new IRTranslator(getOptLevel()));
523   return false;
524 }
525 
526 void RISCVPassConfig::addPreLegalizeMachineIR() {
527   if (getOptLevel() == CodeGenOptLevel::None) {
528     addPass(createRISCVO0PreLegalizerCombiner());
529   } else {
530     addPass(createRISCVPreLegalizerCombiner());
531   }
532 }
533 
534 bool RISCVPassConfig::addLegalizeMachineIR() {
535   addPass(new Legalizer());
536   return false;
537 }
538 
539 void RISCVPassConfig::addPreRegBankSelect() {
540   if (getOptLevel() != CodeGenOptLevel::None)
541     addPass(createRISCVPostLegalizerCombiner());
542 }
543 
544 bool RISCVPassConfig::addRegBankSelect() {
545   addPass(new RegBankSelect());
546   return false;
547 }
548 
549 bool RISCVPassConfig::addGlobalInstructionSelect() {
550   addPass(new InstructionSelect(getOptLevel()));
551   return false;
552 }
553 
554 void RISCVPassConfig::addPreSched2() {
555   addPass(createRISCVPostRAExpandPseudoPass());
556 
557   // Emit KCFI checks for indirect calls.
558   addPass(createKCFIPass());
559   if (TM->getOptLevel() != CodeGenOptLevel::None)
560     addPass(createRISCVLoadStoreOptPass());
561 }
562 
563 void RISCVPassConfig::addPreEmitPass() {
564   // TODO: It would potentially be better to schedule copy propagation after
565   // expanding pseudos (in addPreEmitPass2). However, performing copy
566   // propagation after the machine outliner (which runs after addPreEmitPass)
567   // currently leads to incorrect code-gen, where copies to registers within
568   // outlined functions are removed erroneously.
569   if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
570       EnableRISCVCopyPropagation)
571     addPass(createMachineCopyPropagationPass(true));
572   if (TM->getOptLevel() >= CodeGenOptLevel::Default)
573     addPass(createRISCVLateBranchOptPass());
574   // The IndirectBranchTrackingPass inserts lpad and could have changed the
575   // basic block alignment. It must be done before Branch Relaxation to
576   // prevent the adjusted offset exceeding the branch range.
577   addPass(createRISCVIndirectBranchTrackingPass());
578   addPass(&BranchRelaxationPassID);
579   addPass(createRISCVMakeCompressibleOptPass());
580 }
581 
582 void RISCVPassConfig::addPreEmitPass2() {
583   if (TM->getOptLevel() != CodeGenOptLevel::None) {
584     addPass(createRISCVMoveMergePass());
585     // Schedule PushPop Optimization before expansion of Pseudo instruction,
586     // ensuring return instruction is detected correctly.
587     addPass(createRISCVPushPopOptimizationPass());
588   }
589   addPass(createRISCVExpandPseudoPass());
590 
591   // Schedule the expansion of AMOs at the last possible moment, avoiding the
592   // possibility for other passes to break the requirements for forward
593   // progress in the LR/SC block.
594   addPass(createRISCVExpandAtomicPseudoPass());
595 
596   // KCFI indirect call checks are lowered to a bundle.
597   addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
598     return MF.getFunction().getParent()->getModuleFlag("kcfi");
599   }));
600 }
601 
602 void RISCVPassConfig::addMachineSSAOptimization() {
603   addPass(createRISCVVectorPeepholePass());
604   addPass(createRISCVFoldMemOffsetPass());
605 
606   TargetPassConfig::addMachineSSAOptimization();
607 
608   if (EnableMachineCombiner)
609     addPass(&MachineCombinerID);
610 
611   if (TM->getTargetTriple().isRISCV64()) {
612     addPass(createRISCVOptWInstrsPass());
613   }
614 }
615 
616 void RISCVPassConfig::addPreRegAlloc() {
617   addPass(createRISCVPreRAExpandPseudoPass());
618   if (TM->getOptLevel() != CodeGenOptLevel::None) {
619     addPass(createRISCVMergeBaseOffsetOptPass());
620     if (EnableVLOptimizer)
621       addPass(createRISCVVLOptimizerPass());
622   }
623 
624   addPass(createRISCVInsertReadWriteCSRPass());
625   addPass(createRISCVInsertWriteVXRMPass());
626   addPass(createRISCVLandingPadSetupPass());
627 
628   if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
629     addPass(&MachinePipelinerID);
630 
631   addPass(createRISCVVMV0EliminationPass());
632 }
633 
634 void RISCVPassConfig::addFastRegAlloc() {
635   addPass(&InitUndefID);
636   TargetPassConfig::addFastRegAlloc();
637 }
638 
639 
640 void RISCVPassConfig::addPostRegAlloc() {
641   if (TM->getOptLevel() != CodeGenOptLevel::None &&
642       EnableRedundantCopyElimination)
643     addPass(createRISCVRedundantCopyEliminationPass());
644 }
645 
646 void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
647   PB.registerLateLoopOptimizationsEPCallback([=](LoopPassManager &LPM,
648                                                  OptimizationLevel Level) {
649     LPM.addPass(LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
650   });
651 
652   PB.registerVectorizerEndEPCallback(
653       [](FunctionPassManager &FPM, OptimizationLevel Level) {
654         if (Level.isOptimizingForSpeed())
655           FPM.addPass(createFunctionToLoopPassAdaptor(EVLIndVarSimplifyPass()));
656       });
657 }
658 
659 yaml::MachineFunctionInfo *
660 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
661   return new yaml::RISCVMachineFunctionInfo();
662 }
663 
664 yaml::MachineFunctionInfo *
665 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
666   const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
667   return new yaml::RISCVMachineFunctionInfo(*MFI);
668 }
669 
670 bool RISCVTargetMachine::parseMachineFunctionInfo(
671     const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
672     SMDiagnostic &Error, SMRange &SourceRange) const {
673   const auto &YamlMFI =
674       static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
675   PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
676   return false;
677 }
678