xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp (revision a90b9d0159070121c221b966469c3e36d912bf82) 
1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implements the info about RISC-V target spec.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVTargetMachine.h"
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVTargetObjectFile.h"
18 #include "RISCVTargetTransformInfo.h"
19 #include "TargetInfo/RISCVTargetInfo.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/Analysis/TargetTransformInfo.h"
22 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
25 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26 #include "llvm/CodeGen/MIRParser/MIParser.h"
27 #include "llvm/CodeGen/MIRYamlMapping.h"
28 #include "llvm/CodeGen/MachineScheduler.h"
29 #include "llvm/CodeGen/MacroFusion.h"
30 #include "llvm/CodeGen/Passes.h"
31 #include "llvm/CodeGen/RegAllocRegistry.h"
32 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
33 #include "llvm/CodeGen/TargetPassConfig.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/MC/TargetRegistry.h"
36 #include "llvm/Support/FormattedStream.h"
37 #include "llvm/Target/TargetOptions.h"
38 #include "llvm/Transforms/IPO.h"
39 #include "llvm/Transforms/Scalar.h"
40 #include <optional>
41 using namespace llvm;
42 
43 static cl::opt<bool> EnableRedundantCopyElimination(
44     "riscv-enable-copyelim",
45     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
46     cl::Hidden);
47 
48 // FIXME: Unify control over GlobalMerge.
49 static cl::opt<cl::boolOrDefault>
50     EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
51                       cl::desc("Enable the global merge pass"));
52 
53 static cl::opt<bool>
54     EnableMachineCombiner("riscv-enable-machine-combiner",
55                           cl::desc("Enable the machine combiner pass"),
56                           cl::init(true), cl::Hidden);
57 
58 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
59     "riscv-v-vector-bits-max",
60     cl::desc("Assume V extension vector registers are at most this big, "
61              "with zero meaning no maximum size is assumed."),
62     cl::init(0), cl::Hidden);
63 
64 static cl::opt<int> RVVVectorBitsMinOpt(
65     "riscv-v-vector-bits-min",
66     cl::desc("Assume V extension vector registers are at least this big, "
67              "with zero meaning no minimum size is assumed. A value of -1 "
68              "means use Zvl*b extension. This is primarily used to enable "
69              "autovectorization with fixed width vectors."),
70     cl::init(-1), cl::Hidden);
71 
72 static cl::opt<bool> EnableRISCVCopyPropagation(
73     "riscv-enable-copy-propagation",
74     cl::desc("Enable the copy propagation with RISC-V copy instr"),
75     cl::init(true), cl::Hidden);
76 
77 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
78     "riscv-enable-dead-defs", cl::Hidden,
79     cl::desc("Enable the pass that removes dead"
80              " definitons and replaces stores to"
81              " them with stores to x0"),
82     cl::init(true));
83 
84 static cl::opt<bool>
85     EnableSinkFold("riscv-enable-sink-fold",
86                    cl::desc("Enable sinking and folding of instruction copies"),
87                    cl::init(false), cl::Hidden);
88 
89 static cl::opt<bool>
90     EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
91                            cl::desc("Enable the loop data prefetch pass"),
92                            cl::init(true));
93 
94 static cl::opt<bool>
95     EnableSplitRegAlloc("riscv-split-regalloc", cl::Hidden,
96                         cl::desc("Enable Split RegisterAlloc for RVV"),
97                         cl::init(true));
98 
99 static cl::opt<bool> EnableMISchedLoadClustering(
100     "riscv-misched-load-clustering", cl::Hidden,
101     cl::desc("Enable load clustering in the machine scheduler"),
102     cl::init(false));
103 
104 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
105   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
106   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
107   auto *PR = PassRegistry::getPassRegistry();
108   initializeGlobalISel(*PR);
109   initializeRISCVO0PreLegalizerCombinerPass(*PR);
110   initializeRISCVPreLegalizerCombinerPass(*PR);
111   initializeRISCVPostLegalizerCombinerPass(*PR);
112   initializeKCFIPass(*PR);
113   initializeRISCVDeadRegisterDefinitionsPass(*PR);
114   initializeRISCVMakeCompressibleOptPass(*PR);
115   initializeRISCVGatherScatterLoweringPass(*PR);
116   initializeRISCVCodeGenPreparePass(*PR);
117   initializeRISCVPostRAExpandPseudoPass(*PR);
118   initializeRISCVMergeBaseOffsetOptPass(*PR);
119   initializeRISCVOptWInstrsPass(*PR);
120   initializeRISCVPreRAExpandPseudoPass(*PR);
121   initializeRISCVExpandPseudoPass(*PR);
122   initializeRISCVFoldMasksPass(*PR);
123   initializeRISCVInsertVSETVLIPass(*PR);
124   initializeRISCVInsertReadWriteCSRPass(*PR);
125   initializeRISCVInsertWriteVXRMPass(*PR);
126   initializeRISCVDAGToDAGISelPass(*PR);
127   initializeRISCVInitUndefPass(*PR);
128   initializeRISCVMoveMergePass(*PR);
129   initializeRISCVPushPopOptPass(*PR);
130 }
131 
132 static StringRef computeDataLayout(const Triple &TT,
133                                    const TargetOptions &Options) {
134   StringRef ABIName = Options.MCOptions.getABIName();
135   if (TT.isArch64Bit()) {
136     if (ABIName == "lp64e")
137       return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
138 
139     return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
140   }
141   assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
142 
143   if (ABIName == "ilp32e")
144     return "e-m:e-p:32:32-i64:64-n32-S32";
145 
146   return "e-m:e-p:32:32-i64:64-n32-S128";
147 }
148 
149 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
150                                            std::optional<Reloc::Model> RM) {
151   return RM.value_or(Reloc::Static);
152 }
153 
154 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
155                                        StringRef CPU, StringRef FS,
156                                        const TargetOptions &Options,
157                                        std::optional<Reloc::Model> RM,
158                                        std::optional<CodeModel::Model> CM,
159                                        CodeGenOptLevel OL, bool JIT)
160     : LLVMTargetMachine(T, computeDataLayout(TT, Options), TT, CPU, FS, Options,
161                         getEffectiveRelocModel(TT, RM),
162                         getEffectiveCodeModel(CM, CodeModel::Small), OL),
163       TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
164   initAsmInfo();
165 
166   // RISC-V supports the MachineOutliner.
167   setMachineOutliner(true);
168   setSupportsDefaultOutlining(true);
169 
170   if (TT.isOSFuchsia() && !TT.isArch64Bit())
171     report_fatal_error("Fuchsia is only supported for 64-bit");
172 }
173 
174 const RISCVSubtarget *
175 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
176   Attribute CPUAttr = F.getFnAttribute("target-cpu");
177   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
178   Attribute FSAttr = F.getFnAttribute("target-features");
179 
180   std::string CPU =
181       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
182   std::string TuneCPU =
183       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
184   std::string FS =
185       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
186 
187   unsigned RVVBitsMin = RVVVectorBitsMinOpt;
188   unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
189 
190   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
191   if (VScaleRangeAttr.isValid()) {
192     if (!RVVVectorBitsMinOpt.getNumOccurrences())
193       RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
194     std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
195     if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
196       RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
197   }
198 
199   if (RVVBitsMin != -1U) {
200     // FIXME: Change to >= 32 when VLEN = 32 is supported.
201     assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
202                                 isPowerOf2_32(RVVBitsMin))) &&
203            "V or Zve* extension requires vector length to be in the range of "
204            "64 to 65536 and a power 2!");
205     assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
206            "Minimum V extension vector length should not be larger than its "
207            "maximum!");
208   }
209   assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
210                               isPowerOf2_32(RVVBitsMax))) &&
211          "V or Zve* extension requires vector length to be in the range of "
212          "64 to 65536 and a power 2!");
213 
214   if (RVVBitsMin != -1U) {
215     if (RVVBitsMax != 0) {
216       RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
217       RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
218     }
219 
220     RVVBitsMin = llvm::bit_floor(
221         (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
222   }
223   RVVBitsMax =
224       llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
225 
226   SmallString<512> Key;
227   raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
228                            << CPU << TuneCPU << FS;
229   auto &I = SubtargetMap[Key];
230   if (!I) {
231     // This needs to be done before we create a new subtarget since any
232     // creation will depend on the TM and the code generation flags on the
233     // function that reside in TargetOptions.
234     resetTargetOptions(F);
235     auto ABIName = Options.MCOptions.getABIName();
236     if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
237             F.getParent()->getModuleFlag("target-abi"))) {
238       auto TargetABI = RISCVABI::getTargetABI(ABIName);
239       if (TargetABI != RISCVABI::ABI_Unknown &&
240           ModuleTargetABI->getString() != ABIName) {
241         report_fatal_error("-target-abi option != target-abi module flag");
242       }
243       ABIName = ModuleTargetABI->getString();
244     }
245     I = std::make_unique<RISCVSubtarget>(
246         TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
247   }
248   return I.get();
249 }
250 
251 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
252     BumpPtrAllocator &Allocator, const Function &F,
253     const TargetSubtargetInfo *STI) const {
254   return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(Allocator,
255                                                                     F, STI);
256 }
257 
258 TargetTransformInfo
259 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
260   return TargetTransformInfo(RISCVTTIImpl(this, F));
261 }
262 
263 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
264 // for all memory accesses, so it is reasonable to assume that an
265 // implementation has no-op address space casts. If an implementation makes a
266 // change to this, they can override it here.
267 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
268                                              unsigned DstAS) const {
269   return true;
270 }
271 
272 namespace {
273 
274 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
275 public:
276   RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
277       : RegisterRegAllocBase(N, D, C) {}
278 };
279 
280 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
281                                const TargetRegisterClass &RC) {
282   return RISCV::VRRegClass.hasSubClassEq(&RC) ||
283          RISCV::VRM2RegClass.hasSubClassEq(&RC) ||
284          RISCV::VRM4RegClass.hasSubClassEq(&RC) ||
285          RISCV::VRM8RegClass.hasSubClassEq(&RC) ||
286          RISCV::VRN2M1RegClass.hasSubClassEq(&RC) ||
287          RISCV::VRN2M2RegClass.hasSubClassEq(&RC) ||
288          RISCV::VRN2M4RegClass.hasSubClassEq(&RC) ||
289          RISCV::VRN3M1RegClass.hasSubClassEq(&RC) ||
290          RISCV::VRN3M2RegClass.hasSubClassEq(&RC) ||
291          RISCV::VRN4M1RegClass.hasSubClassEq(&RC) ||
292          RISCV::VRN4M2RegClass.hasSubClassEq(&RC) ||
293          RISCV::VRN5M1RegClass.hasSubClassEq(&RC) ||
294          RISCV::VRN6M1RegClass.hasSubClassEq(&RC) ||
295          RISCV::VRN7M1RegClass.hasSubClassEq(&RC) ||
296          RISCV::VRN8M1RegClass.hasSubClassEq(&RC);
297 }
298 
299 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
300 
301 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
302 
303 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
304 /// This option could designate the rvv register allocator only.
305 /// For example: -riscv-rvv-regalloc=basic
306 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
307                RegisterPassParser<RVVRegisterRegAlloc>>
308     RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
309                 cl::init(&useDefaultRegisterAllocator),
310                 cl::desc("Register allocator to use for RVV register."));
311 
312 static void initializeDefaultRVVRegisterAllocatorOnce() {
313   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
314 
315   if (!Ctor) {
316     Ctor = RVVRegAlloc;
317     RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
318   }
319 }
320 
321 static FunctionPass *createBasicRVVRegisterAllocator() {
322   return createBasicRegisterAllocator(onlyAllocateRVVReg);
323 }
324 
325 static FunctionPass *createGreedyRVVRegisterAllocator() {
326   return createGreedyRegisterAllocator(onlyAllocateRVVReg);
327 }
328 
329 static FunctionPass *createFastRVVRegisterAllocator() {
330   return createFastRegisterAllocator(onlyAllocateRVVReg, false);
331 }
332 
333 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
334                                                "basic register allocator",
335                                                createBasicRVVRegisterAllocator);
336 static RVVRegisterRegAlloc
337     greedyRegAllocRVVReg("greedy", "greedy register allocator",
338                          createGreedyRVVRegisterAllocator);
339 
340 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
341                                               createFastRVVRegisterAllocator);
342 
343 class RISCVPassConfig : public TargetPassConfig {
344 public:
345   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
346       : TargetPassConfig(TM, PM) {
347     if (TM.getOptLevel() != CodeGenOptLevel::None)
348       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
349     setEnableSinkAndFold(EnableSinkFold);
350   }
351 
352   RISCVTargetMachine &getRISCVTargetMachine() const {
353     return getTM<RISCVTargetMachine>();
354   }
355 
356   ScheduleDAGInstrs *
357   createMachineScheduler(MachineSchedContext *C) const override {
358     const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
359     ScheduleDAGMILive *DAG = nullptr;
360     if (EnableMISchedLoadClustering) {
361       DAG = createGenericSchedLive(C);
362       DAG->addMutation(createLoadClusterDAGMutation(
363           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
364     }
365     const auto &MacroFusions = ST.getMacroFusions();
366     if (!MacroFusions.empty()) {
367       DAG = DAG ? DAG : createGenericSchedLive(C);
368       DAG->addMutation(createMacroFusionDAGMutation(MacroFusions));
369     }
370     return DAG;
371   }
372 
373   ScheduleDAGInstrs *
374   createPostMachineScheduler(MachineSchedContext *C) const override {
375     const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
376     const auto &MacroFusions = ST.getMacroFusions();
377     if (!MacroFusions.empty()) {
378       ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
379       DAG->addMutation(createMacroFusionDAGMutation(MacroFusions));
380       return DAG;
381     }
382     return nullptr;
383   }
384 
385   void addIRPasses() override;
386   bool addPreISel() override;
387   bool addInstSelector() override;
388   bool addIRTranslator() override;
389   void addPreLegalizeMachineIR() override;
390   bool addLegalizeMachineIR() override;
391   void addPreRegBankSelect() override;
392   bool addRegBankSelect() override;
393   bool addGlobalInstructionSelect() override;
394   void addPreEmitPass() override;
395   void addPreEmitPass2() override;
396   void addPreSched2() override;
397   void addMachineSSAOptimization() override;
398   FunctionPass *createRVVRegAllocPass(bool Optimized);
399   bool addRegAssignAndRewriteFast() override;
400   bool addRegAssignAndRewriteOptimized() override;
401   void addPreRegAlloc() override;
402   void addPostRegAlloc() override;
403   void addOptimizedRegAlloc() override;
404   void addFastRegAlloc() override;
405 };
406 } // namespace
407 
408 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
409   return new RISCVPassConfig(*this, PM);
410 }
411 
412 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
413   // Initialize the global default.
414   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
415                   initializeDefaultRVVRegisterAllocatorOnce);
416 
417   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
418   if (Ctor != useDefaultRegisterAllocator)
419     return Ctor();
420 
421   if (Optimized)
422     return createGreedyRVVRegisterAllocator();
423 
424   return createFastRVVRegisterAllocator();
425 }
426 
427 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
428   if (EnableSplitRegAlloc)
429     addPass(createRVVRegAllocPass(false));
430   return TargetPassConfig::addRegAssignAndRewriteFast();
431 }
432 
433 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
434   if (EnableSplitRegAlloc) {
435     addPass(createRVVRegAllocPass(true));
436     addPass(createVirtRegRewriter(false));
437   }
438   return TargetPassConfig::addRegAssignAndRewriteOptimized();
439 }
440 
441 void RISCVPassConfig::addIRPasses() {
442   addPass(createAtomicExpandPass());
443 
444   if (getOptLevel() != CodeGenOptLevel::None) {
445     if (EnableLoopDataPrefetch)
446       addPass(createLoopDataPrefetchPass());
447 
448     addPass(createRISCVGatherScatterLoweringPass());
449     addPass(createInterleavedAccessPass());
450     addPass(createRISCVCodeGenPreparePass());
451   }
452 
453   TargetPassConfig::addIRPasses();
454 }
455 
456 bool RISCVPassConfig::addPreISel() {
457   if (TM->getOptLevel() != CodeGenOptLevel::None) {
458     // Add a barrier before instruction selection so that we will not get
459     // deleted block address after enabling default outlining. See D99707 for
460     // more details.
461     addPass(createBarrierNoopPass());
462   }
463 
464   if (EnableGlobalMerge == cl::BOU_TRUE) {
465     addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
466                                   /* OnlyOptimizeForSize */ false,
467                                   /* MergeExternalByDefault */ true));
468   }
469 
470   return false;
471 }
472 
473 bool RISCVPassConfig::addInstSelector() {
474   addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
475 
476   return false;
477 }
478 
479 bool RISCVPassConfig::addIRTranslator() {
480   addPass(new IRTranslator(getOptLevel()));
481   return false;
482 }
483 
484 void RISCVPassConfig::addPreLegalizeMachineIR() {
485   if (getOptLevel() == CodeGenOptLevel::None) {
486     addPass(createRISCVO0PreLegalizerCombiner());
487   } else {
488     addPass(createRISCVPreLegalizerCombiner());
489   }
490 }
491 
492 bool RISCVPassConfig::addLegalizeMachineIR() {
493   addPass(new Legalizer());
494   return false;
495 }
496 
497 void RISCVPassConfig::addPreRegBankSelect() {
498   if (getOptLevel() != CodeGenOptLevel::None)
499     addPass(createRISCVPostLegalizerCombiner());
500 }
501 
502 bool RISCVPassConfig::addRegBankSelect() {
503   addPass(new RegBankSelect());
504   return false;
505 }
506 
507 bool RISCVPassConfig::addGlobalInstructionSelect() {
508   addPass(new InstructionSelect(getOptLevel()));
509   return false;
510 }
511 
512 void RISCVPassConfig::addPreSched2() {
513   addPass(createRISCVPostRAExpandPseudoPass());
514 
515   // Emit KCFI checks for indirect calls.
516   addPass(createKCFIPass());
517 }
518 
519 void RISCVPassConfig::addPreEmitPass() {
520   addPass(&BranchRelaxationPassID);
521   addPass(createRISCVMakeCompressibleOptPass());
522 
523   // TODO: It would potentially be better to schedule copy propagation after
524   // expanding pseudos (in addPreEmitPass2). However, performing copy
525   // propagation after the machine outliner (which runs after addPreEmitPass)
526   // currently leads to incorrect code-gen, where copies to registers within
527   // outlined functions are removed erroneously.
528   if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
529       EnableRISCVCopyPropagation)
530     addPass(createMachineCopyPropagationPass(true));
531 }
532 
533 void RISCVPassConfig::addPreEmitPass2() {
534   if (TM->getOptLevel() != CodeGenOptLevel::None) {
535     addPass(createRISCVMoveMergePass());
536     // Schedule PushPop Optimization before expansion of Pseudo instruction,
537     // ensuring return instruction is detected correctly.
538     addPass(createRISCVPushPopOptimizationPass());
539   }
540   addPass(createRISCVExpandPseudoPass());
541 
542   // Schedule the expansion of AMOs at the last possible moment, avoiding the
543   // possibility for other passes to break the requirements for forward
544   // progress in the LR/SC block.
545   addPass(createRISCVExpandAtomicPseudoPass());
546 
547   // KCFI indirect call checks are lowered to a bundle.
548   addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
549     return MF.getFunction().getParent()->getModuleFlag("kcfi");
550   }));
551 }
552 
553 void RISCVPassConfig::addMachineSSAOptimization() {
554   addPass(createRISCVFoldMasksPass());
555 
556   TargetPassConfig::addMachineSSAOptimization();
557 
558   if (EnableMachineCombiner)
559     addPass(&MachineCombinerID);
560 
561   if (TM->getTargetTriple().isRISCV64()) {
562     addPass(createRISCVOptWInstrsPass());
563   }
564 }
565 
566 void RISCVPassConfig::addPreRegAlloc() {
567   addPass(createRISCVPreRAExpandPseudoPass());
568   if (TM->getOptLevel() != CodeGenOptLevel::None)
569     addPass(createRISCVMergeBaseOffsetOptPass());
570   addPass(createRISCVInsertVSETVLIPass());
571   if (TM->getOptLevel() != CodeGenOptLevel::None &&
572       EnableRISCVDeadRegisterElimination)
573     addPass(createRISCVDeadRegisterDefinitionsPass());
574   addPass(createRISCVInsertReadWriteCSRPass());
575   addPass(createRISCVInsertWriteVXRMPass());
576 }
577 
578 void RISCVPassConfig::addOptimizedRegAlloc() {
579   insertPass(&DetectDeadLanesID, &RISCVInitUndefID);
580 
581   TargetPassConfig::addOptimizedRegAlloc();
582 }
583 
584 void RISCVPassConfig::addFastRegAlloc() {
585   addPass(createRISCVInitUndefPass());
586   TargetPassConfig::addFastRegAlloc();
587 }
588 
589 
590 void RISCVPassConfig::addPostRegAlloc() {
591   if (TM->getOptLevel() != CodeGenOptLevel::None &&
592       EnableRedundantCopyElimination)
593     addPass(createRISCVRedundantCopyEliminationPass());
594 }
595 
596 yaml::MachineFunctionInfo *
597 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
598   return new yaml::RISCVMachineFunctionInfo();
599 }
600 
601 yaml::MachineFunctionInfo *
602 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
603   const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
604   return new yaml::RISCVMachineFunctionInfo(*MFI);
605 }
606 
607 bool RISCVTargetMachine::parseMachineFunctionInfo(
608     const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
609     SMDiagnostic &Error, SMRange &SourceRange) const {
610   const auto &YamlMFI =
611       static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
612   PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
613   return false;
614 }
615