1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implements the info about RISC-V target spec.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "RISCVTargetMachine.h"
14 #include "MCTargetDesc/RISCVBaseInfo.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVTargetObjectFile.h"
18 #include "RISCVTargetTransformInfo.h"
19 #include "TargetInfo/RISCVTargetInfo.h"
20 #include "llvm/Analysis/TargetTransformInfo.h"
21 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
22 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
24 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
25 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
26 #include "llvm/CodeGen/MIRParser/MIParser.h"
27 #include "llvm/CodeGen/MIRYamlMapping.h"
28 #include "llvm/CodeGen/MachineScheduler.h"
29 #include "llvm/CodeGen/MacroFusion.h"
30 #include "llvm/CodeGen/Passes.h"
31 #include "llvm/CodeGen/RegAllocRegistry.h"
32 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
33 #include "llvm/CodeGen/TargetPassConfig.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/MC/TargetRegistry.h"
36 #include "llvm/Passes/PassBuilder.h"
37 #include "llvm/Support/Compiler.h"
38 #include "llvm/Target/TargetOptions.h"
39 #include "llvm/Transforms/IPO.h"
40 #include "llvm/Transforms/Scalar.h"
41 #include "llvm/Transforms/Vectorize/EVLIndVarSimplify.h"
42 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
43 #include <optional>
44 using namespace llvm;
45
46 static cl::opt<bool> EnableRedundantCopyElimination(
47 "riscv-enable-copyelim",
48 cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
49 cl::Hidden);
50
51 // FIXME: Unify control over GlobalMerge.
52 static cl::opt<cl::boolOrDefault>
53 EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
54 cl::desc("Enable the global merge pass"));
55
56 static cl::opt<bool>
57 EnableMachineCombiner("riscv-enable-machine-combiner",
58 cl::desc("Enable the machine combiner pass"),
59 cl::init(true), cl::Hidden);
60
61 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
62 "riscv-v-vector-bits-max",
63 cl::desc("Assume V extension vector registers are at most this big, "
64 "with zero meaning no maximum size is assumed."),
65 cl::init(0), cl::Hidden);
66
67 static cl::opt<int> RVVVectorBitsMinOpt(
68 "riscv-v-vector-bits-min",
69 cl::desc("Assume V extension vector registers are at least this big, "
70 "with zero meaning no minimum size is assumed. A value of -1 "
71 "means use Zvl*b extension. This is primarily used to enable "
72 "autovectorization with fixed width vectors."),
73 cl::init(-1), cl::Hidden);
74
75 static cl::opt<bool> EnableRISCVCopyPropagation(
76 "riscv-enable-copy-propagation",
77 cl::desc("Enable the copy propagation with RISC-V copy instr"),
78 cl::init(true), cl::Hidden);
79
80 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
81 "riscv-enable-dead-defs", cl::Hidden,
82 cl::desc("Enable the pass that removes dead"
83 " definitions and replaces stores to"
84 " them with stores to x0"),
85 cl::init(true));
86
87 static cl::opt<bool>
88 EnableSinkFold("riscv-enable-sink-fold",
89 cl::desc("Enable sinking and folding of instruction copies"),
90 cl::init(true), cl::Hidden);
91
92 static cl::opt<bool>
93 EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
94 cl::desc("Enable the loop data prefetch pass"),
95 cl::init(true));
96
97 static cl::opt<bool> EnableMISchedLoadStoreClustering(
98 "riscv-misched-load-store-clustering", cl::Hidden,
99 cl::desc("Enable load and store clustering in the machine scheduler"),
100 cl::init(true));
101
102 static cl::opt<bool> EnablePostMISchedLoadStoreClustering(
103 "riscv-postmisched-load-store-clustering", cl::Hidden,
104 cl::desc("Enable PostRA load and store clustering in the machine scheduler"),
105 cl::init(true));
106
107 static cl::opt<bool>
108 EnableVLOptimizer("riscv-enable-vl-optimizer",
109 cl::desc("Enable the RISC-V VL Optimizer pass"),
110 cl::init(true), cl::Hidden);
111
112 static cl::opt<bool> DisableVectorMaskMutation(
113 "riscv-disable-vector-mask-mutation",
114 cl::desc("Disable the vector mask scheduling mutation"), cl::init(false),
115 cl::Hidden);
116
117 static cl::opt<bool>
118 EnableMachinePipeliner("riscv-enable-pipeliner",
119 cl::desc("Enable Machine Pipeliner for RISC-V"),
120 cl::init(false), cl::Hidden);
121
LLVMInitializeRISCVTarget()122 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
123 RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
124 RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
125 auto *PR = PassRegistry::getPassRegistry();
126 initializeGlobalISel(*PR);
127 initializeRISCVO0PreLegalizerCombinerPass(*PR);
128 initializeRISCVPreLegalizerCombinerPass(*PR);
129 initializeRISCVPostLegalizerCombinerPass(*PR);
130 initializeKCFIPass(*PR);
131 initializeRISCVDeadRegisterDefinitionsPass(*PR);
132 initializeRISCVLateBranchOptPass(*PR);
133 initializeRISCVMakeCompressibleOptPass(*PR);
134 initializeRISCVGatherScatterLoweringPass(*PR);
135 initializeRISCVCodeGenPreparePass(*PR);
136 initializeRISCVPostRAExpandPseudoPass(*PR);
137 initializeRISCVMergeBaseOffsetOptPass(*PR);
138 initializeRISCVOptWInstrsPass(*PR);
139 initializeRISCVFoldMemOffsetPass(*PR);
140 initializeRISCVPreRAExpandPseudoPass(*PR);
141 initializeRISCVExpandPseudoPass(*PR);
142 initializeRISCVVectorPeepholePass(*PR);
143 initializeRISCVVLOptimizerPass(*PR);
144 initializeRISCVVMV0EliminationPass(*PR);
145 initializeRISCVInsertVSETVLIPass(*PR);
146 initializeRISCVInsertReadWriteCSRPass(*PR);
147 initializeRISCVInsertWriteVXRMPass(*PR);
148 initializeRISCVDAGToDAGISelLegacyPass(*PR);
149 initializeRISCVMoveMergePass(*PR);
150 initializeRISCVPushPopOptPass(*PR);
151 initializeRISCVIndirectBranchTrackingPass(*PR);
152 initializeRISCVLoadStoreOptPass(*PR);
153 initializeRISCVExpandAtomicPseudoPass(*PR);
154 initializeRISCVRedundantCopyEliminationPass(*PR);
155 initializeRISCVAsmPrinterPass(*PR);
156 }
157
computeDataLayout(const Triple & TT,const TargetOptions & Options)158 static StringRef computeDataLayout(const Triple &TT,
159 const TargetOptions &Options) {
160 StringRef ABIName = Options.MCOptions.getABIName();
161 if (TT.isArch64Bit()) {
162 if (ABIName == "lp64e")
163 return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
164
165 return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
166 }
167 assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
168
169 if (ABIName == "ilp32e")
170 return "e-m:e-p:32:32-i64:64-n32-S32";
171
172 return "e-m:e-p:32:32-i64:64-n32-S128";
173 }
174
getEffectiveRelocModel(const Triple & TT,std::optional<Reloc::Model> RM)175 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
176 std::optional<Reloc::Model> RM) {
177 return RM.value_or(Reloc::Static);
178 }
179
RISCVTargetMachine(const Target & T,const Triple & TT,StringRef CPU,StringRef FS,const TargetOptions & Options,std::optional<Reloc::Model> RM,std::optional<CodeModel::Model> CM,CodeGenOptLevel OL,bool JIT)180 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
181 StringRef CPU, StringRef FS,
182 const TargetOptions &Options,
183 std::optional<Reloc::Model> RM,
184 std::optional<CodeModel::Model> CM,
185 CodeGenOptLevel OL, bool JIT)
186 : CodeGenTargetMachineImpl(T, computeDataLayout(TT, Options), TT, CPU, FS,
187 Options, getEffectiveRelocModel(TT, RM),
188 getEffectiveCodeModel(CM, CodeModel::Small), OL),
189 TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
190 initAsmInfo();
191
192 // RISC-V supports the MachineOutliner.
193 setMachineOutliner(true);
194 setSupportsDefaultOutlining(true);
195
196 if (TT.isOSFuchsia() && !TT.isArch64Bit())
197 report_fatal_error("Fuchsia is only supported for 64-bit");
198
199 setCFIFixup(true);
200 }
201
202 const RISCVSubtarget *
getSubtargetImpl(const Function & F) const203 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
204 Attribute CPUAttr = F.getFnAttribute("target-cpu");
205 Attribute TuneAttr = F.getFnAttribute("tune-cpu");
206 Attribute FSAttr = F.getFnAttribute("target-features");
207
208 std::string CPU =
209 CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
210 std::string TuneCPU =
211 TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
212 std::string FS =
213 FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
214
215 unsigned RVVBitsMin = RVVVectorBitsMinOpt;
216 unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
217
218 Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
219 if (VScaleRangeAttr.isValid()) {
220 if (!RVVVectorBitsMinOpt.getNumOccurrences())
221 RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
222 std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
223 if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
224 RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
225 }
226
227 if (RVVBitsMin != -1U) {
228 // FIXME: Change to >= 32 when VLEN = 32 is supported.
229 assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
230 isPowerOf2_32(RVVBitsMin))) &&
231 "V or Zve* extension requires vector length to be in the range of "
232 "64 to 65536 and a power 2!");
233 assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
234 "Minimum V extension vector length should not be larger than its "
235 "maximum!");
236 }
237 assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
238 isPowerOf2_32(RVVBitsMax))) &&
239 "V or Zve* extension requires vector length to be in the range of "
240 "64 to 65536 and a power 2!");
241
242 if (RVVBitsMin != -1U) {
243 if (RVVBitsMax != 0) {
244 RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
245 RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
246 }
247
248 RVVBitsMin = llvm::bit_floor(
249 (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
250 }
251 RVVBitsMax =
252 llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
253
254 SmallString<512> Key;
255 raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
256 << CPU << TuneCPU << FS;
257 auto &I = SubtargetMap[Key];
258 if (!I) {
259 // This needs to be done before we create a new subtarget since any
260 // creation will depend on the TM and the code generation flags on the
261 // function that reside in TargetOptions.
262 resetTargetOptions(F);
263 auto ABIName = Options.MCOptions.getABIName();
264 if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
265 F.getParent()->getModuleFlag("target-abi"))) {
266 auto TargetABI = RISCVABI::getTargetABI(ABIName);
267 if (TargetABI != RISCVABI::ABI_Unknown &&
268 ModuleTargetABI->getString() != ABIName) {
269 report_fatal_error("-target-abi option != target-abi module flag");
270 }
271 ABIName = ModuleTargetABI->getString();
272 }
273 I = std::make_unique<RISCVSubtarget>(
274 TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
275 }
276 return I.get();
277 }
278
createMachineFunctionInfo(BumpPtrAllocator & Allocator,const Function & F,const TargetSubtargetInfo * STI) const279 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
280 BumpPtrAllocator &Allocator, const Function &F,
281 const TargetSubtargetInfo *STI) const {
282 return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(
283 Allocator, F, static_cast<const RISCVSubtarget *>(STI));
284 }
285
286 TargetTransformInfo
getTargetTransformInfo(const Function & F) const287 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
288 return TargetTransformInfo(std::make_unique<RISCVTTIImpl>(this, F));
289 }
290
291 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
292 // for all memory accesses, so it is reasonable to assume that an
293 // implementation has no-op address space casts. If an implementation makes a
294 // change to this, they can override it here.
isNoopAddrSpaceCast(unsigned SrcAS,unsigned DstAS) const295 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
296 unsigned DstAS) const {
297 return true;
298 }
299
300 ScheduleDAGInstrs *
createMachineScheduler(MachineSchedContext * C) const301 RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
302 ScheduleDAGMILive *DAG = createSchedLive(C);
303 if (EnableMISchedLoadStoreClustering) {
304 DAG->addMutation(createLoadClusterDAGMutation(
305 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
306 DAG->addMutation(createStoreClusterDAGMutation(
307 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
308 }
309
310 const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
311 if (!DisableVectorMaskMutation && ST.hasVInstructions())
312 DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
313
314 return DAG;
315 }
316
317 ScheduleDAGInstrs *
createPostMachineScheduler(MachineSchedContext * C) const318 RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
319 ScheduleDAGMI *DAG = createSchedPostRA(C);
320 if (EnablePostMISchedLoadStoreClustering) {
321 DAG->addMutation(createLoadClusterDAGMutation(
322 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
323 DAG->addMutation(createStoreClusterDAGMutation(
324 DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
325 }
326
327 return DAG;
328 }
329
330 namespace {
331
332 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
333 public:
RVVRegisterRegAlloc(const char * N,const char * D,FunctionPassCtor C)334 RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
335 : RegisterRegAllocBase(N, D, C) {}
336 };
337
onlyAllocateRVVReg(const TargetRegisterInfo & TRI,const MachineRegisterInfo & MRI,const Register Reg)338 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
339 const MachineRegisterInfo &MRI,
340 const Register Reg) {
341 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
342 return RISCVRegisterInfo::isRVVRegClass(RC);
343 }
344
useDefaultRegisterAllocator()345 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
346
347 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
348
349 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
350 /// This option could designate the rvv register allocator only.
351 /// For example: -riscv-rvv-regalloc=basic
352 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
353 RegisterPassParser<RVVRegisterRegAlloc>>
354 RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
355 cl::init(&useDefaultRegisterAllocator),
356 cl::desc("Register allocator to use for RVV register."));
357
initializeDefaultRVVRegisterAllocatorOnce()358 static void initializeDefaultRVVRegisterAllocatorOnce() {
359 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
360
361 if (!Ctor) {
362 Ctor = RVVRegAlloc;
363 RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
364 }
365 }
366
createBasicRVVRegisterAllocator()367 static FunctionPass *createBasicRVVRegisterAllocator() {
368 return createBasicRegisterAllocator(onlyAllocateRVVReg);
369 }
370
createGreedyRVVRegisterAllocator()371 static FunctionPass *createGreedyRVVRegisterAllocator() {
372 return createGreedyRegisterAllocator(onlyAllocateRVVReg);
373 }
374
createFastRVVRegisterAllocator()375 static FunctionPass *createFastRVVRegisterAllocator() {
376 return createFastRegisterAllocator(onlyAllocateRVVReg, false);
377 }
378
379 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
380 "basic register allocator",
381 createBasicRVVRegisterAllocator);
382 static RVVRegisterRegAlloc
383 greedyRegAllocRVVReg("greedy", "greedy register allocator",
384 createGreedyRVVRegisterAllocator);
385
386 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
387 createFastRVVRegisterAllocator);
388
389 class RISCVPassConfig : public TargetPassConfig {
390 public:
RISCVPassConfig(RISCVTargetMachine & TM,PassManagerBase & PM)391 RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
392 : TargetPassConfig(TM, PM) {
393 if (TM.getOptLevel() != CodeGenOptLevel::None)
394 substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
395 setEnableSinkAndFold(EnableSinkFold);
396 EnableLoopTermFold = true;
397 }
398
getRISCVTargetMachine() const399 RISCVTargetMachine &getRISCVTargetMachine() const {
400 return getTM<RISCVTargetMachine>();
401 }
402
403 void addIRPasses() override;
404 bool addPreISel() override;
405 void addCodeGenPrepare() override;
406 bool addInstSelector() override;
407 bool addIRTranslator() override;
408 void addPreLegalizeMachineIR() override;
409 bool addLegalizeMachineIR() override;
410 void addPreRegBankSelect() override;
411 bool addRegBankSelect() override;
412 bool addGlobalInstructionSelect() override;
413 void addPreEmitPass() override;
414 void addPreEmitPass2() override;
415 void addPreSched2() override;
416 void addMachineSSAOptimization() override;
417 FunctionPass *createRVVRegAllocPass(bool Optimized);
418 bool addRegAssignAndRewriteFast() override;
419 bool addRegAssignAndRewriteOptimized() override;
420 void addPreRegAlloc() override;
421 void addPostRegAlloc() override;
422 void addFastRegAlloc() override;
423
424 std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
425 };
426 } // namespace
427
createPassConfig(PassManagerBase & PM)428 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
429 return new RISCVPassConfig(*this, PM);
430 }
431
getCSEConfig() const432 std::unique_ptr<CSEConfigBase> RISCVPassConfig::getCSEConfig() const {
433 return getStandardCSEConfigForOpt(TM->getOptLevel());
434 }
435
createRVVRegAllocPass(bool Optimized)436 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
437 // Initialize the global default.
438 llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
439 initializeDefaultRVVRegisterAllocatorOnce);
440
441 RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
442 if (Ctor != useDefaultRegisterAllocator)
443 return Ctor();
444
445 if (Optimized)
446 return createGreedyRVVRegisterAllocator();
447
448 return createFastRVVRegisterAllocator();
449 }
450
addRegAssignAndRewriteFast()451 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
452 addPass(createRVVRegAllocPass(false));
453 addPass(createRISCVInsertVSETVLIPass());
454 if (TM->getOptLevel() != CodeGenOptLevel::None &&
455 EnableRISCVDeadRegisterElimination)
456 addPass(createRISCVDeadRegisterDefinitionsPass());
457 return TargetPassConfig::addRegAssignAndRewriteFast();
458 }
459
addRegAssignAndRewriteOptimized()460 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
461 addPass(createRVVRegAllocPass(true));
462 addPass(createVirtRegRewriter(false));
463 addPass(createRISCVInsertVSETVLIPass());
464 if (TM->getOptLevel() != CodeGenOptLevel::None &&
465 EnableRISCVDeadRegisterElimination)
466 addPass(createRISCVDeadRegisterDefinitionsPass());
467 return TargetPassConfig::addRegAssignAndRewriteOptimized();
468 }
469
addIRPasses()470 void RISCVPassConfig::addIRPasses() {
471 addPass(createAtomicExpandLegacyPass());
472 addPass(createRISCVZacasABIFixPass());
473
474 if (getOptLevel() != CodeGenOptLevel::None) {
475 if (EnableLoopDataPrefetch)
476 addPass(createLoopDataPrefetchPass());
477
478 addPass(createRISCVGatherScatterLoweringPass());
479 addPass(createInterleavedAccessPass());
480 addPass(createRISCVCodeGenPreparePass());
481 }
482
483 TargetPassConfig::addIRPasses();
484 }
485
addPreISel()486 bool RISCVPassConfig::addPreISel() {
487 if (TM->getOptLevel() != CodeGenOptLevel::None) {
488 // Add a barrier before instruction selection so that we will not get
489 // deleted block address after enabling default outlining. See D99707 for
490 // more details.
491 addPass(createBarrierNoopPass());
492 }
493
494 if ((TM->getOptLevel() != CodeGenOptLevel::None &&
495 EnableGlobalMerge == cl::BOU_UNSET) ||
496 EnableGlobalMerge == cl::BOU_TRUE) {
497 // FIXME: Like AArch64, we disable extern global merging by default due to
498 // concerns it might regress some workloads. Unlike AArch64, we don't
499 // currently support enabling the pass in an "OnlyOptimizeForSize" mode.
500 // Investigating and addressing both items are TODO.
501 addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
502 /* OnlyOptimizeForSize */ false,
503 /* MergeExternalByDefault */ true));
504 }
505
506 return false;
507 }
508
addCodeGenPrepare()509 void RISCVPassConfig::addCodeGenPrepare() {
510 if (getOptLevel() != CodeGenOptLevel::None)
511 addPass(createTypePromotionLegacyPass());
512 TargetPassConfig::addCodeGenPrepare();
513 }
514
addInstSelector()515 bool RISCVPassConfig::addInstSelector() {
516 addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
517
518 return false;
519 }
520
addIRTranslator()521 bool RISCVPassConfig::addIRTranslator() {
522 addPass(new IRTranslator(getOptLevel()));
523 return false;
524 }
525
addPreLegalizeMachineIR()526 void RISCVPassConfig::addPreLegalizeMachineIR() {
527 if (getOptLevel() == CodeGenOptLevel::None) {
528 addPass(createRISCVO0PreLegalizerCombiner());
529 } else {
530 addPass(createRISCVPreLegalizerCombiner());
531 }
532 }
533
addLegalizeMachineIR()534 bool RISCVPassConfig::addLegalizeMachineIR() {
535 addPass(new Legalizer());
536 return false;
537 }
538
addPreRegBankSelect()539 void RISCVPassConfig::addPreRegBankSelect() {
540 if (getOptLevel() != CodeGenOptLevel::None)
541 addPass(createRISCVPostLegalizerCombiner());
542 }
543
addRegBankSelect()544 bool RISCVPassConfig::addRegBankSelect() {
545 addPass(new RegBankSelect());
546 return false;
547 }
548
addGlobalInstructionSelect()549 bool RISCVPassConfig::addGlobalInstructionSelect() {
550 addPass(new InstructionSelect(getOptLevel()));
551 return false;
552 }
553
addPreSched2()554 void RISCVPassConfig::addPreSched2() {
555 addPass(createRISCVPostRAExpandPseudoPass());
556
557 // Emit KCFI checks for indirect calls.
558 addPass(createKCFIPass());
559 if (TM->getOptLevel() != CodeGenOptLevel::None)
560 addPass(createRISCVLoadStoreOptPass());
561 }
562
addPreEmitPass()563 void RISCVPassConfig::addPreEmitPass() {
564 // TODO: It would potentially be better to schedule copy propagation after
565 // expanding pseudos (in addPreEmitPass2). However, performing copy
566 // propagation after the machine outliner (which runs after addPreEmitPass)
567 // currently leads to incorrect code-gen, where copies to registers within
568 // outlined functions are removed erroneously.
569 if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
570 EnableRISCVCopyPropagation)
571 addPass(createMachineCopyPropagationPass(true));
572 if (TM->getOptLevel() >= CodeGenOptLevel::Default)
573 addPass(createRISCVLateBranchOptPass());
574 // The IndirectBranchTrackingPass inserts lpad and could have changed the
575 // basic block alignment. It must be done before Branch Relaxation to
576 // prevent the adjusted offset exceeding the branch range.
577 addPass(createRISCVIndirectBranchTrackingPass());
578 addPass(&BranchRelaxationPassID);
579 addPass(createRISCVMakeCompressibleOptPass());
580 }
581
addPreEmitPass2()582 void RISCVPassConfig::addPreEmitPass2() {
583 if (TM->getOptLevel() != CodeGenOptLevel::None) {
584 addPass(createRISCVMoveMergePass());
585 // Schedule PushPop Optimization before expansion of Pseudo instruction,
586 // ensuring return instruction is detected correctly.
587 addPass(createRISCVPushPopOptimizationPass());
588 }
589 addPass(createRISCVExpandPseudoPass());
590
591 // Schedule the expansion of AMOs at the last possible moment, avoiding the
592 // possibility for other passes to break the requirements for forward
593 // progress in the LR/SC block.
594 addPass(createRISCVExpandAtomicPseudoPass());
595
596 // KCFI indirect call checks are lowered to a bundle.
597 addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
598 return MF.getFunction().getParent()->getModuleFlag("kcfi");
599 }));
600 }
601
addMachineSSAOptimization()602 void RISCVPassConfig::addMachineSSAOptimization() {
603 addPass(createRISCVVectorPeepholePass());
604 addPass(createRISCVFoldMemOffsetPass());
605
606 TargetPassConfig::addMachineSSAOptimization();
607
608 if (EnableMachineCombiner)
609 addPass(&MachineCombinerID);
610
611 if (TM->getTargetTriple().isRISCV64()) {
612 addPass(createRISCVOptWInstrsPass());
613 }
614 }
615
addPreRegAlloc()616 void RISCVPassConfig::addPreRegAlloc() {
617 addPass(createRISCVPreRAExpandPseudoPass());
618 if (TM->getOptLevel() != CodeGenOptLevel::None) {
619 addPass(createRISCVMergeBaseOffsetOptPass());
620 if (EnableVLOptimizer)
621 addPass(createRISCVVLOptimizerPass());
622 }
623
624 addPass(createRISCVInsertReadWriteCSRPass());
625 addPass(createRISCVInsertWriteVXRMPass());
626 addPass(createRISCVLandingPadSetupPass());
627
628 if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
629 addPass(&MachinePipelinerID);
630
631 addPass(createRISCVVMV0EliminationPass());
632 }
633
addFastRegAlloc()634 void RISCVPassConfig::addFastRegAlloc() {
635 addPass(&InitUndefID);
636 TargetPassConfig::addFastRegAlloc();
637 }
638
639
addPostRegAlloc()640 void RISCVPassConfig::addPostRegAlloc() {
641 if (TM->getOptLevel() != CodeGenOptLevel::None &&
642 EnableRedundantCopyElimination)
643 addPass(createRISCVRedundantCopyEliminationPass());
644 }
645
registerPassBuilderCallbacks(PassBuilder & PB)646 void RISCVTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
647 PB.registerLateLoopOptimizationsEPCallback([=](LoopPassManager &LPM,
648 OptimizationLevel Level) {
649 LPM.addPass(LoopIdiomVectorizePass(LoopIdiomVectorizeStyle::Predicated));
650 });
651
652 PB.registerVectorizerEndEPCallback(
653 [](FunctionPassManager &FPM, OptimizationLevel Level) {
654 if (Level.isOptimizingForSpeed())
655 FPM.addPass(createFunctionToLoopPassAdaptor(EVLIndVarSimplifyPass()));
656 });
657 }
658
659 yaml::MachineFunctionInfo *
createDefaultFuncInfoYAML() const660 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
661 return new yaml::RISCVMachineFunctionInfo();
662 }
663
664 yaml::MachineFunctionInfo *
convertFuncInfoToYAML(const MachineFunction & MF) const665 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
666 const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
667 return new yaml::RISCVMachineFunctionInfo(*MFI);
668 }
669
parseMachineFunctionInfo(const yaml::MachineFunctionInfo & MFI,PerFunctionMIParsingState & PFS,SMDiagnostic & Error,SMRange & SourceRange) const670 bool RISCVTargetMachine::parseMachineFunctionInfo(
671 const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
672 SMDiagnostic &Error, SMRange &SourceRange) const {
673 const auto &YamlMFI =
674 static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
675 PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
676 return false;
677 }
678