1 //===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// RISC-V.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13
14 #include "MCTargetDesc/RISCVMatInt.h"
15 #include "RISCVRegisterBankInfo.h"
16 #include "RISCVSubtarget.h"
17 #include "RISCVTargetMachine.h"
18 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
20 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24 #include "llvm/CodeGen/MachineJumpTableInfo.h"
25 #include "llvm/IR/IntrinsicsRISCV.h"
26 #include "llvm/Support/Debug.h"
27
28 #define DEBUG_TYPE "riscv-isel"
29
30 using namespace llvm;
31 using namespace MIPatternMatch;
32
33 #define GET_GLOBALISEL_PREDICATE_BITSET
34 #include "RISCVGenGlobalISel.inc"
35 #undef GET_GLOBALISEL_PREDICATE_BITSET
36
37 namespace {
38
39 class RISCVInstructionSelector : public InstructionSelector {
40 public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
getName()46 static const char *getName() { return DEBUG_TYPE; }
47
48 private:
49 const TargetRegisterClass *
50 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
51
52 bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
53 bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
54
55 // tblgen-erated 'select' implementation, used as the initial selector for
56 // the patterns that don't require complex C++.
57 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
58
59 // A lowering phase that runs before any selection attempts.
60 // Returns true if the instruction was modified.
61 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
62 MachineRegisterInfo &MRI);
63
64 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
65 MachineRegisterInfo &MRI);
66
67 // Custom selection methods
68 bool selectCopy(MachineInstr &MI, MachineRegisterInfo &MRI) const;
69 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
70 MachineRegisterInfo &MRI) const;
71 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
72 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
73 MachineRegisterInfo &MRI, bool IsLocal = true,
74 bool IsExternWeak = false) const;
75 bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
76 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
77 MachineRegisterInfo &MRI) const;
78 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
79 MachineRegisterInfo &MRI) const;
80 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
81 MachineIRBuilder &MIB) const;
82 bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
83 MachineRegisterInfo &MRI) const;
84 bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
85 MachineRegisterInfo &MRI) const;
86
87 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
88 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
89
90 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
91 template <unsigned ShAmt>
selectSHXADDOp(MachineOperand & Root) const92 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
93 return selectSHXADDOp(Root, ShAmt);
94 }
95
96 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
97 unsigned ShAmt) const;
98 template <unsigned ShAmt>
selectSHXADD_UWOp(MachineOperand & Root) const99 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
100 return selectSHXADD_UWOp(Root, ShAmt);
101 }
102
103 // Custom renderers for tablegen
104 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
105 int OpIdx) const;
106 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
107 int OpIdx) const;
108 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
109 int OpIdx) const;
110 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
111 int OpIdx) const;
112 void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
113 int OpIdx) const;
114
115 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
116 int OpIdx) const;
117
118 const RISCVSubtarget &STI;
119 const RISCVInstrInfo &TII;
120 const RISCVRegisterInfo &TRI;
121 const RISCVRegisterBankInfo &RBI;
122 const RISCVTargetMachine &TM;
123
124 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
125 // uses "STI." in the code generated by TableGen. We need to unify the name of
126 // Subtarget variable.
127 const RISCVSubtarget *Subtarget = &STI;
128
129 #define GET_GLOBALISEL_PREDICATES_DECL
130 #include "RISCVGenGlobalISel.inc"
131 #undef GET_GLOBALISEL_PREDICATES_DECL
132
133 #define GET_GLOBALISEL_TEMPORARIES_DECL
134 #include "RISCVGenGlobalISel.inc"
135 #undef GET_GLOBALISEL_TEMPORARIES_DECL
136 };
137
138 } // end anonymous namespace
139
140 #define GET_GLOBALISEL_IMPL
141 #include "RISCVGenGlobalISel.inc"
142 #undef GET_GLOBALISEL_IMPL
143
RISCVInstructionSelector(const RISCVTargetMachine & TM,const RISCVSubtarget & STI,const RISCVRegisterBankInfo & RBI)144 RISCVInstructionSelector::RISCVInstructionSelector(
145 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
146 const RISCVRegisterBankInfo &RBI)
147 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
148 TM(TM),
149
150 #define GET_GLOBALISEL_PREDICATES_INIT
151 #include "RISCVGenGlobalISel.inc"
152 #undef GET_GLOBALISEL_PREDICATES_INIT
153 #define GET_GLOBALISEL_TEMPORARIES_INIT
154 #include "RISCVGenGlobalISel.inc"
155 #undef GET_GLOBALISEL_TEMPORARIES_INIT
156 {
157 }
158
159 InstructionSelector::ComplexRendererFns
selectShiftMask(MachineOperand & Root) const160 RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
161 if (!Root.isReg())
162 return std::nullopt;
163
164 using namespace llvm::MIPatternMatch;
165 MachineRegisterInfo &MRI = MF->getRegInfo();
166
167 Register RootReg = Root.getReg();
168 Register ShAmtReg = RootReg;
169 const LLT ShiftLLT = MRI.getType(RootReg);
170 unsigned ShiftWidth = ShiftLLT.getSizeInBits();
171 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
172 // Peek through zext.
173 Register ZExtSrcReg;
174 if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
175 ShAmtReg = ZExtSrcReg;
176 }
177
178 APInt AndMask;
179 Register AndSrcReg;
180 // Try to combine the following pattern (applicable to other shift
181 // instructions as well as 32-bit ones):
182 //
183 // %4:gprb(s64) = G_AND %3, %2
184 // %5:gprb(s64) = G_LSHR %1, %4(s64)
185 //
186 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
187 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
188 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
189 // then it can be eliminated. Given register rs1 or rs2 holding a constant
190 // (the and mask), there are two cases G_AND can be erased:
191 //
192 // 1. the lowest log2(XLEN) bits of the and mask are all set
193 // 2. the bits of the register being masked are already unset (zero set)
194 if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
195 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
196 if (ShMask.isSubsetOf(AndMask)) {
197 ShAmtReg = AndSrcReg;
198 } else {
199 // SimplifyDemandedBits may have optimized the mask so try restoring any
200 // bits that are known zero.
201 KnownBits Known = KB->getKnownBits(AndSrcReg);
202 if (ShMask.isSubsetOf(AndMask | Known.Zero))
203 ShAmtReg = AndSrcReg;
204 }
205 }
206
207 APInt Imm;
208 Register Reg;
209 if (mi_match(ShAmtReg, MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
210 if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
211 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
212 // to avoid the ADD.
213 ShAmtReg = Reg;
214 } else if (mi_match(ShAmtReg, MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
215 if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
216 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
217 // to generate a NEG instead of a SUB of a constant.
218 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
219 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
220 return {{[=](MachineInstrBuilder &MIB) {
221 MachineIRBuilder(*MIB.getInstr())
222 .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
223 MIB.addReg(ShAmtReg);
224 }}};
225 }
226 if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
227 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
228 // to generate a NOT instead of a SUB of a constant.
229 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
230 return {{[=](MachineInstrBuilder &MIB) {
231 MachineIRBuilder(*MIB.getInstr())
232 .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
233 .addImm(-1);
234 MIB.addReg(ShAmtReg);
235 }}};
236 }
237 }
238
239 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
240 }
241
242 InstructionSelector::ComplexRendererFns
selectSHXADDOp(MachineOperand & Root,unsigned ShAmt) const243 RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
244 unsigned ShAmt) const {
245 using namespace llvm::MIPatternMatch;
246 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
247 MachineRegisterInfo &MRI = MF.getRegInfo();
248
249 if (!Root.isReg())
250 return std::nullopt;
251 Register RootReg = Root.getReg();
252
253 const unsigned XLen = STI.getXLen();
254 APInt Mask, C2;
255 Register RegY;
256 std::optional<bool> LeftShift;
257 // (and (shl y, c2), mask)
258 if (mi_match(RootReg, MRI,
259 m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
260 LeftShift = true;
261 // (and (lshr y, c2), mask)
262 else if (mi_match(RootReg, MRI,
263 m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
264 LeftShift = false;
265
266 if (LeftShift.has_value()) {
267 if (*LeftShift)
268 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
269 else
270 Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
271
272 if (Mask.isShiftedMask()) {
273 unsigned Leading = XLen - Mask.getActiveBits();
274 unsigned Trailing = Mask.countr_zero();
275 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
276 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
277 if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
278 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
279 return {{[=](MachineInstrBuilder &MIB) {
280 MachineIRBuilder(*MIB.getInstr())
281 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
282 .addImm(Trailing - C2.getLimitedValue());
283 MIB.addReg(DstReg);
284 }}};
285 }
286
287 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
288 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
289 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
290 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
291 return {{[=](MachineInstrBuilder &MIB) {
292 MachineIRBuilder(*MIB.getInstr())
293 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
294 .addImm(Leading + Trailing);
295 MIB.addReg(DstReg);
296 }}};
297 }
298 }
299 }
300
301 LeftShift.reset();
302
303 // (shl (and y, mask), c2)
304 if (mi_match(RootReg, MRI,
305 m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
306 m_ICst(C2))))
307 LeftShift = true;
308 // (lshr (and y, mask), c2)
309 else if (mi_match(RootReg, MRI,
310 m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
311 m_ICst(C2))))
312 LeftShift = false;
313
314 if (LeftShift.has_value() && Mask.isShiftedMask()) {
315 unsigned Leading = XLen - Mask.getActiveBits();
316 unsigned Trailing = Mask.countr_zero();
317
318 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
319 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
320 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
321 (Trailing + C2.getLimitedValue()) == ShAmt;
322 if (!Cond)
323 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
324 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
325 Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
326 (Trailing - C2.getLimitedValue()) == ShAmt;
327
328 if (Cond) {
329 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
330 return {{[=](MachineInstrBuilder &MIB) {
331 MachineIRBuilder(*MIB.getInstr())
332 .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
333 .addImm(Trailing);
334 MIB.addReg(DstReg);
335 }}};
336 }
337 }
338
339 return std::nullopt;
340 }
341
342 InstructionSelector::ComplexRendererFns
selectSHXADD_UWOp(MachineOperand & Root,unsigned ShAmt) const343 RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
344 unsigned ShAmt) const {
345 using namespace llvm::MIPatternMatch;
346 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
347 MachineRegisterInfo &MRI = MF.getRegInfo();
348
349 if (!Root.isReg())
350 return std::nullopt;
351 Register RootReg = Root.getReg();
352
353 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
354 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
355 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
356 APInt Mask, C2;
357 Register RegX;
358 if (mi_match(
359 RootReg, MRI,
360 m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
361 m_ICst(Mask))))) {
362 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
363
364 if (Mask.isShiftedMask()) {
365 unsigned Leading = Mask.countl_zero();
366 unsigned Trailing = Mask.countr_zero();
367 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
368 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
369 return {{[=](MachineInstrBuilder &MIB) {
370 MachineIRBuilder(*MIB.getInstr())
371 .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
372 .addImm(C2.getLimitedValue() - ShAmt);
373 MIB.addReg(DstReg);
374 }}};
375 }
376 }
377 }
378
379 return std::nullopt;
380 }
381
382 InstructionSelector::ComplexRendererFns
selectAddrRegImm(MachineOperand & Root) const383 RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
384 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
385 MachineRegisterInfo &MRI = MF.getRegInfo();
386
387 if (!Root.isReg())
388 return std::nullopt;
389
390 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
391 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
392 return {{
393 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
394 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
395 }};
396 }
397
398 if (isBaseWithConstantOffset(Root, MRI)) {
399 MachineOperand &LHS = RootDef->getOperand(1);
400 MachineOperand &RHS = RootDef->getOperand(2);
401 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
402 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
403
404 int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
405 if (isInt<12>(RHSC)) {
406 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
407 return {{
408 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
409 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
410 }};
411
412 return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
413 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
414 }
415 }
416
417 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
418 // the combiner?
419 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
420 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
421 }
422
423 /// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
424 /// CC Must be an ICMP Predicate.
getRISCVCCFromICmp(CmpInst::Predicate CC)425 static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
426 switch (CC) {
427 default:
428 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
429 case CmpInst::Predicate::ICMP_EQ:
430 return RISCVCC::COND_EQ;
431 case CmpInst::Predicate::ICMP_NE:
432 return RISCVCC::COND_NE;
433 case CmpInst::Predicate::ICMP_ULT:
434 return RISCVCC::COND_LTU;
435 case CmpInst::Predicate::ICMP_SLT:
436 return RISCVCC::COND_LT;
437 case CmpInst::Predicate::ICMP_UGE:
438 return RISCVCC::COND_GEU;
439 case CmpInst::Predicate::ICMP_SGE:
440 return RISCVCC::COND_GE;
441 }
442 }
443
getOperandsForBranch(Register CondReg,MachineRegisterInfo & MRI,RISCVCC::CondCode & CC,Register & LHS,Register & RHS)444 static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI,
445 RISCVCC::CondCode &CC, Register &LHS,
446 Register &RHS) {
447 // Try to fold an ICmp. If that fails, use a NE compare with X0.
448 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
449 if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
450 LHS = CondReg;
451 RHS = RISCV::X0;
452 CC = RISCVCC::COND_NE;
453 return;
454 }
455
456 // We found an ICmp, do some canonicalizations.
457
458 // Adjust comparisons to use comparison with 0 if possible.
459 if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
460 switch (Pred) {
461 case CmpInst::Predicate::ICMP_SGT:
462 // Convert X > -1 to X >= 0
463 if (*Constant == -1) {
464 CC = RISCVCC::COND_GE;
465 RHS = RISCV::X0;
466 return;
467 }
468 break;
469 case CmpInst::Predicate::ICMP_SLT:
470 // Convert X < 1 to 0 >= X
471 if (*Constant == 1) {
472 CC = RISCVCC::COND_GE;
473 RHS = LHS;
474 LHS = RISCV::X0;
475 return;
476 }
477 break;
478 default:
479 break;
480 }
481 }
482
483 switch (Pred) {
484 default:
485 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
486 case CmpInst::Predicate::ICMP_EQ:
487 case CmpInst::Predicate::ICMP_NE:
488 case CmpInst::Predicate::ICMP_ULT:
489 case CmpInst::Predicate::ICMP_SLT:
490 case CmpInst::Predicate::ICMP_UGE:
491 case CmpInst::Predicate::ICMP_SGE:
492 // These CCs are supported directly by RISC-V branches.
493 break;
494 case CmpInst::Predicate::ICMP_SGT:
495 case CmpInst::Predicate::ICMP_SLE:
496 case CmpInst::Predicate::ICMP_UGT:
497 case CmpInst::Predicate::ICMP_ULE:
498 // These CCs are not supported directly by RISC-V branches, but changing the
499 // direction of the CC and swapping LHS and RHS are.
500 Pred = CmpInst::getSwappedPredicate(Pred);
501 std::swap(LHS, RHS);
502 break;
503 }
504
505 CC = getRISCVCCFromICmp(Pred);
506 return;
507 }
508
select(MachineInstr & MI)509 bool RISCVInstructionSelector::select(MachineInstr &MI) {
510 MachineBasicBlock &MBB = *MI.getParent();
511 MachineFunction &MF = *MBB.getParent();
512 MachineRegisterInfo &MRI = MF.getRegInfo();
513 MachineIRBuilder MIB(MI);
514
515 preISelLower(MI, MIB, MRI);
516 const unsigned Opc = MI.getOpcode();
517
518 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
519 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
520 const Register DefReg = MI.getOperand(0).getReg();
521 const LLT DefTy = MRI.getType(DefReg);
522
523 const RegClassOrRegBank &RegClassOrBank =
524 MRI.getRegClassOrRegBank(DefReg);
525
526 const TargetRegisterClass *DefRC =
527 RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
528 if (!DefRC) {
529 if (!DefTy.isValid()) {
530 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
531 return false;
532 }
533
534 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
535 DefRC = getRegClassForTypeOnBank(DefTy, RB);
536 if (!DefRC) {
537 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
538 return false;
539 }
540 }
541
542 MI.setDesc(TII.get(TargetOpcode::PHI));
543 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
544 }
545
546 // Certain non-generic instructions also need some special handling.
547 if (MI.isCopy())
548 return selectCopy(MI, MRI);
549
550 return true;
551 }
552
553 if (selectImpl(MI, *CoverageInfo))
554 return true;
555
556 switch (Opc) {
557 case TargetOpcode::G_ANYEXT:
558 case TargetOpcode::G_PTRTOINT:
559 case TargetOpcode::G_INTTOPTR:
560 case TargetOpcode::G_TRUNC:
561 case TargetOpcode::G_FREEZE:
562 return selectCopy(MI, MRI);
563 case TargetOpcode::G_CONSTANT: {
564 Register DstReg = MI.getOperand(0).getReg();
565 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
566
567 if (!materializeImm(DstReg, Imm, MIB))
568 return false;
569
570 MI.eraseFromParent();
571 return true;
572 }
573 case TargetOpcode::G_FCONSTANT: {
574 // TODO: Use constant pool for complext constants.
575 // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
576 Register DstReg = MI.getOperand(0).getReg();
577 const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
578 APInt Imm = FPimm.bitcastToAPInt();
579 unsigned Size = MRI.getType(DstReg).getSizeInBits();
580 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
581 Register GPRReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
582 if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
583 return false;
584
585 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
586 : Size == 32 ? RISCV::FMV_W_X
587 : RISCV::FMV_H_X;
588 auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
589 if (!FMV.constrainAllUses(TII, TRI, RBI))
590 return false;
591 } else {
592 assert(Size == 64 && !Subtarget->is64Bit() &&
593 "Unexpected size or subtarget");
594 // Split into two pieces and build through the stack.
595 Register GPRRegHigh = MRI.createVirtualRegister(&RISCV::GPRRegClass);
596 Register GPRRegLow = MRI.createVirtualRegister(&RISCV::GPRRegClass);
597 if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
598 MIB))
599 return false;
600 if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
601 return false;
602 MachineInstrBuilder PairF64 = MIB.buildInstr(
603 RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
604 if (!PairF64.constrainAllUses(TII, TRI, RBI))
605 return false;
606 }
607
608 MI.eraseFromParent();
609 return true;
610 }
611 case TargetOpcode::G_GLOBAL_VALUE: {
612 auto *GV = MI.getOperand(1).getGlobal();
613 if (GV->isThreadLocal()) {
614 // TODO: implement this case.
615 return false;
616 }
617
618 return selectAddr(MI, MIB, MRI, GV->isDSOLocal(),
619 GV->hasExternalWeakLinkage());
620 }
621 case TargetOpcode::G_JUMP_TABLE:
622 case TargetOpcode::G_CONSTANT_POOL:
623 return selectAddr(MI, MIB, MRI);
624 case TargetOpcode::G_BRCOND: {
625 Register LHS, RHS;
626 RISCVCC::CondCode CC;
627 getOperandsForBranch(MI.getOperand(0).getReg(), MRI, CC, LHS, RHS);
628
629 auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
630 .addMBB(MI.getOperand(1).getMBB());
631 MI.eraseFromParent();
632 return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
633 }
634 case TargetOpcode::G_BRJT: {
635 // FIXME: Move to legalization?
636 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
637 unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());
638 assert((EntrySize == 4 || (Subtarget->is64Bit() && EntrySize == 8)) &&
639 "Unsupported jump-table entry size");
640 assert(
641 (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 ||
642 MJTI->getEntryKind() == MachineJumpTableInfo::EK_Custom32 ||
643 MJTI->getEntryKind() == MachineJumpTableInfo::EK_BlockAddress) &&
644 "Unexpected jump-table entry kind");
645
646 auto SLL =
647 MIB.buildInstr(RISCV::SLLI, {&RISCV::GPRRegClass}, {MI.getOperand(2)})
648 .addImm(Log2_32(EntrySize));
649 if (!SLL.constrainAllUses(TII, TRI, RBI))
650 return false;
651
652 // TODO: Use SHXADD. Moving to legalization would fix this automatically.
653 auto ADD = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
654 {MI.getOperand(0), SLL.getReg(0)});
655 if (!ADD.constrainAllUses(TII, TRI, RBI))
656 return false;
657
658 unsigned LdOpc = EntrySize == 8 ? RISCV::LD : RISCV::LW;
659 auto Dest =
660 MIB.buildInstr(LdOpc, {&RISCV::GPRRegClass}, {ADD.getReg(0)})
661 .addImm(0)
662 .addMemOperand(MF.getMachineMemOperand(
663 MachinePointerInfo::getJumpTable(MF), MachineMemOperand::MOLoad,
664 EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout()))));
665 if (!Dest.constrainAllUses(TII, TRI, RBI))
666 return false;
667
668 // If the Kind is EK_LabelDifference32, the table stores an offset from
669 // the location of the table. Add the table address to get an absolute
670 // address.
671 if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32) {
672 Dest = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
673 {Dest.getReg(0), MI.getOperand(0)});
674 if (!Dest.constrainAllUses(TII, TRI, RBI))
675 return false;
676 }
677
678 auto Branch =
679 MIB.buildInstr(RISCV::PseudoBRIND, {}, {Dest.getReg(0)}).addImm(0);
680 if (!Branch.constrainAllUses(TII, TRI, RBI))
681 return false;
682
683 MI.eraseFromParent();
684 return true;
685 }
686 case TargetOpcode::G_BRINDIRECT:
687 MI.setDesc(TII.get(RISCV::PseudoBRIND));
688 MI.addOperand(MachineOperand::CreateImm(0));
689 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
690 case TargetOpcode::G_SEXT_INREG:
691 return selectSExtInreg(MI, MIB);
692 case TargetOpcode::G_FRAME_INDEX: {
693 // TODO: We may want to replace this code with the SelectionDAG patterns,
694 // which fail to get imported because it uses FrameAddrRegImm, which is a
695 // ComplexPattern
696 MI.setDesc(TII.get(RISCV::ADDI));
697 MI.addOperand(MachineOperand::CreateImm(0));
698 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
699 }
700 case TargetOpcode::G_SELECT:
701 return selectSelect(MI, MIB, MRI);
702 case TargetOpcode::G_FCMP:
703 return selectFPCompare(MI, MIB, MRI);
704 case TargetOpcode::G_FENCE: {
705 AtomicOrdering FenceOrdering =
706 static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
707 SyncScope::ID FenceSSID =
708 static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
709 emitFence(FenceOrdering, FenceSSID, MIB);
710 MI.eraseFromParent();
711 return true;
712 }
713 case TargetOpcode::G_IMPLICIT_DEF:
714 return selectImplicitDef(MI, MIB, MRI);
715 case TargetOpcode::G_MERGE_VALUES:
716 return selectMergeValues(MI, MIB, MRI);
717 case TargetOpcode::G_UNMERGE_VALUES:
718 return selectUnmergeValues(MI, MIB, MRI);
719 default:
720 return false;
721 }
722 }
723
selectMergeValues(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI) const724 bool RISCVInstructionSelector::selectMergeValues(
725 MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
726 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
727
728 // Build a F64 Pair from operands
729 if (MI.getNumOperands() != 3)
730 return false;
731 Register Dst = MI.getOperand(0).getReg();
732 Register Lo = MI.getOperand(1).getReg();
733 Register Hi = MI.getOperand(2).getReg();
734 if (!isRegInFprb(Dst, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
735 return false;
736 MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
737 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
738 }
739
selectUnmergeValues(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI) const740 bool RISCVInstructionSelector::selectUnmergeValues(
741 MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
742 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
743
744 // Split F64 Src into two s32 parts
745 if (MI.getNumOperands() != 3)
746 return false;
747 Register Src = MI.getOperand(2).getReg();
748 Register Lo = MI.getOperand(0).getReg();
749 Register Hi = MI.getOperand(1).getReg();
750 if (!isRegInFprb(Src, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
751 return false;
752 MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
753 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
754 }
755
replacePtrWithInt(MachineOperand & Op,MachineIRBuilder & MIB,MachineRegisterInfo & MRI)756 bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
757 MachineIRBuilder &MIB,
758 MachineRegisterInfo &MRI) {
759 Register PtrReg = Op.getReg();
760 assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
761
762 const LLT sXLen = LLT::scalar(STI.getXLen());
763 auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
764 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
765 Op.setReg(PtrToInt.getReg(0));
766 return select(*PtrToInt);
767 }
768
preISelLower(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI)769 void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
770 MachineIRBuilder &MIB,
771 MachineRegisterInfo &MRI) {
772 switch (MI.getOpcode()) {
773 case TargetOpcode::G_PTR_ADD: {
774 Register DstReg = MI.getOperand(0).getReg();
775 const LLT sXLen = LLT::scalar(STI.getXLen());
776
777 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
778 MI.setDesc(TII.get(TargetOpcode::G_ADD));
779 MRI.setType(DstReg, sXLen);
780 break;
781 }
782 case TargetOpcode::G_PTRMASK: {
783 Register DstReg = MI.getOperand(0).getReg();
784 const LLT sXLen = LLT::scalar(STI.getXLen());
785 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
786 MI.setDesc(TII.get(TargetOpcode::G_AND));
787 MRI.setType(DstReg, sXLen);
788 }
789 }
790 }
791
renderNegImm(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const792 void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
793 const MachineInstr &MI,
794 int OpIdx) const {
795 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
796 "Expected G_CONSTANT");
797 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
798 MIB.addImm(-CstVal);
799 }
800
renderImmSubFromXLen(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const801 void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
802 const MachineInstr &MI,
803 int OpIdx) const {
804 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
805 "Expected G_CONSTANT");
806 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
807 MIB.addImm(STI.getXLen() - CstVal);
808 }
809
renderImmSubFrom32(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const810 void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
811 const MachineInstr &MI,
812 int OpIdx) const {
813 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
814 "Expected G_CONSTANT");
815 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
816 MIB.addImm(32 - CstVal);
817 }
818
renderImmPlus1(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const819 void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
820 const MachineInstr &MI,
821 int OpIdx) const {
822 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
823 "Expected G_CONSTANT");
824 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
825 MIB.addImm(CstVal + 1);
826 }
827
renderImm(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const828 void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
829 const MachineInstr &MI,
830 int OpIdx) const {
831 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
832 "Expected G_CONSTANT");
833 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
834 MIB.addImm(CstVal);
835 }
836
renderTrailingZeros(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const837 void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
838 const MachineInstr &MI,
839 int OpIdx) const {
840 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
841 "Expected G_CONSTANT");
842 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
843 MIB.addImm(llvm::countr_zero(C));
844 }
845
getRegClassForTypeOnBank(LLT Ty,const RegisterBank & RB) const846 const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
847 LLT Ty, const RegisterBank &RB) const {
848 if (RB.getID() == RISCV::GPRBRegBankID) {
849 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
850 return &RISCV::GPRRegClass;
851 }
852
853 if (RB.getID() == RISCV::FPRBRegBankID) {
854 if (Ty.getSizeInBits() == 16)
855 return &RISCV::FPR16RegClass;
856 if (Ty.getSizeInBits() == 32)
857 return &RISCV::FPR32RegClass;
858 if (Ty.getSizeInBits() == 64)
859 return &RISCV::FPR64RegClass;
860 }
861
862 if (RB.getID() == RISCV::VRBRegBankID) {
863 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
864 return &RISCV::VRRegClass;
865
866 if (Ty.getSizeInBits().getKnownMinValue() == 128)
867 return &RISCV::VRM2RegClass;
868
869 if (Ty.getSizeInBits().getKnownMinValue() == 256)
870 return &RISCV::VRM4RegClass;
871
872 if (Ty.getSizeInBits().getKnownMinValue() == 512)
873 return &RISCV::VRM8RegClass;
874 }
875
876 return nullptr;
877 }
878
isRegInGprb(Register Reg,MachineRegisterInfo & MRI) const879 bool RISCVInstructionSelector::isRegInGprb(Register Reg,
880 MachineRegisterInfo &MRI) const {
881 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
882 }
883
isRegInFprb(Register Reg,MachineRegisterInfo & MRI) const884 bool RISCVInstructionSelector::isRegInFprb(Register Reg,
885 MachineRegisterInfo &MRI) const {
886 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
887 }
888
selectCopy(MachineInstr & MI,MachineRegisterInfo & MRI) const889 bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
890 MachineRegisterInfo &MRI) const {
891 Register DstReg = MI.getOperand(0).getReg();
892
893 if (DstReg.isPhysical())
894 return true;
895
896 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
897 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
898 assert(DstRC &&
899 "Register class not available for LLT, register bank combination");
900
901 // No need to constrain SrcReg. It will get constrained when
902 // we hit another of its uses or its defs.
903 // Copies do not have constraints.
904 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
905 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
906 << " operand\n");
907 return false;
908 }
909
910 MI.setDesc(TII.get(RISCV::COPY));
911 return true;
912 }
913
selectImplicitDef(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI) const914 bool RISCVInstructionSelector::selectImplicitDef(
915 MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
916 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
917
918 const Register DstReg = MI.getOperand(0).getReg();
919 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
920 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
921
922 assert(DstRC &&
923 "Register class not available for LLT, register bank combination");
924
925 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
926 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
927 << " operand\n");
928 }
929 MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
930 return true;
931 }
932
materializeImm(Register DstReg,int64_t Imm,MachineIRBuilder & MIB) const933 bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
934 MachineIRBuilder &MIB) const {
935 MachineRegisterInfo &MRI = *MIB.getMRI();
936
937 if (Imm == 0) {
938 MIB.buildCopy(DstReg, Register(RISCV::X0));
939 RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, MRI);
940 return true;
941 }
942
943 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
944 unsigned NumInsts = Seq.size();
945 Register SrcReg = RISCV::X0;
946
947 for (unsigned i = 0; i < NumInsts; i++) {
948 Register TmpReg = i < NumInsts - 1
949 ? MRI.createVirtualRegister(&RISCV::GPRRegClass)
950 : DstReg;
951 const RISCVMatInt::Inst &I = Seq[i];
952 MachineInstr *Result;
953
954 switch (I.getOpndKind()) {
955 case RISCVMatInt::Imm:
956 // clang-format off
957 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
958 .addImm(I.getImm());
959 // clang-format on
960 break;
961 case RISCVMatInt::RegX0:
962 Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
963 {SrcReg, Register(RISCV::X0)});
964 break;
965 case RISCVMatInt::RegReg:
966 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
967 break;
968 case RISCVMatInt::RegImm:
969 Result =
970 MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
971 break;
972 }
973
974 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
975 return false;
976
977 SrcReg = TmpReg;
978 }
979
980 return true;
981 }
982
selectAddr(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI,bool IsLocal,bool IsExternWeak) const983 bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
984 MachineIRBuilder &MIB,
985 MachineRegisterInfo &MRI,
986 bool IsLocal,
987 bool IsExternWeak) const {
988 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
989 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
990 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
991 "Unexpected opcode");
992
993 const MachineOperand &DispMO = MI.getOperand(1);
994
995 Register DefReg = MI.getOperand(0).getReg();
996 const LLT DefTy = MRI.getType(DefReg);
997
998 // When HWASAN is used and tagging of global variables is enabled
999 // they should be accessed via the GOT, since the tagged address of a global
1000 // is incompatible with existing code models. This also applies to non-pic
1001 // mode.
1002 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1003 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1004 // Use PC-relative addressing to access the symbol. This generates the
1005 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1006 // %pcrel_lo(auipc)).
1007 MI.setDesc(TII.get(RISCV::PseudoLLA));
1008 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1009 }
1010
1011 // Use PC-relative addressing to access the GOT for this symbol, then
1012 // load the address from the GOT. This generates the pattern (PseudoLGA
1013 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1014 // %pcrel_lo(auipc))).
1015 MachineFunction &MF = *MI.getParent()->getParent();
1016 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1017 MachinePointerInfo::getGOT(MF),
1018 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1019 MachineMemOperand::MOInvariant,
1020 DefTy, Align(DefTy.getSizeInBits() / 8));
1021
1022 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1023 .addDisp(DispMO, 0)
1024 .addMemOperand(MemOp);
1025
1026 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1027 return false;
1028
1029 MI.eraseFromParent();
1030 return true;
1031 }
1032
1033 switch (TM.getCodeModel()) {
1034 default: {
1035 reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1036 getName(), "Unsupported code model for lowering", MI);
1037 return false;
1038 }
1039 case CodeModel::Small: {
1040 // Must lie within a single 2 GiB address range and must lie between
1041 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1042 // (lui %hi(sym)) %lo(sym)).
1043 Register AddrHiDest = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1044 MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1045 .addDisp(DispMO, 0, RISCVII::MO_HI);
1046
1047 if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1048 return false;
1049
1050 auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1051 .addDisp(DispMO, 0, RISCVII::MO_LO);
1052
1053 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1054 return false;
1055
1056 MI.eraseFromParent();
1057 return true;
1058 }
1059 case CodeModel::Medium:
1060 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1061 // relocation needs to reference a label that points to the auipc
1062 // instruction itself, not the global. This cannot be done inside the
1063 // instruction selector.
1064 if (IsExternWeak) {
1065 // An extern weak symbol may be undefined, i.e. have value 0, which may
1066 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1067 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1068 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1069 MachineFunction &MF = *MI.getParent()->getParent();
1070 MachineMemOperand *MemOp = MF.getMachineMemOperand(
1071 MachinePointerInfo::getGOT(MF),
1072 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1073 MachineMemOperand::MOInvariant,
1074 DefTy, Align(DefTy.getSizeInBits() / 8));
1075
1076 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1077 .addDisp(DispMO, 0)
1078 .addMemOperand(MemOp);
1079
1080 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1081 return false;
1082
1083 MI.eraseFromParent();
1084 return true;
1085 }
1086
1087 // Generate a sequence for accessing addresses within any 2GiB range
1088 // within the address space. This generates the pattern (PseudoLLA sym),
1089 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1090 MI.setDesc(TII.get(RISCV::PseudoLLA));
1091 return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1092 }
1093
1094 return false;
1095 }
1096
selectSExtInreg(MachineInstr & MI,MachineIRBuilder & MIB) const1097 bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
1098 MachineIRBuilder &MIB) const {
1099 if (!STI.isRV64())
1100 return false;
1101
1102 const MachineOperand &Size = MI.getOperand(2);
1103 // Only Size == 32 (i.e. shift by 32 bits) is acceptable at this point.
1104 if (!Size.isImm() || Size.getImm() != 32)
1105 return false;
1106
1107 const MachineOperand &Src = MI.getOperand(1);
1108 const MachineOperand &Dst = MI.getOperand(0);
1109 // addiw rd, rs, 0 (i.e. sext.w rd, rs)
1110 MachineInstr *NewMI =
1111 MIB.buildInstr(RISCV::ADDIW, {Dst.getReg()}, {Src.getReg()}).addImm(0U);
1112
1113 if (!constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI))
1114 return false;
1115
1116 MI.eraseFromParent();
1117 return true;
1118 }
1119
selectSelect(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI) const1120 bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1121 MachineIRBuilder &MIB,
1122 MachineRegisterInfo &MRI) const {
1123 auto &SelectMI = cast<GSelect>(MI);
1124
1125 Register LHS, RHS;
1126 RISCVCC::CondCode CC;
1127 getOperandsForBranch(SelectMI.getCondReg(), MRI, CC, LHS, RHS);
1128
1129 Register DstReg = SelectMI.getReg(0);
1130
1131 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1132 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1133 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1134 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1135 : RISCV::Select_FPR64_Using_CC_GPR;
1136 }
1137
1138 MachineInstr *Result = MIB.buildInstr(Opc)
1139 .addDef(DstReg)
1140 .addReg(LHS)
1141 .addReg(RHS)
1142 .addImm(CC)
1143 .addReg(SelectMI.getTrueReg())
1144 .addReg(SelectMI.getFalseReg());
1145 MI.eraseFromParent();
1146 return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1147 }
1148
1149 // Convert an FCMP predicate to one of the supported F or D instructions.
getFCmpOpcode(CmpInst::Predicate Pred,unsigned Size)1150 static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1151 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1152 switch (Pred) {
1153 default:
1154 llvm_unreachable("Unsupported predicate");
1155 case CmpInst::FCMP_OLT:
1156 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1157 case CmpInst::FCMP_OLE:
1158 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1159 case CmpInst::FCMP_OEQ:
1160 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1161 }
1162 }
1163
1164 // Try legalizing an FCMP by swapping or inverting the predicate to one that
1165 // is supported.
legalizeFCmpPredicate(Register & LHS,Register & RHS,CmpInst::Predicate & Pred,bool & NeedInvert)1166 static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1167 CmpInst::Predicate &Pred, bool &NeedInvert) {
1168 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1169 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1170 Pred == CmpInst::FCMP_OEQ;
1171 };
1172
1173 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1174
1175 CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);
1176 if (isLegalFCmpPredicate(InvPred)) {
1177 Pred = InvPred;
1178 std::swap(LHS, RHS);
1179 return true;
1180 }
1181
1182 InvPred = CmpInst::getInversePredicate(Pred);
1183 NeedInvert = true;
1184 if (isLegalFCmpPredicate(InvPred)) {
1185 Pred = InvPred;
1186 return true;
1187 }
1188 InvPred = CmpInst::getSwappedPredicate(InvPred);
1189 if (isLegalFCmpPredicate(InvPred)) {
1190 Pred = InvPred;
1191 std::swap(LHS, RHS);
1192 return true;
1193 }
1194
1195 return false;
1196 }
1197
1198 // Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1199 // the result in DstReg.
1200 // FIXME: Maybe we should expand this earlier.
selectFPCompare(MachineInstr & MI,MachineIRBuilder & MIB,MachineRegisterInfo & MRI) const1201 bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1202 MachineIRBuilder &MIB,
1203 MachineRegisterInfo &MRI) const {
1204 auto &CmpMI = cast<GFCmp>(MI);
1205 CmpInst::Predicate Pred = CmpMI.getCond();
1206
1207 Register DstReg = CmpMI.getReg(0);
1208 Register LHS = CmpMI.getLHSReg();
1209 Register RHS = CmpMI.getRHSReg();
1210
1211 unsigned Size = MRI.getType(LHS).getSizeInBits();
1212 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1213
1214 Register TmpReg = DstReg;
1215
1216 bool NeedInvert = false;
1217 // First try swapping operands or inverting.
1218 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1219 if (NeedInvert)
1220 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1221 auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1222 if (!Cmp.constrainAllUses(TII, TRI, RBI))
1223 return false;
1224 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1225 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1226 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1227 auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1228 {&RISCV::GPRRegClass}, {LHS, RHS});
1229 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1230 return false;
1231 auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1232 {&RISCV::GPRRegClass}, {RHS, LHS});
1233 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1234 return false;
1235 if (NeedInvert)
1236 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1237 auto Or =
1238 MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1239 if (!Or.constrainAllUses(TII, TRI, RBI))
1240 return false;
1241 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1242 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1243 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1244 NeedInvert = Pred == CmpInst::FCMP_UNO;
1245 auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1246 {&RISCV::GPRRegClass}, {LHS, LHS});
1247 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1248 return false;
1249 auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1250 {&RISCV::GPRRegClass}, {RHS, RHS});
1251 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1252 return false;
1253 if (NeedInvert)
1254 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1255 auto And =
1256 MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1257 if (!And.constrainAllUses(TII, TRI, RBI))
1258 return false;
1259 } else
1260 llvm_unreachable("Unhandled predicate");
1261
1262 // Emit an XORI to invert the result if needed.
1263 if (NeedInvert) {
1264 auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1265 if (!Xor.constrainAllUses(TII, TRI, RBI))
1266 return false;
1267 }
1268
1269 MI.eraseFromParent();
1270 return true;
1271 }
1272
emitFence(AtomicOrdering FenceOrdering,SyncScope::ID FenceSSID,MachineIRBuilder & MIB) const1273 void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1274 SyncScope::ID FenceSSID,
1275 MachineIRBuilder &MIB) const {
1276 if (STI.hasStdExtZtso()) {
1277 // The only fence that needs an instruction is a sequentially-consistent
1278 // cross-thread fence.
1279 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1280 FenceSSID == SyncScope::System) {
1281 // fence rw, rw
1282 MIB.buildInstr(RISCV::FENCE, {}, {})
1283 .addImm(RISCVFenceField::R | RISCVFenceField::W)
1284 .addImm(RISCVFenceField::R | RISCVFenceField::W);
1285 return;
1286 }
1287
1288 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1289 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1290 return;
1291 }
1292
1293 // singlethread fences only synchronize with signal handlers on the same
1294 // thread and thus only need to preserve instruction order, not actually
1295 // enforce memory ordering.
1296 if (FenceSSID == SyncScope::SingleThread) {
1297 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1298 return;
1299 }
1300
1301 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1302 // Manual: Volume I.
1303 unsigned Pred, Succ;
1304 switch (FenceOrdering) {
1305 default:
1306 llvm_unreachable("Unexpected ordering");
1307 case AtomicOrdering::AcquireRelease:
1308 // fence acq_rel -> fence.tso
1309 MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1310 return;
1311 case AtomicOrdering::Acquire:
1312 // fence acquire -> fence r, rw
1313 Pred = RISCVFenceField::R;
1314 Succ = RISCVFenceField::R | RISCVFenceField::W;
1315 break;
1316 case AtomicOrdering::Release:
1317 // fence release -> fence rw, w
1318 Pred = RISCVFenceField::R | RISCVFenceField::W;
1319 Succ = RISCVFenceField::W;
1320 break;
1321 case AtomicOrdering::SequentiallyConsistent:
1322 // fence seq_cst -> fence rw, rw
1323 Pred = RISCVFenceField::R | RISCVFenceField::W;
1324 Succ = RISCVFenceField::R | RISCVFenceField::W;
1325 break;
1326 }
1327 MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1328 }
1329
1330 namespace llvm {
1331 InstructionSelector *
createRISCVInstructionSelector(const RISCVTargetMachine & TM,const RISCVSubtarget & Subtarget,const RISCVRegisterBankInfo & RBI)1332 createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1333 const RISCVSubtarget &Subtarget,
1334 const RISCVRegisterBankInfo &RBI) {
1335 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1336 }
1337 } // end namespace llvm
1338