xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp (revision 700637cbb5e582861067a11aaca4d053546871d2) 
1 //===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// RISC-V.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/RISCVMatInt.h"
15 #include "RISCVRegisterBankInfo.h"
16 #include "RISCVSubtarget.h"
17 #include "RISCVTargetMachine.h"
18 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
19 #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
20 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
23 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24 #include "llvm/CodeGen/MachineJumpTableInfo.h"
25 #include "llvm/IR/IntrinsicsRISCV.h"
26 #include "llvm/Support/Debug.h"
27 
28 #define DEBUG_TYPE "riscv-isel"
29 
30 using namespace llvm;
31 using namespace MIPatternMatch;
32 
33 #define GET_GLOBALISEL_PREDICATE_BITSET
34 #include "RISCVGenGlobalISel.inc"
35 #undef GET_GLOBALISEL_PREDICATE_BITSET
36 
37 namespace {
38 
39 class RISCVInstructionSelector : public InstructionSelector {
40 public:
41   RISCVInstructionSelector(const RISCVTargetMachine &TM,
42                            const RISCVSubtarget &STI,
43                            const RISCVRegisterBankInfo &RBI);
44 
45   bool select(MachineInstr &MI) override;
46 
setupMF(MachineFunction & MF,GISelValueTracking * VT,CodeGenCoverage * CoverageInfo,ProfileSummaryInfo * PSI,BlockFrequencyInfo * BFI)47   void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48                CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49                BlockFrequencyInfo *BFI) override {
50     InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
51     MRI = &MF.getRegInfo();
52   }
53 
getName()54   static const char *getName() { return DEBUG_TYPE; }
55 
56 private:
57   const TargetRegisterClass *
58   getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
59 
60   static constexpr unsigned MaxRecursionDepth = 6;
61 
62   bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
63                        const unsigned Depth = 0) const;
hasAllHUsers(const MachineInstr & MI) const64   bool hasAllHUsers(const MachineInstr &MI) const {
65     return hasAllNBitUsers(MI, 16);
66   }
hasAllWUsers(const MachineInstr & MI) const67   bool hasAllWUsers(const MachineInstr &MI) const {
68     return hasAllNBitUsers(MI, 32);
69   }
70 
71   bool isRegInGprb(Register Reg) const;
72   bool isRegInFprb(Register Reg) const;
73 
74   // tblgen-erated 'select' implementation, used as the initial selector for
75   // the patterns that don't require complex C++.
76   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
77 
78   // A lowering phase that runs before any selection attempts.
79   // Returns true if the instruction was modified.
80   void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB);
81 
82   bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB);
83 
84   // Custom selection methods
85   bool selectCopy(MachineInstr &MI) const;
86   bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB) const;
87   bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
88   bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB, bool IsLocal = true,
89                   bool IsExternWeak = false) const;
90   bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB) const;
91   bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB) const;
92   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
93                  MachineIRBuilder &MIB) const;
94   bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
95 
96   ComplexRendererFns selectShiftMask(MachineOperand &Root,
97                                      unsigned ShiftWidth) const;
selectShiftMaskXLen(MachineOperand & Root) const98   ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
99     return selectShiftMask(Root, STI.getXLen());
100   }
selectShiftMask32(MachineOperand & Root) const101   ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
102     return selectShiftMask(Root, 32);
103   }
104   ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
105 
106   ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
107   template <unsigned Bits>
selectSExtBits(MachineOperand & Root) const108   ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
109     return selectSExtBits(Root, Bits);
110   }
111 
112   ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
113   template <unsigned Bits>
selectZExtBits(MachineOperand & Root) const114   ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
115     return selectZExtBits(Root, Bits);
116   }
117 
118   ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
119   template <unsigned ShAmt>
selectSHXADDOp(MachineOperand & Root) const120   ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
121     return selectSHXADDOp(Root, ShAmt);
122   }
123 
124   ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
125                                        unsigned ShAmt) const;
126   template <unsigned ShAmt>
selectSHXADD_UWOp(MachineOperand & Root) const127   ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
128     return selectSHXADD_UWOp(Root, ShAmt);
129   }
130 
131   ComplexRendererFns renderVLOp(MachineOperand &Root) const;
132 
133   // Custom renderers for tablegen
134   void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
135                     int OpIdx) const;
136   void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
137                             int OpIdx) const;
138   void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
139                           int OpIdx) const;
140   void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
141                       int OpIdx) const;
142   void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
143                         int OpIdx) const;
144 
145   void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
146                            int OpIdx) const;
147   void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
148                                  const MachineInstr &MI, int OpIdx) const;
149 
150   void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
151                               int OpIdx) const;
152   void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
153                               int OpIdx) const;
154 
155   const RISCVSubtarget &STI;
156   const RISCVInstrInfo &TII;
157   const RISCVRegisterInfo &TRI;
158   const RISCVRegisterBankInfo &RBI;
159   const RISCVTargetMachine &TM;
160 
161   MachineRegisterInfo *MRI = nullptr;
162 
163   // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
164   // uses "STI." in the code generated by TableGen. We need to unify the name of
165   // Subtarget variable.
166   const RISCVSubtarget *Subtarget = &STI;
167 
168 #define GET_GLOBALISEL_PREDICATES_DECL
169 #include "RISCVGenGlobalISel.inc"
170 #undef GET_GLOBALISEL_PREDICATES_DECL
171 
172 #define GET_GLOBALISEL_TEMPORARIES_DECL
173 #include "RISCVGenGlobalISel.inc"
174 #undef GET_GLOBALISEL_TEMPORARIES_DECL
175 };
176 
177 } // end anonymous namespace
178 
179 #define GET_GLOBALISEL_IMPL
180 #include "RISCVGenGlobalISel.inc"
181 #undef GET_GLOBALISEL_IMPL
182 
RISCVInstructionSelector(const RISCVTargetMachine & TM,const RISCVSubtarget & STI,const RISCVRegisterBankInfo & RBI)183 RISCVInstructionSelector::RISCVInstructionSelector(
184     const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
185     const RISCVRegisterBankInfo &RBI)
186     : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
187       TM(TM),
188 
189 #define GET_GLOBALISEL_PREDICATES_INIT
190 #include "RISCVGenGlobalISel.inc"
191 #undef GET_GLOBALISEL_PREDICATES_INIT
192 #define GET_GLOBALISEL_TEMPORARIES_INIT
193 #include "RISCVGenGlobalISel.inc"
194 #undef GET_GLOBALISEL_TEMPORARIES_INIT
195 {
196 }
197 
198 // Mimics optimizations in ISel and RISCVOptWInst Pass
hasAllNBitUsers(const MachineInstr & MI,unsigned Bits,const unsigned Depth) const199 bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
200                                                unsigned Bits,
201                                                const unsigned Depth) const {
202 
203   assert((MI.getOpcode() == TargetOpcode::G_ADD ||
204           MI.getOpcode() == TargetOpcode::G_SUB ||
205           MI.getOpcode() == TargetOpcode::G_MUL ||
206           MI.getOpcode() == TargetOpcode::G_SHL ||
207           MI.getOpcode() == TargetOpcode::G_LSHR ||
208           MI.getOpcode() == TargetOpcode::G_AND ||
209           MI.getOpcode() == TargetOpcode::G_OR ||
210           MI.getOpcode() == TargetOpcode::G_XOR ||
211           MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
212          "Unexpected opcode");
213 
214   if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
215     return false;
216 
217   auto DestReg = MI.getOperand(0).getReg();
218   for (auto &UserOp : MRI->use_nodbg_operands(DestReg)) {
219     assert(UserOp.getParent() && "UserOp must have a parent");
220     const MachineInstr &UserMI = *UserOp.getParent();
221     unsigned OpIdx = UserOp.getOperandNo();
222 
223     switch (UserMI.getOpcode()) {
224     default:
225       return false;
226     case RISCV::ADDW:
227     case RISCV::ADDIW:
228     case RISCV::SUBW:
229       if (Bits >= 32)
230         break;
231       return false;
232     case RISCV::SLL:
233     case RISCV::SRA:
234     case RISCV::SRL:
235       // Shift amount operands only use log2(Xlen) bits.
236       if (OpIdx == 2 && Bits >= Log2_32(Subtarget->getXLen()))
237         break;
238       return false;
239     case RISCV::SLLI:
240       // SLLI only uses the lower (XLen - ShAmt) bits.
241       if (Bits >= Subtarget->getXLen() - UserMI.getOperand(2).getImm())
242         break;
243       return false;
244     case RISCV::ANDI:
245       if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
246                       (uint64_t)UserMI.getOperand(2).getImm()))
247         break;
248       goto RecCheck;
249     case RISCV::AND:
250     case RISCV::OR:
251     case RISCV::XOR:
252     RecCheck:
253       if (hasAllNBitUsers(UserMI, Bits, Depth + 1))
254         break;
255       return false;
256     case RISCV::SRLI: {
257       unsigned ShAmt = UserMI.getOperand(2).getImm();
258       // If we are shifting right by less than Bits, and users don't demand any
259       // bits that were shifted into [Bits-1:0], then we can consider this as an
260       // N-Bit user.
261       if (Bits > ShAmt && hasAllNBitUsers(UserMI, Bits - ShAmt, Depth + 1))
262         break;
263       return false;
264     }
265     }
266   }
267 
268   return true;
269 }
270 
271 InstructionSelector::ComplexRendererFns
selectShiftMask(MachineOperand & Root,unsigned ShiftWidth) const272 RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
273                                           unsigned ShiftWidth) const {
274   if (!Root.isReg())
275     return std::nullopt;
276 
277   using namespace llvm::MIPatternMatch;
278 
279   Register ShAmtReg = Root.getReg();
280   // Peek through zext.
281   Register ZExtSrcReg;
282   if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))
283     ShAmtReg = ZExtSrcReg;
284 
285   APInt AndMask;
286   Register AndSrcReg;
287   // Try to combine the following pattern (applicable to other shift
288   // instructions as well as 32-bit ones):
289   //
290   //   %4:gprb(s64) = G_AND %3, %2
291   //   %5:gprb(s64) = G_LSHR %1, %4(s64)
292   //
293   // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
294   // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
295   // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
296   // then it can be eliminated. Given register rs1 or rs2 holding a constant
297   // (the and mask), there are two cases G_AND can be erased:
298   //
299   // 1. the lowest log2(XLEN) bits of the and mask are all set
300   // 2. the bits of the register being masked are already unset (zero set)
301   if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
302     APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
303     if (ShMask.isSubsetOf(AndMask)) {
304       ShAmtReg = AndSrcReg;
305     } else {
306       // SimplifyDemandedBits may have optimized the mask so try restoring any
307       // bits that are known zero.
308       KnownBits Known = VT->getKnownBits(AndSrcReg);
309       if (ShMask.isSubsetOf(AndMask | Known.Zero))
310         ShAmtReg = AndSrcReg;
311     }
312   }
313 
314   APInt Imm;
315   Register Reg;
316   if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
317     if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
318       // If we are shifting by X+N where N == 0 mod Size, then just shift by X
319       // to avoid the ADD.
320       ShAmtReg = Reg;
321   } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
322     if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
323       // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
324       // to generate a NEG instead of a SUB of a constant.
325       ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
326       unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
327       return {{[=](MachineInstrBuilder &MIB) {
328         MachineIRBuilder(*MIB.getInstr())
329             .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
330         MIB.addReg(ShAmtReg);
331       }}};
332     }
333     if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
334       // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
335       // to generate a NOT instead of a SUB of a constant.
336       ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
337       return {{[=](MachineInstrBuilder &MIB) {
338         MachineIRBuilder(*MIB.getInstr())
339             .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
340             .addImm(-1);
341         MIB.addReg(ShAmtReg);
342       }}};
343     }
344   }
345 
346   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
347 }
348 
349 InstructionSelector::ComplexRendererFns
selectSExtBits(MachineOperand & Root,unsigned Bits) const350 RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
351                                          unsigned Bits) const {
352   if (!Root.isReg())
353     return std::nullopt;
354   Register RootReg = Root.getReg();
355   MachineInstr *RootDef = MRI->getVRegDef(RootReg);
356 
357   if (RootDef->getOpcode() == TargetOpcode::G_SEXT_INREG &&
358       RootDef->getOperand(2).getImm() == Bits) {
359     return {
360         {[=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); }}};
361   }
362 
363   unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
364   if ((Size - VT->computeNumSignBits(RootReg)) < Bits)
365     return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
366 
367   return std::nullopt;
368 }
369 
370 InstructionSelector::ComplexRendererFns
selectZExtBits(MachineOperand & Root,unsigned Bits) const371 RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
372                                          unsigned Bits) const {
373   if (!Root.isReg())
374     return std::nullopt;
375   Register RootReg = Root.getReg();
376 
377   Register RegX;
378   uint64_t Mask = maskTrailingOnes<uint64_t>(Bits);
379   if (mi_match(RootReg, *MRI, m_GAnd(m_Reg(RegX), m_SpecificICst(Mask)))) {
380     return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
381   }
382 
383   if (mi_match(RootReg, *MRI, m_GZExt(m_Reg(RegX))) &&
384       MRI->getType(RegX).getScalarSizeInBits() == Bits)
385     return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
386 
387   unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
388   if (VT->maskedValueIsZero(RootReg, APInt::getBitsSetFrom(Size, Bits)))
389     return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
390 
391   return std::nullopt;
392 }
393 
394 InstructionSelector::ComplexRendererFns
selectSHXADDOp(MachineOperand & Root,unsigned ShAmt) const395 RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
396                                          unsigned ShAmt) const {
397   using namespace llvm::MIPatternMatch;
398 
399   if (!Root.isReg())
400     return std::nullopt;
401   Register RootReg = Root.getReg();
402 
403   const unsigned XLen = STI.getXLen();
404   APInt Mask, C2;
405   Register RegY;
406   std::optional<bool> LeftShift;
407   // (and (shl y, c2), mask)
408   if (mi_match(RootReg, *MRI,
409                m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
410     LeftShift = true;
411   // (and (lshr y, c2), mask)
412   else if (mi_match(RootReg, *MRI,
413                     m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
414     LeftShift = false;
415 
416   if (LeftShift.has_value()) {
417     if (*LeftShift)
418       Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
419     else
420       Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
421 
422     if (Mask.isShiftedMask()) {
423       unsigned Leading = XLen - Mask.getActiveBits();
424       unsigned Trailing = Mask.countr_zero();
425       // Given (and (shl y, c2), mask) in which mask has no leading zeros and
426       // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
427       if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
428         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
429         return {{[=](MachineInstrBuilder &MIB) {
430           MachineIRBuilder(*MIB.getInstr())
431               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
432               .addImm(Trailing - C2.getLimitedValue());
433           MIB.addReg(DstReg);
434         }}};
435       }
436 
437       // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
438       // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
439       if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
440         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
441         return {{[=](MachineInstrBuilder &MIB) {
442           MachineIRBuilder(*MIB.getInstr())
443               .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
444               .addImm(Leading + Trailing);
445           MIB.addReg(DstReg);
446         }}};
447       }
448     }
449   }
450 
451   LeftShift.reset();
452 
453   // (shl (and y, mask), c2)
454   if (mi_match(RootReg, *MRI,
455                m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
456                       m_ICst(C2))))
457     LeftShift = true;
458   // (lshr (and y, mask), c2)
459   else if (mi_match(RootReg, *MRI,
460                     m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
461                             m_ICst(C2))))
462     LeftShift = false;
463 
464   if (LeftShift.has_value() && Mask.isShiftedMask()) {
465     unsigned Leading = XLen - Mask.getActiveBits();
466     unsigned Trailing = Mask.countr_zero();
467 
468     // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
469     // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
470     bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
471                 (Trailing + C2.getLimitedValue()) == ShAmt;
472     if (!Cond)
473       // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
474       // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
475       Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
476              (Trailing - C2.getLimitedValue()) == ShAmt;
477 
478     if (Cond) {
479       Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
480       return {{[=](MachineInstrBuilder &MIB) {
481         MachineIRBuilder(*MIB.getInstr())
482             .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
483             .addImm(Trailing);
484         MIB.addReg(DstReg);
485       }}};
486     }
487   }
488 
489   return std::nullopt;
490 }
491 
492 InstructionSelector::ComplexRendererFns
selectSHXADD_UWOp(MachineOperand & Root,unsigned ShAmt) const493 RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
494                                             unsigned ShAmt) const {
495   using namespace llvm::MIPatternMatch;
496 
497   if (!Root.isReg())
498     return std::nullopt;
499   Register RootReg = Root.getReg();
500 
501   // Given (and (shl x, c2), mask) in which mask is a shifted mask with
502   // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
503   // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
504   APInt Mask, C2;
505   Register RegX;
506   if (mi_match(
507           RootReg, *MRI,
508           m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
509                                 m_ICst(Mask))))) {
510     Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
511 
512     if (Mask.isShiftedMask()) {
513       unsigned Leading = Mask.countl_zero();
514       unsigned Trailing = Mask.countr_zero();
515       if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
516         Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
517         return {{[=](MachineInstrBuilder &MIB) {
518           MachineIRBuilder(*MIB.getInstr())
519               .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
520               .addImm(C2.getLimitedValue() - ShAmt);
521           MIB.addReg(DstReg);
522         }}};
523       }
524     }
525   }
526 
527   return std::nullopt;
528 }
529 
530 InstructionSelector::ComplexRendererFns
renderVLOp(MachineOperand & Root) const531 RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
532   assert(Root.isReg() && "Expected operand to be a Register");
533   MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
534 
535   if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
536     auto C = RootDef->getOperand(1).getCImm();
537     if (C->getValue().isAllOnes())
538       // If the operand is a G_CONSTANT with value of all ones it is larger than
539       // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
540       // recognized specially by the vsetvli insertion pass.
541       return {{[=](MachineInstrBuilder &MIB) {
542         MIB.addImm(RISCV::VLMaxSentinel);
543       }}};
544 
545     if (isUInt<5>(C->getZExtValue())) {
546       uint64_t ZExtC = C->getZExtValue();
547       return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};
548     }
549   }
550   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};
551 }
552 
553 InstructionSelector::ComplexRendererFns
selectAddrRegImm(MachineOperand & Root) const554 RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
555   if (!Root.isReg())
556     return std::nullopt;
557 
558   MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
559   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
560     return {{
561         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
562         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
563     }};
564   }
565 
566   if (isBaseWithConstantOffset(Root, *MRI)) {
567     MachineOperand &LHS = RootDef->getOperand(1);
568     MachineOperand &RHS = RootDef->getOperand(2);
569     MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
570     MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
571 
572     int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
573     if (isInt<12>(RHSC)) {
574       if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
575         return {{
576             [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
577             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
578         }};
579 
580       return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
581                [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
582     }
583   }
584 
585   // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
586   // the combiner?
587   return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
588            [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
589 }
590 
591 /// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
592 /// CC Must be an ICMP Predicate.
getRISCVCCFromICmp(CmpInst::Predicate CC)593 static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
594   switch (CC) {
595   default:
596     llvm_unreachable("Expected ICMP CmpInst::Predicate.");
597   case CmpInst::Predicate::ICMP_EQ:
598     return RISCVCC::COND_EQ;
599   case CmpInst::Predicate::ICMP_NE:
600     return RISCVCC::COND_NE;
601   case CmpInst::Predicate::ICMP_ULT:
602     return RISCVCC::COND_LTU;
603   case CmpInst::Predicate::ICMP_SLT:
604     return RISCVCC::COND_LT;
605   case CmpInst::Predicate::ICMP_UGE:
606     return RISCVCC::COND_GEU;
607   case CmpInst::Predicate::ICMP_SGE:
608     return RISCVCC::COND_GE;
609   }
610 }
611 
getOperandsForBranch(Register CondReg,RISCVCC::CondCode & CC,Register & LHS,Register & RHS,MachineRegisterInfo & MRI)612 static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
613                                  Register &LHS, Register &RHS,
614                                  MachineRegisterInfo &MRI) {
615   // Try to fold an ICmp. If that fails, use a NE compare with X0.
616   CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
617   if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
618     LHS = CondReg;
619     RHS = RISCV::X0;
620     CC = RISCVCC::COND_NE;
621     return;
622   }
623 
624   // We found an ICmp, do some canonicalization.
625 
626   // Adjust comparisons to use comparison with 0 if possible.
627   if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
628     switch (Pred) {
629     case CmpInst::Predicate::ICMP_SGT:
630       // Convert X > -1 to X >= 0
631       if (*Constant == -1) {
632         CC = RISCVCC::COND_GE;
633         RHS = RISCV::X0;
634         return;
635       }
636       break;
637     case CmpInst::Predicate::ICMP_SLT:
638       // Convert X < 1 to 0 >= X
639       if (*Constant == 1) {
640         CC = RISCVCC::COND_GE;
641         RHS = LHS;
642         LHS = RISCV::X0;
643         return;
644       }
645       break;
646     default:
647       break;
648     }
649   }
650 
651   switch (Pred) {
652   default:
653     llvm_unreachable("Expected ICMP CmpInst::Predicate.");
654   case CmpInst::Predicate::ICMP_EQ:
655   case CmpInst::Predicate::ICMP_NE:
656   case CmpInst::Predicate::ICMP_ULT:
657   case CmpInst::Predicate::ICMP_SLT:
658   case CmpInst::Predicate::ICMP_UGE:
659   case CmpInst::Predicate::ICMP_SGE:
660     // These CCs are supported directly by RISC-V branches.
661     break;
662   case CmpInst::Predicate::ICMP_SGT:
663   case CmpInst::Predicate::ICMP_SLE:
664   case CmpInst::Predicate::ICMP_UGT:
665   case CmpInst::Predicate::ICMP_ULE:
666     // These CCs are not supported directly by RISC-V branches, but changing the
667     // direction of the CC and swapping LHS and RHS are.
668     Pred = CmpInst::getSwappedPredicate(Pred);
669     std::swap(LHS, RHS);
670     break;
671   }
672 
673   CC = getRISCVCCFromICmp(Pred);
674 }
675 
select(MachineInstr & MI)676 bool RISCVInstructionSelector::select(MachineInstr &MI) {
677   MachineIRBuilder MIB(MI);
678 
679   preISelLower(MI, MIB);
680   const unsigned Opc = MI.getOpcode();
681 
682   if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
683     if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
684       const Register DefReg = MI.getOperand(0).getReg();
685       const LLT DefTy = MRI->getType(DefReg);
686 
687       const RegClassOrRegBank &RegClassOrBank =
688           MRI->getRegClassOrRegBank(DefReg);
689 
690       const TargetRegisterClass *DefRC =
691           dyn_cast<const TargetRegisterClass *>(RegClassOrBank);
692       if (!DefRC) {
693         if (!DefTy.isValid()) {
694           LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
695           return false;
696         }
697 
698         const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
699         DefRC = getRegClassForTypeOnBank(DefTy, RB);
700         if (!DefRC) {
701           LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
702           return false;
703         }
704       }
705 
706       MI.setDesc(TII.get(TargetOpcode::PHI));
707       return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
708     }
709 
710     // Certain non-generic instructions also need some special handling.
711     if (MI.isCopy())
712       return selectCopy(MI);
713 
714     return true;
715   }
716 
717   if (selectImpl(MI, *CoverageInfo))
718     return true;
719 
720   switch (Opc) {
721   case TargetOpcode::G_ANYEXT:
722   case TargetOpcode::G_PTRTOINT:
723   case TargetOpcode::G_INTTOPTR:
724   case TargetOpcode::G_TRUNC:
725   case TargetOpcode::G_FREEZE:
726     return selectCopy(MI);
727   case TargetOpcode::G_CONSTANT: {
728     Register DstReg = MI.getOperand(0).getReg();
729     int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
730 
731     if (!materializeImm(DstReg, Imm, MIB))
732       return false;
733 
734     MI.eraseFromParent();
735     return true;
736   }
737   case TargetOpcode::G_FCONSTANT: {
738     // TODO: Use constant pool for complex constants.
739     // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
740     Register DstReg = MI.getOperand(0).getReg();
741     const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
742     APInt Imm = FPimm.bitcastToAPInt();
743     unsigned Size = MRI->getType(DstReg).getSizeInBits();
744     if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
745       Register GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
746       if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
747         return false;
748 
749       unsigned Opcode = Size == 64   ? RISCV::FMV_D_X
750                         : Size == 32 ? RISCV::FMV_W_X
751                                      : RISCV::FMV_H_X;
752       auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
753       if (!FMV.constrainAllUses(TII, TRI, RBI))
754         return false;
755     } else {
756       assert(Size == 64 && !Subtarget->is64Bit() &&
757              "Unexpected size or subtarget");
758       // Split into two pieces and build through the stack.
759       Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
760       Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
761       if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
762                           MIB))
763         return false;
764       if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
765         return false;
766       MachineInstrBuilder PairF64 = MIB.buildInstr(
767           RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
768       if (!PairF64.constrainAllUses(TII, TRI, RBI))
769         return false;
770     }
771 
772     MI.eraseFromParent();
773     return true;
774   }
775   case TargetOpcode::G_GLOBAL_VALUE: {
776     auto *GV = MI.getOperand(1).getGlobal();
777     if (GV->isThreadLocal()) {
778       // TODO: implement this case.
779       return false;
780     }
781 
782     return selectAddr(MI, MIB, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
783   }
784   case TargetOpcode::G_JUMP_TABLE:
785   case TargetOpcode::G_CONSTANT_POOL:
786     return selectAddr(MI, MIB, MRI);
787   case TargetOpcode::G_BRCOND: {
788     Register LHS, RHS;
789     RISCVCC::CondCode CC;
790     getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
791 
792     auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
793                    .addMBB(MI.getOperand(1).getMBB());
794     MI.eraseFromParent();
795     return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
796   }
797   case TargetOpcode::G_BRINDIRECT:
798     MI.setDesc(TII.get(RISCV::PseudoBRIND));
799     MI.addOperand(MachineOperand::CreateImm(0));
800     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
801   case TargetOpcode::G_SELECT:
802     return selectSelect(MI, MIB);
803   case TargetOpcode::G_FCMP:
804     return selectFPCompare(MI, MIB);
805   case TargetOpcode::G_FENCE: {
806     AtomicOrdering FenceOrdering =
807         static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
808     SyncScope::ID FenceSSID =
809         static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
810     emitFence(FenceOrdering, FenceSSID, MIB);
811     MI.eraseFromParent();
812     return true;
813   }
814   case TargetOpcode::G_IMPLICIT_DEF:
815     return selectImplicitDef(MI, MIB);
816   case TargetOpcode::G_UNMERGE_VALUES:
817     return selectUnmergeValues(MI, MIB);
818   default:
819     return false;
820   }
821 }
822 
selectUnmergeValues(MachineInstr & MI,MachineIRBuilder & MIB) const823 bool RISCVInstructionSelector::selectUnmergeValues(
824     MachineInstr &MI, MachineIRBuilder &MIB) const {
825   assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
826 
827   if (!Subtarget->hasStdExtZfa())
828     return false;
829 
830   // Split F64 Src into two s32 parts
831   if (MI.getNumOperands() != 3)
832     return false;
833   Register Src = MI.getOperand(2).getReg();
834   Register Lo = MI.getOperand(0).getReg();
835   Register Hi = MI.getOperand(1).getReg();
836   if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
837     return false;
838 
839   MachineInstr *ExtractLo = MIB.buildInstr(RISCV::FMV_X_W_FPR64, {Lo}, {Src});
840   if (!constrainSelectedInstRegOperands(*ExtractLo, TII, TRI, RBI))
841     return false;
842 
843   MachineInstr *ExtractHi = MIB.buildInstr(RISCV::FMVH_X_D, {Hi}, {Src});
844   if (!constrainSelectedInstRegOperands(*ExtractHi, TII, TRI, RBI))
845     return false;
846 
847   MI.eraseFromParent();
848   return true;
849 }
850 
replacePtrWithInt(MachineOperand & Op,MachineIRBuilder & MIB)851 bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
852                                                  MachineIRBuilder &MIB) {
853   Register PtrReg = Op.getReg();
854   assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
855 
856   const LLT sXLen = LLT::scalar(STI.getXLen());
857   auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
858   MRI->setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
859   Op.setReg(PtrToInt.getReg(0));
860   return select(*PtrToInt);
861 }
862 
preISelLower(MachineInstr & MI,MachineIRBuilder & MIB)863 void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
864                                             MachineIRBuilder &MIB) {
865   switch (MI.getOpcode()) {
866   case TargetOpcode::G_PTR_ADD: {
867     Register DstReg = MI.getOperand(0).getReg();
868     const LLT sXLen = LLT::scalar(STI.getXLen());
869 
870     replacePtrWithInt(MI.getOperand(1), MIB);
871     MI.setDesc(TII.get(TargetOpcode::G_ADD));
872     MRI->setType(DstReg, sXLen);
873     break;
874   }
875   case TargetOpcode::G_PTRMASK: {
876     Register DstReg = MI.getOperand(0).getReg();
877     const LLT sXLen = LLT::scalar(STI.getXLen());
878     replacePtrWithInt(MI.getOperand(1), MIB);
879     MI.setDesc(TII.get(TargetOpcode::G_AND));
880     MRI->setType(DstReg, sXLen);
881     break;
882   }
883   }
884 }
885 
renderNegImm(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const886 void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
887                                             const MachineInstr &MI,
888                                             int OpIdx) const {
889   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
890          "Expected G_CONSTANT");
891   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
892   MIB.addImm(-CstVal);
893 }
894 
renderImmSubFromXLen(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const895 void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
896                                                     const MachineInstr &MI,
897                                                     int OpIdx) const {
898   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
899          "Expected G_CONSTANT");
900   uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
901   MIB.addImm(STI.getXLen() - CstVal);
902 }
903 
renderImmSubFrom32(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const904 void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
905                                                   const MachineInstr &MI,
906                                                   int OpIdx) const {
907   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
908          "Expected G_CONSTANT");
909   uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
910   MIB.addImm(32 - CstVal);
911 }
912 
renderImmPlus1(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const913 void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
914                                               const MachineInstr &MI,
915                                               int OpIdx) const {
916   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
917          "Expected G_CONSTANT");
918   int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
919   MIB.addImm(CstVal + 1);
920 }
921 
renderFrameIndex(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const922 void RISCVInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
923                                                 const MachineInstr &MI,
924                                                 int OpIdx) const {
925   assert(MI.getOpcode() == TargetOpcode::G_FRAME_INDEX && OpIdx == -1 &&
926          "Expected G_FRAME_INDEX");
927   MIB.add(MI.getOperand(1));
928 }
929 
renderTrailingZeros(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const930 void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
931                                                    const MachineInstr &MI,
932                                                    int OpIdx) const {
933   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
934          "Expected G_CONSTANT");
935   uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
936   MIB.addImm(llvm::countr_zero(C));
937 }
938 
renderXLenSubTrailingOnes(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const939 void RISCVInstructionSelector::renderXLenSubTrailingOnes(
940     MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
941   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
942          "Expected G_CONSTANT");
943   uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
944   MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));
945 }
946 
renderAddiPairImmSmall(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const947 void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
948                                                       const MachineInstr &MI,
949                                                       int OpIdx) const {
950   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
951          "Expected G_CONSTANT");
952   int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
953   int64_t Adj = Imm < 0 ? -2048 : 2047;
954   MIB.addImm(Imm - Adj);
955 }
956 
renderAddiPairImmLarge(MachineInstrBuilder & MIB,const MachineInstr & MI,int OpIdx) const957 void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
958                                                       const MachineInstr &MI,
959                                                       int OpIdx) const {
960   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
961          "Expected G_CONSTANT");
962   int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
963   MIB.addImm(Imm);
964 }
965 
getRegClassForTypeOnBank(LLT Ty,const RegisterBank & RB) const966 const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
967     LLT Ty, const RegisterBank &RB) const {
968   if (RB.getID() == RISCV::GPRBRegBankID) {
969     if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
970       return &RISCV::GPRRegClass;
971   }
972 
973   if (RB.getID() == RISCV::FPRBRegBankID) {
974     if (Ty.getSizeInBits() == 16)
975       return &RISCV::FPR16RegClass;
976     if (Ty.getSizeInBits() == 32)
977       return &RISCV::FPR32RegClass;
978     if (Ty.getSizeInBits() == 64)
979       return &RISCV::FPR64RegClass;
980   }
981 
982   if (RB.getID() == RISCV::VRBRegBankID) {
983     if (Ty.getSizeInBits().getKnownMinValue() <= 64)
984       return &RISCV::VRRegClass;
985 
986     if (Ty.getSizeInBits().getKnownMinValue() == 128)
987       return &RISCV::VRM2RegClass;
988 
989     if (Ty.getSizeInBits().getKnownMinValue() == 256)
990       return &RISCV::VRM4RegClass;
991 
992     if (Ty.getSizeInBits().getKnownMinValue() == 512)
993       return &RISCV::VRM8RegClass;
994   }
995 
996   return nullptr;
997 }
998 
isRegInGprb(Register Reg) const999 bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1000   return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1001 }
1002 
isRegInFprb(Register Reg) const1003 bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1004   return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1005 }
1006 
selectCopy(MachineInstr & MI) const1007 bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1008   Register DstReg = MI.getOperand(0).getReg();
1009 
1010   if (DstReg.isPhysical())
1011     return true;
1012 
1013   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1014       MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
1015   assert(DstRC &&
1016          "Register class not available for LLT, register bank combination");
1017 
1018   // No need to constrain SrcReg. It will get constrained when
1019   // we hit another of its uses or its defs.
1020   // Copies do not have constraints.
1021   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
1022     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1023                       << " operand\n");
1024     return false;
1025   }
1026 
1027   MI.setDesc(TII.get(RISCV::COPY));
1028   return true;
1029 }
1030 
selectImplicitDef(MachineInstr & MI,MachineIRBuilder & MIB) const1031 bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI,
1032                                                  MachineIRBuilder &MIB) const {
1033   assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1034 
1035   const Register DstReg = MI.getOperand(0).getReg();
1036   const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
1037       MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI));
1038 
1039   assert(DstRC &&
1040          "Register class not available for LLT, register bank combination");
1041 
1042   if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
1043     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1044                       << " operand\n");
1045   }
1046   MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1047   return true;
1048 }
1049 
materializeImm(Register DstReg,int64_t Imm,MachineIRBuilder & MIB) const1050 bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1051                                               MachineIRBuilder &MIB) const {
1052   if (Imm == 0) {
1053     MIB.buildCopy(DstReg, Register(RISCV::X0));
1054     RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
1055     return true;
1056   }
1057 
1058   RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
1059   unsigned NumInsts = Seq.size();
1060   Register SrcReg = RISCV::X0;
1061 
1062   for (unsigned i = 0; i < NumInsts; i++) {
1063     Register TmpReg = i < NumInsts - 1
1064                           ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
1065                           : DstReg;
1066     const RISCVMatInt::Inst &I = Seq[i];
1067     MachineInstr *Result;
1068 
1069     switch (I.getOpndKind()) {
1070     case RISCVMatInt::Imm:
1071       // clang-format off
1072       Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
1073                    .addImm(I.getImm());
1074       // clang-format on
1075       break;
1076     case RISCVMatInt::RegX0:
1077       Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
1078                               {SrcReg, Register(RISCV::X0)});
1079       break;
1080     case RISCVMatInt::RegReg:
1081       Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
1082       break;
1083     case RISCVMatInt::RegImm:
1084       Result =
1085           MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
1086       break;
1087     }
1088 
1089     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1090       return false;
1091 
1092     SrcReg = TmpReg;
1093   }
1094 
1095   return true;
1096 }
1097 
selectAddr(MachineInstr & MI,MachineIRBuilder & MIB,bool IsLocal,bool IsExternWeak) const1098 bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
1099                                           MachineIRBuilder &MIB, bool IsLocal,
1100                                           bool IsExternWeak) const {
1101   assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1102           MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
1103           MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
1104          "Unexpected opcode");
1105 
1106   const MachineOperand &DispMO = MI.getOperand(1);
1107 
1108   Register DefReg = MI.getOperand(0).getReg();
1109   const LLT DefTy = MRI->getType(DefReg);
1110 
1111   // When HWASAN is used and tagging of global variables is enabled
1112   // they should be accessed via the GOT, since the tagged address of a global
1113   // is incompatible with existing code models. This also applies to non-pic
1114   // mode.
1115   if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1116     if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1117       // Use PC-relative addressing to access the symbol. This generates the
1118       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1119       // %pcrel_lo(auipc)).
1120       MI.setDesc(TII.get(RISCV::PseudoLLA));
1121       return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1122     }
1123 
1124     // Use PC-relative addressing to access the GOT for this symbol, then
1125     // load the address from the GOT. This generates the pattern (PseudoLGA
1126     // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1127     // %pcrel_lo(auipc))).
1128     MachineFunction &MF = *MI.getParent()->getParent();
1129     MachineMemOperand *MemOp = MF.getMachineMemOperand(
1130         MachinePointerInfo::getGOT(MF),
1131         MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1132             MachineMemOperand::MOInvariant,
1133         DefTy, Align(DefTy.getSizeInBits() / 8));
1134 
1135     auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1136                       .addDisp(DispMO, 0)
1137                       .addMemOperand(MemOp);
1138 
1139     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1140       return false;
1141 
1142     MI.eraseFromParent();
1143     return true;
1144   }
1145 
1146   switch (TM.getCodeModel()) {
1147   default: {
1148     reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1149                        getName(), "Unsupported code model for lowering", MI);
1150     return false;
1151   }
1152   case CodeModel::Small: {
1153     // Must lie within a single 2 GiB address range and must lie between
1154     // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1155     // (lui %hi(sym)) %lo(sym)).
1156     Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1157     MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1158                                .addDisp(DispMO, 0, RISCVII::MO_HI);
1159 
1160     if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1161       return false;
1162 
1163     auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1164                       .addDisp(DispMO, 0, RISCVII::MO_LO);
1165 
1166     if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1167       return false;
1168 
1169     MI.eraseFromParent();
1170     return true;
1171   }
1172   case CodeModel::Medium:
1173     // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1174     // relocation needs to reference a label that points to the auipc
1175     // instruction itself, not the global. This cannot be done inside the
1176     // instruction selector.
1177     if (IsExternWeak) {
1178       // An extern weak symbol may be undefined, i.e. have value 0, which may
1179       // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1180       // symbol. This generates the pattern (PseudoLGA sym), which expands to
1181       // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1182       MachineFunction &MF = *MI.getParent()->getParent();
1183       MachineMemOperand *MemOp = MF.getMachineMemOperand(
1184           MachinePointerInfo::getGOT(MF),
1185           MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1186               MachineMemOperand::MOInvariant,
1187           DefTy, Align(DefTy.getSizeInBits() / 8));
1188 
1189       auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1190                         .addDisp(DispMO, 0)
1191                         .addMemOperand(MemOp);
1192 
1193       if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1194         return false;
1195 
1196       MI.eraseFromParent();
1197       return true;
1198     }
1199 
1200     // Generate a sequence for accessing addresses within any 2GiB range
1201     // within the address space. This generates the pattern (PseudoLLA sym),
1202     // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1203     MI.setDesc(TII.get(RISCV::PseudoLLA));
1204     return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
1205   }
1206 
1207   return false;
1208 }
1209 
selectSelect(MachineInstr & MI,MachineIRBuilder & MIB) const1210 bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1211                                             MachineIRBuilder &MIB) const {
1212   auto &SelectMI = cast<GSelect>(MI);
1213 
1214   Register LHS, RHS;
1215   RISCVCC::CondCode CC;
1216   getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);
1217 
1218   Register DstReg = SelectMI.getReg(0);
1219 
1220   unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1221   if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1222     unsigned Size = MRI->getType(DstReg).getSizeInBits();
1223     Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1224                      : RISCV::Select_FPR64_Using_CC_GPR;
1225   }
1226 
1227   MachineInstr *Result = MIB.buildInstr(Opc)
1228                              .addDef(DstReg)
1229                              .addReg(LHS)
1230                              .addReg(RHS)
1231                              .addImm(CC)
1232                              .addReg(SelectMI.getTrueReg())
1233                              .addReg(SelectMI.getFalseReg());
1234   MI.eraseFromParent();
1235   return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1236 }
1237 
1238 // Convert an FCMP predicate to one of the supported F or D instructions.
getFCmpOpcode(CmpInst::Predicate Pred,unsigned Size)1239 static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1240   assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1241   switch (Pred) {
1242   default:
1243     llvm_unreachable("Unsupported predicate");
1244   case CmpInst::FCMP_OLT:
1245     return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1246   case CmpInst::FCMP_OLE:
1247     return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1248   case CmpInst::FCMP_OEQ:
1249     return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1250   }
1251 }
1252 
1253 // Try legalizing an FCMP by swapping or inverting the predicate to one that
1254 // is supported.
legalizeFCmpPredicate(Register & LHS,Register & RHS,CmpInst::Predicate & Pred,bool & NeedInvert)1255 static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
1256                                   CmpInst::Predicate &Pred, bool &NeedInvert) {
1257   auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1258     return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1259            Pred == CmpInst::FCMP_OEQ;
1260   };
1261 
1262   assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1263 
1264   CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);
1265   if (isLegalFCmpPredicate(InvPred)) {
1266     Pred = InvPred;
1267     std::swap(LHS, RHS);
1268     return true;
1269   }
1270 
1271   InvPred = CmpInst::getInversePredicate(Pred);
1272   NeedInvert = true;
1273   if (isLegalFCmpPredicate(InvPred)) {
1274     Pred = InvPred;
1275     return true;
1276   }
1277   InvPred = CmpInst::getSwappedPredicate(InvPred);
1278   if (isLegalFCmpPredicate(InvPred)) {
1279     Pred = InvPred;
1280     std::swap(LHS, RHS);
1281     return true;
1282   }
1283 
1284   return false;
1285 }
1286 
1287 // Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1288 // the result in DstReg.
1289 // FIXME: Maybe we should expand this earlier.
selectFPCompare(MachineInstr & MI,MachineIRBuilder & MIB) const1290 bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1291                                                MachineIRBuilder &MIB) const {
1292   auto &CmpMI = cast<GFCmp>(MI);
1293   CmpInst::Predicate Pred = CmpMI.getCond();
1294 
1295   Register DstReg = CmpMI.getReg(0);
1296   Register LHS = CmpMI.getLHSReg();
1297   Register RHS = CmpMI.getRHSReg();
1298 
1299   unsigned Size = MRI->getType(LHS).getSizeInBits();
1300   assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1301 
1302   Register TmpReg = DstReg;
1303 
1304   bool NeedInvert = false;
1305   // First try swapping operands or inverting.
1306   if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1307     if (NeedInvert)
1308       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1309     auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1310     if (!Cmp.constrainAllUses(TII, TRI, RBI))
1311       return false;
1312   } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1313     // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1314     NeedInvert = Pred == CmpInst::FCMP_UEQ;
1315     auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1316                                {&RISCV::GPRRegClass}, {LHS, RHS});
1317     if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1318       return false;
1319     auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
1320                                {&RISCV::GPRRegClass}, {RHS, LHS});
1321     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1322       return false;
1323     if (NeedInvert)
1324       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1325     auto Or =
1326         MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1327     if (!Or.constrainAllUses(TII, TRI, RBI))
1328       return false;
1329   } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1330     // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1331     // FIXME: If LHS and RHS are the same we can use a single FEQ.
1332     NeedInvert = Pred == CmpInst::FCMP_UNO;
1333     auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1334                                {&RISCV::GPRRegClass}, {LHS, LHS});
1335     if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1336       return false;
1337     auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
1338                                {&RISCV::GPRRegClass}, {RHS, RHS});
1339     if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1340       return false;
1341     if (NeedInvert)
1342       TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1343     auto And =
1344         MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1345     if (!And.constrainAllUses(TII, TRI, RBI))
1346       return false;
1347   } else
1348     llvm_unreachable("Unhandled predicate");
1349 
1350   // Emit an XORI to invert the result if needed.
1351   if (NeedInvert) {
1352     auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1353     if (!Xor.constrainAllUses(TII, TRI, RBI))
1354       return false;
1355   }
1356 
1357   MI.eraseFromParent();
1358   return true;
1359 }
1360 
emitFence(AtomicOrdering FenceOrdering,SyncScope::ID FenceSSID,MachineIRBuilder & MIB) const1361 void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1362                                          SyncScope::ID FenceSSID,
1363                                          MachineIRBuilder &MIB) const {
1364   if (STI.hasStdExtZtso()) {
1365     // The only fence that needs an instruction is a sequentially-consistent
1366     // cross-thread fence.
1367     if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1368         FenceSSID == SyncScope::System) {
1369       // fence rw, rw
1370       MIB.buildInstr(RISCV::FENCE, {}, {})
1371           .addImm(RISCVFenceField::R | RISCVFenceField::W)
1372           .addImm(RISCVFenceField::R | RISCVFenceField::W);
1373       return;
1374     }
1375 
1376     // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1377     MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1378     return;
1379   }
1380 
1381   // singlethread fences only synchronize with signal handlers on the same
1382   // thread and thus only need to preserve instruction order, not actually
1383   // enforce memory ordering.
1384   if (FenceSSID == SyncScope::SingleThread) {
1385     MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1386     return;
1387   }
1388 
1389   // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1390   // Manual: Volume I.
1391   unsigned Pred, Succ;
1392   switch (FenceOrdering) {
1393   default:
1394     llvm_unreachable("Unexpected ordering");
1395   case AtomicOrdering::AcquireRelease:
1396     // fence acq_rel -> fence.tso
1397     MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1398     return;
1399   case AtomicOrdering::Acquire:
1400     // fence acquire -> fence r, rw
1401     Pred = RISCVFenceField::R;
1402     Succ = RISCVFenceField::R | RISCVFenceField::W;
1403     break;
1404   case AtomicOrdering::Release:
1405     // fence release -> fence rw, w
1406     Pred = RISCVFenceField::R | RISCVFenceField::W;
1407     Succ = RISCVFenceField::W;
1408     break;
1409   case AtomicOrdering::SequentiallyConsistent:
1410     // fence seq_cst -> fence rw, rw
1411     Pred = RISCVFenceField::R | RISCVFenceField::W;
1412     Succ = RISCVFenceField::R | RISCVFenceField::W;
1413     break;
1414   }
1415   MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1416 }
1417 
1418 namespace llvm {
1419 InstructionSelector *
createRISCVInstructionSelector(const RISCVTargetMachine & TM,const RISCVSubtarget & Subtarget,const RISCVRegisterBankInfo & RBI)1420 createRISCVInstructionSelector(const RISCVTargetMachine &TM,
1421                                const RISCVSubtarget &Subtarget,
1422                                const RISCVRegisterBankInfo &RBI) {
1423   return new RISCVInstructionSelector(TM, Subtarget, RBI);
1424 }
1425 } // end namespace llvm
1426