xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp (revision 9f23cbd6cae82fd77edfad7173432fa8dccd0a95)
1 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the RISCV implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVInstrInfo.h"
14 #include "MCTargetDesc/RISCVMatInt.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVSubtarget.h"
18 #include "RISCVTargetMachine.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/Analysis/MemoryLocation.h"
22 #include "llvm/CodeGen/LiveIntervals.h"
23 #include "llvm/CodeGen/LiveVariables.h"
24 #include "llvm/CodeGen/MachineCombinerPattern.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/RegisterScavenging.h"
29 #include "llvm/IR/DebugInfoMetadata.h"
30 #include "llvm/MC/MCInstBuilder.h"
31 #include "llvm/MC/TargetRegistry.h"
32 #include "llvm/Support/ErrorHandling.h"
33 
34 using namespace llvm;
35 
36 #define GEN_CHECK_COMPRESS_INSTR
37 #include "RISCVGenCompressInstEmitter.inc"
38 
39 #define GET_INSTRINFO_CTOR_DTOR
40 #define GET_INSTRINFO_NAMED_OPS
41 #include "RISCVGenInstrInfo.inc"
42 
43 static cl::opt<bool> PreferWholeRegisterMove(
44     "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden,
45     cl::desc("Prefer whole register move for vector registers."));
46 
47 namespace llvm::RISCVVPseudosTable {
48 
49 using namespace RISCV;
50 
51 #define GET_RISCVVPseudosTable_IMPL
52 #include "RISCVGenSearchableTables.inc"
53 
54 } // namespace llvm::RISCVVPseudosTable
55 
56 RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI)
57     : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
58       STI(STI) {}
59 
60 MCInst RISCVInstrInfo::getNop() const {
61   if (STI.hasStdExtCOrZca())
62     return MCInstBuilder(RISCV::C_NOP);
63   return MCInstBuilder(RISCV::ADDI)
64       .addReg(RISCV::X0)
65       .addReg(RISCV::X0)
66       .addImm(0);
67 }
68 
69 unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
70                                              int &FrameIndex) const {
71   switch (MI.getOpcode()) {
72   default:
73     return 0;
74   case RISCV::LB:
75   case RISCV::LBU:
76   case RISCV::LH:
77   case RISCV::LHU:
78   case RISCV::FLH:
79   case RISCV::LW:
80   case RISCV::FLW:
81   case RISCV::LWU:
82   case RISCV::LD:
83   case RISCV::FLD:
84     break;
85   }
86 
87   if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
88       MI.getOperand(2).getImm() == 0) {
89     FrameIndex = MI.getOperand(1).getIndex();
90     return MI.getOperand(0).getReg();
91   }
92 
93   return 0;
94 }
95 
96 unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
97                                             int &FrameIndex) const {
98   switch (MI.getOpcode()) {
99   default:
100     return 0;
101   case RISCV::SB:
102   case RISCV::SH:
103   case RISCV::SW:
104   case RISCV::FSH:
105   case RISCV::FSW:
106   case RISCV::SD:
107   case RISCV::FSD:
108     break;
109   }
110 
111   if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
112       MI.getOperand(2).getImm() == 0) {
113     FrameIndex = MI.getOperand(1).getIndex();
114     return MI.getOperand(0).getReg();
115   }
116 
117   return 0;
118 }
119 
120 static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
121                                         unsigned NumRegs) {
122   return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
123 }
124 
125 static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
126                                    const MachineBasicBlock &MBB,
127                                    MachineBasicBlock::const_iterator MBBI,
128                                    MachineBasicBlock::const_iterator &DefMBBI,
129                                    RISCVII::VLMUL LMul) {
130   if (PreferWholeRegisterMove)
131     return false;
132 
133   assert(MBBI->getOpcode() == TargetOpcode::COPY &&
134          "Unexpected COPY instruction.");
135   Register SrcReg = MBBI->getOperand(1).getReg();
136   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
137 
138   bool FoundDef = false;
139   bool FirstVSetVLI = false;
140   unsigned FirstSEW = 0;
141   while (MBBI != MBB.begin()) {
142     --MBBI;
143     if (MBBI->isMetaInstruction())
144       continue;
145 
146     if (MBBI->getOpcode() == RISCV::PseudoVSETVLI ||
147         MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||
148         MBBI->getOpcode() == RISCV::PseudoVSETIVLI) {
149       // There is a vsetvli between COPY and source define instruction.
150       // vy = def_vop ...  (producing instruction)
151       // ...
152       // vsetvli
153       // ...
154       // vx = COPY vy
155       if (!FoundDef) {
156         if (!FirstVSetVLI) {
157           FirstVSetVLI = true;
158           unsigned FirstVType = MBBI->getOperand(2).getImm();
159           RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType);
160           FirstSEW = RISCVVType::getSEW(FirstVType);
161           // The first encountered vsetvli must have the same lmul as the
162           // register class of COPY.
163           if (FirstLMul != LMul)
164             return false;
165         }
166         // Only permit `vsetvli x0, x0, vtype` between COPY and the source
167         // define instruction.
168         if (MBBI->getOperand(0).getReg() != RISCV::X0)
169           return false;
170         if (MBBI->getOperand(1).isImm())
171           return false;
172         if (MBBI->getOperand(1).getReg() != RISCV::X0)
173           return false;
174         continue;
175       }
176 
177       // MBBI is the first vsetvli before the producing instruction.
178       unsigned VType = MBBI->getOperand(2).getImm();
179       // If there is a vsetvli between COPY and the producing instruction.
180       if (FirstVSetVLI) {
181         // If SEW is different, return false.
182         if (RISCVVType::getSEW(VType) != FirstSEW)
183           return false;
184       }
185 
186       // If the vsetvli is tail undisturbed, keep the whole register move.
187       if (!RISCVVType::isTailAgnostic(VType))
188         return false;
189 
190       // The checking is conservative. We only have register classes for
191       // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
192       // for fractional LMUL operations. However, we could not use the vsetvli
193       // lmul for widening operations. The result of widening operation is
194       // 2 x LMUL.
195       return LMul == RISCVVType::getVLMUL(VType);
196     } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
197       return false;
198     } else if (MBBI->getNumDefs()) {
199       // Check all the instructions which will change VL.
200       // For example, vleff has implicit def VL.
201       if (MBBI->modifiesRegister(RISCV::VL))
202         return false;
203 
204       // Only converting whole register copies to vmv.v.v when the defining
205       // value appears in the explicit operands.
206       for (const MachineOperand &MO : MBBI->explicit_operands()) {
207         if (!MO.isReg() || !MO.isDef())
208           continue;
209         if (!FoundDef && TRI->isSubRegisterEq(MO.getReg(), SrcReg)) {
210           // We only permit the source of COPY has the same LMUL as the defined
211           // operand.
212           // There are cases we need to keep the whole register copy if the LMUL
213           // is different.
214           // For example,
215           // $x0 = PseudoVSETIVLI 4, 73   // vsetivli zero, 4, e16,m2,ta,m
216           // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
217           // # The COPY may be created by vlmul_trunc intrinsic.
218           // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
219           //
220           // After widening, the valid value will be 4 x e32 elements. If we
221           // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
222           // FIXME: The COPY of subregister of Zvlsseg register will not be able
223           // to convert to vmv.v.[v|i] under the constraint.
224           if (MO.getReg() != SrcReg)
225             return false;
226 
227           // In widening reduction instructions with LMUL_1 input vector case,
228           // only checking the LMUL is insufficient due to reduction result is
229           // always LMUL_1.
230           // For example,
231           // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
232           // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
233           // $v26 = COPY killed renamable $v8
234           // After widening, The valid value will be 1 x e16 elements. If we
235           // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
236           uint64_t TSFlags = MBBI->getDesc().TSFlags;
237           if (RISCVII::isRVVWideningReduction(TSFlags))
238             return false;
239 
240           // If the producing instruction does not depend on vsetvli, do not
241           // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
242           if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
243             return false;
244 
245           // Found the definition.
246           FoundDef = true;
247           DefMBBI = MBBI;
248           break;
249         }
250       }
251     }
252   }
253 
254   return false;
255 }
256 
257 void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
258                                  MachineBasicBlock::iterator MBBI,
259                                  const DebugLoc &DL, MCRegister DstReg,
260                                  MCRegister SrcReg, bool KillSrc) const {
261   if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
262     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
263         .addReg(SrcReg, getKillRegState(KillSrc))
264         .addImm(0);
265     return;
266   }
267 
268   // Handle copy from csr
269   if (RISCV::VCSRRegClass.contains(SrcReg) &&
270       RISCV::GPRRegClass.contains(DstReg)) {
271     const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
272     BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)
273       .addImm(RISCVSysReg::lookupSysRegByName(TRI.getName(SrcReg))->Encoding)
274       .addReg(RISCV::X0);
275     return;
276   }
277 
278   // FPR->FPR copies and VR->VR copies.
279   unsigned Opc;
280   bool IsScalableVector = true;
281   unsigned NF = 1;
282   RISCVII::VLMUL LMul = RISCVII::LMUL_1;
283   unsigned SubRegIdx = RISCV::sub_vrm1_0;
284   if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
285     if (!STI.hasStdExtZfh() && STI.hasStdExtZfhmin()) {
286       // Zfhmin subset doesn't have FSGNJ_H, replaces FSGNJ_H with FSGNJ_S.
287       const TargetRegisterInfo *TRI = STI.getRegisterInfo();
288       DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
289                                         &RISCV::FPR32RegClass);
290       SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
291                                         &RISCV::FPR32RegClass);
292       Opc = RISCV::FSGNJ_S;
293     } else {
294       Opc = RISCV::FSGNJ_H;
295     }
296     IsScalableVector = false;
297   } else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
298     Opc = RISCV::FSGNJ_S;
299     IsScalableVector = false;
300   } else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
301     Opc = RISCV::FSGNJ_D;
302     IsScalableVector = false;
303   } else if (RISCV::VRRegClass.contains(DstReg, SrcReg)) {
304     Opc = RISCV::VMV1R_V;
305     LMul = RISCVII::LMUL_1;
306   } else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) {
307     Opc = RISCV::VMV2R_V;
308     LMul = RISCVII::LMUL_2;
309   } else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) {
310     Opc = RISCV::VMV4R_V;
311     LMul = RISCVII::LMUL_4;
312   } else if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) {
313     Opc = RISCV::VMV8R_V;
314     LMul = RISCVII::LMUL_8;
315   } else if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) {
316     Opc = RISCV::VMV1R_V;
317     SubRegIdx = RISCV::sub_vrm1_0;
318     NF = 2;
319     LMul = RISCVII::LMUL_1;
320   } else if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) {
321     Opc = RISCV::VMV2R_V;
322     SubRegIdx = RISCV::sub_vrm2_0;
323     NF = 2;
324     LMul = RISCVII::LMUL_2;
325   } else if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) {
326     Opc = RISCV::VMV4R_V;
327     SubRegIdx = RISCV::sub_vrm4_0;
328     NF = 2;
329     LMul = RISCVII::LMUL_4;
330   } else if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) {
331     Opc = RISCV::VMV1R_V;
332     SubRegIdx = RISCV::sub_vrm1_0;
333     NF = 3;
334     LMul = RISCVII::LMUL_1;
335   } else if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) {
336     Opc = RISCV::VMV2R_V;
337     SubRegIdx = RISCV::sub_vrm2_0;
338     NF = 3;
339     LMul = RISCVII::LMUL_2;
340   } else if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) {
341     Opc = RISCV::VMV1R_V;
342     SubRegIdx = RISCV::sub_vrm1_0;
343     NF = 4;
344     LMul = RISCVII::LMUL_1;
345   } else if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) {
346     Opc = RISCV::VMV2R_V;
347     SubRegIdx = RISCV::sub_vrm2_0;
348     NF = 4;
349     LMul = RISCVII::LMUL_2;
350   } else if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) {
351     Opc = RISCV::VMV1R_V;
352     SubRegIdx = RISCV::sub_vrm1_0;
353     NF = 5;
354     LMul = RISCVII::LMUL_1;
355   } else if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) {
356     Opc = RISCV::VMV1R_V;
357     SubRegIdx = RISCV::sub_vrm1_0;
358     NF = 6;
359     LMul = RISCVII::LMUL_1;
360   } else if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) {
361     Opc = RISCV::VMV1R_V;
362     SubRegIdx = RISCV::sub_vrm1_0;
363     NF = 7;
364     LMul = RISCVII::LMUL_1;
365   } else if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) {
366     Opc = RISCV::VMV1R_V;
367     SubRegIdx = RISCV::sub_vrm1_0;
368     NF = 8;
369     LMul = RISCVII::LMUL_1;
370   } else {
371     llvm_unreachable("Impossible reg-to-reg copy");
372   }
373 
374   if (IsScalableVector) {
375     bool UseVMV_V_V = false;
376     MachineBasicBlock::const_iterator DefMBBI;
377     unsigned VIOpc;
378     if (isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
379       UseVMV_V_V = true;
380       // We only need to handle LMUL = 1/2/4/8 here because we only define
381       // vector register classes for LMUL = 1/2/4/8.
382       switch (LMul) {
383       default:
384         llvm_unreachable("Impossible LMUL for vector register copy.");
385       case RISCVII::LMUL_1:
386         Opc = RISCV::PseudoVMV_V_V_M1;
387         VIOpc = RISCV::PseudoVMV_V_I_M1;
388         break;
389       case RISCVII::LMUL_2:
390         Opc = RISCV::PseudoVMV_V_V_M2;
391         VIOpc = RISCV::PseudoVMV_V_I_M2;
392         break;
393       case RISCVII::LMUL_4:
394         Opc = RISCV::PseudoVMV_V_V_M4;
395         VIOpc = RISCV::PseudoVMV_V_I_M4;
396         break;
397       case RISCVII::LMUL_8:
398         Opc = RISCV::PseudoVMV_V_V_M8;
399         VIOpc = RISCV::PseudoVMV_V_I_M8;
400         break;
401       }
402     }
403 
404     bool UseVMV_V_I = false;
405     if (UseVMV_V_V && (DefMBBI->getOpcode() == VIOpc)) {
406       UseVMV_V_I = true;
407       Opc = VIOpc;
408     }
409 
410     if (NF == 1) {
411       auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), DstReg);
412       if (UseVMV_V_I)
413         MIB = MIB.add(DefMBBI->getOperand(1));
414       else
415         MIB = MIB.addReg(SrcReg, getKillRegState(KillSrc));
416       if (UseVMV_V_V) {
417         const MCInstrDesc &Desc = DefMBBI->getDesc();
418         MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
419         MIB.add(DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc))); // SEW
420         MIB.addReg(RISCV::VL, RegState::Implicit);
421         MIB.addReg(RISCV::VTYPE, RegState::Implicit);
422       }
423     } else {
424       const TargetRegisterInfo *TRI = STI.getRegisterInfo();
425 
426       int I = 0, End = NF, Incr = 1;
427       unsigned SrcEncoding = TRI->getEncodingValue(SrcReg);
428       unsigned DstEncoding = TRI->getEncodingValue(DstReg);
429       unsigned LMulVal;
430       bool Fractional;
431       std::tie(LMulVal, Fractional) = RISCVVType::decodeVLMUL(LMul);
432       assert(!Fractional && "It is impossible be fractional lmul here.");
433       if (forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NF * LMulVal)) {
434         I = NF - 1;
435         End = -1;
436         Incr = -1;
437       }
438 
439       for (; I != End; I += Incr) {
440         auto MIB = BuildMI(MBB, MBBI, DL, get(Opc),
441                            TRI->getSubReg(DstReg, SubRegIdx + I));
442         if (UseVMV_V_I)
443           MIB = MIB.add(DefMBBI->getOperand(1));
444         else
445           MIB = MIB.addReg(TRI->getSubReg(SrcReg, SubRegIdx + I),
446                            getKillRegState(KillSrc));
447         if (UseVMV_V_V) {
448           const MCInstrDesc &Desc = DefMBBI->getDesc();
449           MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
450           MIB.add(DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc))); // SEW
451           MIB.addReg(RISCV::VL, RegState::Implicit);
452           MIB.addReg(RISCV::VTYPE, RegState::Implicit);
453         }
454       }
455     }
456   } else {
457     BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
458         .addReg(SrcReg, getKillRegState(KillSrc))
459         .addReg(SrcReg, getKillRegState(KillSrc));
460   }
461 }
462 
463 void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
464                                          MachineBasicBlock::iterator I,
465                                          Register SrcReg, bool IsKill, int FI,
466                                          const TargetRegisterClass *RC,
467                                          const TargetRegisterInfo *TRI,
468                                          Register VReg) const {
469   DebugLoc DL;
470   if (I != MBB.end())
471     DL = I->getDebugLoc();
472 
473   MachineFunction *MF = MBB.getParent();
474   MachineFrameInfo &MFI = MF->getFrameInfo();
475 
476   unsigned Opcode;
477   bool IsScalableVector = true;
478   if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
479     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
480              RISCV::SW : RISCV::SD;
481     IsScalableVector = false;
482   } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
483     Opcode = RISCV::FSH;
484     IsScalableVector = false;
485   } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
486     Opcode = RISCV::FSW;
487     IsScalableVector = false;
488   } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
489     Opcode = RISCV::FSD;
490     IsScalableVector = false;
491   } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
492     Opcode = RISCV::VS1R_V;
493   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
494     Opcode = RISCV::VS2R_V;
495   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
496     Opcode = RISCV::VS4R_V;
497   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
498     Opcode = RISCV::VS8R_V;
499   } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
500     Opcode = RISCV::PseudoVSPILL2_M1;
501   else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
502     Opcode = RISCV::PseudoVSPILL2_M2;
503   else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
504     Opcode = RISCV::PseudoVSPILL2_M4;
505   else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
506     Opcode = RISCV::PseudoVSPILL3_M1;
507   else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
508     Opcode = RISCV::PseudoVSPILL3_M2;
509   else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
510     Opcode = RISCV::PseudoVSPILL4_M1;
511   else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
512     Opcode = RISCV::PseudoVSPILL4_M2;
513   else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
514     Opcode = RISCV::PseudoVSPILL5_M1;
515   else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
516     Opcode = RISCV::PseudoVSPILL6_M1;
517   else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
518     Opcode = RISCV::PseudoVSPILL7_M1;
519   else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
520     Opcode = RISCV::PseudoVSPILL8_M1;
521   else
522     llvm_unreachable("Can't store this register to stack slot");
523 
524   if (IsScalableVector) {
525     MachineMemOperand *MMO = MF->getMachineMemOperand(
526         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
527         MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
528 
529     MFI.setStackID(FI, TargetStackID::ScalableVector);
530     BuildMI(MBB, I, DL, get(Opcode))
531         .addReg(SrcReg, getKillRegState(IsKill))
532         .addFrameIndex(FI)
533         .addMemOperand(MMO);
534   } else {
535     MachineMemOperand *MMO = MF->getMachineMemOperand(
536         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
537         MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
538 
539     BuildMI(MBB, I, DL, get(Opcode))
540         .addReg(SrcReg, getKillRegState(IsKill))
541         .addFrameIndex(FI)
542         .addImm(0)
543         .addMemOperand(MMO);
544   }
545 }
546 
547 void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
548                                           MachineBasicBlock::iterator I,
549                                           Register DstReg, int FI,
550                                           const TargetRegisterClass *RC,
551                                           const TargetRegisterInfo *TRI,
552                                           Register VReg) const {
553   DebugLoc DL;
554   if (I != MBB.end())
555     DL = I->getDebugLoc();
556 
557   MachineFunction *MF = MBB.getParent();
558   MachineFrameInfo &MFI = MF->getFrameInfo();
559 
560   unsigned Opcode;
561   bool IsScalableVector = true;
562   if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
563     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
564              RISCV::LW : RISCV::LD;
565     IsScalableVector = false;
566   } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
567     Opcode = RISCV::FLH;
568     IsScalableVector = false;
569   } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
570     Opcode = RISCV::FLW;
571     IsScalableVector = false;
572   } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
573     Opcode = RISCV::FLD;
574     IsScalableVector = false;
575   } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
576     Opcode = RISCV::VL1RE8_V;
577   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
578     Opcode = RISCV::VL2RE8_V;
579   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
580     Opcode = RISCV::VL4RE8_V;
581   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
582     Opcode = RISCV::VL8RE8_V;
583   } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
584     Opcode = RISCV::PseudoVRELOAD2_M1;
585   else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
586     Opcode = RISCV::PseudoVRELOAD2_M2;
587   else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
588     Opcode = RISCV::PseudoVRELOAD2_M4;
589   else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
590     Opcode = RISCV::PseudoVRELOAD3_M1;
591   else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
592     Opcode = RISCV::PseudoVRELOAD3_M2;
593   else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
594     Opcode = RISCV::PseudoVRELOAD4_M1;
595   else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
596     Opcode = RISCV::PseudoVRELOAD4_M2;
597   else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
598     Opcode = RISCV::PseudoVRELOAD5_M1;
599   else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
600     Opcode = RISCV::PseudoVRELOAD6_M1;
601   else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
602     Opcode = RISCV::PseudoVRELOAD7_M1;
603   else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
604     Opcode = RISCV::PseudoVRELOAD8_M1;
605   else
606     llvm_unreachable("Can't load this register from stack slot");
607 
608   if (IsScalableVector) {
609     MachineMemOperand *MMO = MF->getMachineMemOperand(
610         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
611         MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
612 
613     MFI.setStackID(FI, TargetStackID::ScalableVector);
614     BuildMI(MBB, I, DL, get(Opcode), DstReg)
615         .addFrameIndex(FI)
616         .addMemOperand(MMO);
617   } else {
618     MachineMemOperand *MMO = MF->getMachineMemOperand(
619         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
620         MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
621 
622     BuildMI(MBB, I, DL, get(Opcode), DstReg)
623         .addFrameIndex(FI)
624         .addImm(0)
625         .addMemOperand(MMO);
626   }
627 }
628 
629 MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
630     MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
631     MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
632     VirtRegMap *VRM) const {
633   const MachineFrameInfo &MFI = MF.getFrameInfo();
634 
635   // The below optimizations narrow the load so they are only valid for little
636   // endian.
637   // TODO: Support big endian by adding an offset into the frame object?
638   if (MF.getDataLayout().isBigEndian())
639     return nullptr;
640 
641   // Fold load from stack followed by sext.w into lw.
642   // TODO: Fold with sext.b, sext.h, zext.b, zext.h, zext.w?
643   if (Ops.size() != 1 || Ops[0] != 1)
644    return nullptr;
645 
646   unsigned LoadOpc;
647   switch (MI.getOpcode()) {
648   default:
649     if (RISCV::isSEXT_W(MI)) {
650       LoadOpc = RISCV::LW;
651       break;
652     }
653     if (RISCV::isZEXT_W(MI)) {
654       LoadOpc = RISCV::LWU;
655       break;
656     }
657     if (RISCV::isZEXT_B(MI)) {
658       LoadOpc = RISCV::LBU;
659       break;
660     }
661     return nullptr;
662   case RISCV::SEXT_H:
663     LoadOpc = RISCV::LH;
664     break;
665   case RISCV::SEXT_B:
666     LoadOpc = RISCV::LB;
667     break;
668   case RISCV::ZEXT_H_RV32:
669   case RISCV::ZEXT_H_RV64:
670     LoadOpc = RISCV::LHU;
671     break;
672   }
673 
674   MachineMemOperand *MMO = MF.getMachineMemOperand(
675       MachinePointerInfo::getFixedStack(MF, FrameIndex),
676       MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIndex),
677       MFI.getObjectAlign(FrameIndex));
678 
679   Register DstReg = MI.getOperand(0).getReg();
680   return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(LoadOpc),
681                  DstReg)
682       .addFrameIndex(FrameIndex)
683       .addImm(0)
684       .addMemOperand(MMO);
685 }
686 
687 void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
688                             MachineBasicBlock::iterator MBBI,
689                             const DebugLoc &DL, Register DstReg, uint64_t Val,
690                             MachineInstr::MIFlag Flag) const {
691   Register SrcReg = RISCV::X0;
692 
693   if (!STI.is64Bit() && !isInt<32>(Val))
694     report_fatal_error("Should only materialize 32-bit constants for RV32");
695 
696   RISCVMatInt::InstSeq Seq =
697       RISCVMatInt::generateInstSeq(Val, STI.getFeatureBits());
698   assert(!Seq.empty());
699 
700   for (RISCVMatInt::Inst &Inst : Seq) {
701     switch (Inst.getOpndKind()) {
702     case RISCVMatInt::Imm:
703       BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
704           .addImm(Inst.getImm())
705           .setMIFlag(Flag);
706       break;
707     case RISCVMatInt::RegX0:
708       BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
709           .addReg(SrcReg, RegState::Kill)
710           .addReg(RISCV::X0)
711           .setMIFlag(Flag);
712       break;
713     case RISCVMatInt::RegReg:
714       BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
715           .addReg(SrcReg, RegState::Kill)
716           .addReg(SrcReg, RegState::Kill)
717           .setMIFlag(Flag);
718       break;
719     case RISCVMatInt::RegImm:
720       BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
721           .addReg(SrcReg, RegState::Kill)
722           .addImm(Inst.getImm())
723           .setMIFlag(Flag);
724       break;
725     }
726 
727     // Only the first instruction has X0 as its source.
728     SrcReg = DstReg;
729   }
730 }
731 
732 static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) {
733   switch (Opc) {
734   default:
735     return RISCVCC::COND_INVALID;
736   case RISCV::BEQ:
737     return RISCVCC::COND_EQ;
738   case RISCV::BNE:
739     return RISCVCC::COND_NE;
740   case RISCV::BLT:
741     return RISCVCC::COND_LT;
742   case RISCV::BGE:
743     return RISCVCC::COND_GE;
744   case RISCV::BLTU:
745     return RISCVCC::COND_LTU;
746   case RISCV::BGEU:
747     return RISCVCC::COND_GEU;
748   }
749 }
750 
751 // The contents of values added to Cond are not examined outside of
752 // RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
753 // push BranchOpcode, Reg1, Reg2.
754 static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
755                             SmallVectorImpl<MachineOperand> &Cond) {
756   // Block ends with fall-through condbranch.
757   assert(LastInst.getDesc().isConditionalBranch() &&
758          "Unknown conditional branch");
759   Target = LastInst.getOperand(2).getMBB();
760   unsigned CC = getCondFromBranchOpc(LastInst.getOpcode());
761   Cond.push_back(MachineOperand::CreateImm(CC));
762   Cond.push_back(LastInst.getOperand(0));
763   Cond.push_back(LastInst.getOperand(1));
764 }
765 
766 const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const {
767   switch (CC) {
768   default:
769     llvm_unreachable("Unknown condition code!");
770   case RISCVCC::COND_EQ:
771     return get(RISCV::BEQ);
772   case RISCVCC::COND_NE:
773     return get(RISCV::BNE);
774   case RISCVCC::COND_LT:
775     return get(RISCV::BLT);
776   case RISCVCC::COND_GE:
777     return get(RISCV::BGE);
778   case RISCVCC::COND_LTU:
779     return get(RISCV::BLTU);
780   case RISCVCC::COND_GEU:
781     return get(RISCV::BGEU);
782   }
783 }
784 
785 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
786   switch (CC) {
787   default:
788     llvm_unreachable("Unrecognized conditional branch");
789   case RISCVCC::COND_EQ:
790     return RISCVCC::COND_NE;
791   case RISCVCC::COND_NE:
792     return RISCVCC::COND_EQ;
793   case RISCVCC::COND_LT:
794     return RISCVCC::COND_GE;
795   case RISCVCC::COND_GE:
796     return RISCVCC::COND_LT;
797   case RISCVCC::COND_LTU:
798     return RISCVCC::COND_GEU;
799   case RISCVCC::COND_GEU:
800     return RISCVCC::COND_LTU;
801   }
802 }
803 
804 bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
805                                    MachineBasicBlock *&TBB,
806                                    MachineBasicBlock *&FBB,
807                                    SmallVectorImpl<MachineOperand> &Cond,
808                                    bool AllowModify) const {
809   TBB = FBB = nullptr;
810   Cond.clear();
811 
812   // If the block has no terminators, it just falls into the block after it.
813   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
814   if (I == MBB.end() || !isUnpredicatedTerminator(*I))
815     return false;
816 
817   // Count the number of terminators and find the first unconditional or
818   // indirect branch.
819   MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
820   int NumTerminators = 0;
821   for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
822        J++) {
823     NumTerminators++;
824     if (J->getDesc().isUnconditionalBranch() ||
825         J->getDesc().isIndirectBranch()) {
826       FirstUncondOrIndirectBr = J.getReverse();
827     }
828   }
829 
830   // If AllowModify is true, we can erase any terminators after
831   // FirstUncondOrIndirectBR.
832   if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
833     while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
834       std::next(FirstUncondOrIndirectBr)->eraseFromParent();
835       NumTerminators--;
836     }
837     I = FirstUncondOrIndirectBr;
838   }
839 
840   // We can't handle blocks that end in an indirect branch.
841   if (I->getDesc().isIndirectBranch())
842     return true;
843 
844   // We can't handle blocks with more than 2 terminators.
845   if (NumTerminators > 2)
846     return true;
847 
848   // Handle a single unconditional branch.
849   if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
850     TBB = getBranchDestBlock(*I);
851     return false;
852   }
853 
854   // Handle a single conditional branch.
855   if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
856     parseCondBranch(*I, TBB, Cond);
857     return false;
858   }
859 
860   // Handle a conditional branch followed by an unconditional branch.
861   if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
862       I->getDesc().isUnconditionalBranch()) {
863     parseCondBranch(*std::prev(I), TBB, Cond);
864     FBB = getBranchDestBlock(*I);
865     return false;
866   }
867 
868   // Otherwise, we can't handle this.
869   return true;
870 }
871 
872 unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
873                                       int *BytesRemoved) const {
874   if (BytesRemoved)
875     *BytesRemoved = 0;
876   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
877   if (I == MBB.end())
878     return 0;
879 
880   if (!I->getDesc().isUnconditionalBranch() &&
881       !I->getDesc().isConditionalBranch())
882     return 0;
883 
884   // Remove the branch.
885   if (BytesRemoved)
886     *BytesRemoved += getInstSizeInBytes(*I);
887   I->eraseFromParent();
888 
889   I = MBB.end();
890 
891   if (I == MBB.begin())
892     return 1;
893   --I;
894   if (!I->getDesc().isConditionalBranch())
895     return 1;
896 
897   // Remove the branch.
898   if (BytesRemoved)
899     *BytesRemoved += getInstSizeInBytes(*I);
900   I->eraseFromParent();
901   return 2;
902 }
903 
904 // Inserts a branch into the end of the specific MachineBasicBlock, returning
905 // the number of instructions inserted.
906 unsigned RISCVInstrInfo::insertBranch(
907     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
908     ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
909   if (BytesAdded)
910     *BytesAdded = 0;
911 
912   // Shouldn't be a fall through.
913   assert(TBB && "insertBranch must not be told to insert a fallthrough");
914   assert((Cond.size() == 3 || Cond.size() == 0) &&
915          "RISCV branch conditions have two components!");
916 
917   // Unconditional branch.
918   if (Cond.empty()) {
919     MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
920     if (BytesAdded)
921       *BytesAdded += getInstSizeInBytes(MI);
922     return 1;
923   }
924 
925   // Either a one or two-way conditional branch.
926   auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
927   MachineInstr &CondMI =
928       *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB);
929   if (BytesAdded)
930     *BytesAdded += getInstSizeInBytes(CondMI);
931 
932   // One-way conditional branch.
933   if (!FBB)
934     return 1;
935 
936   // Two-way conditional branch.
937   MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
938   if (BytesAdded)
939     *BytesAdded += getInstSizeInBytes(MI);
940   return 2;
941 }
942 
943 void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
944                                           MachineBasicBlock &DestBB,
945                                           MachineBasicBlock &RestoreBB,
946                                           const DebugLoc &DL, int64_t BrOffset,
947                                           RegScavenger *RS) const {
948   assert(RS && "RegScavenger required for long branching");
949   assert(MBB.empty() &&
950          "new block should be inserted for expanding unconditional branch");
951   assert(MBB.pred_size() == 1);
952   assert(RestoreBB.empty() &&
953          "restore block should be inserted for restoring clobbered registers");
954 
955   MachineFunction *MF = MBB.getParent();
956   MachineRegisterInfo &MRI = MF->getRegInfo();
957   RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
958   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
959 
960   if (!isInt<32>(BrOffset))
961     report_fatal_error(
962         "Branch offsets outside of the signed 32-bit range not supported");
963 
964   // FIXME: A virtual register must be used initially, as the register
965   // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
966   // uses the same workaround).
967   Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
968   auto II = MBB.end();
969   // We may also update the jump target to RestoreBB later.
970   MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
971                           .addReg(ScratchReg, RegState::Define | RegState::Dead)
972                           .addMBB(&DestBB, RISCVII::MO_CALL);
973 
974   RS->enterBasicBlockEnd(MBB);
975   Register TmpGPR =
976       RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MI.getIterator(),
977                                     /*RestoreAfter=*/false, /*SpAdj=*/0,
978                                     /*AllowSpill=*/false);
979   if (TmpGPR != RISCV::NoRegister)
980     RS->setRegUsed(TmpGPR);
981   else {
982     // The case when there is no scavenged register needs special handling.
983 
984     // Pick s11 because it doesn't make a difference.
985     TmpGPR = RISCV::X27;
986 
987     int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
988     if (FrameIndex == -1)
989       report_fatal_error("underestimated function size");
990 
991     storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,
992                         &RISCV::GPRRegClass, TRI, Register());
993     TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
994                              /*SpAdj=*/0, /*FIOperandNum=*/1);
995 
996     MI.getOperand(1).setMBB(&RestoreBB);
997 
998     loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,
999                          &RISCV::GPRRegClass, TRI, Register());
1000     TRI->eliminateFrameIndex(RestoreBB.back(),
1001                              /*SpAdj=*/0, /*FIOperandNum=*/1);
1002   }
1003 
1004   MRI.replaceRegWith(ScratchReg, TmpGPR);
1005   MRI.clearVirtRegs();
1006 }
1007 
1008 bool RISCVInstrInfo::reverseBranchCondition(
1009     SmallVectorImpl<MachineOperand> &Cond) const {
1010   assert((Cond.size() == 3) && "Invalid branch condition!");
1011   auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1012   Cond[0].setImm(getOppositeBranchCondition(CC));
1013   return false;
1014 }
1015 
1016 MachineBasicBlock *
1017 RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1018   assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1019   // The branch target is always the last operand.
1020   int NumOp = MI.getNumExplicitOperands();
1021   return MI.getOperand(NumOp - 1).getMBB();
1022 }
1023 
1024 bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1025                                            int64_t BrOffset) const {
1026   unsigned XLen = STI.getXLen();
1027   // Ideally we could determine the supported branch offset from the
1028   // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1029   // PseudoBR.
1030   switch (BranchOp) {
1031   default:
1032     llvm_unreachable("Unexpected opcode!");
1033   case RISCV::BEQ:
1034   case RISCV::BNE:
1035   case RISCV::BLT:
1036   case RISCV::BGE:
1037   case RISCV::BLTU:
1038   case RISCV::BGEU:
1039     return isIntN(13, BrOffset);
1040   case RISCV::JAL:
1041   case RISCV::PseudoBR:
1042     return isIntN(21, BrOffset);
1043   case RISCV::PseudoJump:
1044     return isIntN(32, SignExtend64(BrOffset + 0x800, XLen));
1045   }
1046 }
1047 
1048 // If the operation has a predicated pseudo instruction, return the pseudo
1049 // instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1050 // TODO: Support more operations.
1051 unsigned getPredicatedOpcode(unsigned Opcode) {
1052   switch (Opcode) {
1053   case RISCV::ADD:   return RISCV::PseudoCCADD;   break;
1054   case RISCV::SUB:   return RISCV::PseudoCCSUB;   break;
1055   case RISCV::AND:   return RISCV::PseudoCCAND;   break;
1056   case RISCV::OR:    return RISCV::PseudoCCOR;    break;
1057   case RISCV::XOR:   return RISCV::PseudoCCXOR;   break;
1058 
1059   case RISCV::ADDW:  return RISCV::PseudoCCADDW;  break;
1060   case RISCV::SUBW:  return RISCV::PseudoCCSUBW;  break;
1061   }
1062 
1063   return RISCV::INSTRUCTION_LIST_END;
1064 }
1065 
1066 /// Identify instructions that can be folded into a CCMOV instruction, and
1067 /// return the defining instruction.
1068 static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1069                                            const MachineRegisterInfo &MRI,
1070                                            const TargetInstrInfo *TII) {
1071   if (!Reg.isVirtual())
1072     return nullptr;
1073   if (!MRI.hasOneNonDBGUse(Reg))
1074     return nullptr;
1075   MachineInstr *MI = MRI.getVRegDef(Reg);
1076   if (!MI)
1077     return nullptr;
1078   // Check if MI can be predicated and folded into the CCMOV.
1079   if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1080     return nullptr;
1081   // Check if MI has any other defs or physreg uses.
1082   for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) {
1083     const MachineOperand &MO = MI->getOperand(i);
1084     // Reject frame index operands, PEI can't handle the predicated pseudos.
1085     if (MO.isFI() || MO.isCPI() || MO.isJTI())
1086       return nullptr;
1087     if (!MO.isReg())
1088       continue;
1089     // MI can't have any tied operands, that would conflict with predication.
1090     if (MO.isTied())
1091       return nullptr;
1092     if (MO.isDef())
1093       return nullptr;
1094     // Allow constant physregs.
1095     if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(MO.getReg()))
1096       return nullptr;
1097   }
1098   bool DontMoveAcrossStores = true;
1099   if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
1100     return nullptr;
1101   return MI;
1102 }
1103 
1104 bool RISCVInstrInfo::analyzeSelect(const MachineInstr &MI,
1105                                    SmallVectorImpl<MachineOperand> &Cond,
1106                                    unsigned &TrueOp, unsigned &FalseOp,
1107                                    bool &Optimizable) const {
1108   assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1109          "Unknown select instruction");
1110   // CCMOV operands:
1111   // 0: Def.
1112   // 1: LHS of compare.
1113   // 2: RHS of compare.
1114   // 3: Condition code.
1115   // 4: False use.
1116   // 5: True use.
1117   TrueOp = 5;
1118   FalseOp = 4;
1119   Cond.push_back(MI.getOperand(1));
1120   Cond.push_back(MI.getOperand(2));
1121   Cond.push_back(MI.getOperand(3));
1122   // We can only fold when we support short forward branch opt.
1123   Optimizable = STI.hasShortForwardBranchOpt();
1124   return false;
1125 }
1126 
1127 MachineInstr *
1128 RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
1129                                SmallPtrSetImpl<MachineInstr *> &SeenMIs,
1130                                bool PreferFalse) const {
1131   assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1132          "Unknown select instruction");
1133   if (!STI.hasShortForwardBranchOpt())
1134     return nullptr;
1135 
1136   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1137   MachineInstr *DefMI =
1138       canFoldAsPredicatedOp(MI.getOperand(5).getReg(), MRI, this);
1139   bool Invert = !DefMI;
1140   if (!DefMI)
1141     DefMI = canFoldAsPredicatedOp(MI.getOperand(4).getReg(), MRI, this);
1142   if (!DefMI)
1143     return nullptr;
1144 
1145   // Find new register class to use.
1146   MachineOperand FalseReg = MI.getOperand(Invert ? 5 : 4);
1147   Register DestReg = MI.getOperand(0).getReg();
1148   const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
1149   if (!MRI.constrainRegClass(DestReg, PreviousClass))
1150     return nullptr;
1151 
1152   unsigned PredOpc = getPredicatedOpcode(DefMI->getOpcode());
1153   assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1154 
1155   // Create a new predicated version of DefMI.
1156   MachineInstrBuilder NewMI =
1157       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1158 
1159   // Copy the condition portion.
1160   NewMI.add(MI.getOperand(1));
1161   NewMI.add(MI.getOperand(2));
1162 
1163   // Add condition code, inverting if necessary.
1164   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
1165   if (Invert)
1166     CC = RISCVCC::getOppositeBranchCondition(CC);
1167   NewMI.addImm(CC);
1168 
1169   // Copy the false register.
1170   NewMI.add(FalseReg);
1171 
1172   // Copy all the DefMI operands.
1173   const MCInstrDesc &DefDesc = DefMI->getDesc();
1174   for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1175     NewMI.add(DefMI->getOperand(i));
1176 
1177   // Update SeenMIs set: register newly created MI and erase removed DefMI.
1178   SeenMIs.insert(NewMI);
1179   SeenMIs.erase(DefMI);
1180 
1181   // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1182   // DefMI would be invalid when tranferred inside the loop.  Checking for a
1183   // loop is expensive, but at least remove kill flags if they are in different
1184   // BBs.
1185   if (DefMI->getParent() != MI.getParent())
1186     NewMI->clearKillInfo();
1187 
1188   // The caller will erase MI, but not DefMI.
1189   DefMI->eraseFromParent();
1190   return NewMI;
1191 }
1192 
1193 unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1194   if (MI.isMetaInstruction())
1195     return 0;
1196 
1197   unsigned Opcode = MI.getOpcode();
1198 
1199   if (Opcode == TargetOpcode::INLINEASM ||
1200       Opcode == TargetOpcode::INLINEASM_BR) {
1201     const MachineFunction &MF = *MI.getParent()->getParent();
1202     const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget());
1203     return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
1204                               *TM.getMCAsmInfo());
1205   }
1206 
1207   if (MI.getParent() && MI.getParent()->getParent()) {
1208     if (isCompressibleInst(MI, STI))
1209       return 2;
1210   }
1211   return get(Opcode).getSize();
1212 }
1213 
1214 bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1215   const unsigned Opcode = MI.getOpcode();
1216   switch (Opcode) {
1217   default:
1218     break;
1219   case RISCV::FSGNJ_D:
1220   case RISCV::FSGNJ_S:
1221   case RISCV::FSGNJ_H:
1222     // The canonical floating-point move is fsgnj rd, rs, rs.
1223     return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1224            MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
1225   case RISCV::ADDI:
1226   case RISCV::ORI:
1227   case RISCV::XORI:
1228     return (MI.getOperand(1).isReg() &&
1229             MI.getOperand(1).getReg() == RISCV::X0) ||
1230            (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1231   }
1232   return MI.isAsCheapAsAMove();
1233 }
1234 
1235 std::optional<DestSourcePair>
1236 RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
1237   if (MI.isMoveReg())
1238     return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1239   switch (MI.getOpcode()) {
1240   default:
1241     break;
1242   case RISCV::ADDI:
1243     // Operand 1 can be a frameindex but callers expect registers
1244     if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
1245         MI.getOperand(2).getImm() == 0)
1246       return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1247     break;
1248   case RISCV::FSGNJ_D:
1249   case RISCV::FSGNJ_S:
1250   case RISCV::FSGNJ_H:
1251     // The canonical floating-point move is fsgnj rd, rs, rs.
1252     if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1253         MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
1254       return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1255     break;
1256   }
1257   return std::nullopt;
1258 }
1259 
1260 void RISCVInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
1261                                            MachineInstr &OldMI2,
1262                                            MachineInstr &NewMI1,
1263                                            MachineInstr &NewMI2) const {
1264   uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
1265   NewMI1.setFlags(IntersectedFlags);
1266   NewMI2.setFlags(IntersectedFlags);
1267 }
1268 
1269 void RISCVInstrInfo::finalizeInsInstrs(
1270     MachineInstr &Root, MachineCombinerPattern &P,
1271     SmallVectorImpl<MachineInstr *> &InsInstrs) const {
1272   int16_t FrmOpIdx =
1273       RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);
1274   if (FrmOpIdx < 0) {
1275     assert(all_of(InsInstrs,
1276                   [](MachineInstr *MI) {
1277                     return RISCV::getNamedOperandIdx(MI->getOpcode(),
1278                                                      RISCV::OpName::frm) < 0;
1279                   }) &&
1280            "New instructions require FRM whereas the old one does not have it");
1281     return;
1282   }
1283 
1284   const MachineOperand &FRM = Root.getOperand(FrmOpIdx);
1285   MachineFunction &MF = *Root.getMF();
1286 
1287   for (auto *NewMI : InsInstrs) {
1288     assert(static_cast<unsigned>(RISCV::getNamedOperandIdx(
1289                NewMI->getOpcode(), RISCV::OpName::frm)) ==
1290                NewMI->getNumOperands() &&
1291            "Instruction has unexpected number of operands");
1292     MachineInstrBuilder MIB(MF, NewMI);
1293     MIB.add(FRM);
1294     if (FRM.getImm() == RISCVFPRndMode::DYN)
1295       MIB.addUse(RISCV::FRM, RegState::Implicit);
1296   }
1297 }
1298 
1299 static bool isFADD(unsigned Opc) {
1300   switch (Opc) {
1301   default:
1302     return false;
1303   case RISCV::FADD_H:
1304   case RISCV::FADD_S:
1305   case RISCV::FADD_D:
1306     return true;
1307   }
1308 }
1309 
1310 static bool isFSUB(unsigned Opc) {
1311   switch (Opc) {
1312   default:
1313     return false;
1314   case RISCV::FSUB_H:
1315   case RISCV::FSUB_S:
1316   case RISCV::FSUB_D:
1317     return true;
1318   }
1319 }
1320 
1321 static bool isFMUL(unsigned Opc) {
1322   switch (Opc) {
1323   default:
1324     return false;
1325   case RISCV::FMUL_H:
1326   case RISCV::FMUL_S:
1327   case RISCV::FMUL_D:
1328     return true;
1329   }
1330 }
1331 
1332 bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
1333                                             bool &Commuted) const {
1334   if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
1335     return false;
1336 
1337   const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
1338   unsigned OperandIdx = Commuted ? 2 : 1;
1339   const MachineInstr &Sibling =
1340       *MRI.getVRegDef(Inst.getOperand(OperandIdx).getReg());
1341 
1342   int16_t InstFrmOpIdx =
1343       RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);
1344   int16_t SiblingFrmOpIdx =
1345       RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);
1346 
1347   return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
1348          RISCV::hasEqualFRM(Inst, Sibling);
1349 }
1350 
1351 bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
1352                                                  bool Invert) const {
1353   unsigned Opc = Inst.getOpcode();
1354   if (Invert) {
1355     auto InverseOpcode = getInverseOpcode(Opc);
1356     if (!InverseOpcode)
1357       return false;
1358     Opc = *InverseOpcode;
1359   }
1360 
1361   if (isFADD(Opc) || isFMUL(Opc))
1362     return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
1363            Inst.getFlag(MachineInstr::MIFlag::FmNsz);
1364 
1365   switch (Opc) {
1366   default:
1367     return false;
1368   case RISCV::ADD:
1369   case RISCV::ADDW:
1370   case RISCV::AND:
1371   case RISCV::OR:
1372   case RISCV::XOR:
1373   // From RISC-V ISA spec, if both the high and low bits of the same product
1374   // are required, then the recommended code sequence is:
1375   //
1376   // MULH[[S]U] rdh, rs1, rs2
1377   // MUL        rdl, rs1, rs2
1378   // (source register specifiers must be in same order and rdh cannot be the
1379   //  same as rs1 or rs2)
1380   //
1381   // Microarchitectures can then fuse these into a single multiply operation
1382   // instead of performing two separate multiplies.
1383   // MachineCombiner may reassociate MUL operands and lose the fusion
1384   // opportunity.
1385   case RISCV::MUL:
1386   case RISCV::MULW:
1387   case RISCV::MIN:
1388   case RISCV::MINU:
1389   case RISCV::MAX:
1390   case RISCV::MAXU:
1391   case RISCV::FMIN_H:
1392   case RISCV::FMIN_S:
1393   case RISCV::FMIN_D:
1394   case RISCV::FMAX_H:
1395   case RISCV::FMAX_S:
1396   case RISCV::FMAX_D:
1397     return true;
1398   }
1399 
1400   return false;
1401 }
1402 
1403 std::optional<unsigned>
1404 RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
1405   switch (Opcode) {
1406   default:
1407     return std::nullopt;
1408   case RISCV::FADD_H:
1409     return RISCV::FSUB_H;
1410   case RISCV::FADD_S:
1411     return RISCV::FSUB_S;
1412   case RISCV::FADD_D:
1413     return RISCV::FSUB_D;
1414   case RISCV::FSUB_H:
1415     return RISCV::FADD_H;
1416   case RISCV::FSUB_S:
1417     return RISCV::FADD_S;
1418   case RISCV::FSUB_D:
1419     return RISCV::FADD_D;
1420   case RISCV::ADD:
1421     return RISCV::SUB;
1422   case RISCV::SUB:
1423     return RISCV::ADD;
1424   case RISCV::ADDW:
1425     return RISCV::SUBW;
1426   case RISCV::SUBW:
1427     return RISCV::ADDW;
1428   }
1429 }
1430 
1431 static bool canCombineFPFusedMultiply(const MachineInstr &Root,
1432                                       const MachineOperand &MO,
1433                                       bool DoRegPressureReduce) {
1434   if (!MO.isReg() || !MO.getReg().isVirtual())
1435     return false;
1436   const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1437   MachineInstr *MI = MRI.getVRegDef(MO.getReg());
1438   if (!MI || !isFMUL(MI->getOpcode()))
1439     return false;
1440 
1441   if (!Root.getFlag(MachineInstr::MIFlag::FmContract) ||
1442       !MI->getFlag(MachineInstr::MIFlag::FmContract))
1443     return false;
1444 
1445   // Try combining even if fmul has more than one use as it eliminates
1446   // dependency between fadd(fsub) and fmul. However, it can extend liveranges
1447   // for fmul operands, so reject the transformation in register pressure
1448   // reduction mode.
1449   if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
1450     return false;
1451 
1452   // Do not combine instructions from different basic blocks.
1453   if (Root.getParent() != MI->getParent())
1454     return false;
1455   return RISCV::hasEqualFRM(Root, *MI);
1456 }
1457 
1458 static bool
1459 getFPFusedMultiplyPatterns(MachineInstr &Root,
1460                            SmallVectorImpl<MachineCombinerPattern> &Patterns,
1461                            bool DoRegPressureReduce) {
1462   unsigned Opc = Root.getOpcode();
1463   bool IsFAdd = isFADD(Opc);
1464   if (!IsFAdd && !isFSUB(Opc))
1465     return false;
1466   bool Added = false;
1467   if (canCombineFPFusedMultiply(Root, Root.getOperand(1),
1468                                 DoRegPressureReduce)) {
1469     Patterns.push_back(IsFAdd ? MachineCombinerPattern::FMADD_AX
1470                               : MachineCombinerPattern::FMSUB);
1471     Added = true;
1472   }
1473   if (canCombineFPFusedMultiply(Root, Root.getOperand(2),
1474                                 DoRegPressureReduce)) {
1475     Patterns.push_back(IsFAdd ? MachineCombinerPattern::FMADD_XA
1476                               : MachineCombinerPattern::FNMSUB);
1477     Added = true;
1478   }
1479   return Added;
1480 }
1481 
1482 static bool getFPPatterns(MachineInstr &Root,
1483                           SmallVectorImpl<MachineCombinerPattern> &Patterns,
1484                           bool DoRegPressureReduce) {
1485   return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
1486 }
1487 
1488 bool RISCVInstrInfo::getMachineCombinerPatterns(
1489     MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
1490     bool DoRegPressureReduce) const {
1491 
1492   if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
1493     return true;
1494 
1495   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
1496                                                      DoRegPressureReduce);
1497 }
1498 
1499 static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc,
1500                                          MachineCombinerPattern Pattern) {
1501   switch (RootOpc) {
1502   default:
1503     llvm_unreachable("Unexpected opcode");
1504   case RISCV::FADD_H:
1505     return RISCV::FMADD_H;
1506   case RISCV::FADD_S:
1507     return RISCV::FMADD_S;
1508   case RISCV::FADD_D:
1509     return RISCV::FMADD_D;
1510   case RISCV::FSUB_H:
1511     return Pattern == MachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
1512                                                     : RISCV::FNMSUB_H;
1513   case RISCV::FSUB_S:
1514     return Pattern == MachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
1515                                                     : RISCV::FNMSUB_S;
1516   case RISCV::FSUB_D:
1517     return Pattern == MachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
1518                                                     : RISCV::FNMSUB_D;
1519   }
1520 }
1521 
1522 static unsigned getAddendOperandIdx(MachineCombinerPattern Pattern) {
1523   switch (Pattern) {
1524   default:
1525     llvm_unreachable("Unexpected pattern");
1526   case MachineCombinerPattern::FMADD_AX:
1527   case MachineCombinerPattern::FMSUB:
1528     return 2;
1529   case MachineCombinerPattern::FMADD_XA:
1530   case MachineCombinerPattern::FNMSUB:
1531     return 1;
1532   }
1533 }
1534 
1535 static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
1536                                    MachineCombinerPattern Pattern,
1537                                    SmallVectorImpl<MachineInstr *> &InsInstrs,
1538                                    SmallVectorImpl<MachineInstr *> &DelInstrs) {
1539   MachineFunction *MF = Root.getMF();
1540   MachineRegisterInfo &MRI = MF->getRegInfo();
1541   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1542 
1543   MachineOperand &Mul1 = Prev.getOperand(1);
1544   MachineOperand &Mul2 = Prev.getOperand(2);
1545   MachineOperand &Dst = Root.getOperand(0);
1546   MachineOperand &Addend = Root.getOperand(getAddendOperandIdx(Pattern));
1547 
1548   Register DstReg = Dst.getReg();
1549   unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);
1550   auto IntersectedFlags = Root.getFlags() & Prev.getFlags();
1551   DebugLoc MergedLoc =
1552       DILocation::getMergedLocation(Root.getDebugLoc(), Prev.getDebugLoc());
1553 
1554   MachineInstrBuilder MIB =
1555       BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
1556           .addReg(Mul1.getReg(), getKillRegState(Mul1.isKill()))
1557           .addReg(Mul2.getReg(), getKillRegState(Mul2.isKill()))
1558           .addReg(Addend.getReg(), getKillRegState(Addend.isKill()))
1559           .setMIFlags(IntersectedFlags);
1560 
1561   // Mul operands are not killed anymore.
1562   Mul1.setIsKill(false);
1563   Mul2.setIsKill(false);
1564 
1565   InsInstrs.push_back(MIB);
1566   if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg()))
1567     DelInstrs.push_back(&Prev);
1568   DelInstrs.push_back(&Root);
1569 }
1570 
1571 void RISCVInstrInfo::genAlternativeCodeSequence(
1572     MachineInstr &Root, MachineCombinerPattern Pattern,
1573     SmallVectorImpl<MachineInstr *> &InsInstrs,
1574     SmallVectorImpl<MachineInstr *> &DelInstrs,
1575     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
1576   MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1577   switch (Pattern) {
1578   default:
1579     TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
1580                                                 DelInstrs, InstrIdxForVirtReg);
1581     return;
1582   case MachineCombinerPattern::FMADD_AX:
1583   case MachineCombinerPattern::FMSUB: {
1584     MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(1).getReg());
1585     combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
1586     return;
1587   }
1588   case MachineCombinerPattern::FMADD_XA:
1589   case MachineCombinerPattern::FNMSUB: {
1590     MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(2).getReg());
1591     combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
1592     return;
1593   }
1594   }
1595 }
1596 
1597 bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
1598                                        StringRef &ErrInfo) const {
1599   MCInstrDesc const &Desc = MI.getDesc();
1600 
1601   for (auto &OI : enumerate(Desc.operands())) {
1602     unsigned OpType = OI.value().OperandType;
1603     if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
1604         OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
1605       const MachineOperand &MO = MI.getOperand(OI.index());
1606       if (MO.isImm()) {
1607         int64_t Imm = MO.getImm();
1608         bool Ok;
1609         switch (OpType) {
1610         default:
1611           llvm_unreachable("Unexpected operand type");
1612 
1613           // clang-format off
1614 #define CASE_OPERAND_UIMM(NUM)                                                 \
1615   case RISCVOp::OPERAND_UIMM##NUM:                                             \
1616     Ok = isUInt<NUM>(Imm);                                                     \
1617     break;
1618         CASE_OPERAND_UIMM(2)
1619         CASE_OPERAND_UIMM(3)
1620         CASE_OPERAND_UIMM(4)
1621         CASE_OPERAND_UIMM(5)
1622         CASE_OPERAND_UIMM(7)
1623         case RISCVOp::OPERAND_UIMM7_LSB00:
1624           Ok = isShiftedUInt<5, 2>(Imm);
1625           break;
1626         case RISCVOp::OPERAND_UIMM8_LSB00:
1627           Ok = isShiftedUInt<6, 2>(Imm);
1628           break;
1629         case RISCVOp::OPERAND_UIMM8_LSB000:
1630           Ok = isShiftedUInt<5, 3>(Imm);
1631           break;
1632         CASE_OPERAND_UIMM(12)
1633         CASE_OPERAND_UIMM(20)
1634           // clang-format on
1635         case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
1636           Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
1637           break;
1638         case RISCVOp::OPERAND_ZERO:
1639           Ok = Imm == 0;
1640           break;
1641         case RISCVOp::OPERAND_SIMM5:
1642           Ok = isInt<5>(Imm);
1643           break;
1644         case RISCVOp::OPERAND_SIMM5_PLUS1:
1645           Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;
1646           break;
1647         case RISCVOp::OPERAND_SIMM6:
1648           Ok = isInt<6>(Imm);
1649           break;
1650         case RISCVOp::OPERAND_SIMM6_NONZERO:
1651           Ok = Imm != 0 && isInt<6>(Imm);
1652           break;
1653         case RISCVOp::OPERAND_VTYPEI10:
1654           Ok = isUInt<10>(Imm);
1655           break;
1656         case RISCVOp::OPERAND_VTYPEI11:
1657           Ok = isUInt<11>(Imm);
1658           break;
1659         case RISCVOp::OPERAND_SIMM12:
1660           Ok = isInt<12>(Imm);
1661           break;
1662         case RISCVOp::OPERAND_SIMM12_LSB00000:
1663           Ok = isShiftedInt<7, 5>(Imm);
1664           break;
1665         case RISCVOp::OPERAND_UIMMLOG2XLEN:
1666           Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
1667           break;
1668         case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
1669           Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
1670           Ok = Ok && Imm != 0;
1671           break;
1672         case RISCVOp::OPERAND_UIMM_SHFL:
1673           Ok = STI.is64Bit() ? isUInt<5>(Imm) : isUInt<4>(Imm);
1674           break;
1675         case RISCVOp::OPERAND_RVKRNUM:
1676           Ok = Imm >= 0 && Imm <= 10;
1677           break;
1678         }
1679         if (!Ok) {
1680           ErrInfo = "Invalid immediate";
1681           return false;
1682         }
1683       }
1684     }
1685   }
1686 
1687   const uint64_t TSFlags = Desc.TSFlags;
1688   if (RISCVII::hasMergeOp(TSFlags)) {
1689     unsigned OpIdx = RISCVII::getMergeOpNum(Desc);
1690     if (MI.findTiedOperandIdx(0) != OpIdx) {
1691       ErrInfo = "Merge op improperly tied";
1692       return false;
1693     }
1694   }
1695   if (RISCVII::hasVLOp(TSFlags)) {
1696     const MachineOperand &Op = MI.getOperand(RISCVII::getVLOpNum(Desc));
1697     if (!Op.isImm() && !Op.isReg())  {
1698       ErrInfo = "Invalid operand type for VL operand";
1699       return false;
1700     }
1701     if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
1702       const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1703       auto *RC = MRI.getRegClass(Op.getReg());
1704       if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
1705         ErrInfo = "Invalid register class for VL operand";
1706         return false;
1707       }
1708     }
1709     if (!RISCVII::hasSEWOp(TSFlags)) {
1710       ErrInfo = "VL operand w/o SEW operand?";
1711       return false;
1712     }
1713   }
1714   if (RISCVII::hasSEWOp(TSFlags)) {
1715     unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
1716     uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
1717     if (Log2SEW > 31) {
1718       ErrInfo = "Unexpected SEW value";
1719       return false;
1720     }
1721     unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
1722     if (!RISCVVType::isValidSEW(SEW)) {
1723       ErrInfo = "Unexpected SEW value";
1724       return false;
1725     }
1726   }
1727   if (RISCVII::hasVecPolicyOp(TSFlags)) {
1728     unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
1729     uint64_t Policy = MI.getOperand(OpIdx).getImm();
1730     if (Policy > (RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC)) {
1731       ErrInfo = "Invalid Policy Value";
1732       return false;
1733     }
1734     if (!RISCVII::hasVLOp(TSFlags)) {
1735       ErrInfo = "policy operand w/o VL operand?";
1736       return false;
1737     }
1738 
1739     // VecPolicy operands can only exist on instructions with passthru/merge
1740     // arguments. Note that not all arguments with passthru have vec policy
1741     // operands- some instructions have implicit policies.
1742     unsigned UseOpIdx;
1743     if (!MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
1744       ErrInfo = "policy operand w/o tied operand?";
1745       return false;
1746     }
1747   }
1748 
1749   return true;
1750 }
1751 
1752 // Return true if get the base operand, byte offset of an instruction and the
1753 // memory width. Width is the size of memory that is being loaded/stored.
1754 bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
1755     const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
1756     unsigned &Width, const TargetRegisterInfo *TRI) const {
1757   if (!LdSt.mayLoadOrStore())
1758     return false;
1759 
1760   // Here we assume the standard RISC-V ISA, which uses a base+offset
1761   // addressing mode. You'll need to relax these conditions to support custom
1762   // load/stores instructions.
1763   if (LdSt.getNumExplicitOperands() != 3)
1764     return false;
1765   if (!LdSt.getOperand(1).isReg() || !LdSt.getOperand(2).isImm())
1766     return false;
1767 
1768   if (!LdSt.hasOneMemOperand())
1769     return false;
1770 
1771   Width = (*LdSt.memoperands_begin())->getSize();
1772   BaseReg = &LdSt.getOperand(1);
1773   Offset = LdSt.getOperand(2).getImm();
1774   return true;
1775 }
1776 
1777 bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
1778     const MachineInstr &MIa, const MachineInstr &MIb) const {
1779   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1780   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1781 
1782   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1783       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1784     return false;
1785 
1786   // Retrieve the base register, offset from the base register and width. Width
1787   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4).  If
1788   // base registers are identical, and the offset of a lower memory access +
1789   // the width doesn't overlap the offset of a higher memory access,
1790   // then the memory accesses are different.
1791   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
1792   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1793   int64_t OffsetA = 0, OffsetB = 0;
1794   unsigned int WidthA = 0, WidthB = 0;
1795   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
1796       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
1797     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
1798       int LowOffset = std::min(OffsetA, OffsetB);
1799       int HighOffset = std::max(OffsetA, OffsetB);
1800       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1801       if (LowOffset + LowWidth <= HighOffset)
1802         return true;
1803     }
1804   }
1805   return false;
1806 }
1807 
1808 std::pair<unsigned, unsigned>
1809 RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
1810   const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
1811   return std::make_pair(TF & Mask, TF & ~Mask);
1812 }
1813 
1814 ArrayRef<std::pair<unsigned, const char *>>
1815 RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
1816   using namespace RISCVII;
1817   static const std::pair<unsigned, const char *> TargetFlags[] = {
1818       {MO_CALL, "riscv-call"},
1819       {MO_PLT, "riscv-plt"},
1820       {MO_LO, "riscv-lo"},
1821       {MO_HI, "riscv-hi"},
1822       {MO_PCREL_LO, "riscv-pcrel-lo"},
1823       {MO_PCREL_HI, "riscv-pcrel-hi"},
1824       {MO_GOT_HI, "riscv-got-hi"},
1825       {MO_TPREL_LO, "riscv-tprel-lo"},
1826       {MO_TPREL_HI, "riscv-tprel-hi"},
1827       {MO_TPREL_ADD, "riscv-tprel-add"},
1828       {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
1829       {MO_TLS_GD_HI, "riscv-tls-gd-hi"}};
1830   return ArrayRef(TargetFlags);
1831 }
1832 bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
1833     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
1834   const Function &F = MF.getFunction();
1835 
1836   // Can F be deduplicated by the linker? If it can, don't outline from it.
1837   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
1838     return false;
1839 
1840   // Don't outline from functions with section markings; the program could
1841   // expect that all the code is in the named section.
1842   if (F.hasSection())
1843     return false;
1844 
1845   // It's safe to outline from MF.
1846   return true;
1847 }
1848 
1849 bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
1850                                             unsigned &Flags) const {
1851   // More accurate safety checking is done in getOutliningCandidateInfo.
1852   return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
1853 }
1854 
1855 // Enum values indicating how an outlined call should be constructed.
1856 enum MachineOutlinerConstructionID {
1857   MachineOutlinerDefault
1858 };
1859 
1860 bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
1861     MachineFunction &MF) const {
1862   return MF.getFunction().hasMinSize();
1863 }
1864 
1865 outliner::OutlinedFunction RISCVInstrInfo::getOutliningCandidateInfo(
1866     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
1867 
1868   // First we need to filter out candidates where the X5 register (IE t0) can't
1869   // be used to setup the function call.
1870   auto CannotInsertCall = [](outliner::Candidate &C) {
1871     const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();
1872     return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
1873   };
1874 
1875   llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall);
1876 
1877   // If the sequence doesn't have enough candidates left, then we're done.
1878   if (RepeatedSequenceLocs.size() < 2)
1879     return outliner::OutlinedFunction();
1880 
1881   unsigned SequenceSize = 0;
1882 
1883   auto I = RepeatedSequenceLocs[0].front();
1884   auto E = std::next(RepeatedSequenceLocs[0].back());
1885   for (; I != E; ++I)
1886     SequenceSize += getInstSizeInBytes(*I);
1887 
1888   // call t0, function = 8 bytes.
1889   unsigned CallOverhead = 8;
1890   for (auto &C : RepeatedSequenceLocs)
1891     C.setCallInfo(MachineOutlinerDefault, CallOverhead);
1892 
1893   // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
1894   unsigned FrameOverhead = 4;
1895   if (RepeatedSequenceLocs[0]
1896           .getMF()
1897           ->getSubtarget<RISCVSubtarget>()
1898           .hasStdExtCOrZca())
1899     FrameOverhead = 2;
1900 
1901   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
1902                                     FrameOverhead, MachineOutlinerDefault);
1903 }
1904 
1905 outliner::InstrType
1906 RISCVInstrInfo::getOutliningType(MachineBasicBlock::iterator &MBBI,
1907                                  unsigned Flags) const {
1908   MachineInstr &MI = *MBBI;
1909   MachineBasicBlock *MBB = MI.getParent();
1910   const TargetRegisterInfo *TRI =
1911       MBB->getParent()->getSubtarget().getRegisterInfo();
1912   const auto &F = MI.getMF()->getFunction();
1913 
1914   // Positions generally can't safely be outlined.
1915   if (MI.isPosition()) {
1916     // We can manually strip out CFI instructions later.
1917     if (MI.isCFIInstruction())
1918       // If current function has exception handling code, we can't outline &
1919       // strip these CFI instructions since it may break .eh_frame section
1920       // needed in unwinding.
1921       return F.needsUnwindTableEntry() ? outliner::InstrType::Illegal
1922                                        : outliner::InstrType::Invisible;
1923 
1924     return outliner::InstrType::Illegal;
1925   }
1926 
1927   // Don't trust the user to write safe inline assembly.
1928   if (MI.isInlineAsm())
1929     return outliner::InstrType::Illegal;
1930 
1931   // We can't outline branches to other basic blocks.
1932   if (MI.isTerminator() && !MBB->succ_empty())
1933     return outliner::InstrType::Illegal;
1934 
1935   // We need support for tail calls to outlined functions before return
1936   // statements can be allowed.
1937   if (MI.isReturn())
1938     return outliner::InstrType::Illegal;
1939 
1940   // Don't allow modifying the X5 register which we use for return addresses for
1941   // these outlined functions.
1942   if (MI.modifiesRegister(RISCV::X5, TRI) ||
1943       MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5))
1944     return outliner::InstrType::Illegal;
1945 
1946   // Make sure the operands don't reference something unsafe.
1947   for (const auto &MO : MI.operands()) {
1948     if (MO.isMBB() || MO.isBlockAddress() || MO.isCPI() || MO.isJTI())
1949       return outliner::InstrType::Illegal;
1950 
1951     // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
1952     // if any possible.
1953     if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
1954         (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
1955          F.hasSection()))
1956       return outliner::InstrType::Illegal;
1957   }
1958 
1959   // Don't allow instructions which won't be materialized to impact outlining
1960   // analysis.
1961   if (MI.isMetaInstruction())
1962     return outliner::InstrType::Invisible;
1963 
1964   return outliner::InstrType::Legal;
1965 }
1966 
1967 void RISCVInstrInfo::buildOutlinedFrame(
1968     MachineBasicBlock &MBB, MachineFunction &MF,
1969     const outliner::OutlinedFunction &OF) const {
1970 
1971   // Strip out any CFI instructions
1972   bool Changed = true;
1973   while (Changed) {
1974     Changed = false;
1975     auto I = MBB.begin();
1976     auto E = MBB.end();
1977     for (; I != E; ++I) {
1978       if (I->isCFIInstruction()) {
1979         I->removeFromParent();
1980         Changed = true;
1981         break;
1982       }
1983     }
1984   }
1985 
1986   MBB.addLiveIn(RISCV::X5);
1987 
1988   // Add in a return instruction to the end of the outlined frame.
1989   MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))
1990       .addReg(RISCV::X0, RegState::Define)
1991       .addReg(RISCV::X5)
1992       .addImm(0));
1993 }
1994 
1995 MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
1996     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
1997     MachineFunction &MF, outliner::Candidate &C) const {
1998 
1999   // Add in a call instruction to the outlined function at the given location.
2000   It = MBB.insert(It,
2001                   BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
2002                       .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
2003                                         RISCVII::MO_CALL));
2004   return It;
2005 }
2006 
2007 // MIR printer helper function to annotate Operands with a comment.
2008 std::string RISCVInstrInfo::createMIROperandComment(
2009     const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
2010     const TargetRegisterInfo *TRI) const {
2011   // Print a generic comment for this operand if there is one.
2012   std::string GenericComment =
2013       TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
2014   if (!GenericComment.empty())
2015     return GenericComment;
2016 
2017   // If not, we must have an immediate operand.
2018   if (!Op.isImm())
2019     return std::string();
2020 
2021   std::string Comment;
2022   raw_string_ostream OS(Comment);
2023 
2024   uint64_t TSFlags = MI.getDesc().TSFlags;
2025 
2026   // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
2027   // operand of vector codegen pseudos.
2028   if ((MI.getOpcode() == RISCV::VSETVLI || MI.getOpcode() == RISCV::VSETIVLI ||
2029        MI.getOpcode() == RISCV::PseudoVSETVLI ||
2030        MI.getOpcode() == RISCV::PseudoVSETIVLI ||
2031        MI.getOpcode() == RISCV::PseudoVSETVLIX0) &&
2032       OpIdx == 2) {
2033     unsigned Imm = MI.getOperand(OpIdx).getImm();
2034     RISCVVType::printVType(Imm, OS);
2035   } else if (RISCVII::hasSEWOp(TSFlags) &&
2036              OpIdx == RISCVII::getSEWOpNum(MI.getDesc())) {
2037     unsigned Log2SEW = MI.getOperand(OpIdx).getImm();
2038     unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2039     assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
2040     OS << "e" << SEW;
2041   } else if (RISCVII::hasVecPolicyOp(TSFlags) &&
2042              OpIdx == RISCVII::getVecPolicyOpNum(MI.getDesc())) {
2043     unsigned Policy = MI.getOperand(OpIdx).getImm();
2044     assert(Policy <= (RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC) &&
2045            "Invalid Policy Value");
2046     OS << (Policy & RISCVII::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
2047        << (Policy & RISCVII::MASK_AGNOSTIC ? "ma" : "mu");
2048   }
2049 
2050   OS.flush();
2051   return Comment;
2052 }
2053 
2054 // clang-format off
2055 #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL)                                \
2056   RISCV::PseudoV##OP##_##TYPE##_##LMUL
2057 
2058 #define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE)                                    \
2059   CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1):                                       \
2060   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2):                                  \
2061   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4):                                  \
2062   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8)
2063 
2064 #define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE)                                   \
2065   CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2):                                      \
2066   case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE)
2067 
2068 #define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE)                                   \
2069   CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4):                                      \
2070   case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE)
2071 
2072 #define CASE_VFMA_OPCODE_LMULS(OP, TYPE)                                       \
2073   CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8):                                      \
2074   case CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE)
2075 
2076 #define CASE_VFMA_SPLATS(OP)                                                   \
2077   CASE_VFMA_OPCODE_LMULS_MF4(OP, VF16):                                        \
2078   case CASE_VFMA_OPCODE_LMULS_MF2(OP, VF32):                                   \
2079   case CASE_VFMA_OPCODE_LMULS_M1(OP, VF64)
2080 // clang-format on
2081 
2082 bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2083                                            unsigned &SrcOpIdx1,
2084                                            unsigned &SrcOpIdx2) const {
2085   const MCInstrDesc &Desc = MI.getDesc();
2086   if (!Desc.isCommutable())
2087     return false;
2088 
2089   switch (MI.getOpcode()) {
2090   case RISCV::PseudoCCMOVGPR:
2091     // Operands 4 and 5 are commutable.
2092     return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
2093   case CASE_VFMA_SPLATS(FMADD):
2094   case CASE_VFMA_SPLATS(FMSUB):
2095   case CASE_VFMA_SPLATS(FMACC):
2096   case CASE_VFMA_SPLATS(FMSAC):
2097   case CASE_VFMA_SPLATS(FNMADD):
2098   case CASE_VFMA_SPLATS(FNMSUB):
2099   case CASE_VFMA_SPLATS(FNMACC):
2100   case CASE_VFMA_SPLATS(FNMSAC):
2101   case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV):
2102   case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV):
2103   case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV):
2104   case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV):
2105   case CASE_VFMA_OPCODE_LMULS(MADD, VX):
2106   case CASE_VFMA_OPCODE_LMULS(NMSUB, VX):
2107   case CASE_VFMA_OPCODE_LMULS(MACC, VX):
2108   case CASE_VFMA_OPCODE_LMULS(NMSAC, VX):
2109   case CASE_VFMA_OPCODE_LMULS(MACC, VV):
2110   case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): {
2111     // If the tail policy is undisturbed we can't commute.
2112     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
2113     if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
2114       return false;
2115 
2116     // For these instructions we can only swap operand 1 and operand 3 by
2117     // changing the opcode.
2118     unsigned CommutableOpIdx1 = 1;
2119     unsigned CommutableOpIdx2 = 3;
2120     if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2121                               CommutableOpIdx2))
2122       return false;
2123     return true;
2124   }
2125   case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV):
2126   case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV):
2127   case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV):
2128   case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV):
2129   case CASE_VFMA_OPCODE_LMULS(MADD, VV):
2130   case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): {
2131     // If the tail policy is undisturbed we can't commute.
2132     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
2133     if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
2134       return false;
2135 
2136     // For these instructions we have more freedom. We can commute with the
2137     // other multiplicand or with the addend/subtrahend/minuend.
2138 
2139     // Any fixed operand must be from source 1, 2 or 3.
2140     if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
2141       return false;
2142     if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
2143       return false;
2144 
2145     // It both ops are fixed one must be the tied source.
2146     if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2147         SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
2148       return false;
2149 
2150     // Look for two different register operands assumed to be commutable
2151     // regardless of the FMA opcode. The FMA opcode is adjusted later if
2152     // needed.
2153     if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2154         SrcOpIdx2 == CommuteAnyOperandIndex) {
2155       // At least one of operands to be commuted is not specified and
2156       // this method is free to choose appropriate commutable operands.
2157       unsigned CommutableOpIdx1 = SrcOpIdx1;
2158       if (SrcOpIdx1 == SrcOpIdx2) {
2159         // Both of operands are not fixed. Set one of commutable
2160         // operands to the tied source.
2161         CommutableOpIdx1 = 1;
2162       } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
2163         // Only one of the operands is not fixed.
2164         CommutableOpIdx1 = SrcOpIdx2;
2165       }
2166 
2167       // CommutableOpIdx1 is well defined now. Let's choose another commutable
2168       // operand and assign its index to CommutableOpIdx2.
2169       unsigned CommutableOpIdx2;
2170       if (CommutableOpIdx1 != 1) {
2171         // If we haven't already used the tied source, we must use it now.
2172         CommutableOpIdx2 = 1;
2173       } else {
2174         Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
2175 
2176         // The commuted operands should have different registers.
2177         // Otherwise, the commute transformation does not change anything and
2178         // is useless. We use this as a hint to make our decision.
2179         if (Op1Reg != MI.getOperand(2).getReg())
2180           CommutableOpIdx2 = 2;
2181         else
2182           CommutableOpIdx2 = 3;
2183       }
2184 
2185       // Assign the found pair of commutable indices to SrcOpIdx1 and
2186       // SrcOpIdx2 to return those values.
2187       if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2188                                 CommutableOpIdx2))
2189         return false;
2190     }
2191 
2192     return true;
2193   }
2194   }
2195 
2196   return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2197 }
2198 
2199 #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL)               \
2200   case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL:                                \
2201     Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL;                             \
2202     break;
2203 
2204 #define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)                   \
2205   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1)                       \
2206   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2)                       \
2207   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4)                       \
2208   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
2209 
2210 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)                  \
2211   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                      \
2212   CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
2213 
2214 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)                  \
2215   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                      \
2216   CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
2217 
2218 #define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                      \
2219   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                      \
2220   CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
2221 
2222 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                           \
2223   CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VF16)                        \
2224   CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VF32)                        \
2225   CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VF64)
2226 
2227 MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2228                                                      bool NewMI,
2229                                                      unsigned OpIdx1,
2230                                                      unsigned OpIdx2) const {
2231   auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
2232     if (NewMI)
2233       return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
2234     return MI;
2235   };
2236 
2237   switch (MI.getOpcode()) {
2238   case RISCV::PseudoCCMOVGPR: {
2239     // CCMOV can be commuted by inverting the condition.
2240     auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
2241     CC = RISCVCC::getOppositeBranchCondition(CC);
2242     auto &WorkingMI = cloneIfNew(MI);
2243     WorkingMI.getOperand(3).setImm(CC);
2244     return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,
2245                                                    OpIdx1, OpIdx2);
2246   }
2247   case CASE_VFMA_SPLATS(FMACC):
2248   case CASE_VFMA_SPLATS(FMADD):
2249   case CASE_VFMA_SPLATS(FMSAC):
2250   case CASE_VFMA_SPLATS(FMSUB):
2251   case CASE_VFMA_SPLATS(FNMACC):
2252   case CASE_VFMA_SPLATS(FNMADD):
2253   case CASE_VFMA_SPLATS(FNMSAC):
2254   case CASE_VFMA_SPLATS(FNMSUB):
2255   case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV):
2256   case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV):
2257   case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV):
2258   case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV):
2259   case CASE_VFMA_OPCODE_LMULS(MADD, VX):
2260   case CASE_VFMA_OPCODE_LMULS(NMSUB, VX):
2261   case CASE_VFMA_OPCODE_LMULS(MACC, VX):
2262   case CASE_VFMA_OPCODE_LMULS(NMSAC, VX):
2263   case CASE_VFMA_OPCODE_LMULS(MACC, VV):
2264   case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): {
2265     // It only make sense to toggle these between clobbering the
2266     // addend/subtrahend/minuend one of the multiplicands.
2267     assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
2268     assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
2269     unsigned Opc;
2270     switch (MI.getOpcode()) {
2271       default:
2272         llvm_unreachable("Unexpected opcode");
2273       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
2274       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
2275       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
2276       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
2277       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
2278       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
2279       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
2280       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
2281       CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMACC, FMADD, VV)
2282       CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSAC, FMSUB, VV)
2283       CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMACC, FNMADD, VV)
2284       CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSAC, FNMSUB, VV)
2285       CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
2286       CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
2287       CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
2288       CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
2289       CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
2290       CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
2291     }
2292 
2293     auto &WorkingMI = cloneIfNew(MI);
2294     WorkingMI.setDesc(get(Opc));
2295     return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
2296                                                    OpIdx1, OpIdx2);
2297   }
2298   case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV):
2299   case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV):
2300   case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV):
2301   case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV):
2302   case CASE_VFMA_OPCODE_LMULS(MADD, VV):
2303   case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): {
2304     assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
2305     // If one of the operands, is the addend we need to change opcode.
2306     // Otherwise we're just swapping 2 of the multiplicands.
2307     if (OpIdx1 == 3 || OpIdx2 == 3) {
2308       unsigned Opc;
2309       switch (MI.getOpcode()) {
2310         default:
2311           llvm_unreachable("Unexpected opcode");
2312         CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMADD, FMACC, VV)
2313         CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSUB, FMSAC, VV)
2314         CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMADD, FNMACC, VV)
2315         CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSUB, FNMSAC, VV)
2316         CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
2317         CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
2318       }
2319 
2320       auto &WorkingMI = cloneIfNew(MI);
2321       WorkingMI.setDesc(get(Opc));
2322       return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
2323                                                      OpIdx1, OpIdx2);
2324     }
2325     // Let the default code handle it.
2326     break;
2327   }
2328   }
2329 
2330   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2331 }
2332 
2333 #undef CASE_VFMA_CHANGE_OPCODE_SPLATS
2334 #undef CASE_VFMA_CHANGE_OPCODE_LMULS
2335 #undef CASE_VFMA_CHANGE_OPCODE_COMMON
2336 #undef CASE_VFMA_SPLATS
2337 #undef CASE_VFMA_OPCODE_LMULS
2338 #undef CASE_VFMA_OPCODE_COMMON
2339 
2340 // clang-format off
2341 #define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL)                                    \
2342   RISCV::PseudoV##OP##_##LMUL##_TIED
2343 
2344 #define CASE_WIDEOP_OPCODE_LMULS_MF4(OP)                                       \
2345   CASE_WIDEOP_OPCODE_COMMON(OP, MF4):                                          \
2346   case CASE_WIDEOP_OPCODE_COMMON(OP, MF2):                                     \
2347   case CASE_WIDEOP_OPCODE_COMMON(OP, M1):                                      \
2348   case CASE_WIDEOP_OPCODE_COMMON(OP, M2):                                      \
2349   case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
2350 
2351 #define CASE_WIDEOP_OPCODE_LMULS(OP)                                           \
2352   CASE_WIDEOP_OPCODE_COMMON(OP, MF8):                                          \
2353   case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
2354 // clang-format on
2355 
2356 #define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL)                             \
2357   case RISCV::PseudoV##OP##_##LMUL##_TIED:                                     \
2358     NewOpc = RISCV::PseudoV##OP##_##LMUL;                                      \
2359     break;
2360 
2361 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)                                 \
2362   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4)                                    \
2363   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2)                                    \
2364   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1)                                     \
2365   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2)                                     \
2366   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
2367 
2368 #define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                    \
2369   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8)                                    \
2370   CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
2371 
2372 MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
2373                                                     LiveVariables *LV,
2374                                                     LiveIntervals *LIS) const {
2375   switch (MI.getOpcode()) {
2376   default:
2377     break;
2378   case CASE_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
2379   case CASE_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV):
2380   case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
2381   case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
2382   case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
2383   case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
2384     // If the tail policy is undisturbed we can't convert.
2385     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
2386            MI.getNumExplicitOperands() == 6);
2387     if ((MI.getOperand(5).getImm() & 1) == 0)
2388       return nullptr;
2389 
2390     // clang-format off
2391     unsigned NewOpc;
2392     switch (MI.getOpcode()) {
2393     default:
2394       llvm_unreachable("Unexpected opcode");
2395     CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV)
2396     CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV)
2397     CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
2398     CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
2399     CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
2400     CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
2401     }
2402     // clang-format on
2403 
2404     MachineBasicBlock &MBB = *MI.getParent();
2405     MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
2406                                   .add(MI.getOperand(0))
2407                                   .add(MI.getOperand(1))
2408                                   .add(MI.getOperand(2))
2409                                   .add(MI.getOperand(3))
2410                                   .add(MI.getOperand(4));
2411     MIB.copyImplicitOps(MI);
2412 
2413     if (LV) {
2414       unsigned NumOps = MI.getNumOperands();
2415       for (unsigned I = 1; I < NumOps; ++I) {
2416         MachineOperand &Op = MI.getOperand(I);
2417         if (Op.isReg() && Op.isKill())
2418           LV->replaceKillInstruction(Op.getReg(), MI, *MIB);
2419       }
2420     }
2421 
2422     if (LIS) {
2423       SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB);
2424 
2425       if (MI.getOperand(0).isEarlyClobber()) {
2426         // Use operand 1 was tied to early-clobber def operand 0, so its live
2427         // interval could have ended at an early-clobber slot. Now they are not
2428         // tied we need to update it to the normal register slot.
2429         LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg());
2430         LiveRange::Segment *S = LI.getSegmentContaining(Idx);
2431         if (S->end == Idx.getRegSlot(true))
2432           S->end = Idx.getRegSlot();
2433       }
2434     }
2435 
2436     return MIB;
2437   }
2438   }
2439 
2440   return nullptr;
2441 }
2442 
2443 #undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
2444 #undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
2445 #undef CASE_WIDEOP_OPCODE_LMULS
2446 #undef CASE_WIDEOP_OPCODE_COMMON
2447 
2448 void RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF,
2449                                            MachineBasicBlock &MBB,
2450                                            MachineBasicBlock::iterator II,
2451                                            const DebugLoc &DL, Register DestReg,
2452                                            int64_t Amount,
2453                                            MachineInstr::MIFlag Flag) const {
2454   assert(Amount > 0 && "There is no need to get VLEN scaled value.");
2455   assert(Amount % 8 == 0 &&
2456          "Reserve the stack by the multiple of one vector size.");
2457 
2458   MachineRegisterInfo &MRI = MF.getRegInfo();
2459   int64_t NumOfVReg = Amount / 8;
2460 
2461   BuildMI(MBB, II, DL, get(RISCV::PseudoReadVLENB), DestReg).setMIFlag(Flag);
2462   assert(isInt<32>(NumOfVReg) &&
2463          "Expect the number of vector registers within 32-bits.");
2464   if (isPowerOf2_32(NumOfVReg)) {
2465     uint32_t ShiftAmount = Log2_32(NumOfVReg);
2466     if (ShiftAmount == 0)
2467       return;
2468     BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
2469         .addReg(DestReg, RegState::Kill)
2470         .addImm(ShiftAmount)
2471         .setMIFlag(Flag);
2472   } else if (STI.hasStdExtZba() &&
2473              ((NumOfVReg % 3 == 0 && isPowerOf2_64(NumOfVReg / 3)) ||
2474               (NumOfVReg % 5 == 0 && isPowerOf2_64(NumOfVReg / 5)) ||
2475               (NumOfVReg % 9 == 0 && isPowerOf2_64(NumOfVReg / 9)))) {
2476     // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
2477     unsigned Opc;
2478     uint32_t ShiftAmount;
2479     if (NumOfVReg % 9 == 0) {
2480       Opc = RISCV::SH3ADD;
2481       ShiftAmount = Log2_64(NumOfVReg / 9);
2482     } else if (NumOfVReg % 5 == 0) {
2483       Opc = RISCV::SH2ADD;
2484       ShiftAmount = Log2_64(NumOfVReg / 5);
2485     } else if (NumOfVReg % 3 == 0) {
2486       Opc = RISCV::SH1ADD;
2487       ShiftAmount = Log2_64(NumOfVReg / 3);
2488     } else {
2489       llvm_unreachable("Unexpected number of vregs");
2490     }
2491     if (ShiftAmount)
2492       BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
2493           .addReg(DestReg, RegState::Kill)
2494           .addImm(ShiftAmount)
2495           .setMIFlag(Flag);
2496     BuildMI(MBB, II, DL, get(Opc), DestReg)
2497         .addReg(DestReg, RegState::Kill)
2498         .addReg(DestReg)
2499         .setMIFlag(Flag);
2500   } else if (isPowerOf2_32(NumOfVReg - 1)) {
2501     Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2502     uint32_t ShiftAmount = Log2_32(NumOfVReg - 1);
2503     BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
2504         .addReg(DestReg)
2505         .addImm(ShiftAmount)
2506         .setMIFlag(Flag);
2507     BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
2508         .addReg(ScaledRegister, RegState::Kill)
2509         .addReg(DestReg, RegState::Kill)
2510         .setMIFlag(Flag);
2511   } else if (isPowerOf2_32(NumOfVReg + 1)) {
2512     Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2513     uint32_t ShiftAmount = Log2_32(NumOfVReg + 1);
2514     BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
2515         .addReg(DestReg)
2516         .addImm(ShiftAmount)
2517         .setMIFlag(Flag);
2518     BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)
2519         .addReg(ScaledRegister, RegState::Kill)
2520         .addReg(DestReg, RegState::Kill)
2521         .setMIFlag(Flag);
2522   } else {
2523     Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2524     movImm(MBB, II, DL, N, NumOfVReg, Flag);
2525     if (!STI.hasStdExtM() && !STI.hasStdExtZmmul())
2526       MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
2527           MF.getFunction(),
2528           "M- or Zmmul-extension must be enabled to calculate the vscaled size/"
2529           "offset."});
2530     BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)
2531         .addReg(DestReg, RegState::Kill)
2532         .addReg(N, RegState::Kill)
2533         .setMIFlag(Flag);
2534   }
2535 }
2536 
2537 // Checks if all users only demand the lower \p OrigBits of the original
2538 // instruction's result.
2539 // TODO: handle multiple interdependent transformations
2540 bool RISCVInstrInfo::hasAllNBitUsers(const MachineInstr &OrigMI,
2541                                      const MachineRegisterInfo &MRI,
2542                                      unsigned OrigBits) const {
2543 
2544   SmallSet<std::pair<const MachineInstr *, unsigned>, 4> Visited;
2545   SmallVector<std::pair<const MachineInstr *, unsigned>, 4> Worklist;
2546 
2547   Worklist.push_back(std::make_pair(&OrigMI, OrigBits));
2548 
2549   while (!Worklist.empty()) {
2550     auto P = Worklist.pop_back_val();
2551     const MachineInstr *MI = P.first;
2552     unsigned Bits = P.second;
2553 
2554     if (!Visited.insert(P).second)
2555       continue;
2556 
2557     // Only handle instructions with one def.
2558     if (MI->getNumExplicitDefs() != 1)
2559       return false;
2560 
2561     for (auto &UserOp : MRI.use_operands(MI->getOperand(0).getReg())) {
2562       const MachineInstr *UserMI = UserOp.getParent();
2563       unsigned OpIdx = UserMI->getOperandNo(&UserOp);
2564 
2565       switch (UserMI->getOpcode()) {
2566       default:
2567         return false;
2568 
2569       case RISCV::ADDIW:
2570       case RISCV::ADDW:
2571       case RISCV::DIVUW:
2572       case RISCV::DIVW:
2573       case RISCV::MULW:
2574       case RISCV::REMUW:
2575       case RISCV::REMW:
2576       case RISCV::SLLIW:
2577       case RISCV::SLLW:
2578       case RISCV::SRAIW:
2579       case RISCV::SRAW:
2580       case RISCV::SRLIW:
2581       case RISCV::SRLW:
2582       case RISCV::SUBW:
2583       case RISCV::ROLW:
2584       case RISCV::RORW:
2585       case RISCV::RORIW:
2586       case RISCV::CLZW:
2587       case RISCV::CTZW:
2588       case RISCV::CPOPW:
2589       case RISCV::SLLI_UW:
2590       case RISCV::FMV_W_X:
2591       case RISCV::FCVT_H_W:
2592       case RISCV::FCVT_H_WU:
2593       case RISCV::FCVT_S_W:
2594       case RISCV::FCVT_S_WU:
2595       case RISCV::FCVT_D_W:
2596       case RISCV::FCVT_D_WU:
2597         if (Bits >= 32)
2598           break;
2599         return false;
2600       case RISCV::SEXT_B:
2601       case RISCV::PACKH:
2602         if (Bits >= 8)
2603           break;
2604         return false;
2605       case RISCV::SEXT_H:
2606       case RISCV::FMV_H_X:
2607       case RISCV::ZEXT_H_RV32:
2608       case RISCV::ZEXT_H_RV64:
2609       case RISCV::PACKW:
2610         if (Bits >= 16)
2611           break;
2612         return false;
2613 
2614       case RISCV::PACK:
2615         if (Bits >= (STI.getXLen() / 2))
2616           break;
2617         return false;
2618 
2619       case RISCV::SRLI: {
2620         // If we are shifting right by less than Bits, and users don't demand
2621         // any bits that were shifted into [Bits-1:0], then we can consider this
2622         // as an N-Bit user.
2623         unsigned ShAmt = UserMI->getOperand(2).getImm();
2624         if (Bits > ShAmt) {
2625           Worklist.push_back(std::make_pair(UserMI, Bits - ShAmt));
2626           break;
2627         }
2628         return false;
2629       }
2630 
2631       // these overwrite higher input bits, otherwise the lower word of output
2632       // depends only on the lower word of input. So check their uses read W.
2633       case RISCV::SLLI:
2634         if (Bits >= (STI.getXLen() - UserMI->getOperand(2).getImm()))
2635           break;
2636         Worklist.push_back(std::make_pair(UserMI, Bits));
2637         break;
2638       case RISCV::ANDI: {
2639         uint64_t Imm = UserMI->getOperand(2).getImm();
2640         if (Bits >= (unsigned)llvm::bit_width(Imm))
2641           break;
2642         Worklist.push_back(std::make_pair(UserMI, Bits));
2643         break;
2644       }
2645       case RISCV::ORI: {
2646         uint64_t Imm = UserMI->getOperand(2).getImm();
2647         if (Bits >= (unsigned)llvm::bit_width<uint64_t>(~Imm))
2648           break;
2649         Worklist.push_back(std::make_pair(UserMI, Bits));
2650         break;
2651       }
2652 
2653       case RISCV::SLL:
2654       case RISCV::BSET:
2655       case RISCV::BCLR:
2656       case RISCV::BINV:
2657         // Operand 2 is the shift amount which uses log2(xlen) bits.
2658         if (OpIdx == 2) {
2659           if (Bits >= Log2_32(STI.getXLen()))
2660             break;
2661           return false;
2662         }
2663         Worklist.push_back(std::make_pair(UserMI, Bits));
2664         break;
2665 
2666       case RISCV::SRA:
2667       case RISCV::SRL:
2668       case RISCV::ROL:
2669       case RISCV::ROR:
2670         // Operand 2 is the shift amount which uses 6 bits.
2671         if (OpIdx == 2 && Bits >= Log2_32(STI.getXLen()))
2672           break;
2673         return false;
2674 
2675       case RISCV::ADD_UW:
2676       case RISCV::SH1ADD_UW:
2677       case RISCV::SH2ADD_UW:
2678       case RISCV::SH3ADD_UW:
2679         // Operand 1 is implicitly zero extended.
2680         if (OpIdx == 1 && Bits >= 32)
2681           break;
2682         Worklist.push_back(std::make_pair(UserMI, Bits));
2683         break;
2684 
2685       case RISCV::BEXTI:
2686         if (UserMI->getOperand(2).getImm() >= Bits)
2687           return false;
2688         break;
2689 
2690       case RISCV::SB:
2691         // The first argument is the value to store.
2692         if (OpIdx == 0 && Bits >= 8)
2693           break;
2694         return false;
2695       case RISCV::SH:
2696         // The first argument is the value to store.
2697         if (OpIdx == 0 && Bits >= 16)
2698           break;
2699         return false;
2700       case RISCV::SW:
2701         // The first argument is the value to store.
2702         if (OpIdx == 0 && Bits >= 32)
2703           break;
2704         return false;
2705 
2706       // For these, lower word of output in these operations, depends only on
2707       // the lower word of input. So, we check all uses only read lower word.
2708       case RISCV::COPY:
2709       case RISCV::PHI:
2710 
2711       case RISCV::ADD:
2712       case RISCV::ADDI:
2713       case RISCV::AND:
2714       case RISCV::MUL:
2715       case RISCV::OR:
2716       case RISCV::SUB:
2717       case RISCV::XOR:
2718       case RISCV::XORI:
2719 
2720       case RISCV::ANDN:
2721       case RISCV::BREV8:
2722       case RISCV::CLMUL:
2723       case RISCV::ORC_B:
2724       case RISCV::ORN:
2725       case RISCV::SH1ADD:
2726       case RISCV::SH2ADD:
2727       case RISCV::SH3ADD:
2728       case RISCV::XNOR:
2729       case RISCV::BSETI:
2730       case RISCV::BCLRI:
2731       case RISCV::BINVI:
2732         Worklist.push_back(std::make_pair(UserMI, Bits));
2733         break;
2734 
2735       case RISCV::PseudoCCMOVGPR:
2736         // Either operand 4 or operand 5 is returned by this instruction. If
2737         // only the lower word of the result is used, then only the lower word
2738         // of operand 4 and 5 is used.
2739         if (OpIdx != 4 && OpIdx != 5)
2740           return false;
2741         Worklist.push_back(std::make_pair(UserMI, Bits));
2742         break;
2743 
2744       case RISCV::VT_MASKC:
2745       case RISCV::VT_MASKCN:
2746         if (OpIdx != 1)
2747           return false;
2748         Worklist.push_back(std::make_pair(UserMI, Bits));
2749         break;
2750       }
2751     }
2752   }
2753 
2754   return true;
2755 }
2756 
2757 // Returns true if this is the sext.w pattern, addiw rd, rs1, 0.
2758 bool RISCV::isSEXT_W(const MachineInstr &MI) {
2759   return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
2760          MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0;
2761 }
2762 
2763 // Returns true if this is the zext.w pattern, adduw rd, rs1, x0.
2764 bool RISCV::isZEXT_W(const MachineInstr &MI) {
2765   return MI.getOpcode() == RISCV::ADD_UW && MI.getOperand(1).isReg() &&
2766          MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0;
2767 }
2768 
2769 // Returns true if this is the zext.b pattern, andi rd, rs1, 255.
2770 bool RISCV::isZEXT_B(const MachineInstr &MI) {
2771   return MI.getOpcode() == RISCV::ANDI && MI.getOperand(1).isReg() &&
2772          MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 255;
2773 }
2774 
2775 static bool isRVVWholeLoadStore(unsigned Opcode) {
2776   switch (Opcode) {
2777   default:
2778     return false;
2779   case RISCV::VS1R_V:
2780   case RISCV::VS2R_V:
2781   case RISCV::VS4R_V:
2782   case RISCV::VS8R_V:
2783   case RISCV::VL1RE8_V:
2784   case RISCV::VL2RE8_V:
2785   case RISCV::VL4RE8_V:
2786   case RISCV::VL8RE8_V:
2787   case RISCV::VL1RE16_V:
2788   case RISCV::VL2RE16_V:
2789   case RISCV::VL4RE16_V:
2790   case RISCV::VL8RE16_V:
2791   case RISCV::VL1RE32_V:
2792   case RISCV::VL2RE32_V:
2793   case RISCV::VL4RE32_V:
2794   case RISCV::VL8RE32_V:
2795   case RISCV::VL1RE64_V:
2796   case RISCV::VL2RE64_V:
2797   case RISCV::VL4RE64_V:
2798   case RISCV::VL8RE64_V:
2799     return true;
2800   }
2801 }
2802 
2803 bool RISCV::isRVVSpill(const MachineInstr &MI) {
2804   // RVV lacks any support for immediate addressing for stack addresses, so be
2805   // conservative.
2806   unsigned Opcode = MI.getOpcode();
2807   if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
2808       !isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
2809     return false;
2810   return true;
2811 }
2812 
2813 std::optional<std::pair<unsigned, unsigned>>
2814 RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
2815   switch (Opcode) {
2816   default:
2817     return std::nullopt;
2818   case RISCV::PseudoVSPILL2_M1:
2819   case RISCV::PseudoVRELOAD2_M1:
2820     return std::make_pair(2u, 1u);
2821   case RISCV::PseudoVSPILL2_M2:
2822   case RISCV::PseudoVRELOAD2_M2:
2823     return std::make_pair(2u, 2u);
2824   case RISCV::PseudoVSPILL2_M4:
2825   case RISCV::PseudoVRELOAD2_M4:
2826     return std::make_pair(2u, 4u);
2827   case RISCV::PseudoVSPILL3_M1:
2828   case RISCV::PseudoVRELOAD3_M1:
2829     return std::make_pair(3u, 1u);
2830   case RISCV::PseudoVSPILL3_M2:
2831   case RISCV::PseudoVRELOAD3_M2:
2832     return std::make_pair(3u, 2u);
2833   case RISCV::PseudoVSPILL4_M1:
2834   case RISCV::PseudoVRELOAD4_M1:
2835     return std::make_pair(4u, 1u);
2836   case RISCV::PseudoVSPILL4_M2:
2837   case RISCV::PseudoVRELOAD4_M2:
2838     return std::make_pair(4u, 2u);
2839   case RISCV::PseudoVSPILL5_M1:
2840   case RISCV::PseudoVRELOAD5_M1:
2841     return std::make_pair(5u, 1u);
2842   case RISCV::PseudoVSPILL6_M1:
2843   case RISCV::PseudoVRELOAD6_M1:
2844     return std::make_pair(6u, 1u);
2845   case RISCV::PseudoVSPILL7_M1:
2846   case RISCV::PseudoVRELOAD7_M1:
2847     return std::make_pair(7u, 1u);
2848   case RISCV::PseudoVSPILL8_M1:
2849   case RISCV::PseudoVRELOAD8_M1:
2850     return std::make_pair(8u, 1u);
2851   }
2852 }
2853 
2854 bool RISCV::isFaultFirstLoad(const MachineInstr &MI) {
2855   return MI.getNumExplicitDefs() == 2 && MI.modifiesRegister(RISCV::VL) &&
2856          !MI.isInlineAsm();
2857 }
2858 
2859 bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
2860   int16_t MI1FrmOpIdx =
2861       RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
2862   int16_t MI2FrmOpIdx =
2863       RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);
2864   if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
2865     return false;
2866   MachineOperand FrmOp1 = MI1.getOperand(MI1FrmOpIdx);
2867   MachineOperand FrmOp2 = MI2.getOperand(MI2FrmOpIdx);
2868   return FrmOp1.getImm() == FrmOp2.getImm();
2869 }
2870