xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelLowering.cpp (revision 3e8eb5c7f4909209c042403ddee340b2ee7003a5) 
1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineJumpTableInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/IntrinsicsRISCV.h"
35 #include "llvm/IR/PatternMatch.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/KnownBits.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/raw_ostream.h"
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "riscv-lower"
45 
46 STATISTIC(NumTailCalls, "Number of tail calls");
47 
48 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
49                                          const RISCVSubtarget &STI)
50     : TargetLowering(TM), Subtarget(STI) {
51 
52   if (Subtarget.isRV32E())
53     report_fatal_error("Codegen not yet implemented for RV32E");
54 
55   RISCVABI::ABI ABI = Subtarget.getTargetABI();
56   assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
57 
58   if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
59       !Subtarget.hasStdExtF()) {
60     errs() << "Hard-float 'f' ABI can't be used for a target that "
61                 "doesn't support the F instruction set extension (ignoring "
62                           "target-abi)\n";
63     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
64   } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
65              !Subtarget.hasStdExtD()) {
66     errs() << "Hard-float 'd' ABI can't be used for a target that "
67               "doesn't support the D instruction set extension (ignoring "
68               "target-abi)\n";
69     ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
70   }
71 
72   switch (ABI) {
73   default:
74     report_fatal_error("Don't know how to lower this ABI");
75   case RISCVABI::ABI_ILP32:
76   case RISCVABI::ABI_ILP32F:
77   case RISCVABI::ABI_ILP32D:
78   case RISCVABI::ABI_LP64:
79   case RISCVABI::ABI_LP64F:
80   case RISCVABI::ABI_LP64D:
81     break;
82   }
83 
84   MVT XLenVT = Subtarget.getXLenVT();
85 
86   // Set up the register classes.
87   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
88 
89   if (Subtarget.hasStdExtZfh())
90     addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
91   if (Subtarget.hasStdExtF())
92     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
93   if (Subtarget.hasStdExtD())
94     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
95 
96   static const MVT::SimpleValueType BoolVecVTs[] = {
97       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
98       MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
99   static const MVT::SimpleValueType IntVecVTs[] = {
100       MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
101       MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
102       MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
103       MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
104       MVT::nxv4i64, MVT::nxv8i64};
105   static const MVT::SimpleValueType F16VecVTs[] = {
106       MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
107       MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
108   static const MVT::SimpleValueType F32VecVTs[] = {
109       MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
110   static const MVT::SimpleValueType F64VecVTs[] = {
111       MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
112 
113   if (Subtarget.hasVInstructions()) {
114     auto addRegClassForRVV = [this](MVT VT) {
115       unsigned Size = VT.getSizeInBits().getKnownMinValue();
116       assert(Size <= 512 && isPowerOf2_32(Size));
117       const TargetRegisterClass *RC;
118       if (Size <= 64)
119         RC = &RISCV::VRRegClass;
120       else if (Size == 128)
121         RC = &RISCV::VRM2RegClass;
122       else if (Size == 256)
123         RC = &RISCV::VRM4RegClass;
124       else
125         RC = &RISCV::VRM8RegClass;
126 
127       addRegisterClass(VT, RC);
128     };
129 
130     for (MVT VT : BoolVecVTs)
131       addRegClassForRVV(VT);
132     for (MVT VT : IntVecVTs) {
133       if (VT.getVectorElementType() == MVT::i64 &&
134           !Subtarget.hasVInstructionsI64())
135         continue;
136       addRegClassForRVV(VT);
137     }
138 
139     if (Subtarget.hasVInstructionsF16())
140       for (MVT VT : F16VecVTs)
141         addRegClassForRVV(VT);
142 
143     if (Subtarget.hasVInstructionsF32())
144       for (MVT VT : F32VecVTs)
145         addRegClassForRVV(VT);
146 
147     if (Subtarget.hasVInstructionsF64())
148       for (MVT VT : F64VecVTs)
149         addRegClassForRVV(VT);
150 
151     if (Subtarget.useRVVForFixedLengthVectors()) {
152       auto addRegClassForFixedVectors = [this](MVT VT) {
153         MVT ContainerVT = getContainerForFixedLengthVector(VT);
154         unsigned RCID = getRegClassIDForVecVT(ContainerVT);
155         const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
156         addRegisterClass(VT, TRI.getRegClass(RCID));
157       };
158       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
159         if (useRVVForFixedLengthVectorVT(VT))
160           addRegClassForFixedVectors(VT);
161 
162       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
163         if (useRVVForFixedLengthVectorVT(VT))
164           addRegClassForFixedVectors(VT);
165     }
166   }
167 
168   // Compute derived properties from the register classes.
169   computeRegisterProperties(STI.getRegisterInfo());
170 
171   setStackPointerRegisterToSaveRestore(RISCV::X2);
172 
173   for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
174     setLoadExtAction(N, XLenVT, MVT::i1, Promote);
175 
176   // TODO: add all necessary setOperationAction calls.
177   setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
178 
179   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
180   setOperationAction(ISD::BR_CC, XLenVT, Expand);
181   setOperationAction(ISD::BRCOND, MVT::Other, Custom);
182   setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
183 
184   setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
185   setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
186 
187   setOperationAction(ISD::VASTART, MVT::Other, Custom);
188   setOperationAction(ISD::VAARG, MVT::Other, Expand);
189   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
190   setOperationAction(ISD::VAEND, MVT::Other, Expand);
191 
192   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
193   if (!Subtarget.hasStdExtZbb()) {
194     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
195     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
196   }
197 
198   if (Subtarget.is64Bit()) {
199     setOperationAction(ISD::ADD, MVT::i32, Custom);
200     setOperationAction(ISD::SUB, MVT::i32, Custom);
201     setOperationAction(ISD::SHL, MVT::i32, Custom);
202     setOperationAction(ISD::SRA, MVT::i32, Custom);
203     setOperationAction(ISD::SRL, MVT::i32, Custom);
204 
205     setOperationAction(ISD::UADDO, MVT::i32, Custom);
206     setOperationAction(ISD::USUBO, MVT::i32, Custom);
207     setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
208     setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
209   } else {
210     setLibcallName(RTLIB::SHL_I128, nullptr);
211     setLibcallName(RTLIB::SRL_I128, nullptr);
212     setLibcallName(RTLIB::SRA_I128, nullptr);
213     setLibcallName(RTLIB::MUL_I128, nullptr);
214     setLibcallName(RTLIB::MULO_I64, nullptr);
215   }
216 
217   if (!Subtarget.hasStdExtM()) {
218     setOperationAction(ISD::MUL, XLenVT, Expand);
219     setOperationAction(ISD::MULHS, XLenVT, Expand);
220     setOperationAction(ISD::MULHU, XLenVT, Expand);
221     setOperationAction(ISD::SDIV, XLenVT, Expand);
222     setOperationAction(ISD::UDIV, XLenVT, Expand);
223     setOperationAction(ISD::SREM, XLenVT, Expand);
224     setOperationAction(ISD::UREM, XLenVT, Expand);
225   } else {
226     if (Subtarget.is64Bit()) {
227       setOperationAction(ISD::MUL, MVT::i32, Custom);
228       setOperationAction(ISD::MUL, MVT::i128, Custom);
229 
230       setOperationAction(ISD::SDIV, MVT::i8, Custom);
231       setOperationAction(ISD::UDIV, MVT::i8, Custom);
232       setOperationAction(ISD::UREM, MVT::i8, Custom);
233       setOperationAction(ISD::SDIV, MVT::i16, Custom);
234       setOperationAction(ISD::UDIV, MVT::i16, Custom);
235       setOperationAction(ISD::UREM, MVT::i16, Custom);
236       setOperationAction(ISD::SDIV, MVT::i32, Custom);
237       setOperationAction(ISD::UDIV, MVT::i32, Custom);
238       setOperationAction(ISD::UREM, MVT::i32, Custom);
239     } else {
240       setOperationAction(ISD::MUL, MVT::i64, Custom);
241     }
242   }
243 
244   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
245   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
246   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
247   setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
248 
249   setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
250   setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
251   setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
252 
253   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
254       Subtarget.hasStdExtZbkb()) {
255     if (Subtarget.is64Bit()) {
256       setOperationAction(ISD::ROTL, MVT::i32, Custom);
257       setOperationAction(ISD::ROTR, MVT::i32, Custom);
258     }
259   } else {
260     setOperationAction(ISD::ROTL, XLenVT, Expand);
261     setOperationAction(ISD::ROTR, XLenVT, Expand);
262   }
263 
264   if (Subtarget.hasStdExtZbp()) {
265     // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
266     // more combining.
267     setOperationAction(ISD::BITREVERSE, XLenVT,   Custom);
268     setOperationAction(ISD::BSWAP,      XLenVT,   Custom);
269     setOperationAction(ISD::BITREVERSE, MVT::i8,  Custom);
270     // BSWAP i8 doesn't exist.
271     setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
272     setOperationAction(ISD::BSWAP,      MVT::i16, Custom);
273 
274     if (Subtarget.is64Bit()) {
275       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
276       setOperationAction(ISD::BSWAP,      MVT::i32, Custom);
277     }
278   } else {
279     // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
280     // pattern match it directly in isel.
281     setOperationAction(ISD::BSWAP, XLenVT,
282                        (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb())
283                            ? Legal
284                            : Expand);
285     // Zbkb can use rev8+brev8 to implement bitreverse.
286     setOperationAction(ISD::BITREVERSE, XLenVT,
287                        Subtarget.hasStdExtZbkb() ? Custom : Expand);
288   }
289 
290   if (Subtarget.hasStdExtZbb()) {
291     setOperationAction(ISD::SMIN, XLenVT, Legal);
292     setOperationAction(ISD::SMAX, XLenVT, Legal);
293     setOperationAction(ISD::UMIN, XLenVT, Legal);
294     setOperationAction(ISD::UMAX, XLenVT, Legal);
295 
296     if (Subtarget.is64Bit()) {
297       setOperationAction(ISD::CTTZ, MVT::i32, Custom);
298       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
299       setOperationAction(ISD::CTLZ, MVT::i32, Custom);
300       setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
301     }
302   } else {
303     setOperationAction(ISD::CTTZ, XLenVT, Expand);
304     setOperationAction(ISD::CTLZ, XLenVT, Expand);
305     setOperationAction(ISD::CTPOP, XLenVT, Expand);
306   }
307 
308   if (Subtarget.hasStdExtZbt()) {
309     setOperationAction(ISD::FSHL, XLenVT, Custom);
310     setOperationAction(ISD::FSHR, XLenVT, Custom);
311     setOperationAction(ISD::SELECT, XLenVT, Legal);
312 
313     if (Subtarget.is64Bit()) {
314       setOperationAction(ISD::FSHL, MVT::i32, Custom);
315       setOperationAction(ISD::FSHR, MVT::i32, Custom);
316     }
317   } else {
318     setOperationAction(ISD::SELECT, XLenVT, Custom);
319   }
320 
321   static const ISD::CondCode FPCCToExpand[] = {
322       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
323       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
324       ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
325 
326   static const ISD::NodeType FPOpToExpand[] = {
327       ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,
328       ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
329 
330   if (Subtarget.hasStdExtZfh())
331     setOperationAction(ISD::BITCAST, MVT::i16, Custom);
332 
333   if (Subtarget.hasStdExtZfh()) {
334     setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
335     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
336     setOperationAction(ISD::LRINT, MVT::f16, Legal);
337     setOperationAction(ISD::LLRINT, MVT::f16, Legal);
338     setOperationAction(ISD::LROUND, MVT::f16, Legal);
339     setOperationAction(ISD::LLROUND, MVT::f16, Legal);
340     setOperationAction(ISD::STRICT_LRINT, MVT::f16, Legal);
341     setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Legal);
342     setOperationAction(ISD::STRICT_LROUND, MVT::f16, Legal);
343     setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Legal);
344     setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal);
345     setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal);
346     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal);
347     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal);
348     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal);
349     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
350     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
351     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal);
352     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Legal);
353     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Legal);
354     for (auto CC : FPCCToExpand)
355       setCondCodeAction(CC, MVT::f16, Expand);
356     setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
357     setOperationAction(ISD::SELECT, MVT::f16, Custom);
358     setOperationAction(ISD::BR_CC, MVT::f16, Expand);
359 
360     setOperationAction(ISD::FREM,       MVT::f16, Promote);
361     setOperationAction(ISD::FCEIL,      MVT::f16, Promote);
362     setOperationAction(ISD::FFLOOR,     MVT::f16, Promote);
363     setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
364     setOperationAction(ISD::FRINT,      MVT::f16, Promote);
365     setOperationAction(ISD::FROUND,     MVT::f16, Promote);
366     setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
367     setOperationAction(ISD::FTRUNC,     MVT::f16, Promote);
368     setOperationAction(ISD::FPOW,       MVT::f16, Promote);
369     setOperationAction(ISD::FPOWI,      MVT::f16, Promote);
370     setOperationAction(ISD::FCOS,       MVT::f16, Promote);
371     setOperationAction(ISD::FSIN,       MVT::f16, Promote);
372     setOperationAction(ISD::FSINCOS,    MVT::f16, Promote);
373     setOperationAction(ISD::FEXP,       MVT::f16, Promote);
374     setOperationAction(ISD::FEXP2,      MVT::f16, Promote);
375     setOperationAction(ISD::FLOG,       MVT::f16, Promote);
376     setOperationAction(ISD::FLOG2,      MVT::f16, Promote);
377     setOperationAction(ISD::FLOG10,     MVT::f16, Promote);
378 
379     // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
380     // complete support for all operations in LegalizeDAG.
381 
382     // We need to custom promote this.
383     if (Subtarget.is64Bit())
384       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
385   }
386 
387   if (Subtarget.hasStdExtF()) {
388     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
389     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
390     setOperationAction(ISD::LRINT, MVT::f32, Legal);
391     setOperationAction(ISD::LLRINT, MVT::f32, Legal);
392     setOperationAction(ISD::LROUND, MVT::f32, Legal);
393     setOperationAction(ISD::LLROUND, MVT::f32, Legal);
394     setOperationAction(ISD::STRICT_LRINT, MVT::f32, Legal);
395     setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Legal);
396     setOperationAction(ISD::STRICT_LROUND, MVT::f32, Legal);
397     setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Legal);
398     setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
399     setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
400     setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
401     setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
402     setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
403     setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
404     setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
405     setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
406     for (auto CC : FPCCToExpand)
407       setCondCodeAction(CC, MVT::f32, Expand);
408     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
409     setOperationAction(ISD::SELECT, MVT::f32, Custom);
410     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
411     for (auto Op : FPOpToExpand)
412       setOperationAction(Op, MVT::f32, Expand);
413     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
414     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
415   }
416 
417   if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
418     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
419 
420   if (Subtarget.hasStdExtD()) {
421     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
422     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
423     setOperationAction(ISD::LRINT, MVT::f64, Legal);
424     setOperationAction(ISD::LLRINT, MVT::f64, Legal);
425     setOperationAction(ISD::LROUND, MVT::f64, Legal);
426     setOperationAction(ISD::LLROUND, MVT::f64, Legal);
427     setOperationAction(ISD::STRICT_LRINT, MVT::f64, Legal);
428     setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Legal);
429     setOperationAction(ISD::STRICT_LROUND, MVT::f64, Legal);
430     setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Legal);
431     setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
432     setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
433     setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
434     setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
435     setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
436     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
437     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
438     setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
439     setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
440     setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
441     for (auto CC : FPCCToExpand)
442       setCondCodeAction(CC, MVT::f64, Expand);
443     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
444     setOperationAction(ISD::SELECT, MVT::f64, Custom);
445     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
446     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
447     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
448     for (auto Op : FPOpToExpand)
449       setOperationAction(Op, MVT::f64, Expand);
450     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
451     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
452   }
453 
454   if (Subtarget.is64Bit()) {
455     setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
456     setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
457     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
458     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
459   }
460 
461   if (Subtarget.hasStdExtF()) {
462     setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
463     setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
464 
465     setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal);
466     setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal);
467     setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal);
468     setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal);
469 
470     setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
471     setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
472   }
473 
474   setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
475   setOperationAction(ISD::BlockAddress, XLenVT, Custom);
476   setOperationAction(ISD::ConstantPool, XLenVT, Custom);
477   setOperationAction(ISD::JumpTable, XLenVT, Custom);
478 
479   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
480 
481   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
482   // Unfortunately this can't be determined just from the ISA naming string.
483   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
484                      Subtarget.is64Bit() ? Legal : Custom);
485 
486   setOperationAction(ISD::TRAP, MVT::Other, Legal);
487   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
488   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
489   if (Subtarget.is64Bit())
490     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
491 
492   if (Subtarget.hasStdExtA()) {
493     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
494     setMinCmpXchgSizeInBits(32);
495   } else {
496     setMaxAtomicSizeInBitsSupported(0);
497   }
498 
499   setBooleanContents(ZeroOrOneBooleanContent);
500 
501   if (Subtarget.hasVInstructions()) {
502     setBooleanVectorContents(ZeroOrOneBooleanContent);
503 
504     setOperationAction(ISD::VSCALE, XLenVT, Custom);
505 
506     // RVV intrinsics may have illegal operands.
507     // We also need to custom legalize vmv.x.s.
508     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
509     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
510     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
511     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
512     if (Subtarget.is64Bit()) {
513       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
514     } else {
515       setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
516       setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
517     }
518 
519     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
520     setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
521 
522     static const unsigned IntegerVPOps[] = {
523         ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,
524         ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
525         ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,
526         ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,
527         ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
528         ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
529         ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
530         ISD::VP_MERGE,       ISD::VP_SELECT};
531 
532     static const unsigned FloatingPointVPOps[] = {
533         ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,
534         ISD::VP_FDIV,        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
535         ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
536         ISD::VP_SELECT};
537 
538     if (!Subtarget.is64Bit()) {
539       // We must custom-lower certain vXi64 operations on RV32 due to the vector
540       // element type being illegal.
541       setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
542       setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
543 
544       setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
545       setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
546       setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
547       setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
548       setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
549       setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
550       setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
551       setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
552 
553       setOperationAction(ISD::VP_REDUCE_ADD, MVT::i64, Custom);
554       setOperationAction(ISD::VP_REDUCE_AND, MVT::i64, Custom);
555       setOperationAction(ISD::VP_REDUCE_OR, MVT::i64, Custom);
556       setOperationAction(ISD::VP_REDUCE_XOR, MVT::i64, Custom);
557       setOperationAction(ISD::VP_REDUCE_SMAX, MVT::i64, Custom);
558       setOperationAction(ISD::VP_REDUCE_SMIN, MVT::i64, Custom);
559       setOperationAction(ISD::VP_REDUCE_UMAX, MVT::i64, Custom);
560       setOperationAction(ISD::VP_REDUCE_UMIN, MVT::i64, Custom);
561     }
562 
563     for (MVT VT : BoolVecVTs) {
564       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
565 
566       // Mask VTs are custom-expanded into a series of standard nodes
567       setOperationAction(ISD::TRUNCATE, VT, Custom);
568       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
569       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
570       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
571 
572       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
573       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
574 
575       setOperationAction(ISD::SELECT, VT, Custom);
576       setOperationAction(ISD::SELECT_CC, VT, Expand);
577       setOperationAction(ISD::VSELECT, VT, Expand);
578       setOperationAction(ISD::VP_MERGE, VT, Expand);
579       setOperationAction(ISD::VP_SELECT, VT, Expand);
580 
581       setOperationAction(ISD::VP_AND, VT, Custom);
582       setOperationAction(ISD::VP_OR, VT, Custom);
583       setOperationAction(ISD::VP_XOR, VT, Custom);
584 
585       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
586       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
587       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
588 
589       setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
590       setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
591       setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
592 
593       // RVV has native int->float & float->int conversions where the
594       // element type sizes are within one power-of-two of each other. Any
595       // wider distances between type sizes have to be lowered as sequences
596       // which progressively narrow the gap in stages.
597       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
598       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
599       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
600       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
601 
602       // Expand all extending loads to types larger than this, and truncating
603       // stores from types larger than this.
604       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
605         setTruncStoreAction(OtherVT, VT, Expand);
606         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
607         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
608         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
609       }
610     }
611 
612     for (MVT VT : IntVecVTs) {
613       if (VT.getVectorElementType() == MVT::i64 &&
614           !Subtarget.hasVInstructionsI64())
615         continue;
616 
617       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
618       setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
619 
620       // Vectors implement MULHS/MULHU.
621       setOperationAction(ISD::SMUL_LOHI, VT, Expand);
622       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
623 
624       // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
625       if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV()) {
626         setOperationAction(ISD::MULHU, VT, Expand);
627         setOperationAction(ISD::MULHS, VT, Expand);
628       }
629 
630       setOperationAction(ISD::SMIN, VT, Legal);
631       setOperationAction(ISD::SMAX, VT, Legal);
632       setOperationAction(ISD::UMIN, VT, Legal);
633       setOperationAction(ISD::UMAX, VT, Legal);
634 
635       setOperationAction(ISD::ROTL, VT, Expand);
636       setOperationAction(ISD::ROTR, VT, Expand);
637 
638       setOperationAction(ISD::CTTZ, VT, Expand);
639       setOperationAction(ISD::CTLZ, VT, Expand);
640       setOperationAction(ISD::CTPOP, VT, Expand);
641 
642       setOperationAction(ISD::BSWAP, VT, Expand);
643 
644       // Custom-lower extensions and truncations from/to mask types.
645       setOperationAction(ISD::ANY_EXTEND, VT, Custom);
646       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
647       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
648 
649       // RVV has native int->float & float->int conversions where the
650       // element type sizes are within one power-of-two of each other. Any
651       // wider distances between type sizes have to be lowered as sequences
652       // which progressively narrow the gap in stages.
653       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
654       setOperationAction(ISD::UINT_TO_FP, VT, Custom);
655       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
656       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
657 
658       setOperationAction(ISD::SADDSAT, VT, Legal);
659       setOperationAction(ISD::UADDSAT, VT, Legal);
660       setOperationAction(ISD::SSUBSAT, VT, Legal);
661       setOperationAction(ISD::USUBSAT, VT, Legal);
662 
663       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
664       // nodes which truncate by one power of two at a time.
665       setOperationAction(ISD::TRUNCATE, VT, Custom);
666 
667       // Custom-lower insert/extract operations to simplify patterns.
668       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
669       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
670 
671       // Custom-lower reduction operations to set up the corresponding custom
672       // nodes' operands.
673       setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
674       setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
675       setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
676       setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
677       setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
678       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
679       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
680       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
681 
682       for (unsigned VPOpc : IntegerVPOps)
683         setOperationAction(VPOpc, VT, Custom);
684 
685       setOperationAction(ISD::LOAD, VT, Custom);
686       setOperationAction(ISD::STORE, VT, Custom);
687 
688       setOperationAction(ISD::MLOAD, VT, Custom);
689       setOperationAction(ISD::MSTORE, VT, Custom);
690       setOperationAction(ISD::MGATHER, VT, Custom);
691       setOperationAction(ISD::MSCATTER, VT, Custom);
692 
693       setOperationAction(ISD::VP_LOAD, VT, Custom);
694       setOperationAction(ISD::VP_STORE, VT, Custom);
695       setOperationAction(ISD::VP_GATHER, VT, Custom);
696       setOperationAction(ISD::VP_SCATTER, VT, Custom);
697 
698       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
699       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
700       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
701 
702       setOperationAction(ISD::SELECT, VT, Custom);
703       setOperationAction(ISD::SELECT_CC, VT, Expand);
704 
705       setOperationAction(ISD::STEP_VECTOR, VT, Custom);
706       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
707 
708       for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
709         setTruncStoreAction(VT, OtherVT, Expand);
710         setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
711         setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
712         setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
713       }
714 
715       // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
716       // type that can represent the value exactly.
717       if (VT.getVectorElementType() != MVT::i64) {
718         MVT FloatEltVT =
719             VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
720         EVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
721         if (isTypeLegal(FloatVT)) {
722           setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
723           setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
724         }
725       }
726     }
727 
728     // Expand various CCs to best match the RVV ISA, which natively supports UNE
729     // but no other unordered comparisons, and supports all ordered comparisons
730     // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
731     // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
732     // and we pattern-match those back to the "original", swapping operands once
733     // more. This way we catch both operations and both "vf" and "fv" forms with
734     // fewer patterns.
735     static const ISD::CondCode VFPCCToExpand[] = {
736         ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
737         ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
738         ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
739     };
740 
741     // Sets common operation actions on RVV floating-point vector types.
742     const auto SetCommonVFPActions = [&](MVT VT) {
743       setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
744       // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
745       // sizes are within one power-of-two of each other. Therefore conversions
746       // between vXf16 and vXf64 must be lowered as sequences which convert via
747       // vXf32.
748       setOperationAction(ISD::FP_ROUND, VT, Custom);
749       setOperationAction(ISD::FP_EXTEND, VT, Custom);
750       // Custom-lower insert/extract operations to simplify patterns.
751       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
752       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
753       // Expand various condition codes (explained above).
754       for (auto CC : VFPCCToExpand)
755         setCondCodeAction(CC, VT, Expand);
756 
757       setOperationAction(ISD::FMINNUM, VT, Legal);
758       setOperationAction(ISD::FMAXNUM, VT, Legal);
759 
760       setOperationAction(ISD::FTRUNC, VT, Custom);
761       setOperationAction(ISD::FCEIL, VT, Custom);
762       setOperationAction(ISD::FFLOOR, VT, Custom);
763 
764       setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
765       setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
766       setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
767       setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
768 
769       setOperationAction(ISD::FCOPYSIGN, VT, Legal);
770 
771       setOperationAction(ISD::LOAD, VT, Custom);
772       setOperationAction(ISD::STORE, VT, Custom);
773 
774       setOperationAction(ISD::MLOAD, VT, Custom);
775       setOperationAction(ISD::MSTORE, VT, Custom);
776       setOperationAction(ISD::MGATHER, VT, Custom);
777       setOperationAction(ISD::MSCATTER, VT, Custom);
778 
779       setOperationAction(ISD::VP_LOAD, VT, Custom);
780       setOperationAction(ISD::VP_STORE, VT, Custom);
781       setOperationAction(ISD::VP_GATHER, VT, Custom);
782       setOperationAction(ISD::VP_SCATTER, VT, Custom);
783 
784       setOperationAction(ISD::SELECT, VT, Custom);
785       setOperationAction(ISD::SELECT_CC, VT, Expand);
786 
787       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
788       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
789       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
790 
791       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
792 
793       for (unsigned VPOpc : FloatingPointVPOps)
794         setOperationAction(VPOpc, VT, Custom);
795     };
796 
797     // Sets common extload/truncstore actions on RVV floating-point vector
798     // types.
799     const auto SetCommonVFPExtLoadTruncStoreActions =
800         [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
801           for (auto SmallVT : SmallerVTs) {
802             setTruncStoreAction(VT, SmallVT, Expand);
803             setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
804           }
805         };
806 
807     if (Subtarget.hasVInstructionsF16())
808       for (MVT VT : F16VecVTs)
809         SetCommonVFPActions(VT);
810 
811     for (MVT VT : F32VecVTs) {
812       if (Subtarget.hasVInstructionsF32())
813         SetCommonVFPActions(VT);
814       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
815     }
816 
817     for (MVT VT : F64VecVTs) {
818       if (Subtarget.hasVInstructionsF64())
819         SetCommonVFPActions(VT);
820       SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
821       SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
822     }
823 
824     if (Subtarget.useRVVForFixedLengthVectors()) {
825       for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
826         if (!useRVVForFixedLengthVectorVT(VT))
827           continue;
828 
829         // By default everything must be expanded.
830         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
831           setOperationAction(Op, VT, Expand);
832         for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
833           setTruncStoreAction(VT, OtherVT, Expand);
834           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
835           setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
836           setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
837         }
838 
839         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
840         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
841         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
842 
843         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
844         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
845 
846         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
847         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
848 
849         setOperationAction(ISD::LOAD, VT, Custom);
850         setOperationAction(ISD::STORE, VT, Custom);
851 
852         setOperationAction(ISD::SETCC, VT, Custom);
853 
854         setOperationAction(ISD::SELECT, VT, Custom);
855 
856         setOperationAction(ISD::TRUNCATE, VT, Custom);
857 
858         setOperationAction(ISD::BITCAST, VT, Custom);
859 
860         setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
861         setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
862         setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
863 
864         setOperationAction(ISD::VP_REDUCE_AND, VT, Custom);
865         setOperationAction(ISD::VP_REDUCE_OR, VT, Custom);
866         setOperationAction(ISD::VP_REDUCE_XOR, VT, Custom);
867 
868         setOperationAction(ISD::SINT_TO_FP, VT, Custom);
869         setOperationAction(ISD::UINT_TO_FP, VT, Custom);
870         setOperationAction(ISD::FP_TO_SINT, VT, Custom);
871         setOperationAction(ISD::FP_TO_UINT, VT, Custom);
872 
873         // Operations below are different for between masks and other vectors.
874         if (VT.getVectorElementType() == MVT::i1) {
875           setOperationAction(ISD::VP_AND, VT, Custom);
876           setOperationAction(ISD::VP_OR, VT, Custom);
877           setOperationAction(ISD::VP_XOR, VT, Custom);
878           setOperationAction(ISD::AND, VT, Custom);
879           setOperationAction(ISD::OR, VT, Custom);
880           setOperationAction(ISD::XOR, VT, Custom);
881           continue;
882         }
883 
884         // Use SPLAT_VECTOR to prevent type legalization from destroying the
885         // splats when type legalizing i64 scalar on RV32.
886         // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
887         // improvements first.
888         if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
889           setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
890           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
891         }
892 
893         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
894         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
895 
896         setOperationAction(ISD::MLOAD, VT, Custom);
897         setOperationAction(ISD::MSTORE, VT, Custom);
898         setOperationAction(ISD::MGATHER, VT, Custom);
899         setOperationAction(ISD::MSCATTER, VT, Custom);
900 
901         setOperationAction(ISD::VP_LOAD, VT, Custom);
902         setOperationAction(ISD::VP_STORE, VT, Custom);
903         setOperationAction(ISD::VP_GATHER, VT, Custom);
904         setOperationAction(ISD::VP_SCATTER, VT, Custom);
905 
906         setOperationAction(ISD::ADD, VT, Custom);
907         setOperationAction(ISD::MUL, VT, Custom);
908         setOperationAction(ISD::SUB, VT, Custom);
909         setOperationAction(ISD::AND, VT, Custom);
910         setOperationAction(ISD::OR, VT, Custom);
911         setOperationAction(ISD::XOR, VT, Custom);
912         setOperationAction(ISD::SDIV, VT, Custom);
913         setOperationAction(ISD::SREM, VT, Custom);
914         setOperationAction(ISD::UDIV, VT, Custom);
915         setOperationAction(ISD::UREM, VT, Custom);
916         setOperationAction(ISD::SHL, VT, Custom);
917         setOperationAction(ISD::SRA, VT, Custom);
918         setOperationAction(ISD::SRL, VT, Custom);
919 
920         setOperationAction(ISD::SMIN, VT, Custom);
921         setOperationAction(ISD::SMAX, VT, Custom);
922         setOperationAction(ISD::UMIN, VT, Custom);
923         setOperationAction(ISD::UMAX, VT, Custom);
924         setOperationAction(ISD::ABS,  VT, Custom);
925 
926         // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
927         if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV()) {
928           setOperationAction(ISD::MULHS, VT, Custom);
929           setOperationAction(ISD::MULHU, VT, Custom);
930         }
931 
932         setOperationAction(ISD::SADDSAT, VT, Custom);
933         setOperationAction(ISD::UADDSAT, VT, Custom);
934         setOperationAction(ISD::SSUBSAT, VT, Custom);
935         setOperationAction(ISD::USUBSAT, VT, Custom);
936 
937         setOperationAction(ISD::VSELECT, VT, Custom);
938         setOperationAction(ISD::SELECT_CC, VT, Expand);
939 
940         setOperationAction(ISD::ANY_EXTEND, VT, Custom);
941         setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
942         setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
943 
944         // Custom-lower reduction operations to set up the corresponding custom
945         // nodes' operands.
946         setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
947         setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
948         setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
949         setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
950         setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
951 
952         for (unsigned VPOpc : IntegerVPOps)
953           setOperationAction(VPOpc, VT, Custom);
954 
955         // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if we have a floating point
956         // type that can represent the value exactly.
957         if (VT.getVectorElementType() != MVT::i64) {
958           MVT FloatEltVT =
959               VT.getVectorElementType() == MVT::i32 ? MVT::f64 : MVT::f32;
960           EVT FloatVT =
961               MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
962           if (isTypeLegal(FloatVT)) {
963             setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
964             setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Custom);
965           }
966         }
967       }
968 
969       for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
970         if (!useRVVForFixedLengthVectorVT(VT))
971           continue;
972 
973         // By default everything must be expanded.
974         for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
975           setOperationAction(Op, VT, Expand);
976         for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
977           setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
978           setTruncStoreAction(VT, OtherVT, Expand);
979         }
980 
981         // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
982         setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
983         setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
984 
985         setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
986         setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
987         setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
988         setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
989         setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
990 
991         setOperationAction(ISD::LOAD, VT, Custom);
992         setOperationAction(ISD::STORE, VT, Custom);
993         setOperationAction(ISD::MLOAD, VT, Custom);
994         setOperationAction(ISD::MSTORE, VT, Custom);
995         setOperationAction(ISD::MGATHER, VT, Custom);
996         setOperationAction(ISD::MSCATTER, VT, Custom);
997 
998         setOperationAction(ISD::VP_LOAD, VT, Custom);
999         setOperationAction(ISD::VP_STORE, VT, Custom);
1000         setOperationAction(ISD::VP_GATHER, VT, Custom);
1001         setOperationAction(ISD::VP_SCATTER, VT, Custom);
1002 
1003         setOperationAction(ISD::FADD, VT, Custom);
1004         setOperationAction(ISD::FSUB, VT, Custom);
1005         setOperationAction(ISD::FMUL, VT, Custom);
1006         setOperationAction(ISD::FDIV, VT, Custom);
1007         setOperationAction(ISD::FNEG, VT, Custom);
1008         setOperationAction(ISD::FABS, VT, Custom);
1009         setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1010         setOperationAction(ISD::FSQRT, VT, Custom);
1011         setOperationAction(ISD::FMA, VT, Custom);
1012         setOperationAction(ISD::FMINNUM, VT, Custom);
1013         setOperationAction(ISD::FMAXNUM, VT, Custom);
1014 
1015         setOperationAction(ISD::FP_ROUND, VT, Custom);
1016         setOperationAction(ISD::FP_EXTEND, VT, Custom);
1017 
1018         setOperationAction(ISD::FTRUNC, VT, Custom);
1019         setOperationAction(ISD::FCEIL, VT, Custom);
1020         setOperationAction(ISD::FFLOOR, VT, Custom);
1021 
1022         for (auto CC : VFPCCToExpand)
1023           setCondCodeAction(CC, VT, Expand);
1024 
1025         setOperationAction(ISD::VSELECT, VT, Custom);
1026         setOperationAction(ISD::SELECT, VT, Custom);
1027         setOperationAction(ISD::SELECT_CC, VT, Expand);
1028 
1029         setOperationAction(ISD::BITCAST, VT, Custom);
1030 
1031         setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1032         setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1033         setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1034         setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1035 
1036         for (unsigned VPOpc : FloatingPointVPOps)
1037           setOperationAction(VPOpc, VT, Custom);
1038       }
1039 
1040       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1041       setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1042       setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1043       setOperationAction(ISD::BITCAST, MVT::i32, Custom);
1044       setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1045       if (Subtarget.hasStdExtZfh())
1046         setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1047       if (Subtarget.hasStdExtF())
1048         setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1049       if (Subtarget.hasStdExtD())
1050         setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1051     }
1052   }
1053 
1054   // Function alignments.
1055   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
1056   setMinFunctionAlignment(FunctionAlignment);
1057   setPrefFunctionAlignment(FunctionAlignment);
1058 
1059   setMinimumJumpTableEntries(5);
1060 
1061   // Jumps are expensive, compared to logic
1062   setJumpIsExpensive();
1063 
1064   setTargetDAGCombine(ISD::ADD);
1065   setTargetDAGCombine(ISD::SUB);
1066   setTargetDAGCombine(ISD::AND);
1067   setTargetDAGCombine(ISD::OR);
1068   setTargetDAGCombine(ISD::XOR);
1069   setTargetDAGCombine(ISD::ANY_EXTEND);
1070   if (Subtarget.hasStdExtF()) {
1071     setTargetDAGCombine(ISD::ZERO_EXTEND);
1072     setTargetDAGCombine(ISD::FP_TO_SINT);
1073     setTargetDAGCombine(ISD::FP_TO_UINT);
1074     setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
1075     setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
1076   }
1077   if (Subtarget.hasVInstructions()) {
1078     setTargetDAGCombine(ISD::FCOPYSIGN);
1079     setTargetDAGCombine(ISD::MGATHER);
1080     setTargetDAGCombine(ISD::MSCATTER);
1081     setTargetDAGCombine(ISD::VP_GATHER);
1082     setTargetDAGCombine(ISD::VP_SCATTER);
1083     setTargetDAGCombine(ISD::SRA);
1084     setTargetDAGCombine(ISD::SRL);
1085     setTargetDAGCombine(ISD::SHL);
1086     setTargetDAGCombine(ISD::STORE);
1087   }
1088 
1089   setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1090   setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1091 }
1092 
1093 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1094                                             LLVMContext &Context,
1095                                             EVT VT) const {
1096   if (!VT.isVector())
1097     return getPointerTy(DL);
1098   if (Subtarget.hasVInstructions() &&
1099       (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1100     return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1101   return VT.changeVectorElementTypeToInteger();
1102 }
1103 
1104 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1105   return Subtarget.getXLenVT();
1106 }
1107 
1108 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1109                                              const CallInst &I,
1110                                              MachineFunction &MF,
1111                                              unsigned Intrinsic) const {
1112   auto &DL = I.getModule()->getDataLayout();
1113   switch (Intrinsic) {
1114   default:
1115     return false;
1116   case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1117   case Intrinsic::riscv_masked_atomicrmw_add_i32:
1118   case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1119   case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1120   case Intrinsic::riscv_masked_atomicrmw_max_i32:
1121   case Intrinsic::riscv_masked_atomicrmw_min_i32:
1122   case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1123   case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1124   case Intrinsic::riscv_masked_cmpxchg_i32:
1125     Info.opc = ISD::INTRINSIC_W_CHAIN;
1126     Info.memVT = MVT::i32;
1127     Info.ptrVal = I.getArgOperand(0);
1128     Info.offset = 0;
1129     Info.align = Align(4);
1130     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1131                  MachineMemOperand::MOVolatile;
1132     return true;
1133   case Intrinsic::riscv_masked_strided_load:
1134     Info.opc = ISD::INTRINSIC_W_CHAIN;
1135     Info.ptrVal = I.getArgOperand(1);
1136     Info.memVT = getValueType(DL, I.getType()->getScalarType());
1137     Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1138     Info.size = MemoryLocation::UnknownSize;
1139     Info.flags |= MachineMemOperand::MOLoad;
1140     return true;
1141   case Intrinsic::riscv_masked_strided_store:
1142     Info.opc = ISD::INTRINSIC_VOID;
1143     Info.ptrVal = I.getArgOperand(1);
1144     Info.memVT =
1145         getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1146     Info.align = Align(
1147         DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1148         8);
1149     Info.size = MemoryLocation::UnknownSize;
1150     Info.flags |= MachineMemOperand::MOStore;
1151     return true;
1152   }
1153 }
1154 
1155 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1156                                                 const AddrMode &AM, Type *Ty,
1157                                                 unsigned AS,
1158                                                 Instruction *I) const {
1159   // No global is ever allowed as a base.
1160   if (AM.BaseGV)
1161     return false;
1162 
1163   // Require a 12-bit signed offset.
1164   if (!isInt<12>(AM.BaseOffs))
1165     return false;
1166 
1167   switch (AM.Scale) {
1168   case 0: // "r+i" or just "i", depending on HasBaseReg.
1169     break;
1170   case 1:
1171     if (!AM.HasBaseReg) // allow "r+i".
1172       break;
1173     return false; // disallow "r+r" or "r+r+i".
1174   default:
1175     return false;
1176   }
1177 
1178   return true;
1179 }
1180 
1181 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1182   return isInt<12>(Imm);
1183 }
1184 
1185 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1186   return isInt<12>(Imm);
1187 }
1188 
1189 // On RV32, 64-bit integers are split into their high and low parts and held
1190 // in two different registers, so the trunc is free since the low register can
1191 // just be used.
1192 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1193   if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1194     return false;
1195   unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1196   unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1197   return (SrcBits == 64 && DestBits == 32);
1198 }
1199 
1200 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1201   if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
1202       !SrcVT.isInteger() || !DstVT.isInteger())
1203     return false;
1204   unsigned SrcBits = SrcVT.getSizeInBits();
1205   unsigned DestBits = DstVT.getSizeInBits();
1206   return (SrcBits == 64 && DestBits == 32);
1207 }
1208 
1209 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1210   // Zexts are free if they can be combined with a load.
1211   // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1212   // poorly with type legalization of compares preferring sext.
1213   if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1214     EVT MemVT = LD->getMemoryVT();
1215     if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1216         (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1217          LD->getExtensionType() == ISD::ZEXTLOAD))
1218       return true;
1219   }
1220 
1221   return TargetLowering::isZExtFree(Val, VT2);
1222 }
1223 
1224 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1225   return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1226 }
1227 
1228 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
1229   return Subtarget.hasStdExtZbb();
1230 }
1231 
1232 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
1233   return Subtarget.hasStdExtZbb();
1234 }
1235 
1236 bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1237   EVT VT = Y.getValueType();
1238 
1239   // FIXME: Support vectors once we have tests.
1240   if (VT.isVector())
1241     return false;
1242 
1243   return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp() ||
1244           Subtarget.hasStdExtZbkb()) &&
1245          !isa<ConstantSDNode>(Y);
1246 }
1247 
1248 /// Check if sinking \p I's operands to I's basic block is profitable, because
1249 /// the operands can be folded into a target instruction, e.g.
1250 /// splats of scalars can fold into vector instructions.
1251 bool RISCVTargetLowering::shouldSinkOperands(
1252     Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1253   using namespace llvm::PatternMatch;
1254 
1255   if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1256     return false;
1257 
1258   auto IsSinker = [&](Instruction *I, int Operand) {
1259     switch (I->getOpcode()) {
1260     case Instruction::Add:
1261     case Instruction::Sub:
1262     case Instruction::Mul:
1263     case Instruction::And:
1264     case Instruction::Or:
1265     case Instruction::Xor:
1266     case Instruction::FAdd:
1267     case Instruction::FSub:
1268     case Instruction::FMul:
1269     case Instruction::FDiv:
1270     case Instruction::ICmp:
1271     case Instruction::FCmp:
1272       return true;
1273     case Instruction::Shl:
1274     case Instruction::LShr:
1275     case Instruction::AShr:
1276     case Instruction::UDiv:
1277     case Instruction::SDiv:
1278     case Instruction::URem:
1279     case Instruction::SRem:
1280       return Operand == 1;
1281     case Instruction::Call:
1282       if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1283         switch (II->getIntrinsicID()) {
1284         case Intrinsic::fma:
1285           return Operand == 0 || Operand == 1;
1286         // FIXME: Our patterns can only match vx/vf instructions when the splat
1287         // it on the RHS, because TableGen doesn't recognize our VP operations
1288         // as commutative.
1289         case Intrinsic::vp_add:
1290         case Intrinsic::vp_mul:
1291         case Intrinsic::vp_and:
1292         case Intrinsic::vp_or:
1293         case Intrinsic::vp_xor:
1294         case Intrinsic::vp_fadd:
1295         case Intrinsic::vp_fmul:
1296         case Intrinsic::vp_shl:
1297         case Intrinsic::vp_lshr:
1298         case Intrinsic::vp_ashr:
1299         case Intrinsic::vp_udiv:
1300         case Intrinsic::vp_sdiv:
1301         case Intrinsic::vp_urem:
1302         case Intrinsic::vp_srem:
1303           return Operand == 1;
1304         // ... with the exception of vp.sub/vp.fsub/vp.fdiv, which have
1305         // explicit patterns for both LHS and RHS (as 'vr' versions).
1306         case Intrinsic::vp_sub:
1307         case Intrinsic::vp_fsub:
1308         case Intrinsic::vp_fdiv:
1309           return Operand == 0 || Operand == 1;
1310         default:
1311           return false;
1312         }
1313       }
1314       return false;
1315     default:
1316       return false;
1317     }
1318   };
1319 
1320   for (auto OpIdx : enumerate(I->operands())) {
1321     if (!IsSinker(I, OpIdx.index()))
1322       continue;
1323 
1324     Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1325     // Make sure we are not already sinking this operand
1326     if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1327       continue;
1328 
1329     // We are looking for a splat that can be sunk.
1330     if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1331                              m_Undef(), m_ZeroMask())))
1332       continue;
1333 
1334     // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1335     // and vector registers
1336     for (Use &U : Op->uses()) {
1337       Instruction *Insn = cast<Instruction>(U.getUser());
1338       if (!IsSinker(Insn, U.getOperandNo()))
1339         return false;
1340     }
1341 
1342     Ops.push_back(&Op->getOperandUse(0));
1343     Ops.push_back(&OpIdx.value());
1344   }
1345   return true;
1346 }
1347 
1348 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1349                                        bool ForCodeSize) const {
1350   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1351   if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1352     return false;
1353   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1354     return false;
1355   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1356     return false;
1357   return Imm.isZero();
1358 }
1359 
1360 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1361   return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1362          (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1363          (VT == MVT::f64 && Subtarget.hasStdExtD());
1364 }
1365 
1366 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1367                                                       CallingConv::ID CC,
1368                                                       EVT VT) const {
1369   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1370   // We might still end up using a GPR but that will be decided based on ABI.
1371   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1372   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1373     return MVT::f32;
1374 
1375   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1376 }
1377 
1378 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1379                                                            CallingConv::ID CC,
1380                                                            EVT VT) const {
1381   // Use f32 to pass f16 if it is legal and Zfh is not enabled.
1382   // We might still end up using a GPR but that will be decided based on ABI.
1383   // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin.
1384   if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1385     return 1;
1386 
1387   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1388 }
1389 
1390 // Changes the condition code and swaps operands if necessary, so the SetCC
1391 // operation matches one of the comparisons supported directly by branches
1392 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1393 // with 1/-1.
1394 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1395                                     ISD::CondCode &CC, SelectionDAG &DAG) {
1396   // Convert X > -1 to X >= 0.
1397   if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1398     RHS = DAG.getConstant(0, DL, RHS.getValueType());
1399     CC = ISD::SETGE;
1400     return;
1401   }
1402   // Convert X < 1 to 0 >= X.
1403   if (CC == ISD::SETLT && isOneConstant(RHS)) {
1404     RHS = LHS;
1405     LHS = DAG.getConstant(0, DL, RHS.getValueType());
1406     CC = ISD::SETGE;
1407     return;
1408   }
1409 
1410   switch (CC) {
1411   default:
1412     break;
1413   case ISD::SETGT:
1414   case ISD::SETLE:
1415   case ISD::SETUGT:
1416   case ISD::SETULE:
1417     CC = ISD::getSetCCSwappedOperands(CC);
1418     std::swap(LHS, RHS);
1419     break;
1420   }
1421 }
1422 
1423 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1424   assert(VT.isScalableVector() && "Expecting a scalable vector type");
1425   unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1426   if (VT.getVectorElementType() == MVT::i1)
1427     KnownSize *= 8;
1428 
1429   switch (KnownSize) {
1430   default:
1431     llvm_unreachable("Invalid LMUL.");
1432   case 8:
1433     return RISCVII::VLMUL::LMUL_F8;
1434   case 16:
1435     return RISCVII::VLMUL::LMUL_F4;
1436   case 32:
1437     return RISCVII::VLMUL::LMUL_F2;
1438   case 64:
1439     return RISCVII::VLMUL::LMUL_1;
1440   case 128:
1441     return RISCVII::VLMUL::LMUL_2;
1442   case 256:
1443     return RISCVII::VLMUL::LMUL_4;
1444   case 512:
1445     return RISCVII::VLMUL::LMUL_8;
1446   }
1447 }
1448 
1449 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1450   switch (LMul) {
1451   default:
1452     llvm_unreachable("Invalid LMUL.");
1453   case RISCVII::VLMUL::LMUL_F8:
1454   case RISCVII::VLMUL::LMUL_F4:
1455   case RISCVII::VLMUL::LMUL_F2:
1456   case RISCVII::VLMUL::LMUL_1:
1457     return RISCV::VRRegClassID;
1458   case RISCVII::VLMUL::LMUL_2:
1459     return RISCV::VRM2RegClassID;
1460   case RISCVII::VLMUL::LMUL_4:
1461     return RISCV::VRM4RegClassID;
1462   case RISCVII::VLMUL::LMUL_8:
1463     return RISCV::VRM8RegClassID;
1464   }
1465 }
1466 
1467 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1468   RISCVII::VLMUL LMUL = getLMUL(VT);
1469   if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1470       LMUL == RISCVII::VLMUL::LMUL_F4 ||
1471       LMUL == RISCVII::VLMUL::LMUL_F2 ||
1472       LMUL == RISCVII::VLMUL::LMUL_1) {
1473     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1474                   "Unexpected subreg numbering");
1475     return RISCV::sub_vrm1_0 + Index;
1476   }
1477   if (LMUL == RISCVII::VLMUL::LMUL_2) {
1478     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1479                   "Unexpected subreg numbering");
1480     return RISCV::sub_vrm2_0 + Index;
1481   }
1482   if (LMUL == RISCVII::VLMUL::LMUL_4) {
1483     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1484                   "Unexpected subreg numbering");
1485     return RISCV::sub_vrm4_0 + Index;
1486   }
1487   llvm_unreachable("Invalid vector type.");
1488 }
1489 
1490 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1491   if (VT.getVectorElementType() == MVT::i1)
1492     return RISCV::VRRegClassID;
1493   return getRegClassIDForLMUL(getLMUL(VT));
1494 }
1495 
1496 // Attempt to decompose a subvector insert/extract between VecVT and
1497 // SubVecVT via subregister indices. Returns the subregister index that
1498 // can perform the subvector insert/extract with the given element index, as
1499 // well as the index corresponding to any leftover subvectors that must be
1500 // further inserted/extracted within the register class for SubVecVT.
1501 std::pair<unsigned, unsigned>
1502 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1503     MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1504     const RISCVRegisterInfo *TRI) {
1505   static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1506                  RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1507                  RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1508                 "Register classes not ordered");
1509   unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1510   unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1511   // Try to compose a subregister index that takes us from the incoming
1512   // LMUL>1 register class down to the outgoing one. At each step we half
1513   // the LMUL:
1514   //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1515   // Note that this is not guaranteed to find a subregister index, such as
1516   // when we are extracting from one VR type to another.
1517   unsigned SubRegIdx = RISCV::NoSubRegister;
1518   for (const unsigned RCID :
1519        {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1520     if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1521       VecVT = VecVT.getHalfNumVectorElementsVT();
1522       bool IsHi =
1523           InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1524       SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1525                                             getSubregIndexByMVT(VecVT, IsHi));
1526       if (IsHi)
1527         InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1528     }
1529   return {SubRegIdx, InsertExtractIdx};
1530 }
1531 
1532 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1533 // stores for those types.
1534 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1535   return !Subtarget.useRVVForFixedLengthVectors() ||
1536          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1537 }
1538 
1539 bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1540   if (ScalarTy->isPointerTy())
1541     return true;
1542 
1543   if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1544       ScalarTy->isIntegerTy(32))
1545     return true;
1546 
1547   if (ScalarTy->isIntegerTy(64))
1548     return Subtarget.hasVInstructionsI64();
1549 
1550   if (ScalarTy->isHalfTy())
1551     return Subtarget.hasVInstructionsF16();
1552   if (ScalarTy->isFloatTy())
1553     return Subtarget.hasVInstructionsF32();
1554   if (ScalarTy->isDoubleTy())
1555     return Subtarget.hasVInstructionsF64();
1556 
1557   return false;
1558 }
1559 
1560 static SDValue getVLOperand(SDValue Op) {
1561   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
1562           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
1563          "Unexpected opcode");
1564   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
1565   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
1566   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
1567       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1568   if (!II)
1569     return SDValue();
1570   return Op.getOperand(II->VLOperand + 1 + HasChain);
1571 }
1572 
1573 static bool useRVVForFixedLengthVectorVT(MVT VT,
1574                                          const RISCVSubtarget &Subtarget) {
1575   assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1576   if (!Subtarget.useRVVForFixedLengthVectors())
1577     return false;
1578 
1579   // We only support a set of vector types with a consistent maximum fixed size
1580   // across all supported vector element types to avoid legalization issues.
1581   // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1582   // fixed-length vector type we support is 1024 bytes.
1583   if (VT.getFixedSizeInBits() > 1024 * 8)
1584     return false;
1585 
1586   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1587 
1588   MVT EltVT = VT.getVectorElementType();
1589 
1590   // Don't use RVV for vectors we cannot scalarize if required.
1591   switch (EltVT.SimpleTy) {
1592   // i1 is supported but has different rules.
1593   default:
1594     return false;
1595   case MVT::i1:
1596     // Masks can only use a single register.
1597     if (VT.getVectorNumElements() > MinVLen)
1598       return false;
1599     MinVLen /= 8;
1600     break;
1601   case MVT::i8:
1602   case MVT::i16:
1603   case MVT::i32:
1604     break;
1605   case MVT::i64:
1606     if (!Subtarget.hasVInstructionsI64())
1607       return false;
1608     break;
1609   case MVT::f16:
1610     if (!Subtarget.hasVInstructionsF16())
1611       return false;
1612     break;
1613   case MVT::f32:
1614     if (!Subtarget.hasVInstructionsF32())
1615       return false;
1616     break;
1617   case MVT::f64:
1618     if (!Subtarget.hasVInstructionsF64())
1619       return false;
1620     break;
1621   }
1622 
1623   // Reject elements larger than ELEN.
1624   if (EltVT.getSizeInBits() > Subtarget.getMaxELENForFixedLengthVectors())
1625     return false;
1626 
1627   unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1628   // Don't use RVV for types that don't fit.
1629   if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1630     return false;
1631 
1632   // TODO: Perhaps an artificial restriction, but worth having whilst getting
1633   // the base fixed length RVV support in place.
1634   if (!VT.isPow2VectorType())
1635     return false;
1636 
1637   return true;
1638 }
1639 
1640 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1641   return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1642 }
1643 
1644 // Return the largest legal scalable vector type that matches VT's element type.
1645 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1646                                             const RISCVSubtarget &Subtarget) {
1647   // This may be called before legal types are setup.
1648   assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1649           useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1650          "Expected legal fixed length vector!");
1651 
1652   unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1653   unsigned MaxELen = Subtarget.getMaxELENForFixedLengthVectors();
1654 
1655   MVT EltVT = VT.getVectorElementType();
1656   switch (EltVT.SimpleTy) {
1657   default:
1658     llvm_unreachable("unexpected element type for RVV container");
1659   case MVT::i1:
1660   case MVT::i8:
1661   case MVT::i16:
1662   case MVT::i32:
1663   case MVT::i64:
1664   case MVT::f16:
1665   case MVT::f32:
1666   case MVT::f64: {
1667     // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1668     // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1669     // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1670     unsigned NumElts =
1671         (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1672     NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1673     assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1674     return MVT::getScalableVectorVT(EltVT, NumElts);
1675   }
1676   }
1677 }
1678 
1679 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1680                                             const RISCVSubtarget &Subtarget) {
1681   return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1682                                           Subtarget);
1683 }
1684 
1685 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1686   return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1687 }
1688 
1689 // Grow V to consume an entire RVV register.
1690 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1691                                        const RISCVSubtarget &Subtarget) {
1692   assert(VT.isScalableVector() &&
1693          "Expected to convert into a scalable vector!");
1694   assert(V.getValueType().isFixedLengthVector() &&
1695          "Expected a fixed length vector operand!");
1696   SDLoc DL(V);
1697   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1698   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1699 }
1700 
1701 // Shrink V so it's just big enough to maintain a VT's worth of data.
1702 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1703                                          const RISCVSubtarget &Subtarget) {
1704   assert(VT.isFixedLengthVector() &&
1705          "Expected to convert into a fixed length vector!");
1706   assert(V.getValueType().isScalableVector() &&
1707          "Expected a scalable vector operand!");
1708   SDLoc DL(V);
1709   SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1710   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1711 }
1712 
1713 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1714 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1715 // the vector type that it is contained in.
1716 static std::pair<SDValue, SDValue>
1717 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1718                 const RISCVSubtarget &Subtarget) {
1719   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1720   MVT XLenVT = Subtarget.getXLenVT();
1721   SDValue VL = VecVT.isFixedLengthVector()
1722                    ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1723                    : DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT);
1724   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1725   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1726   return {Mask, VL};
1727 }
1728 
1729 // As above but assuming the given type is a scalable vector type.
1730 static std::pair<SDValue, SDValue>
1731 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1732                         const RISCVSubtarget &Subtarget) {
1733   assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1734   return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1735 }
1736 
1737 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1738 // of either is (currently) supported. This can get us into an infinite loop
1739 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1740 // as a ..., etc.
1741 // Until either (or both) of these can reliably lower any node, reporting that
1742 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1743 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1744 // which is not desirable.
1745 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1746     EVT VT, unsigned DefinedValues) const {
1747   return false;
1748 }
1749 
1750 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
1751   // Only splats are currently supported.
1752   if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
1753     return true;
1754 
1755   return false;
1756 }
1757 
1758 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
1759                                   const RISCVSubtarget &Subtarget) {
1760   // RISCV FP-to-int conversions saturate to the destination register size, but
1761   // don't produce 0 for nan. We can use a conversion instruction and fix the
1762   // nan case with a compare and a select.
1763   SDValue Src = Op.getOperand(0);
1764 
1765   EVT DstVT = Op.getValueType();
1766   EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1767 
1768   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1769   unsigned Opc;
1770   if (SatVT == DstVT)
1771     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
1772   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1773     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
1774   else
1775     return SDValue();
1776   // FIXME: Support other SatVTs by clamping before or after the conversion.
1777 
1778   SDLoc DL(Op);
1779   SDValue FpToInt = DAG.getNode(
1780       Opc, DL, DstVT, Src,
1781       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
1782 
1783   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1784   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1785 }
1786 
1787 // Expand vector FTRUNC, FCEIL, and FFLOOR by converting to the integer domain
1788 // and back. Taking care to avoid converting values that are nan or already
1789 // correct.
1790 // TODO: Floor and ceil could be shorter by changing rounding mode, but we don't
1791 // have FRM dependencies modeled yet.
1792 static SDValue lowerFTRUNC_FCEIL_FFLOOR(SDValue Op, SelectionDAG &DAG) {
1793   MVT VT = Op.getSimpleValueType();
1794   assert(VT.isVector() && "Unexpected type");
1795 
1796   SDLoc DL(Op);
1797 
1798   // Freeze the source since we are increasing the number of uses.
1799   SDValue Src = DAG.getNode(ISD::FREEZE, DL, VT, Op.getOperand(0));
1800 
1801   // Truncate to integer and convert back to FP.
1802   MVT IntVT = VT.changeVectorElementTypeToInteger();
1803   SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src);
1804   Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated);
1805 
1806   MVT SetccVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1807 
1808   if (Op.getOpcode() == ISD::FCEIL) {
1809     // If the truncated value is the greater than or equal to the original
1810     // value, we've computed the ceil. Otherwise, we went the wrong way and
1811     // need to increase by 1.
1812     // FIXME: This should use a masked operation. Handle here or in isel?
1813     SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated,
1814                                  DAG.getConstantFP(1.0, DL, VT));
1815     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT);
1816     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1817   } else if (Op.getOpcode() == ISD::FFLOOR) {
1818     // If the truncated value is the less than or equal to the original value,
1819     // we've computed the floor. Otherwise, we went the wrong way and need to
1820     // decrease by 1.
1821     // FIXME: This should use a masked operation. Handle here or in isel?
1822     SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated,
1823                                  DAG.getConstantFP(1.0, DL, VT));
1824     SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT);
1825     Truncated = DAG.getSelect(DL, VT, NeedAdjust, Adjust, Truncated);
1826   }
1827 
1828   // Restore the original sign so that -0.0 is preserved.
1829   Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src);
1830 
1831   // Determine the largest integer that can be represented exactly. This and
1832   // values larger than it don't have any fractional bits so don't need to
1833   // be converted.
1834   const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
1835   unsigned Precision = APFloat::semanticsPrecision(FltSem);
1836   APFloat MaxVal = APFloat(FltSem);
1837   MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
1838                           /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
1839   SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
1840 
1841   // If abs(Src) was larger than MaxVal or nan, keep it.
1842   SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src);
1843   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT);
1844   return DAG.getSelect(DL, VT, Setcc, Truncated, Src);
1845 }
1846 
1847 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1848                                  const RISCVSubtarget &Subtarget) {
1849   MVT VT = Op.getSimpleValueType();
1850   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1851 
1852   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1853 
1854   SDLoc DL(Op);
1855   SDValue Mask, VL;
1856   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1857 
1858   unsigned Opc =
1859       VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1860   SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1861   return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1862 }
1863 
1864 struct VIDSequence {
1865   int64_t StepNumerator;
1866   unsigned StepDenominator;
1867   int64_t Addend;
1868 };
1869 
1870 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1871 // to the (non-zero) step S and start value X. This can be then lowered as the
1872 // RVV sequence (VID * S) + X, for example.
1873 // The step S is represented as an integer numerator divided by a positive
1874 // denominator. Note that the implementation currently only identifies
1875 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1876 // cannot detect 2/3, for example.
1877 // Note that this method will also match potentially unappealing index
1878 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1879 // determine whether this is worth generating code for.
1880 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1881   unsigned NumElts = Op.getNumOperands();
1882   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1883   if (!Op.getValueType().isInteger())
1884     return None;
1885 
1886   Optional<unsigned> SeqStepDenom;
1887   Optional<int64_t> SeqStepNum, SeqAddend;
1888   Optional<std::pair<uint64_t, unsigned>> PrevElt;
1889   unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1890   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1891     // Assume undef elements match the sequence; we just have to be careful
1892     // when interpolating across them.
1893     if (Op.getOperand(Idx).isUndef())
1894       continue;
1895     // The BUILD_VECTOR must be all constants.
1896     if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1897       return None;
1898 
1899     uint64_t Val = Op.getConstantOperandVal(Idx) &
1900                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1901 
1902     if (PrevElt) {
1903       // Calculate the step since the last non-undef element, and ensure
1904       // it's consistent across the entire sequence.
1905       unsigned IdxDiff = Idx - PrevElt->second;
1906       int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1907 
1908       // A zero-value value difference means that we're somewhere in the middle
1909       // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1910       // step change before evaluating the sequence.
1911       if (ValDiff == 0)
1912         continue;
1913 
1914       int64_t Remainder = ValDiff % IdxDiff;
1915       // Normalize the step if it's greater than 1.
1916       if (Remainder != ValDiff) {
1917         // The difference must cleanly divide the element span.
1918         if (Remainder != 0)
1919           return None;
1920         ValDiff /= IdxDiff;
1921         IdxDiff = 1;
1922       }
1923 
1924       if (!SeqStepNum)
1925         SeqStepNum = ValDiff;
1926       else if (ValDiff != SeqStepNum)
1927         return None;
1928 
1929       if (!SeqStepDenom)
1930         SeqStepDenom = IdxDiff;
1931       else if (IdxDiff != *SeqStepDenom)
1932         return None;
1933     }
1934 
1935     // Record this non-undef element for later.
1936     if (!PrevElt || PrevElt->first != Val)
1937       PrevElt = std::make_pair(Val, Idx);
1938   }
1939 
1940   // We need to have logged a step for this to count as a legal index sequence.
1941   if (!SeqStepNum || !SeqStepDenom)
1942     return None;
1943 
1944   // Loop back through the sequence and validate elements we might have skipped
1945   // while waiting for a valid step. While doing this, log any sequence addend.
1946   for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1947     if (Op.getOperand(Idx).isUndef())
1948       continue;
1949     uint64_t Val = Op.getConstantOperandVal(Idx) &
1950                    maskTrailingOnes<uint64_t>(EltSizeInBits);
1951     uint64_t ExpectedVal =
1952         (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1953     int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1954     if (!SeqAddend)
1955       SeqAddend = Addend;
1956     else if (Addend != SeqAddend)
1957       return None;
1958   }
1959 
1960   assert(SeqAddend && "Must have an addend if we have a step");
1961 
1962   return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1963 }
1964 
1965 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
1966                                  const RISCVSubtarget &Subtarget) {
1967   MVT VT = Op.getSimpleValueType();
1968   assert(VT.isFixedLengthVector() && "Unexpected vector!");
1969 
1970   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1971 
1972   SDLoc DL(Op);
1973   SDValue Mask, VL;
1974   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1975 
1976   MVT XLenVT = Subtarget.getXLenVT();
1977   unsigned NumElts = Op.getNumOperands();
1978 
1979   if (VT.getVectorElementType() == MVT::i1) {
1980     if (ISD::isBuildVectorAllZeros(Op.getNode())) {
1981       SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
1982       return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
1983     }
1984 
1985     if (ISD::isBuildVectorAllOnes(Op.getNode())) {
1986       SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
1987       return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
1988     }
1989 
1990     // Lower constant mask BUILD_VECTORs via an integer vector type, in
1991     // scalar integer chunks whose bit-width depends on the number of mask
1992     // bits and XLEN.
1993     // First, determine the most appropriate scalar integer type to use. This
1994     // is at most XLenVT, but may be shrunk to a smaller vector element type
1995     // according to the size of the final vector - use i8 chunks rather than
1996     // XLenVT if we're producing a v8i1. This results in more consistent
1997     // codegen across RV32 and RV64.
1998     unsigned NumViaIntegerBits =
1999         std::min(std::max(NumElts, 8u), Subtarget.getXLen());
2000     NumViaIntegerBits = std::min(NumViaIntegerBits,
2001                                  Subtarget.getMaxELENForFixedLengthVectors());
2002     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
2003       // If we have to use more than one INSERT_VECTOR_ELT then this
2004       // optimization is likely to increase code size; avoid peforming it in
2005       // such a case. We can use a load from a constant pool in this case.
2006       if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
2007         return SDValue();
2008       // Now we can create our integer vector type. Note that it may be larger
2009       // than the resulting mask type: v4i1 would use v1i8 as its integer type.
2010       MVT IntegerViaVecVT =
2011           MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
2012                            divideCeil(NumElts, NumViaIntegerBits));
2013 
2014       uint64_t Bits = 0;
2015       unsigned BitPos = 0, IntegerEltIdx = 0;
2016       SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
2017 
2018       for (unsigned I = 0; I < NumElts; I++, BitPos++) {
2019         // Once we accumulate enough bits to fill our scalar type, insert into
2020         // our vector and clear our accumulated data.
2021         if (I != 0 && I % NumViaIntegerBits == 0) {
2022           if (NumViaIntegerBits <= 32)
2023             Bits = SignExtend64(Bits, 32);
2024           SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2025           Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
2026                             Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2027           Bits = 0;
2028           BitPos = 0;
2029           IntegerEltIdx++;
2030         }
2031         SDValue V = Op.getOperand(I);
2032         bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2033         Bits |= ((uint64_t)BitValue << BitPos);
2034       }
2035 
2036       // Insert the (remaining) scalar value into position in our integer
2037       // vector type.
2038       if (NumViaIntegerBits <= 32)
2039         Bits = SignExtend64(Bits, 32);
2040       SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2041       Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
2042                         DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2043 
2044       if (NumElts < NumViaIntegerBits) {
2045         // If we're producing a smaller vector than our minimum legal integer
2046         // type, bitcast to the equivalent (known-legal) mask type, and extract
2047         // our final mask.
2048         assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
2049         Vec = DAG.getBitcast(MVT::v8i1, Vec);
2050         Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
2051                           DAG.getConstant(0, DL, XLenVT));
2052       } else {
2053         // Else we must have produced an integer type with the same size as the
2054         // mask type; bitcast for the final result.
2055         assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
2056         Vec = DAG.getBitcast(VT, Vec);
2057       }
2058 
2059       return Vec;
2060     }
2061 
2062     // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
2063     // vector type, we have a legal equivalently-sized i8 type, so we can use
2064     // that.
2065     MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
2066     SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
2067 
2068     SDValue WideVec;
2069     if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2070       // For a splat, perform a scalar truncate before creating the wider
2071       // vector.
2072       assert(Splat.getValueType() == XLenVT &&
2073              "Unexpected type for i1 splat value");
2074       Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2075                           DAG.getConstant(1, DL, XLenVT));
2076       WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2077     } else {
2078       SmallVector<SDValue, 8> Ops(Op->op_values());
2079       WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2080       SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2081       WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2082     }
2083 
2084     return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2085   }
2086 
2087   if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2088     unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2089                                         : RISCVISD::VMV_V_X_VL;
2090     Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
2091     return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2092   }
2093 
2094   // Try and match index sequences, which we can lower to the vid instruction
2095   // with optional modifications. An all-undef vector is matched by
2096   // getSplatValue, above.
2097   if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2098     int64_t StepNumerator = SimpleVID->StepNumerator;
2099     unsigned StepDenominator = SimpleVID->StepDenominator;
2100     int64_t Addend = SimpleVID->Addend;
2101 
2102     assert(StepNumerator != 0 && "Invalid step");
2103     bool Negate = false;
2104     int64_t SplatStepVal = StepNumerator;
2105     unsigned StepOpcode = ISD::MUL;
2106     if (StepNumerator != 1) {
2107       if (isPowerOf2_64(std::abs(StepNumerator))) {
2108         Negate = StepNumerator < 0;
2109         StepOpcode = ISD::SHL;
2110         SplatStepVal = Log2_64(std::abs(StepNumerator));
2111       }
2112     }
2113 
2114     // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2115     // threshold since it's the immediate value many RVV instructions accept.
2116     // There is no vmul.vi instruction so ensure multiply constant can fit in
2117     // a single addi instruction.
2118     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2119          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2120         isPowerOf2_32(StepDenominator) &&
2121         (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
2122       SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
2123       // Convert right out of the scalable type so we can use standard ISD
2124       // nodes for the rest of the computation. If we used scalable types with
2125       // these, we'd lose the fixed-length vector info and generate worse
2126       // vsetvli code.
2127       VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
2128       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2129           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2130         SDValue SplatStep = DAG.getSplatVector(
2131             VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2132         VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep);
2133       }
2134       if (StepDenominator != 1) {
2135         SDValue SplatStep = DAG.getSplatVector(
2136             VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2137         VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
2138       }
2139       if (Addend != 0 || Negate) {
2140         SDValue SplatAddend =
2141             DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
2142         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
2143       }
2144       return VID;
2145     }
2146   }
2147 
2148   // Attempt to detect "hidden" splats, which only reveal themselves as splats
2149   // when re-interpreted as a vector with a larger element type. For example,
2150   //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2151   // could be instead splat as
2152   //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
2153   // TODO: This optimization could also work on non-constant splats, but it
2154   // would require bit-manipulation instructions to construct the splat value.
2155   SmallVector<SDValue> Sequence;
2156   unsigned EltBitSize = VT.getScalarSizeInBits();
2157   const auto *BV = cast<BuildVectorSDNode>(Op);
2158   if (VT.isInteger() && EltBitSize < 64 &&
2159       ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2160       BV->getRepeatedSequence(Sequence) &&
2161       (Sequence.size() * EltBitSize) <= 64) {
2162     unsigned SeqLen = Sequence.size();
2163     MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2164     MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2165     assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2166             ViaIntVT == MVT::i64) &&
2167            "Unexpected sequence type");
2168 
2169     unsigned EltIdx = 0;
2170     uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2171     uint64_t SplatValue = 0;
2172     // Construct the amalgamated value which can be splatted as this larger
2173     // vector type.
2174     for (const auto &SeqV : Sequence) {
2175       if (!SeqV.isUndef())
2176         SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2177                        << (EltIdx * EltBitSize));
2178       EltIdx++;
2179     }
2180 
2181     // On RV64, sign-extend from 32 to 64 bits where possible in order to
2182     // achieve better constant materializion.
2183     if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2184       SplatValue = SignExtend64(SplatValue, 32);
2185 
2186     // Since we can't introduce illegal i64 types at this stage, we can only
2187     // perform an i64 splat on RV32 if it is its own sign-extended value. That
2188     // way we can use RVV instructions to splat.
2189     assert((ViaIntVT.bitsLE(XLenVT) ||
2190             (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2191            "Unexpected bitcast sequence");
2192     if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2193       SDValue ViaVL =
2194           DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2195       MVT ViaContainerVT =
2196           getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2197       SDValue Splat =
2198           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2199                       DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2200       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2201       return DAG.getBitcast(VT, Splat);
2202     }
2203   }
2204 
2205   // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2206   // which constitute a large proportion of the elements. In such cases we can
2207   // splat a vector with the dominant element and make up the shortfall with
2208   // INSERT_VECTOR_ELTs.
2209   // Note that this includes vectors of 2 elements by association. The
2210   // upper-most element is the "dominant" one, allowing us to use a splat to
2211   // "insert" the upper element, and an insert of the lower element at position
2212   // 0, which improves codegen.
2213   SDValue DominantValue;
2214   unsigned MostCommonCount = 0;
2215   DenseMap<SDValue, unsigned> ValueCounts;
2216   unsigned NumUndefElts =
2217       count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2218 
2219   // Track the number of scalar loads we know we'd be inserting, estimated as
2220   // any non-zero floating-point constant. Other kinds of element are either
2221   // already in registers or are materialized on demand. The threshold at which
2222   // a vector load is more desirable than several scalar materializion and
2223   // vector-insertion instructions is not known.
2224   unsigned NumScalarLoads = 0;
2225 
2226   for (SDValue V : Op->op_values()) {
2227     if (V.isUndef())
2228       continue;
2229 
2230     ValueCounts.insert(std::make_pair(V, 0));
2231     unsigned &Count = ValueCounts[V];
2232 
2233     if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2234       NumScalarLoads += !CFP->isExactlyValue(+0.0);
2235 
2236     // Is this value dominant? In case of a tie, prefer the highest element as
2237     // it's cheaper to insert near the beginning of a vector than it is at the
2238     // end.
2239     if (++Count >= MostCommonCount) {
2240       DominantValue = V;
2241       MostCommonCount = Count;
2242     }
2243   }
2244 
2245   assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2246   unsigned NumDefElts = NumElts - NumUndefElts;
2247   unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2248 
2249   // Don't perform this optimization when optimizing for size, since
2250   // materializing elements and inserting them tends to cause code bloat.
2251   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2252       ((MostCommonCount > DominantValueCountThreshold) ||
2253        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2254     // Start by splatting the most common element.
2255     SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2256 
2257     DenseSet<SDValue> Processed{DominantValue};
2258     MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2259     for (const auto &OpIdx : enumerate(Op->ops())) {
2260       const SDValue &V = OpIdx.value();
2261       if (V.isUndef() || !Processed.insert(V).second)
2262         continue;
2263       if (ValueCounts[V] == 1) {
2264         Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2265                           DAG.getConstant(OpIdx.index(), DL, XLenVT));
2266       } else {
2267         // Blend in all instances of this value using a VSELECT, using a
2268         // mask where each bit signals whether that element is the one
2269         // we're after.
2270         SmallVector<SDValue> Ops;
2271         transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2272           return DAG.getConstant(V == V1, DL, XLenVT);
2273         });
2274         Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2275                           DAG.getBuildVector(SelMaskTy, DL, Ops),
2276                           DAG.getSplatBuildVector(VT, DL, V), Vec);
2277       }
2278     }
2279 
2280     return Vec;
2281   }
2282 
2283   return SDValue();
2284 }
2285 
2286 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
2287                                    SDValue Hi, SDValue VL, SelectionDAG &DAG) {
2288   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2289     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2290     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2291     // If Hi constant is all the same sign bit as Lo, lower this as a custom
2292     // node in order to try and match RVV vector/scalar instructions.
2293     if ((LoC >> 31) == HiC)
2294       return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
2295 
2296     // If vl is equal to VLMax and Hi constant is equal to Lo, we could use
2297     // vmv.v.x whose EEW = 32 to lower it.
2298     auto *Const = dyn_cast<ConstantSDNode>(VL);
2299     if (LoC == HiC && Const && Const->getSExtValue() == RISCV::VLMaxSentinel) {
2300       MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2301       // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2302       // access the subtarget here now.
2303       auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT, Lo, VL);
2304       return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2305     }
2306   }
2307 
2308   // Fall back to a stack store and stride x0 vector load.
2309   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
2310 }
2311 
2312 // Called by type legalization to handle splat of i64 on RV32.
2313 // FIXME: We can optimize this when the type has sign or zero bits in one
2314 // of the halves.
2315 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
2316                                    SDValue VL, SelectionDAG &DAG) {
2317   assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2318   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2319                            DAG.getConstant(0, DL, MVT::i32));
2320   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2321                            DAG.getConstant(1, DL, MVT::i32));
2322   return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
2323 }
2324 
2325 // This function lowers a splat of a scalar operand Splat with the vector
2326 // length VL. It ensures the final sequence is type legal, which is useful when
2327 // lowering a splat after type legalization.
2328 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
2329                                 SelectionDAG &DAG,
2330                                 const RISCVSubtarget &Subtarget) {
2331   if (VT.isFloatingPoint()) {
2332     // If VL is 1, we could use vfmv.s.f.
2333     if (isOneConstant(VL))
2334       return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT),
2335                          Scalar, VL);
2336     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
2337   }
2338 
2339   MVT XLenVT = Subtarget.getXLenVT();
2340 
2341   // Simplest case is that the operand needs to be promoted to XLenVT.
2342   if (Scalar.getValueType().bitsLE(XLenVT)) {
2343     // If the operand is a constant, sign extend to increase our chances
2344     // of being able to use a .vi instruction. ANY_EXTEND would become a
2345     // a zero extend and the simm5 check in isel would fail.
2346     // FIXME: Should we ignore the upper bits in isel instead?
2347     unsigned ExtOpc =
2348         isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2349     Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2350     ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2351     // If VL is 1 and the scalar value won't benefit from immediate, we could
2352     // use vmv.s.x.
2353     if (isOneConstant(VL) &&
2354         (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2355       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
2356                          VL);
2357     return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
2358   }
2359 
2360   assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2361          "Unexpected scalar for splat lowering!");
2362 
2363   if (isOneConstant(VL) && isNullConstant(Scalar))
2364     return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT),
2365                        DAG.getConstant(0, DL, XLenVT), VL);
2366 
2367   // Otherwise use the more complicated splatting algorithm.
2368   return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
2369 }
2370 
2371 // Is the mask a slidedown that shifts in undefs.
2372 static int matchShuffleAsSlideDown(ArrayRef<int> Mask) {
2373   int Size = Mask.size();
2374 
2375   // Elements shifted in should be undef.
2376   auto CheckUndefs = [&](int Shift) {
2377     for (int i = Size - Shift; i != Size; ++i)
2378       if (Mask[i] >= 0)
2379         return false;
2380     return true;
2381   };
2382 
2383   // Elements should be shifted or undef.
2384   auto MatchShift = [&](int Shift) {
2385     for (int i = 0; i != Size - Shift; ++i)
2386        if (Mask[i] >= 0 && Mask[i] != Shift + i)
2387          return false;
2388     return true;
2389   };
2390 
2391   // Try all possible shifts.
2392   for (int Shift = 1; Shift != Size; ++Shift)
2393     if (CheckUndefs(Shift) && MatchShift(Shift))
2394       return Shift;
2395 
2396   // No match.
2397   return -1;
2398 }
2399 
2400 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, bool &SwapSources,
2401                                 const RISCVSubtarget &Subtarget) {
2402   // We need to be able to widen elements to the next larger integer type.
2403   if (VT.getScalarSizeInBits() >= Subtarget.getMaxELENForFixedLengthVectors())
2404     return false;
2405 
2406   int Size = Mask.size();
2407   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
2408 
2409   int Srcs[] = {-1, -1};
2410   for (int i = 0; i != Size; ++i) {
2411     // Ignore undef elements.
2412     if (Mask[i] < 0)
2413       continue;
2414 
2415     // Is this an even or odd element.
2416     int Pol = i % 2;
2417 
2418     // Ensure we consistently use the same source for this element polarity.
2419     int Src = Mask[i] / Size;
2420     if (Srcs[Pol] < 0)
2421       Srcs[Pol] = Src;
2422     if (Srcs[Pol] != Src)
2423       return false;
2424 
2425     // Make sure the element within the source is appropriate for this element
2426     // in the destination.
2427     int Elt = Mask[i] % Size;
2428     if (Elt != i / 2)
2429       return false;
2430   }
2431 
2432   // We need to find a source for each polarity and they can't be the same.
2433   if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2434     return false;
2435 
2436   // Swap the sources if the second source was in the even polarity.
2437   SwapSources = Srcs[0] > Srcs[1];
2438 
2439   return true;
2440 }
2441 
2442 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
2443                                    const RISCVSubtarget &Subtarget) {
2444   SDValue V1 = Op.getOperand(0);
2445   SDValue V2 = Op.getOperand(1);
2446   SDLoc DL(Op);
2447   MVT XLenVT = Subtarget.getXLenVT();
2448   MVT VT = Op.getSimpleValueType();
2449   unsigned NumElts = VT.getVectorNumElements();
2450   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2451 
2452   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2453 
2454   SDValue TrueMask, VL;
2455   std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2456 
2457   if (SVN->isSplat()) {
2458     const int Lane = SVN->getSplatIndex();
2459     if (Lane >= 0) {
2460       MVT SVT = VT.getVectorElementType();
2461 
2462       // Turn splatted vector load into a strided load with an X0 stride.
2463       SDValue V = V1;
2464       // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
2465       // with undef.
2466       // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
2467       int Offset = Lane;
2468       if (V.getOpcode() == ISD::CONCAT_VECTORS) {
2469         int OpElements =
2470             V.getOperand(0).getSimpleValueType().getVectorNumElements();
2471         V = V.getOperand(Offset / OpElements);
2472         Offset %= OpElements;
2473       }
2474 
2475       // We need to ensure the load isn't atomic or volatile.
2476       if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
2477         auto *Ld = cast<LoadSDNode>(V);
2478         Offset *= SVT.getStoreSize();
2479         SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
2480                                                    TypeSize::Fixed(Offset), DL);
2481 
2482         // If this is SEW=64 on RV32, use a strided load with a stride of x0.
2483         if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
2484           SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
2485           SDValue IntID =
2486               DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
2487           SDValue Ops[] = {Ld->getChain(),
2488                            IntID,
2489                            DAG.getUNDEF(ContainerVT),
2490                            NewAddr,
2491                            DAG.getRegister(RISCV::X0, XLenVT),
2492                            VL};
2493           SDValue NewLoad = DAG.getMemIntrinsicNode(
2494               ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
2495               DAG.getMachineFunction().getMachineMemOperand(
2496                   Ld->getMemOperand(), Offset, SVT.getStoreSize()));
2497           DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
2498           return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
2499         }
2500 
2501         // Otherwise use a scalar load and splat. This will give the best
2502         // opportunity to fold a splat into the operation. ISel can turn it into
2503         // the x0 strided load if we aren't able to fold away the select.
2504         if (SVT.isFloatingPoint())
2505           V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
2506                           Ld->getPointerInfo().getWithOffset(Offset),
2507                           Ld->getOriginalAlign(),
2508                           Ld->getMemOperand()->getFlags());
2509         else
2510           V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
2511                              Ld->getPointerInfo().getWithOffset(Offset), SVT,
2512                              Ld->getOriginalAlign(),
2513                              Ld->getMemOperand()->getFlags());
2514         DAG.makeEquivalentMemoryOrdering(Ld, V);
2515 
2516         unsigned Opc =
2517             VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
2518         SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
2519         return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2520       }
2521 
2522       V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2523       assert(Lane < (int)NumElts && "Unexpected lane!");
2524       SDValue Gather =
2525           DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
2526                       DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
2527       return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2528     }
2529   }
2530 
2531   ArrayRef<int> Mask = SVN->getMask();
2532 
2533   // Try to match as a slidedown.
2534   int SlideAmt = matchShuffleAsSlideDown(Mask);
2535   if (SlideAmt >= 0) {
2536     // TODO: Should we reduce the VL to account for the upper undef elements?
2537     // Requires additional vsetvlis, but might be faster to execute.
2538     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2539     SDValue SlideDown =
2540         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
2541                     DAG.getUNDEF(ContainerVT), V1,
2542                     DAG.getConstant(SlideAmt, DL, XLenVT),
2543                     TrueMask, VL);
2544     return convertFromScalableVector(VT, SlideDown, DAG, Subtarget);
2545   }
2546 
2547   // Detect an interleave shuffle and lower to
2548   // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
2549   bool SwapSources;
2550   if (isInterleaveShuffle(Mask, VT, SwapSources, Subtarget)) {
2551     // Swap sources if needed.
2552     if (SwapSources)
2553       std::swap(V1, V2);
2554 
2555     // Extract the lower half of the vectors.
2556     MVT HalfVT = VT.getHalfNumVectorElementsVT();
2557     V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
2558                      DAG.getConstant(0, DL, XLenVT));
2559     V2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V2,
2560                      DAG.getConstant(0, DL, XLenVT));
2561 
2562     // Double the element width and halve the number of elements in an int type.
2563     unsigned EltBits = VT.getScalarSizeInBits();
2564     MVT WideIntEltVT = MVT::getIntegerVT(EltBits * 2);
2565     MVT WideIntVT =
2566         MVT::getVectorVT(WideIntEltVT, VT.getVectorNumElements() / 2);
2567     // Convert this to a scalable vector. We need to base this on the
2568     // destination size to ensure there's always a type with a smaller LMUL.
2569     MVT WideIntContainerVT =
2570         getContainerForFixedLengthVector(DAG, WideIntVT, Subtarget);
2571 
2572     // Convert sources to scalable vectors with the same element count as the
2573     // larger type.
2574     MVT HalfContainerVT = MVT::getVectorVT(
2575         VT.getVectorElementType(), WideIntContainerVT.getVectorElementCount());
2576     V1 = convertToScalableVector(HalfContainerVT, V1, DAG, Subtarget);
2577     V2 = convertToScalableVector(HalfContainerVT, V2, DAG, Subtarget);
2578 
2579     // Cast sources to integer.
2580     MVT IntEltVT = MVT::getIntegerVT(EltBits);
2581     MVT IntHalfVT =
2582         MVT::getVectorVT(IntEltVT, HalfContainerVT.getVectorElementCount());
2583     V1 = DAG.getBitcast(IntHalfVT, V1);
2584     V2 = DAG.getBitcast(IntHalfVT, V2);
2585 
2586     // Freeze V2 since we use it twice and we need to be sure that the add and
2587     // multiply see the same value.
2588     V2 = DAG.getNode(ISD::FREEZE, DL, IntHalfVT, V2);
2589 
2590     // Recreate TrueMask using the widened type's element count.
2591     MVT MaskVT =
2592         MVT::getVectorVT(MVT::i1, HalfContainerVT.getVectorElementCount());
2593     TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2594 
2595     // Widen V1 and V2 with 0s and add one copy of V2 to V1.
2596     SDValue Add = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideIntContainerVT, V1,
2597                               V2, TrueMask, VL);
2598     // Create 2^eltbits - 1 copies of V2 by multiplying by the largest integer.
2599     SDValue Multiplier = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntHalfVT,
2600                                      DAG.getAllOnesConstant(DL, XLenVT));
2601     SDValue WidenMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideIntContainerVT,
2602                                    V2, Multiplier, TrueMask, VL);
2603     // Add the new copies to our previous addition giving us 2^eltbits copies of
2604     // V2. This is equivalent to shifting V2 left by eltbits. This should
2605     // combine with the vwmulu.vv above to form vwmaccu.vv.
2606     Add = DAG.getNode(RISCVISD::ADD_VL, DL, WideIntContainerVT, Add, WidenMul,
2607                       TrueMask, VL);
2608     // Cast back to ContainerVT. We need to re-create a new ContainerVT in case
2609     // WideIntContainerVT is a larger fractional LMUL than implied by the fixed
2610     // vector VT.
2611     ContainerVT =
2612         MVT::getVectorVT(VT.getVectorElementType(),
2613                          WideIntContainerVT.getVectorElementCount() * 2);
2614     Add = DAG.getBitcast(ContainerVT, Add);
2615     return convertFromScalableVector(VT, Add, DAG, Subtarget);
2616   }
2617 
2618   // Detect shuffles which can be re-expressed as vector selects; these are
2619   // shuffles in which each element in the destination is taken from an element
2620   // at the corresponding index in either source vectors.
2621   bool IsSelect = all_of(enumerate(Mask), [&](const auto &MaskIdx) {
2622     int MaskIndex = MaskIdx.value();
2623     return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
2624   });
2625 
2626   assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
2627 
2628   SmallVector<SDValue> MaskVals;
2629   // As a backup, shuffles can be lowered via a vrgather instruction, possibly
2630   // merged with a second vrgather.
2631   SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
2632 
2633   // By default we preserve the original operand order, and use a mask to
2634   // select LHS as true and RHS as false. However, since RVV vector selects may
2635   // feature splats but only on the LHS, we may choose to invert our mask and
2636   // instead select between RHS and LHS.
2637   bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
2638   bool InvertMask = IsSelect == SwapOps;
2639 
2640   // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
2641   // half.
2642   DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
2643 
2644   // Now construct the mask that will be used by the vselect or blended
2645   // vrgather operation. For vrgathers, construct the appropriate indices into
2646   // each vector.
2647   for (int MaskIndex : Mask) {
2648     bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
2649     MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
2650     if (!IsSelect) {
2651       bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
2652       GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2653                                      ? DAG.getConstant(MaskIndex, DL, XLenVT)
2654                                      : DAG.getUNDEF(XLenVT));
2655       GatherIndicesRHS.push_back(
2656           IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
2657                             : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
2658       if (IsLHSOrUndefIndex && MaskIndex >= 0)
2659         ++LHSIndexCounts[MaskIndex];
2660       if (!IsLHSOrUndefIndex)
2661         ++RHSIndexCounts[MaskIndex - NumElts];
2662     }
2663   }
2664 
2665   if (SwapOps) {
2666     std::swap(V1, V2);
2667     std::swap(GatherIndicesLHS, GatherIndicesRHS);
2668   }
2669 
2670   assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2671   MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2672   SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2673 
2674   if (IsSelect)
2675     return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2676 
2677   if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2678     // On such a large vector we're unable to use i8 as the index type.
2679     // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2680     // may involve vector splitting if we're already at LMUL=8, or our
2681     // user-supplied maximum fixed-length LMUL.
2682     return SDValue();
2683   }
2684 
2685   unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2686   unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2687   MVT IndexVT = VT.changeTypeToInteger();
2688   // Since we can't introduce illegal index types at this stage, use i16 and
2689   // vrgatherei16 if the corresponding index type for plain vrgather is greater
2690   // than XLenVT.
2691   if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2692     GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2693     IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2694   }
2695 
2696   MVT IndexContainerVT =
2697       ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2698 
2699   SDValue Gather;
2700   // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2701   // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2702   if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2703     Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2704   } else {
2705     V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2706     // If only one index is used, we can use a "splat" vrgather.
2707     // TODO: We can splat the most-common index and fix-up any stragglers, if
2708     // that's beneficial.
2709     if (LHSIndexCounts.size() == 1) {
2710       int SplatIndex = LHSIndexCounts.begin()->getFirst();
2711       Gather =
2712           DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2713                       DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2714     } else {
2715       SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2716       LHSIndices =
2717           convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2718 
2719       Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2720                            TrueMask, VL);
2721     }
2722   }
2723 
2724   // If a second vector operand is used by this shuffle, blend it in with an
2725   // additional vrgather.
2726   if (!V2.isUndef()) {
2727     V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2728     // If only one index is used, we can use a "splat" vrgather.
2729     // TODO: We can splat the most-common index and fix-up any stragglers, if
2730     // that's beneficial.
2731     if (RHSIndexCounts.size() == 1) {
2732       int SplatIndex = RHSIndexCounts.begin()->getFirst();
2733       V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2734                        DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2735     } else {
2736       SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2737       RHSIndices =
2738           convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2739       V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2740                        VL);
2741     }
2742 
2743     MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2744     SelectMask =
2745         convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2746 
2747     Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2748                          Gather, VL);
2749   }
2750 
2751   return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2752 }
2753 
2754 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2755                                      SDLoc DL, SelectionDAG &DAG,
2756                                      const RISCVSubtarget &Subtarget) {
2757   if (VT.isScalableVector())
2758     return DAG.getFPExtendOrRound(Op, DL, VT);
2759   assert(VT.isFixedLengthVector() &&
2760          "Unexpected value type for RVV FP extend/round lowering");
2761   SDValue Mask, VL;
2762   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2763   unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2764                         ? RISCVISD::FP_EXTEND_VL
2765                         : RISCVISD::FP_ROUND_VL;
2766   return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2767 }
2768 
2769 // Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
2770 // the exponent.
2771 static SDValue lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
2772   MVT VT = Op.getSimpleValueType();
2773   unsigned EltSize = VT.getScalarSizeInBits();
2774   SDValue Src = Op.getOperand(0);
2775   SDLoc DL(Op);
2776 
2777   // We need a FP type that can represent the value.
2778   // TODO: Use f16 for i8 when possible?
2779   MVT FloatEltVT = EltSize == 32 ? MVT::f64 : MVT::f32;
2780   MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
2781 
2782   // Legal types should have been checked in the RISCVTargetLowering
2783   // constructor.
2784   // TODO: Splitting may make sense in some cases.
2785   assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
2786          "Expected legal float type!");
2787 
2788   // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
2789   // The trailing zero count is equal to log2 of this single bit value.
2790   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
2791     SDValue Neg =
2792         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
2793     Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
2794   }
2795 
2796   // We have a legal FP type, convert to it.
2797   SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
2798   // Bitcast to integer and shift the exponent to the LSB.
2799   EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
2800   SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
2801   unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
2802   SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
2803                               DAG.getConstant(ShiftAmt, DL, IntVT));
2804   // Truncate back to original type to allow vnsrl.
2805   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift);
2806   // The exponent contains log2 of the value in biased form.
2807   unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
2808 
2809   // For trailing zeros, we just need to subtract the bias.
2810   if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
2811     return DAG.getNode(ISD::SUB, DL, VT, Trunc,
2812                        DAG.getConstant(ExponentBias, DL, VT));
2813 
2814   // For leading zeros, we need to remove the bias and convert from log2 to
2815   // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
2816   unsigned Adjust = ExponentBias + (EltSize - 1);
2817   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc);
2818 }
2819 
2820 // While RVV has alignment restrictions, we should always be able to load as a
2821 // legal equivalently-sized byte-typed vector instead. This method is
2822 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
2823 // the load is already correctly-aligned, it returns SDValue().
2824 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
2825                                                     SelectionDAG &DAG) const {
2826   auto *Load = cast<LoadSDNode>(Op);
2827   assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
2828 
2829   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2830                                      Load->getMemoryVT(),
2831                                      *Load->getMemOperand()))
2832     return SDValue();
2833 
2834   SDLoc DL(Op);
2835   MVT VT = Op.getSimpleValueType();
2836   unsigned EltSizeBits = VT.getScalarSizeInBits();
2837   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2838          "Unexpected unaligned RVV load type");
2839   MVT NewVT =
2840       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2841   assert(NewVT.isValid() &&
2842          "Expecting equally-sized RVV vector types to be legal");
2843   SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
2844                           Load->getPointerInfo(), Load->getOriginalAlign(),
2845                           Load->getMemOperand()->getFlags());
2846   return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
2847 }
2848 
2849 // While RVV has alignment restrictions, we should always be able to store as a
2850 // legal equivalently-sized byte-typed vector instead. This method is
2851 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
2852 // returns SDValue() if the store is already correctly aligned.
2853 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
2854                                                      SelectionDAG &DAG) const {
2855   auto *Store = cast<StoreSDNode>(Op);
2856   assert(Store && Store->getValue().getValueType().isVector() &&
2857          "Expected vector store");
2858 
2859   if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2860                                      Store->getMemoryVT(),
2861                                      *Store->getMemOperand()))
2862     return SDValue();
2863 
2864   SDLoc DL(Op);
2865   SDValue StoredVal = Store->getValue();
2866   MVT VT = StoredVal.getSimpleValueType();
2867   unsigned EltSizeBits = VT.getScalarSizeInBits();
2868   assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2869          "Unexpected unaligned RVV store type");
2870   MVT NewVT =
2871       MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2872   assert(NewVT.isValid() &&
2873          "Expecting equally-sized RVV vector types to be legal");
2874   StoredVal = DAG.getBitcast(NewVT, StoredVal);
2875   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
2876                       Store->getPointerInfo(), Store->getOriginalAlign(),
2877                       Store->getMemOperand()->getFlags());
2878 }
2879 
2880 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
2881                                             SelectionDAG &DAG) const {
2882   switch (Op.getOpcode()) {
2883   default:
2884     report_fatal_error("unimplemented operand");
2885   case ISD::GlobalAddress:
2886     return lowerGlobalAddress(Op, DAG);
2887   case ISD::BlockAddress:
2888     return lowerBlockAddress(Op, DAG);
2889   case ISD::ConstantPool:
2890     return lowerConstantPool(Op, DAG);
2891   case ISD::JumpTable:
2892     return lowerJumpTable(Op, DAG);
2893   case ISD::GlobalTLSAddress:
2894     return lowerGlobalTLSAddress(Op, DAG);
2895   case ISD::SELECT:
2896     return lowerSELECT(Op, DAG);
2897   case ISD::BRCOND:
2898     return lowerBRCOND(Op, DAG);
2899   case ISD::VASTART:
2900     return lowerVASTART(Op, DAG);
2901   case ISD::FRAMEADDR:
2902     return lowerFRAMEADDR(Op, DAG);
2903   case ISD::RETURNADDR:
2904     return lowerRETURNADDR(Op, DAG);
2905   case ISD::SHL_PARTS:
2906     return lowerShiftLeftParts(Op, DAG);
2907   case ISD::SRA_PARTS:
2908     return lowerShiftRightParts(Op, DAG, true);
2909   case ISD::SRL_PARTS:
2910     return lowerShiftRightParts(Op, DAG, false);
2911   case ISD::BITCAST: {
2912     SDLoc DL(Op);
2913     EVT VT = Op.getValueType();
2914     SDValue Op0 = Op.getOperand(0);
2915     EVT Op0VT = Op0.getValueType();
2916     MVT XLenVT = Subtarget.getXLenVT();
2917     if (VT.isFixedLengthVector()) {
2918       // We can handle fixed length vector bitcasts with a simple replacement
2919       // in isel.
2920       if (Op0VT.isFixedLengthVector())
2921         return Op;
2922       // When bitcasting from scalar to fixed-length vector, insert the scalar
2923       // into a one-element vector of the result type, and perform a vector
2924       // bitcast.
2925       if (!Op0VT.isVector()) {
2926         EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
2927         if (!isTypeLegal(BVT))
2928           return SDValue();
2929         return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
2930                                               DAG.getUNDEF(BVT), Op0,
2931                                               DAG.getConstant(0, DL, XLenVT)));
2932       }
2933       return SDValue();
2934     }
2935     // Custom-legalize bitcasts from fixed-length vector types to scalar types
2936     // thus: bitcast the vector to a one-element vector type whose element type
2937     // is the same as the result type, and extract the first element.
2938     if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
2939       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
2940       if (!isTypeLegal(BVT))
2941         return SDValue();
2942       SDValue BVec = DAG.getBitcast(BVT, Op0);
2943       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
2944                          DAG.getConstant(0, DL, XLenVT));
2945     }
2946     if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
2947       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
2948       SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
2949       return FPConv;
2950     }
2951     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
2952         Subtarget.hasStdExtF()) {
2953       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
2954       SDValue FPConv =
2955           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
2956       return FPConv;
2957     }
2958     return SDValue();
2959   }
2960   case ISD::INTRINSIC_WO_CHAIN:
2961     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2962   case ISD::INTRINSIC_W_CHAIN:
2963     return LowerINTRINSIC_W_CHAIN(Op, DAG);
2964   case ISD::INTRINSIC_VOID:
2965     return LowerINTRINSIC_VOID(Op, DAG);
2966   case ISD::BSWAP:
2967   case ISD::BITREVERSE: {
2968     MVT VT = Op.getSimpleValueType();
2969     SDLoc DL(Op);
2970     if (Subtarget.hasStdExtZbp()) {
2971       // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
2972       // Start with the maximum immediate value which is the bitwidth - 1.
2973       unsigned Imm = VT.getSizeInBits() - 1;
2974       // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
2975       if (Op.getOpcode() == ISD::BSWAP)
2976         Imm &= ~0x7U;
2977       return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
2978                          DAG.getConstant(Imm, DL, VT));
2979     }
2980     assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
2981     assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
2982     // Expand bitreverse to a bswap(rev8) followed by brev8.
2983     SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
2984     // We use the Zbp grevi encoding for rev.b/brev8 which will be recognized
2985     // as brev8 by an isel pattern.
2986     return DAG.getNode(RISCVISD::GREV, DL, VT, BSwap,
2987                        DAG.getConstant(7, DL, VT));
2988   }
2989   case ISD::FSHL:
2990   case ISD::FSHR: {
2991     MVT VT = Op.getSimpleValueType();
2992     assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
2993     SDLoc DL(Op);
2994     // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
2995     // use log(XLen) bits. Mask the shift amount accordingly to prevent
2996     // accidentally setting the extra bit.
2997     unsigned ShAmtWidth = Subtarget.getXLen() - 1;
2998     SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
2999                                 DAG.getConstant(ShAmtWidth, DL, VT));
3000     // fshl and fshr concatenate their operands in the same order. fsr and fsl
3001     // instruction use different orders. fshl will return its first operand for
3002     // shift of zero, fshr will return its second operand. fsl and fsr both
3003     // return rs1 so the ISD nodes need to have different operand orders.
3004     // Shift amount is in rs2.
3005     SDValue Op0 = Op.getOperand(0);
3006     SDValue Op1 = Op.getOperand(1);
3007     unsigned Opc = RISCVISD::FSL;
3008     if (Op.getOpcode() == ISD::FSHR) {
3009       std::swap(Op0, Op1);
3010       Opc = RISCVISD::FSR;
3011     }
3012     return DAG.getNode(Opc, DL, VT, Op0, Op1, ShAmt);
3013   }
3014   case ISD::TRUNCATE: {
3015     SDLoc DL(Op);
3016     MVT VT = Op.getSimpleValueType();
3017     // Only custom-lower vector truncates
3018     if (!VT.isVector())
3019       return Op;
3020 
3021     // Truncates to mask types are handled differently
3022     if (VT.getVectorElementType() == MVT::i1)
3023       return lowerVectorMaskTrunc(Op, DAG);
3024 
3025     // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
3026     // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
3027     // truncate by one power of two at a time.
3028     MVT DstEltVT = VT.getVectorElementType();
3029 
3030     SDValue Src = Op.getOperand(0);
3031     MVT SrcVT = Src.getSimpleValueType();
3032     MVT SrcEltVT = SrcVT.getVectorElementType();
3033 
3034     assert(DstEltVT.bitsLT(SrcEltVT) &&
3035            isPowerOf2_64(DstEltVT.getSizeInBits()) &&
3036            isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
3037            "Unexpected vector truncate lowering");
3038 
3039     MVT ContainerVT = SrcVT;
3040     if (SrcVT.isFixedLengthVector()) {
3041       ContainerVT = getContainerForFixedLengthVector(SrcVT);
3042       Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3043     }
3044 
3045     SDValue Result = Src;
3046     SDValue Mask, VL;
3047     std::tie(Mask, VL) =
3048         getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
3049     LLVMContext &Context = *DAG.getContext();
3050     const ElementCount Count = ContainerVT.getVectorElementCount();
3051     do {
3052       SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3053       EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
3054       Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
3055                            Mask, VL);
3056     } while (SrcEltVT != DstEltVT);
3057 
3058     if (SrcVT.isFixedLengthVector())
3059       Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
3060 
3061     return Result;
3062   }
3063   case ISD::ANY_EXTEND:
3064   case ISD::ZERO_EXTEND:
3065     if (Op.getOperand(0).getValueType().isVector() &&
3066         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3067       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
3068     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
3069   case ISD::SIGN_EXTEND:
3070     if (Op.getOperand(0).getValueType().isVector() &&
3071         Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3072       return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
3073     return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
3074   case ISD::SPLAT_VECTOR_PARTS:
3075     return lowerSPLAT_VECTOR_PARTS(Op, DAG);
3076   case ISD::INSERT_VECTOR_ELT:
3077     return lowerINSERT_VECTOR_ELT(Op, DAG);
3078   case ISD::EXTRACT_VECTOR_ELT:
3079     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
3080   case ISD::VSCALE: {
3081     MVT VT = Op.getSimpleValueType();
3082     SDLoc DL(Op);
3083     SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
3084     // We define our scalable vector types for lmul=1 to use a 64 bit known
3085     // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
3086     // vscale as VLENB / 8.
3087     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
3088     if (Subtarget.getMinVLen() < RISCV::RVVBitsPerBlock)
3089       report_fatal_error("Support for VLEN==32 is incomplete.");
3090     if (isa<ConstantSDNode>(Op.getOperand(0))) {
3091       // We assume VLENB is a multiple of 8. We manually choose the best shift
3092       // here because SimplifyDemandedBits isn't always able to simplify it.
3093       uint64_t Val = Op.getConstantOperandVal(0);
3094       if (isPowerOf2_64(Val)) {
3095         uint64_t Log2 = Log2_64(Val);
3096         if (Log2 < 3)
3097           return DAG.getNode(ISD::SRL, DL, VT, VLENB,
3098                              DAG.getConstant(3 - Log2, DL, VT));
3099         if (Log2 > 3)
3100           return DAG.getNode(ISD::SHL, DL, VT, VLENB,
3101                              DAG.getConstant(Log2 - 3, DL, VT));
3102         return VLENB;
3103       }
3104       // If the multiplier is a multiple of 8, scale it down to avoid needing
3105       // to shift the VLENB value.
3106       if ((Val % 8) == 0)
3107         return DAG.getNode(ISD::MUL, DL, VT, VLENB,
3108                            DAG.getConstant(Val / 8, DL, VT));
3109     }
3110 
3111     SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
3112                                  DAG.getConstant(3, DL, VT));
3113     return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
3114   }
3115   case ISD::FPOWI: {
3116     // Custom promote f16 powi with illegal i32 integer type on RV64. Once
3117     // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
3118     if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
3119         Op.getOperand(1).getValueType() == MVT::i32) {
3120       SDLoc DL(Op);
3121       SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
3122       SDValue Powi =
3123           DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
3124       return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
3125                          DAG.getIntPtrConstant(0, DL));
3126     }
3127     return SDValue();
3128   }
3129   case ISD::FP_EXTEND: {
3130     // RVV can only do fp_extend to types double the size as the source. We
3131     // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
3132     // via f32.
3133     SDLoc DL(Op);
3134     MVT VT = Op.getSimpleValueType();
3135     SDValue Src = Op.getOperand(0);
3136     MVT SrcVT = Src.getSimpleValueType();
3137 
3138     // Prepare any fixed-length vector operands.
3139     MVT ContainerVT = VT;
3140     if (SrcVT.isFixedLengthVector()) {
3141       ContainerVT = getContainerForFixedLengthVector(VT);
3142       MVT SrcContainerVT =
3143           ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
3144       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3145     }
3146 
3147     if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
3148         SrcVT.getVectorElementType() != MVT::f16) {
3149       // For scalable vectors, we only need to close the gap between
3150       // vXf16->vXf64.
3151       if (!VT.isFixedLengthVector())
3152         return Op;
3153       // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
3154       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3155       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3156     }
3157 
3158     MVT InterVT = VT.changeVectorElementType(MVT::f32);
3159     MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
3160     SDValue IntermediateExtend = getRVVFPExtendOrRound(
3161         Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
3162 
3163     SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
3164                                            DL, DAG, Subtarget);
3165     if (VT.isFixedLengthVector())
3166       return convertFromScalableVector(VT, Extend, DAG, Subtarget);
3167     return Extend;
3168   }
3169   case ISD::FP_ROUND: {
3170     // RVV can only do fp_round to types half the size as the source. We
3171     // custom-lower f64->f16 rounds via RVV's round-to-odd float
3172     // conversion instruction.
3173     SDLoc DL(Op);
3174     MVT VT = Op.getSimpleValueType();
3175     SDValue Src = Op.getOperand(0);
3176     MVT SrcVT = Src.getSimpleValueType();
3177 
3178     // Prepare any fixed-length vector operands.
3179     MVT ContainerVT = VT;
3180     if (VT.isFixedLengthVector()) {
3181       MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3182       ContainerVT =
3183           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3184       Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3185     }
3186 
3187     if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
3188         SrcVT.getVectorElementType() != MVT::f64) {
3189       // For scalable vectors, we only need to close the gap between
3190       // vXf64<->vXf16.
3191       if (!VT.isFixedLengthVector())
3192         return Op;
3193       // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
3194       Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
3195       return convertFromScalableVector(VT, Src, DAG, Subtarget);
3196     }
3197 
3198     SDValue Mask, VL;
3199     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3200 
3201     MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
3202     SDValue IntermediateRound =
3203         DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
3204     SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
3205                                           DL, DAG, Subtarget);
3206 
3207     if (VT.isFixedLengthVector())
3208       return convertFromScalableVector(VT, Round, DAG, Subtarget);
3209     return Round;
3210   }
3211   case ISD::FP_TO_SINT:
3212   case ISD::FP_TO_UINT:
3213   case ISD::SINT_TO_FP:
3214   case ISD::UINT_TO_FP: {
3215     // RVV can only do fp<->int conversions to types half/double the size as
3216     // the source. We custom-lower any conversions that do two hops into
3217     // sequences.
3218     MVT VT = Op.getSimpleValueType();
3219     if (!VT.isVector())
3220       return Op;
3221     SDLoc DL(Op);
3222     SDValue Src = Op.getOperand(0);
3223     MVT EltVT = VT.getVectorElementType();
3224     MVT SrcVT = Src.getSimpleValueType();
3225     MVT SrcEltVT = SrcVT.getVectorElementType();
3226     unsigned EltSize = EltVT.getSizeInBits();
3227     unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3228     assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
3229            "Unexpected vector element types");
3230 
3231     bool IsInt2FP = SrcEltVT.isInteger();
3232     // Widening conversions
3233     if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
3234       if (IsInt2FP) {
3235         // Do a regular integer sign/zero extension then convert to float.
3236         MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
3237                                       VT.getVectorElementCount());
3238         unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
3239                                  ? ISD::ZERO_EXTEND
3240                                  : ISD::SIGN_EXTEND;
3241         SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
3242         return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
3243       }
3244       // FP2Int
3245       assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
3246       // Do one doubling fp_extend then complete the operation by converting
3247       // to int.
3248       MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3249       SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
3250       return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
3251     }
3252 
3253     // Narrowing conversions
3254     if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
3255       if (IsInt2FP) {
3256         // One narrowing int_to_fp, then an fp_round.
3257         assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
3258         MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
3259         SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
3260         return DAG.getFPExtendOrRound(Int2FP, DL, VT);
3261       }
3262       // FP2Int
3263       // One narrowing fp_to_int, then truncate the integer. If the float isn't
3264       // representable by the integer, the result is poison.
3265       MVT IVecVT =
3266           MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
3267                            VT.getVectorElementCount());
3268       SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
3269       return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
3270     }
3271 
3272     // Scalable vectors can exit here. Patterns will handle equally-sized
3273     // conversions halving/doubling ones.
3274     if (!VT.isFixedLengthVector())
3275       return Op;
3276 
3277     // For fixed-length vectors we lower to a custom "VL" node.
3278     unsigned RVVOpc = 0;
3279     switch (Op.getOpcode()) {
3280     default:
3281       llvm_unreachable("Impossible opcode");
3282     case ISD::FP_TO_SINT:
3283       RVVOpc = RISCVISD::FP_TO_SINT_VL;
3284       break;
3285     case ISD::FP_TO_UINT:
3286       RVVOpc = RISCVISD::FP_TO_UINT_VL;
3287       break;
3288     case ISD::SINT_TO_FP:
3289       RVVOpc = RISCVISD::SINT_TO_FP_VL;
3290       break;
3291     case ISD::UINT_TO_FP:
3292       RVVOpc = RISCVISD::UINT_TO_FP_VL;
3293       break;
3294     }
3295 
3296     MVT ContainerVT, SrcContainerVT;
3297     // Derive the reference container type from the larger vector type.
3298     if (SrcEltSize > EltSize) {
3299       SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
3300       ContainerVT =
3301           SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
3302     } else {
3303       ContainerVT = getContainerForFixedLengthVector(VT);
3304       SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
3305     }
3306 
3307     SDValue Mask, VL;
3308     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3309 
3310     Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3311     Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
3312     return convertFromScalableVector(VT, Src, DAG, Subtarget);
3313   }
3314   case ISD::FP_TO_SINT_SAT:
3315   case ISD::FP_TO_UINT_SAT:
3316     return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
3317   case ISD::FTRUNC:
3318   case ISD::FCEIL:
3319   case ISD::FFLOOR:
3320     return lowerFTRUNC_FCEIL_FFLOOR(Op, DAG);
3321   case ISD::VECREDUCE_ADD:
3322   case ISD::VECREDUCE_UMAX:
3323   case ISD::VECREDUCE_SMAX:
3324   case ISD::VECREDUCE_UMIN:
3325   case ISD::VECREDUCE_SMIN:
3326     return lowerVECREDUCE(Op, DAG);
3327   case ISD::VECREDUCE_AND:
3328   case ISD::VECREDUCE_OR:
3329   case ISD::VECREDUCE_XOR:
3330     if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
3331       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
3332     return lowerVECREDUCE(Op, DAG);
3333   case ISD::VECREDUCE_FADD:
3334   case ISD::VECREDUCE_SEQ_FADD:
3335   case ISD::VECREDUCE_FMIN:
3336   case ISD::VECREDUCE_FMAX:
3337     return lowerFPVECREDUCE(Op, DAG);
3338   case ISD::VP_REDUCE_ADD:
3339   case ISD::VP_REDUCE_UMAX:
3340   case ISD::VP_REDUCE_SMAX:
3341   case ISD::VP_REDUCE_UMIN:
3342   case ISD::VP_REDUCE_SMIN:
3343   case ISD::VP_REDUCE_FADD:
3344   case ISD::VP_REDUCE_SEQ_FADD:
3345   case ISD::VP_REDUCE_FMIN:
3346   case ISD::VP_REDUCE_FMAX:
3347     return lowerVPREDUCE(Op, DAG);
3348   case ISD::VP_REDUCE_AND:
3349   case ISD::VP_REDUCE_OR:
3350   case ISD::VP_REDUCE_XOR:
3351     if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
3352       return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
3353     return lowerVPREDUCE(Op, DAG);
3354   case ISD::INSERT_SUBVECTOR:
3355     return lowerINSERT_SUBVECTOR(Op, DAG);
3356   case ISD::EXTRACT_SUBVECTOR:
3357     return lowerEXTRACT_SUBVECTOR(Op, DAG);
3358   case ISD::STEP_VECTOR:
3359     return lowerSTEP_VECTOR(Op, DAG);
3360   case ISD::VECTOR_REVERSE:
3361     return lowerVECTOR_REVERSE(Op, DAG);
3362   case ISD::BUILD_VECTOR:
3363     return lowerBUILD_VECTOR(Op, DAG, Subtarget);
3364   case ISD::SPLAT_VECTOR:
3365     if (Op.getValueType().getVectorElementType() == MVT::i1)
3366       return lowerVectorMaskSplat(Op, DAG);
3367     return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
3368   case ISD::VECTOR_SHUFFLE:
3369     return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
3370   case ISD::CONCAT_VECTORS: {
3371     // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
3372     // better than going through the stack, as the default expansion does.
3373     SDLoc DL(Op);
3374     MVT VT = Op.getSimpleValueType();
3375     unsigned NumOpElts =
3376         Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3377     SDValue Vec = DAG.getUNDEF(VT);
3378     for (const auto &OpIdx : enumerate(Op->ops())) {
3379       SDValue SubVec = OpIdx.value();
3380       // Don't insert undef subvectors.
3381       if (SubVec.isUndef())
3382         continue;
3383       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
3384                         DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
3385     }
3386     return Vec;
3387   }
3388   case ISD::LOAD:
3389     if (auto V = expandUnalignedRVVLoad(Op, DAG))
3390       return V;
3391     if (Op.getValueType().isFixedLengthVector())
3392       return lowerFixedLengthVectorLoadToRVV(Op, DAG);
3393     return Op;
3394   case ISD::STORE:
3395     if (auto V = expandUnalignedRVVStore(Op, DAG))
3396       return V;
3397     if (Op.getOperand(1).getValueType().isFixedLengthVector())
3398       return lowerFixedLengthVectorStoreToRVV(Op, DAG);
3399     return Op;
3400   case ISD::MLOAD:
3401   case ISD::VP_LOAD:
3402     return lowerMaskedLoad(Op, DAG);
3403   case ISD::MSTORE:
3404   case ISD::VP_STORE:
3405     return lowerMaskedStore(Op, DAG);
3406   case ISD::SETCC:
3407     return lowerFixedLengthVectorSetccToRVV(Op, DAG);
3408   case ISD::ADD:
3409     return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
3410   case ISD::SUB:
3411     return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
3412   case ISD::MUL:
3413     return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
3414   case ISD::MULHS:
3415     return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
3416   case ISD::MULHU:
3417     return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
3418   case ISD::AND:
3419     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
3420                                               RISCVISD::AND_VL);
3421   case ISD::OR:
3422     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
3423                                               RISCVISD::OR_VL);
3424   case ISD::XOR:
3425     return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
3426                                               RISCVISD::XOR_VL);
3427   case ISD::SDIV:
3428     return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
3429   case ISD::SREM:
3430     return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
3431   case ISD::UDIV:
3432     return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
3433   case ISD::UREM:
3434     return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
3435   case ISD::SHL:
3436   case ISD::SRA:
3437   case ISD::SRL:
3438     if (Op.getSimpleValueType().isFixedLengthVector())
3439       return lowerFixedLengthVectorShiftToRVV(Op, DAG);
3440     // This can be called for an i32 shift amount that needs to be promoted.
3441     assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
3442            "Unexpected custom legalisation");
3443     return SDValue();
3444   case ISD::SADDSAT:
3445     return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
3446   case ISD::UADDSAT:
3447     return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
3448   case ISD::SSUBSAT:
3449     return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
3450   case ISD::USUBSAT:
3451     return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
3452   case ISD::FADD:
3453     return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
3454   case ISD::FSUB:
3455     return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
3456   case ISD::FMUL:
3457     return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
3458   case ISD::FDIV:
3459     return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
3460   case ISD::FNEG:
3461     return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
3462   case ISD::FABS:
3463     return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
3464   case ISD::FSQRT:
3465     return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
3466   case ISD::FMA:
3467     return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
3468   case ISD::SMIN:
3469     return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
3470   case ISD::SMAX:
3471     return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
3472   case ISD::UMIN:
3473     return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
3474   case ISD::UMAX:
3475     return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
3476   case ISD::FMINNUM:
3477     return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
3478   case ISD::FMAXNUM:
3479     return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
3480   case ISD::ABS:
3481     return lowerABS(Op, DAG);
3482   case ISD::CTLZ_ZERO_UNDEF:
3483   case ISD::CTTZ_ZERO_UNDEF:
3484     return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
3485   case ISD::VSELECT:
3486     return lowerFixedLengthVectorSelectToRVV(Op, DAG);
3487   case ISD::FCOPYSIGN:
3488     return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
3489   case ISD::MGATHER:
3490   case ISD::VP_GATHER:
3491     return lowerMaskedGather(Op, DAG);
3492   case ISD::MSCATTER:
3493   case ISD::VP_SCATTER:
3494     return lowerMaskedScatter(Op, DAG);
3495   case ISD::FLT_ROUNDS_:
3496     return lowerGET_ROUNDING(Op, DAG);
3497   case ISD::SET_ROUNDING:
3498     return lowerSET_ROUNDING(Op, DAG);
3499   case ISD::VP_SELECT:
3500     return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL);
3501   case ISD::VP_MERGE:
3502     return lowerVPOp(Op, DAG, RISCVISD::VP_MERGE_VL);
3503   case ISD::VP_ADD:
3504     return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
3505   case ISD::VP_SUB:
3506     return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
3507   case ISD::VP_MUL:
3508     return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
3509   case ISD::VP_SDIV:
3510     return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
3511   case ISD::VP_UDIV:
3512     return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
3513   case ISD::VP_SREM:
3514     return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
3515   case ISD::VP_UREM:
3516     return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
3517   case ISD::VP_AND:
3518     return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL);
3519   case ISD::VP_OR:
3520     return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL);
3521   case ISD::VP_XOR:
3522     return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL);
3523   case ISD::VP_ASHR:
3524     return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
3525   case ISD::VP_LSHR:
3526     return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
3527   case ISD::VP_SHL:
3528     return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
3529   case ISD::VP_FADD:
3530     return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
3531   case ISD::VP_FSUB:
3532     return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
3533   case ISD::VP_FMUL:
3534     return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
3535   case ISD::VP_FDIV:
3536     return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
3537   }
3538 }
3539 
3540 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
3541                              SelectionDAG &DAG, unsigned Flags) {
3542   return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
3543 }
3544 
3545 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
3546                              SelectionDAG &DAG, unsigned Flags) {
3547   return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
3548                                    Flags);
3549 }
3550 
3551 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
3552                              SelectionDAG &DAG, unsigned Flags) {
3553   return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
3554                                    N->getOffset(), Flags);
3555 }
3556 
3557 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
3558                              SelectionDAG &DAG, unsigned Flags) {
3559   return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
3560 }
3561 
3562 template <class NodeTy>
3563 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3564                                      bool IsLocal) const {
3565   SDLoc DL(N);
3566   EVT Ty = getPointerTy(DAG.getDataLayout());
3567 
3568   if (isPositionIndependent()) {
3569     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3570     if (IsLocal)
3571       // Use PC-relative addressing to access the symbol. This generates the
3572       // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
3573       // %pcrel_lo(auipc)).
3574       return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3575 
3576     // Use PC-relative addressing to access the GOT for this symbol, then load
3577     // the address from the GOT. This generates the pattern (PseudoLA sym),
3578     // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
3579     return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
3580   }
3581 
3582   switch (getTargetMachine().getCodeModel()) {
3583   default:
3584     report_fatal_error("Unsupported code model for lowering");
3585   case CodeModel::Small: {
3586     // Generate a sequence for accessing addresses within the first 2 GiB of
3587     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
3588     SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
3589     SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
3590     SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3591     return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
3592   }
3593   case CodeModel::Medium: {
3594     // Generate a sequence for accessing addresses within any 2GiB range within
3595     // the address space. This generates the pattern (PseudoLLA sym), which
3596     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
3597     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
3598     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
3599   }
3600   }
3601 }
3602 
3603 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
3604                                                 SelectionDAG &DAG) const {
3605   SDLoc DL(Op);
3606   EVT Ty = Op.getValueType();
3607   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3608   int64_t Offset = N->getOffset();
3609   MVT XLenVT = Subtarget.getXLenVT();
3610 
3611   const GlobalValue *GV = N->getGlobal();
3612   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
3613   SDValue Addr = getAddr(N, DAG, IsLocal);
3614 
3615   // In order to maximise the opportunity for common subexpression elimination,
3616   // emit a separate ADD node for the global address offset instead of folding
3617   // it in the global address node. Later peephole optimisations may choose to
3618   // fold it back in when profitable.
3619   if (Offset != 0)
3620     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3621                        DAG.getConstant(Offset, DL, XLenVT));
3622   return Addr;
3623 }
3624 
3625 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
3626                                                SelectionDAG &DAG) const {
3627   BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
3628 
3629   return getAddr(N, DAG);
3630 }
3631 
3632 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
3633                                                SelectionDAG &DAG) const {
3634   ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
3635 
3636   return getAddr(N, DAG);
3637 }
3638 
3639 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
3640                                             SelectionDAG &DAG) const {
3641   JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
3642 
3643   return getAddr(N, DAG);
3644 }
3645 
3646 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
3647                                               SelectionDAG &DAG,
3648                                               bool UseGOT) const {
3649   SDLoc DL(N);
3650   EVT Ty = getPointerTy(DAG.getDataLayout());
3651   const GlobalValue *GV = N->getGlobal();
3652   MVT XLenVT = Subtarget.getXLenVT();
3653 
3654   if (UseGOT) {
3655     // Use PC-relative addressing to access the GOT for this TLS symbol, then
3656     // load the address from the GOT and add the thread pointer. This generates
3657     // the pattern (PseudoLA_TLS_IE sym), which expands to
3658     // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
3659     SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3660     SDValue Load =
3661         SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
3662 
3663     // Add the thread pointer.
3664     SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3665     return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
3666   }
3667 
3668   // Generate a sequence for accessing the address relative to the thread
3669   // pointer, with the appropriate adjustment for the thread pointer offset.
3670   // This generates the pattern
3671   // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
3672   SDValue AddrHi =
3673       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
3674   SDValue AddrAdd =
3675       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
3676   SDValue AddrLo =
3677       DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
3678 
3679   SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
3680   SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
3681   SDValue MNAdd = SDValue(
3682       DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
3683       0);
3684   return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
3685 }
3686 
3687 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
3688                                                SelectionDAG &DAG) const {
3689   SDLoc DL(N);
3690   EVT Ty = getPointerTy(DAG.getDataLayout());
3691   IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
3692   const GlobalValue *GV = N->getGlobal();
3693 
3694   // Use a PC-relative addressing mode to access the global dynamic GOT address.
3695   // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
3696   // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
3697   SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
3698   SDValue Load =
3699       SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
3700 
3701   // Prepare argument list to generate call.
3702   ArgListTy Args;
3703   ArgListEntry Entry;
3704   Entry.Node = Load;
3705   Entry.Ty = CallTy;
3706   Args.push_back(Entry);
3707 
3708   // Setup call to __tls_get_addr.
3709   TargetLowering::CallLoweringInfo CLI(DAG);
3710   CLI.setDebugLoc(DL)
3711       .setChain(DAG.getEntryNode())
3712       .setLibCallee(CallingConv::C, CallTy,
3713                     DAG.getExternalSymbol("__tls_get_addr", Ty),
3714                     std::move(Args));
3715 
3716   return LowerCallTo(CLI).first;
3717 }
3718 
3719 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
3720                                                    SelectionDAG &DAG) const {
3721   SDLoc DL(Op);
3722   EVT Ty = Op.getValueType();
3723   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
3724   int64_t Offset = N->getOffset();
3725   MVT XLenVT = Subtarget.getXLenVT();
3726 
3727   TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
3728 
3729   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
3730       CallingConv::GHC)
3731     report_fatal_error("In GHC calling convention TLS is not supported");
3732 
3733   SDValue Addr;
3734   switch (Model) {
3735   case TLSModel::LocalExec:
3736     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
3737     break;
3738   case TLSModel::InitialExec:
3739     Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
3740     break;
3741   case TLSModel::LocalDynamic:
3742   case TLSModel::GeneralDynamic:
3743     Addr = getDynamicTLSAddr(N, DAG);
3744     break;
3745   }
3746 
3747   // In order to maximise the opportunity for common subexpression elimination,
3748   // emit a separate ADD node for the global address offset instead of folding
3749   // it in the global address node. Later peephole optimisations may choose to
3750   // fold it back in when profitable.
3751   if (Offset != 0)
3752     return DAG.getNode(ISD::ADD, DL, Ty, Addr,
3753                        DAG.getConstant(Offset, DL, XLenVT));
3754   return Addr;
3755 }
3756 
3757 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3758   SDValue CondV = Op.getOperand(0);
3759   SDValue TrueV = Op.getOperand(1);
3760   SDValue FalseV = Op.getOperand(2);
3761   SDLoc DL(Op);
3762   MVT VT = Op.getSimpleValueType();
3763   MVT XLenVT = Subtarget.getXLenVT();
3764 
3765   // Lower vector SELECTs to VSELECTs by splatting the condition.
3766   if (VT.isVector()) {
3767     MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
3768     SDValue CondSplat = VT.isScalableVector()
3769                             ? DAG.getSplatVector(SplatCondVT, DL, CondV)
3770                             : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
3771     return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
3772   }
3773 
3774   // If the result type is XLenVT and CondV is the output of a SETCC node
3775   // which also operated on XLenVT inputs, then merge the SETCC node into the
3776   // lowered RISCVISD::SELECT_CC to take advantage of the integer
3777   // compare+branch instructions. i.e.:
3778   // (select (setcc lhs, rhs, cc), truev, falsev)
3779   // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
3780   if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
3781       CondV.getOperand(0).getSimpleValueType() == XLenVT) {
3782     SDValue LHS = CondV.getOperand(0);
3783     SDValue RHS = CondV.getOperand(1);
3784     const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
3785     ISD::CondCode CCVal = CC->get();
3786 
3787     // Special case for a select of 2 constants that have a diffence of 1.
3788     // Normally this is done by DAGCombine, but if the select is introduced by
3789     // type legalization or op legalization, we miss it. Restricting to SETLT
3790     // case for now because that is what signed saturating add/sub need.
3791     // FIXME: We don't need the condition to be SETLT or even a SETCC,
3792     // but we would probably want to swap the true/false values if the condition
3793     // is SETGE/SETLE to avoid an XORI.
3794     if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3795         CCVal == ISD::SETLT) {
3796       const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3797       const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3798       if (TrueVal - 1 == FalseVal)
3799         return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3800       if (TrueVal + 1 == FalseVal)
3801         return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3802     }
3803 
3804     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3805 
3806     SDValue TargetCC = DAG.getCondCode(CCVal);
3807     SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3808     return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3809   }
3810 
3811   // Otherwise:
3812   // (select condv, truev, falsev)
3813   // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
3814   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3815   SDValue SetNE = DAG.getCondCode(ISD::SETNE);
3816 
3817   SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3818 
3819   return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3820 }
3821 
3822 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
3823   SDValue CondV = Op.getOperand(1);
3824   SDLoc DL(Op);
3825   MVT XLenVT = Subtarget.getXLenVT();
3826 
3827   if (CondV.getOpcode() == ISD::SETCC &&
3828       CondV.getOperand(0).getValueType() == XLenVT) {
3829     SDValue LHS = CondV.getOperand(0);
3830     SDValue RHS = CondV.getOperand(1);
3831     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
3832 
3833     translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3834 
3835     SDValue TargetCC = DAG.getCondCode(CCVal);
3836     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3837                        LHS, RHS, TargetCC, Op.getOperand(2));
3838   }
3839 
3840   return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3841                      CondV, DAG.getConstant(0, DL, XLenVT),
3842                      DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
3843 }
3844 
3845 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3846   MachineFunction &MF = DAG.getMachineFunction();
3847   RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
3848 
3849   SDLoc DL(Op);
3850   SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3851                                  getPointerTy(MF.getDataLayout()));
3852 
3853   // vastart just stores the address of the VarArgsFrameIndex slot into the
3854   // memory location argument.
3855   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3856   return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
3857                       MachinePointerInfo(SV));
3858 }
3859 
3860 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
3861                                             SelectionDAG &DAG) const {
3862   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3863   MachineFunction &MF = DAG.getMachineFunction();
3864   MachineFrameInfo &MFI = MF.getFrameInfo();
3865   MFI.setFrameAddressIsTaken(true);
3866   Register FrameReg = RI.getFrameRegister(MF);
3867   int XLenInBytes = Subtarget.getXLen() / 8;
3868 
3869   EVT VT = Op.getValueType();
3870   SDLoc DL(Op);
3871   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
3872   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3873   while (Depth--) {
3874     int Offset = -(XLenInBytes * 2);
3875     SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
3876                               DAG.getIntPtrConstant(Offset, DL));
3877     FrameAddr =
3878         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
3879   }
3880   return FrameAddr;
3881 }
3882 
3883 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
3884                                              SelectionDAG &DAG) const {
3885   const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3886   MachineFunction &MF = DAG.getMachineFunction();
3887   MachineFrameInfo &MFI = MF.getFrameInfo();
3888   MFI.setReturnAddressIsTaken(true);
3889   MVT XLenVT = Subtarget.getXLenVT();
3890   int XLenInBytes = Subtarget.getXLen() / 8;
3891 
3892   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3893     return SDValue();
3894 
3895   EVT VT = Op.getValueType();
3896   SDLoc DL(Op);
3897   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3898   if (Depth) {
3899     int Off = -XLenInBytes;
3900     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3901     SDValue Offset = DAG.getConstant(Off, DL, VT);
3902     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3903                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3904                        MachinePointerInfo());
3905   }
3906 
3907   // Return the value of the return address register, marking it an implicit
3908   // live-in.
3909   Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
3910   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
3911 }
3912 
3913 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
3914                                                  SelectionDAG &DAG) const {
3915   SDLoc DL(Op);
3916   SDValue Lo = Op.getOperand(0);
3917   SDValue Hi = Op.getOperand(1);
3918   SDValue Shamt = Op.getOperand(2);
3919   EVT VT = Lo.getValueType();
3920 
3921   // if Shamt-XLEN < 0: // Shamt < XLEN
3922   //   Lo = Lo << Shamt
3923   //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
3924   // else:
3925   //   Lo = 0
3926   //   Hi = Lo << (Shamt-XLEN)
3927 
3928   SDValue Zero = DAG.getConstant(0, DL, VT);
3929   SDValue One = DAG.getConstant(1, DL, VT);
3930   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3931   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3932   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3933   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3934 
3935   SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
3936   SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
3937   SDValue ShiftRightLo =
3938       DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
3939   SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
3940   SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
3941   SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
3942 
3943   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3944 
3945   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
3946   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3947 
3948   SDValue Parts[2] = {Lo, Hi};
3949   return DAG.getMergeValues(Parts, DL);
3950 }
3951 
3952 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
3953                                                   bool IsSRA) const {
3954   SDLoc DL(Op);
3955   SDValue Lo = Op.getOperand(0);
3956   SDValue Hi = Op.getOperand(1);
3957   SDValue Shamt = Op.getOperand(2);
3958   EVT VT = Lo.getValueType();
3959 
3960   // SRA expansion:
3961   //   if Shamt-XLEN < 0: // Shamt < XLEN
3962   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3963   //     Hi = Hi >>s Shamt
3964   //   else:
3965   //     Lo = Hi >>s (Shamt-XLEN);
3966   //     Hi = Hi >>s (XLEN-1)
3967   //
3968   // SRL expansion:
3969   //   if Shamt-XLEN < 0: // Shamt < XLEN
3970   //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3971   //     Hi = Hi >>u Shamt
3972   //   else:
3973   //     Lo = Hi >>u (Shamt-XLEN);
3974   //     Hi = 0;
3975 
3976   unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
3977 
3978   SDValue Zero = DAG.getConstant(0, DL, VT);
3979   SDValue One = DAG.getConstant(1, DL, VT);
3980   SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3981   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3982   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3983   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3984 
3985   SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
3986   SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
3987   SDValue ShiftLeftHi =
3988       DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
3989   SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
3990   SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
3991   SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
3992   SDValue HiFalse =
3993       IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
3994 
3995   SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3996 
3997   Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
3998   Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3999 
4000   SDValue Parts[2] = {Lo, Hi};
4001   return DAG.getMergeValues(Parts, DL);
4002 }
4003 
4004 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
4005 // legal equivalently-sized i8 type, so we can use that as a go-between.
4006 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
4007                                                   SelectionDAG &DAG) const {
4008   SDLoc DL(Op);
4009   MVT VT = Op.getSimpleValueType();
4010   SDValue SplatVal = Op.getOperand(0);
4011   // All-zeros or all-ones splats are handled specially.
4012   if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
4013     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
4014     return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
4015   }
4016   if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
4017     SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
4018     return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
4019   }
4020   MVT XLenVT = Subtarget.getXLenVT();
4021   assert(SplatVal.getValueType() == XLenVT &&
4022          "Unexpected type for i1 splat value");
4023   MVT InterVT = VT.changeVectorElementType(MVT::i8);
4024   SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
4025                          DAG.getConstant(1, DL, XLenVT));
4026   SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
4027   SDValue Zero = DAG.getConstant(0, DL, InterVT);
4028   return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
4029 }
4030 
4031 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
4032 // illegal (currently only vXi64 RV32).
4033 // FIXME: We could also catch non-constant sign-extended i32 values and lower
4034 // them to SPLAT_VECTOR_I64
4035 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
4036                                                      SelectionDAG &DAG) const {
4037   SDLoc DL(Op);
4038   MVT VecVT = Op.getSimpleValueType();
4039   assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
4040          "Unexpected SPLAT_VECTOR_PARTS lowering");
4041 
4042   assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
4043   SDValue Lo = Op.getOperand(0);
4044   SDValue Hi = Op.getOperand(1);
4045 
4046   if (VecVT.isFixedLengthVector()) {
4047     MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4048     SDLoc DL(Op);
4049     SDValue Mask, VL;
4050     std::tie(Mask, VL) =
4051         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4052 
4053     SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
4054     return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
4055   }
4056 
4057   if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4058     int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4059     int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4060     // If Hi constant is all the same sign bit as Lo, lower this as a custom
4061     // node in order to try and match RVV vector/scalar instructions.
4062     if ((LoC >> 31) == HiC)
4063       return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4064   }
4065 
4066   // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
4067   if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4068       isa<ConstantSDNode>(Hi.getOperand(1)) &&
4069       Hi.getConstantOperandVal(1) == 31)
4070     return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
4071 
4072   // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
4073   return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
4074                      DAG.getTargetConstant(RISCV::VLMaxSentinel, DL, MVT::i64));
4075 }
4076 
4077 // Custom-lower extensions from mask vectors by using a vselect either with 1
4078 // for zero/any-extension or -1 for sign-extension:
4079 //   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
4080 // Note that any-extension is lowered identically to zero-extension.
4081 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
4082                                                 int64_t ExtTrueVal) const {
4083   SDLoc DL(Op);
4084   MVT VecVT = Op.getSimpleValueType();
4085   SDValue Src = Op.getOperand(0);
4086   // Only custom-lower extensions from mask types
4087   assert(Src.getValueType().isVector() &&
4088          Src.getValueType().getVectorElementType() == MVT::i1);
4089 
4090   MVT XLenVT = Subtarget.getXLenVT();
4091   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
4092   SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
4093 
4094   if (VecVT.isScalableVector()) {
4095     // Be careful not to introduce illegal scalar types at this stage, and be
4096     // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
4097     // illegal and must be expanded. Since we know that the constants are
4098     // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
4099     bool IsRV32E64 =
4100         !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
4101 
4102     if (!IsRV32E64) {
4103       SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
4104       SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
4105     } else {
4106       SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
4107       SplatTrueVal =
4108           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
4109     }
4110 
4111     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
4112   }
4113 
4114   MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
4115   MVT I1ContainerVT =
4116       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4117 
4118   SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
4119 
4120   SDValue Mask, VL;
4121   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4122 
4123   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
4124   SplatTrueVal =
4125       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
4126   SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
4127                                SplatTrueVal, SplatZero, VL);
4128 
4129   return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
4130 }
4131 
4132 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4133     SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
4134   MVT ExtVT = Op.getSimpleValueType();
4135   // Only custom-lower extensions from fixed-length vector types.
4136   if (!ExtVT.isFixedLengthVector())
4137     return Op;
4138   MVT VT = Op.getOperand(0).getSimpleValueType();
4139   // Grab the canonical container type for the extended type. Infer the smaller
4140   // type from that to ensure the same number of vector elements, as we know
4141   // the LMUL will be sufficient to hold the smaller type.
4142   MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
4143   // Get the extended container type manually to ensure the same number of
4144   // vector elements between source and dest.
4145   MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
4146                                      ContainerExtVT.getVectorElementCount());
4147 
4148   SDValue Op1 =
4149       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
4150 
4151   SDLoc DL(Op);
4152   SDValue Mask, VL;
4153   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4154 
4155   SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
4156 
4157   return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
4158 }
4159 
4160 // Custom-lower truncations from vectors to mask vectors by using a mask and a
4161 // setcc operation:
4162 //   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
4163 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
4164                                                   SelectionDAG &DAG) const {
4165   SDLoc DL(Op);
4166   EVT MaskVT = Op.getValueType();
4167   // Only expect to custom-lower truncations to mask types
4168   assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
4169          "Unexpected type for vector mask lowering");
4170   SDValue Src = Op.getOperand(0);
4171   MVT VecVT = Src.getSimpleValueType();
4172 
4173   // If this is a fixed vector, we need to convert it to a scalable vector.
4174   MVT ContainerVT = VecVT;
4175   if (VecVT.isFixedLengthVector()) {
4176     ContainerVT = getContainerForFixedLengthVector(VecVT);
4177     Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
4178   }
4179 
4180   SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
4181   SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
4182 
4183   SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
4184   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
4185 
4186   if (VecVT.isScalableVector()) {
4187     SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
4188     return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
4189   }
4190 
4191   SDValue Mask, VL;
4192   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4193 
4194   MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
4195   SDValue Trunc =
4196       DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
4197   Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
4198                       DAG.getCondCode(ISD::SETNE), Mask, VL);
4199   return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
4200 }
4201 
4202 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
4203 // first position of a vector, and that vector is slid up to the insert index.
4204 // By limiting the active vector length to index+1 and merging with the
4205 // original vector (with an undisturbed tail policy for elements >= VL), we
4206 // achieve the desired result of leaving all elements untouched except the one
4207 // at VL-1, which is replaced with the desired value.
4208 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4209                                                     SelectionDAG &DAG) const {
4210   SDLoc DL(Op);
4211   MVT VecVT = Op.getSimpleValueType();
4212   SDValue Vec = Op.getOperand(0);
4213   SDValue Val = Op.getOperand(1);
4214   SDValue Idx = Op.getOperand(2);
4215 
4216   if (VecVT.getVectorElementType() == MVT::i1) {
4217     // FIXME: For now we just promote to an i8 vector and insert into that,
4218     // but this is probably not optimal.
4219     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4220     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4221     Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
4222     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
4223   }
4224 
4225   MVT ContainerVT = VecVT;
4226   // If the operand is a fixed-length vector, convert to a scalable one.
4227   if (VecVT.isFixedLengthVector()) {
4228     ContainerVT = getContainerForFixedLengthVector(VecVT);
4229     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4230   }
4231 
4232   MVT XLenVT = Subtarget.getXLenVT();
4233 
4234   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4235   bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
4236   // Even i64-element vectors on RV32 can be lowered without scalar
4237   // legalization if the most-significant 32 bits of the value are not affected
4238   // by the sign-extension of the lower 32 bits.
4239   // TODO: We could also catch sign extensions of a 32-bit value.
4240   if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
4241     const auto *CVal = cast<ConstantSDNode>(Val);
4242     if (isInt<32>(CVal->getSExtValue())) {
4243       IsLegalInsert = true;
4244       Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4245     }
4246   }
4247 
4248   SDValue Mask, VL;
4249   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4250 
4251   SDValue ValInVec;
4252 
4253   if (IsLegalInsert) {
4254     unsigned Opc =
4255         VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
4256     if (isNullConstant(Idx)) {
4257       Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
4258       if (!VecVT.isFixedLengthVector())
4259         return Vec;
4260       return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
4261     }
4262     ValInVec =
4263         DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
4264   } else {
4265     // On RV32, i64-element vectors must be specially handled to place the
4266     // value at element 0, by using two vslide1up instructions in sequence on
4267     // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
4268     // this.
4269     SDValue One = DAG.getConstant(1, DL, XLenVT);
4270     SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
4271     SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
4272     MVT I32ContainerVT =
4273         MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
4274     SDValue I32Mask =
4275         getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
4276     // Limit the active VL to two.
4277     SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
4278     // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
4279     // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
4280     ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
4281                            InsertI64VL);
4282     // First slide in the hi value, then the lo in underneath it.
4283     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4284                            ValHi, I32Mask, InsertI64VL);
4285     ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
4286                            ValLo, I32Mask, InsertI64VL);
4287     // Bitcast back to the right container type.
4288     ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
4289   }
4290 
4291   // Now that the value is in a vector, slide it into position.
4292   SDValue InsertVL =
4293       DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
4294   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4295                                 ValInVec, Idx, Mask, InsertVL);
4296   if (!VecVT.isFixedLengthVector())
4297     return Slideup;
4298   return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4299 }
4300 
4301 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
4302 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
4303 // types this is done using VMV_X_S to allow us to glean information about the
4304 // sign bits of the result.
4305 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4306                                                      SelectionDAG &DAG) const {
4307   SDLoc DL(Op);
4308   SDValue Idx = Op.getOperand(1);
4309   SDValue Vec = Op.getOperand(0);
4310   EVT EltVT = Op.getValueType();
4311   MVT VecVT = Vec.getSimpleValueType();
4312   MVT XLenVT = Subtarget.getXLenVT();
4313 
4314   if (VecVT.getVectorElementType() == MVT::i1) {
4315     if (VecVT.isFixedLengthVector()) {
4316       unsigned NumElts = VecVT.getVectorNumElements();
4317       if (NumElts >= 8) {
4318         MVT WideEltVT;
4319         unsigned WidenVecLen;
4320         SDValue ExtractElementIdx;
4321         SDValue ExtractBitIdx;
4322         unsigned MaxEEW = Subtarget.getMaxELENForFixedLengthVectors();
4323         MVT LargestEltVT = MVT::getIntegerVT(
4324             std::min(MaxEEW, unsigned(XLenVT.getSizeInBits())));
4325         if (NumElts <= LargestEltVT.getSizeInBits()) {
4326           assert(isPowerOf2_32(NumElts) &&
4327                  "the number of elements should be power of 2");
4328           WideEltVT = MVT::getIntegerVT(NumElts);
4329           WidenVecLen = 1;
4330           ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
4331           ExtractBitIdx = Idx;
4332         } else {
4333           WideEltVT = LargestEltVT;
4334           WidenVecLen = NumElts / WideEltVT.getSizeInBits();
4335           // extract element index = index / element width
4336           ExtractElementIdx = DAG.getNode(
4337               ISD::SRL, DL, XLenVT, Idx,
4338               DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
4339           // mask bit index = index % element width
4340           ExtractBitIdx = DAG.getNode(
4341               ISD::AND, DL, XLenVT, Idx,
4342               DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
4343         }
4344         MVT WideVT = MVT::getVectorVT(WideEltVT, WidenVecLen);
4345         Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
4346         SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
4347                                          Vec, ExtractElementIdx);
4348         // Extract the bit from GPR.
4349         SDValue ShiftRight =
4350             DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
4351         return DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
4352                            DAG.getConstant(1, DL, XLenVT));
4353       }
4354     }
4355     // Otherwise, promote to an i8 vector and extract from that.
4356     MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
4357     Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
4358     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
4359   }
4360 
4361   // If this is a fixed vector, we need to convert it to a scalable vector.
4362   MVT ContainerVT = VecVT;
4363   if (VecVT.isFixedLengthVector()) {
4364     ContainerVT = getContainerForFixedLengthVector(VecVT);
4365     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4366   }
4367 
4368   // If the index is 0, the vector is already in the right position.
4369   if (!isNullConstant(Idx)) {
4370     // Use a VL of 1 to avoid processing more elements than we need.
4371     SDValue VL = DAG.getConstant(1, DL, XLenVT);
4372     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4373     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4374     Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4375                       DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
4376   }
4377 
4378   if (!EltVT.isInteger()) {
4379     // Floating-point extracts are handled in TableGen.
4380     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
4381                        DAG.getConstant(0, DL, XLenVT));
4382   }
4383 
4384   SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
4385   return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
4386 }
4387 
4388 // Some RVV intrinsics may claim that they want an integer operand to be
4389 // promoted or expanded.
4390 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
4391                                           const RISCVSubtarget &Subtarget) {
4392   assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4393           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
4394          "Unexpected opcode");
4395 
4396   if (!Subtarget.hasVInstructions())
4397     return SDValue();
4398 
4399   bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
4400   unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
4401   SDLoc DL(Op);
4402 
4403   const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
4404       RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
4405   if (!II || !II->hasSplatOperand())
4406     return SDValue();
4407 
4408   unsigned SplatOp = II->SplatOperand + 1 + HasChain;
4409   assert(SplatOp < Op.getNumOperands());
4410 
4411   SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
4412   SDValue &ScalarOp = Operands[SplatOp];
4413   MVT OpVT = ScalarOp.getSimpleValueType();
4414   MVT XLenVT = Subtarget.getXLenVT();
4415 
4416   // If this isn't a scalar, or its type is XLenVT we're done.
4417   if (!OpVT.isScalarInteger() || OpVT == XLenVT)
4418     return SDValue();
4419 
4420   // Simplest case is that the operand needs to be promoted to XLenVT.
4421   if (OpVT.bitsLT(XLenVT)) {
4422     // If the operand is a constant, sign extend to increase our chances
4423     // of being able to use a .vi instruction. ANY_EXTEND would become a
4424     // a zero extend and the simm5 check in isel would fail.
4425     // FIXME: Should we ignore the upper bits in isel instead?
4426     unsigned ExtOpc =
4427         isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4428     ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
4429     return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4430   }
4431 
4432   // Use the previous operand to get the vXi64 VT. The result might be a mask
4433   // VT for compares. Using the previous operand assumes that the previous
4434   // operand will never have a smaller element size than a scalar operand and
4435   // that a widening operation never uses SEW=64.
4436   // NOTE: If this fails the below assert, we can probably just find the
4437   // element count from any operand or result and use it to construct the VT.
4438   assert(II->SplatOperand > 0 && "Unexpected splat operand!");
4439   MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
4440 
4441   // The more complex case is when the scalar is larger than XLenVT.
4442   assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
4443          VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
4444 
4445   // If this is a sign-extended 32-bit constant, we can truncate it and rely
4446   // on the instruction to sign-extend since SEW>XLEN.
4447   if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
4448     if (isInt<32>(CVal->getSExtValue())) {
4449       ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
4450       return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4451     }
4452   }
4453 
4454   // We need to convert the scalar to a splat vector.
4455   // FIXME: Can we implicitly truncate the scalar if it is known to
4456   // be sign extended?
4457   SDValue VL = getVLOperand(Op);
4458   assert(VL.getValueType() == XLenVT);
4459   ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
4460   return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
4461 }
4462 
4463 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
4464                                                      SelectionDAG &DAG) const {
4465   unsigned IntNo = Op.getConstantOperandVal(0);
4466   SDLoc DL(Op);
4467   MVT XLenVT = Subtarget.getXLenVT();
4468 
4469   switch (IntNo) {
4470   default:
4471     break; // Don't custom lower most intrinsics.
4472   case Intrinsic::thread_pointer: {
4473     EVT PtrVT = getPointerTy(DAG.getDataLayout());
4474     return DAG.getRegister(RISCV::X4, PtrVT);
4475   }
4476   case Intrinsic::riscv_orc_b:
4477   case Intrinsic::riscv_brev8: {
4478     // Lower to the GORCI encoding for orc.b or the GREVI encoding for brev8.
4479     unsigned Opc =
4480         IntNo == Intrinsic::riscv_brev8 ? RISCVISD::GREV : RISCVISD::GORC;
4481     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1),
4482                        DAG.getConstant(7, DL, XLenVT));
4483   }
4484   case Intrinsic::riscv_grev:
4485   case Intrinsic::riscv_gorc: {
4486     unsigned Opc =
4487         IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
4488     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4489   }
4490   case Intrinsic::riscv_zip:
4491   case Intrinsic::riscv_unzip: {
4492     // Lower to the SHFLI encoding for zip or the UNSHFLI encoding for unzip.
4493     // For i32 the immdiate is 15. For i64 the immediate is 31.
4494     unsigned Opc =
4495         IntNo == Intrinsic::riscv_zip ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4496     unsigned BitWidth = Op.getValueSizeInBits();
4497     assert(isPowerOf2_32(BitWidth) && BitWidth >= 2 && "Unexpected bit width");
4498     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1),
4499                        DAG.getConstant((BitWidth / 2) - 1, DL, XLenVT));
4500   }
4501   case Intrinsic::riscv_shfl:
4502   case Intrinsic::riscv_unshfl: {
4503     unsigned Opc =
4504         IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
4505     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4506   }
4507   case Intrinsic::riscv_bcompress:
4508   case Intrinsic::riscv_bdecompress: {
4509     unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
4510                                                        : RISCVISD::BDECOMPRESS;
4511     return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
4512   }
4513   case Intrinsic::riscv_bfp:
4514     return DAG.getNode(RISCVISD::BFP, DL, XLenVT, Op.getOperand(1),
4515                        Op.getOperand(2));
4516   case Intrinsic::riscv_fsl:
4517     return DAG.getNode(RISCVISD::FSL, DL, XLenVT, Op.getOperand(1),
4518                        Op.getOperand(2), Op.getOperand(3));
4519   case Intrinsic::riscv_fsr:
4520     return DAG.getNode(RISCVISD::FSR, DL, XLenVT, Op.getOperand(1),
4521                        Op.getOperand(2), Op.getOperand(3));
4522   case Intrinsic::riscv_vmv_x_s:
4523     assert(Op.getValueType() == XLenVT && "Unexpected VT!");
4524     return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
4525                        Op.getOperand(1));
4526   case Intrinsic::riscv_vmv_v_x:
4527     return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
4528                             Op.getSimpleValueType(), DL, DAG, Subtarget);
4529   case Intrinsic::riscv_vfmv_v_f:
4530     return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
4531                        Op.getOperand(1), Op.getOperand(2));
4532   case Intrinsic::riscv_vmv_s_x: {
4533     SDValue Scalar = Op.getOperand(2);
4534 
4535     if (Scalar.getValueType().bitsLE(XLenVT)) {
4536       Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
4537       return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
4538                          Op.getOperand(1), Scalar, Op.getOperand(3));
4539     }
4540 
4541     assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
4542 
4543     // This is an i64 value that lives in two scalar registers. We have to
4544     // insert this in a convoluted way. First we build vXi64 splat containing
4545     // the/ two values that we assemble using some bit math. Next we'll use
4546     // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
4547     // to merge element 0 from our splat into the source vector.
4548     // FIXME: This is probably not the best way to do this, but it is
4549     // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
4550     // point.
4551     //   sw lo, (a0)
4552     //   sw hi, 4(a0)
4553     //   vlse vX, (a0)
4554     //
4555     //   vid.v      vVid
4556     //   vmseq.vx   mMask, vVid, 0
4557     //   vmerge.vvm vDest, vSrc, vVal, mMask
4558     MVT VT = Op.getSimpleValueType();
4559     SDValue Vec = Op.getOperand(1);
4560     SDValue VL = getVLOperand(Op);
4561 
4562     SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
4563     SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4564                                       DAG.getConstant(0, DL, MVT::i32), VL);
4565 
4566     MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
4567     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4568     SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4569     SDValue SelectCond =
4570         DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
4571                     DAG.getCondCode(ISD::SETEQ), Mask, VL);
4572     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
4573                        Vec, VL);
4574   }
4575   case Intrinsic::riscv_vslide1up:
4576   case Intrinsic::riscv_vslide1down:
4577   case Intrinsic::riscv_vslide1up_mask:
4578   case Intrinsic::riscv_vslide1down_mask: {
4579     // We need to special case these when the scalar is larger than XLen.
4580     unsigned NumOps = Op.getNumOperands();
4581     bool IsMasked = NumOps == 7;
4582     unsigned OpOffset = IsMasked ? 1 : 0;
4583     SDValue Scalar = Op.getOperand(2 + OpOffset);
4584     if (Scalar.getValueType().bitsLE(XLenVT))
4585       break;
4586 
4587     // Splatting a sign extended constant is fine.
4588     if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
4589       if (isInt<32>(CVal->getSExtValue()))
4590         break;
4591 
4592     MVT VT = Op.getSimpleValueType();
4593     assert(VT.getVectorElementType() == MVT::i64 &&
4594            Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
4595 
4596     // Convert the vector source to the equivalent nxvXi32 vector.
4597     MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4598     SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
4599 
4600     SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4601                                    DAG.getConstant(0, DL, XLenVT));
4602     SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
4603                                    DAG.getConstant(1, DL, XLenVT));
4604 
4605     // Double the VL since we halved SEW.
4606     SDValue VL = getVLOperand(Op);
4607     SDValue I32VL =
4608         DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
4609 
4610     MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
4611     SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
4612 
4613     // Shift the two scalar parts in using SEW=32 slide1up/slide1down
4614     // instructions.
4615     if (IntNo == Intrinsic::riscv_vslide1up ||
4616         IntNo == Intrinsic::riscv_vslide1up_mask) {
4617       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
4618                         I32Mask, I32VL);
4619       Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
4620                         I32Mask, I32VL);
4621     } else {
4622       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
4623                         I32Mask, I32VL);
4624       Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
4625                         I32Mask, I32VL);
4626     }
4627 
4628     // Convert back to nxvXi64.
4629     Vec = DAG.getBitcast(VT, Vec);
4630 
4631     if (!IsMasked)
4632       return Vec;
4633 
4634     // Apply mask after the operation.
4635     SDValue Mask = Op.getOperand(NumOps - 3);
4636     SDValue MaskedOff = Op.getOperand(1);
4637     return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
4638   }
4639   }
4640 
4641   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4642 }
4643 
4644 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
4645                                                     SelectionDAG &DAG) const {
4646   unsigned IntNo = Op.getConstantOperandVal(1);
4647   switch (IntNo) {
4648   default:
4649     break;
4650   case Intrinsic::riscv_masked_strided_load: {
4651     SDLoc DL(Op);
4652     MVT XLenVT = Subtarget.getXLenVT();
4653 
4654     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4655     // the selection of the masked intrinsics doesn't do this for us.
4656     SDValue Mask = Op.getOperand(5);
4657     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4658 
4659     MVT VT = Op->getSimpleValueType(0);
4660     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4661 
4662     SDValue PassThru = Op.getOperand(2);
4663     if (!IsUnmasked) {
4664       MVT MaskVT =
4665           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4666       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4667       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4668     }
4669 
4670     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4671 
4672     SDValue IntID = DAG.getTargetConstant(
4673         IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask, DL,
4674         XLenVT);
4675 
4676     auto *Load = cast<MemIntrinsicSDNode>(Op);
4677     SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
4678     if (IsUnmasked)
4679       Ops.push_back(DAG.getUNDEF(ContainerVT));
4680     else
4681       Ops.push_back(PassThru);
4682     Ops.push_back(Op.getOperand(3)); // Ptr
4683     Ops.push_back(Op.getOperand(4)); // Stride
4684     if (!IsUnmasked)
4685       Ops.push_back(Mask);
4686     Ops.push_back(VL);
4687     if (!IsUnmasked) {
4688       SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
4689       Ops.push_back(Policy);
4690     }
4691 
4692     SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4693     SDValue Result =
4694         DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4695                                 Load->getMemoryVT(), Load->getMemOperand());
4696     SDValue Chain = Result.getValue(1);
4697     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4698     return DAG.getMergeValues({Result, Chain}, DL);
4699   }
4700   }
4701 
4702   return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
4703 }
4704 
4705 SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
4706                                                  SelectionDAG &DAG) const {
4707   unsigned IntNo = Op.getConstantOperandVal(1);
4708   switch (IntNo) {
4709   default:
4710     break;
4711   case Intrinsic::riscv_masked_strided_store: {
4712     SDLoc DL(Op);
4713     MVT XLenVT = Subtarget.getXLenVT();
4714 
4715     // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4716     // the selection of the masked intrinsics doesn't do this for us.
4717     SDValue Mask = Op.getOperand(5);
4718     bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4719 
4720     SDValue Val = Op.getOperand(2);
4721     MVT VT = Val.getSimpleValueType();
4722     MVT ContainerVT = getContainerForFixedLengthVector(VT);
4723 
4724     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4725     if (!IsUnmasked) {
4726       MVT MaskVT =
4727           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4728       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4729     }
4730 
4731     SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4732 
4733     SDValue IntID = DAG.getTargetConstant(
4734         IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask, DL,
4735         XLenVT);
4736 
4737     auto *Store = cast<MemIntrinsicSDNode>(Op);
4738     SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
4739     Ops.push_back(Val);
4740     Ops.push_back(Op.getOperand(3)); // Ptr
4741     Ops.push_back(Op.getOperand(4)); // Stride
4742     if (!IsUnmasked)
4743       Ops.push_back(Mask);
4744     Ops.push_back(VL);
4745 
4746     return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
4747                                    Ops, Store->getMemoryVT(),
4748                                    Store->getMemOperand());
4749   }
4750   }
4751 
4752   return SDValue();
4753 }
4754 
4755 static MVT getLMUL1VT(MVT VT) {
4756   assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
4757          "Unexpected vector MVT");
4758   return MVT::getScalableVectorVT(
4759       VT.getVectorElementType(),
4760       RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
4761 }
4762 
4763 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
4764   switch (ISDOpcode) {
4765   default:
4766     llvm_unreachable("Unhandled reduction");
4767   case ISD::VECREDUCE_ADD:
4768     return RISCVISD::VECREDUCE_ADD_VL;
4769   case ISD::VECREDUCE_UMAX:
4770     return RISCVISD::VECREDUCE_UMAX_VL;
4771   case ISD::VECREDUCE_SMAX:
4772     return RISCVISD::VECREDUCE_SMAX_VL;
4773   case ISD::VECREDUCE_UMIN:
4774     return RISCVISD::VECREDUCE_UMIN_VL;
4775   case ISD::VECREDUCE_SMIN:
4776     return RISCVISD::VECREDUCE_SMIN_VL;
4777   case ISD::VECREDUCE_AND:
4778     return RISCVISD::VECREDUCE_AND_VL;
4779   case ISD::VECREDUCE_OR:
4780     return RISCVISD::VECREDUCE_OR_VL;
4781   case ISD::VECREDUCE_XOR:
4782     return RISCVISD::VECREDUCE_XOR_VL;
4783   }
4784 }
4785 
4786 SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
4787                                                          SelectionDAG &DAG,
4788                                                          bool IsVP) const {
4789   SDLoc DL(Op);
4790   SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
4791   MVT VecVT = Vec.getSimpleValueType();
4792   assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
4793           Op.getOpcode() == ISD::VECREDUCE_OR ||
4794           Op.getOpcode() == ISD::VECREDUCE_XOR ||
4795           Op.getOpcode() == ISD::VP_REDUCE_AND ||
4796           Op.getOpcode() == ISD::VP_REDUCE_OR ||
4797           Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
4798          "Unexpected reduction lowering");
4799 
4800   MVT XLenVT = Subtarget.getXLenVT();
4801   assert(Op.getValueType() == XLenVT &&
4802          "Expected reduction output to be legalized to XLenVT");
4803 
4804   MVT ContainerVT = VecVT;
4805   if (VecVT.isFixedLengthVector()) {
4806     ContainerVT = getContainerForFixedLengthVector(VecVT);
4807     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4808   }
4809 
4810   SDValue Mask, VL;
4811   if (IsVP) {
4812     Mask = Op.getOperand(2);
4813     VL = Op.getOperand(3);
4814   } else {
4815     std::tie(Mask, VL) =
4816         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4817   }
4818 
4819   unsigned BaseOpc;
4820   ISD::CondCode CC;
4821   SDValue Zero = DAG.getConstant(0, DL, XLenVT);
4822 
4823   switch (Op.getOpcode()) {
4824   default:
4825     llvm_unreachable("Unhandled reduction");
4826   case ISD::VECREDUCE_AND:
4827   case ISD::VP_REDUCE_AND: {
4828     // vcpop ~x == 0
4829     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4830     Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
4831     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4832     CC = ISD::SETEQ;
4833     BaseOpc = ISD::AND;
4834     break;
4835   }
4836   case ISD::VECREDUCE_OR:
4837   case ISD::VP_REDUCE_OR:
4838     // vcpop x != 0
4839     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4840     CC = ISD::SETNE;
4841     BaseOpc = ISD::OR;
4842     break;
4843   case ISD::VECREDUCE_XOR:
4844   case ISD::VP_REDUCE_XOR: {
4845     // ((vcpop x) & 1) != 0
4846     SDValue One = DAG.getConstant(1, DL, XLenVT);
4847     Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
4848     Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
4849     CC = ISD::SETNE;
4850     BaseOpc = ISD::XOR;
4851     break;
4852   }
4853   }
4854 
4855   SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
4856 
4857   if (!IsVP)
4858     return SetCC;
4859 
4860   // Now include the start value in the operation.
4861   // Note that we must return the start value when no elements are operated
4862   // upon. The vcpop instructions we've emitted in each case above will return
4863   // 0 for an inactive vector, and so we've already received the neutral value:
4864   // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
4865   // can simply include the start value.
4866   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
4867 }
4868 
4869 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
4870                                             SelectionDAG &DAG) const {
4871   SDLoc DL(Op);
4872   SDValue Vec = Op.getOperand(0);
4873   EVT VecEVT = Vec.getValueType();
4874 
4875   unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
4876 
4877   // Due to ordering in legalize types we may have a vector type that needs to
4878   // be split. Do that manually so we can get down to a legal type.
4879   while (getTypeAction(*DAG.getContext(), VecEVT) ==
4880          TargetLowering::TypeSplitVector) {
4881     SDValue Lo, Hi;
4882     std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
4883     VecEVT = Lo.getValueType();
4884     Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
4885   }
4886 
4887   // TODO: The type may need to be widened rather than split. Or widened before
4888   // it can be split.
4889   if (!isTypeLegal(VecEVT))
4890     return SDValue();
4891 
4892   MVT VecVT = VecEVT.getSimpleVT();
4893   MVT VecEltVT = VecVT.getVectorElementType();
4894   unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
4895 
4896   MVT ContainerVT = VecVT;
4897   if (VecVT.isFixedLengthVector()) {
4898     ContainerVT = getContainerForFixedLengthVector(VecVT);
4899     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4900   }
4901 
4902   MVT M1VT = getLMUL1VT(ContainerVT);
4903   MVT XLenVT = Subtarget.getXLenVT();
4904 
4905   SDValue Mask, VL;
4906   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4907 
4908   SDValue NeutralElem =
4909       DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
4910   SDValue IdentitySplat = lowerScalarSplat(
4911       NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4912   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec,
4913                                   IdentitySplat, Mask, VL);
4914   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4915                              DAG.getConstant(0, DL, XLenVT));
4916   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
4917 }
4918 
4919 // Given a reduction op, this function returns the matching reduction opcode,
4920 // the vector SDValue and the scalar SDValue required to lower this to a
4921 // RISCVISD node.
4922 static std::tuple<unsigned, SDValue, SDValue>
4923 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
4924   SDLoc DL(Op);
4925   auto Flags = Op->getFlags();
4926   unsigned Opcode = Op.getOpcode();
4927   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
4928   switch (Opcode) {
4929   default:
4930     llvm_unreachable("Unhandled reduction");
4931   case ISD::VECREDUCE_FADD: {
4932     // Use positive zero if we can. It is cheaper to materialize.
4933     SDValue Zero =
4934         DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
4935     return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
4936   }
4937   case ISD::VECREDUCE_SEQ_FADD:
4938     return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
4939                            Op.getOperand(0));
4940   case ISD::VECREDUCE_FMIN:
4941     return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
4942                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4943   case ISD::VECREDUCE_FMAX:
4944     return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
4945                            DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
4946   }
4947 }
4948 
4949 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
4950                                               SelectionDAG &DAG) const {
4951   SDLoc DL(Op);
4952   MVT VecEltVT = Op.getSimpleValueType();
4953 
4954   unsigned RVVOpcode;
4955   SDValue VectorVal, ScalarVal;
4956   std::tie(RVVOpcode, VectorVal, ScalarVal) =
4957       getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
4958   MVT VecVT = VectorVal.getSimpleValueType();
4959 
4960   MVT ContainerVT = VecVT;
4961   if (VecVT.isFixedLengthVector()) {
4962     ContainerVT = getContainerForFixedLengthVector(VecVT);
4963     VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
4964   }
4965 
4966   MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
4967   MVT XLenVT = Subtarget.getXLenVT();
4968 
4969   SDValue Mask, VL;
4970   std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
4971 
4972   SDValue ScalarSplat = lowerScalarSplat(
4973       ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget);
4974   SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT),
4975                                   VectorVal, ScalarSplat, Mask, VL);
4976   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
4977                      DAG.getConstant(0, DL, XLenVT));
4978 }
4979 
4980 static unsigned getRVVVPReductionOp(unsigned ISDOpcode) {
4981   switch (ISDOpcode) {
4982   default:
4983     llvm_unreachable("Unhandled reduction");
4984   case ISD::VP_REDUCE_ADD:
4985     return RISCVISD::VECREDUCE_ADD_VL;
4986   case ISD::VP_REDUCE_UMAX:
4987     return RISCVISD::VECREDUCE_UMAX_VL;
4988   case ISD::VP_REDUCE_SMAX:
4989     return RISCVISD::VECREDUCE_SMAX_VL;
4990   case ISD::VP_REDUCE_UMIN:
4991     return RISCVISD::VECREDUCE_UMIN_VL;
4992   case ISD::VP_REDUCE_SMIN:
4993     return RISCVISD::VECREDUCE_SMIN_VL;
4994   case ISD::VP_REDUCE_AND:
4995     return RISCVISD::VECREDUCE_AND_VL;
4996   case ISD::VP_REDUCE_OR:
4997     return RISCVISD::VECREDUCE_OR_VL;
4998   case ISD::VP_REDUCE_XOR:
4999     return RISCVISD::VECREDUCE_XOR_VL;
5000   case ISD::VP_REDUCE_FADD:
5001     return RISCVISD::VECREDUCE_FADD_VL;
5002   case ISD::VP_REDUCE_SEQ_FADD:
5003     return RISCVISD::VECREDUCE_SEQ_FADD_VL;
5004   case ISD::VP_REDUCE_FMAX:
5005     return RISCVISD::VECREDUCE_FMAX_VL;
5006   case ISD::VP_REDUCE_FMIN:
5007     return RISCVISD::VECREDUCE_FMIN_VL;
5008   }
5009 }
5010 
5011 SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
5012                                            SelectionDAG &DAG) const {
5013   SDLoc DL(Op);
5014   SDValue Vec = Op.getOperand(1);
5015   EVT VecEVT = Vec.getValueType();
5016 
5017   // TODO: The type may need to be widened rather than split. Or widened before
5018   // it can be split.
5019   if (!isTypeLegal(VecEVT))
5020     return SDValue();
5021 
5022   MVT VecVT = VecEVT.getSimpleVT();
5023   MVT VecEltVT = VecVT.getVectorElementType();
5024   unsigned RVVOpcode = getRVVVPReductionOp(Op.getOpcode());
5025 
5026   MVT ContainerVT = VecVT;
5027   if (VecVT.isFixedLengthVector()) {
5028     ContainerVT = getContainerForFixedLengthVector(VecVT);
5029     Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5030   }
5031 
5032   SDValue VL = Op.getOperand(3);
5033   SDValue Mask = Op.getOperand(2);
5034 
5035   MVT M1VT = getLMUL1VT(ContainerVT);
5036   MVT XLenVT = Subtarget.getXLenVT();
5037   MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT;
5038 
5039   SDValue StartSplat =
5040       lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT,
5041                        DL, DAG, Subtarget);
5042   SDValue Reduction =
5043       DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL);
5044   SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
5045                              DAG.getConstant(0, DL, XLenVT));
5046   if (!VecVT.isInteger())
5047     return Elt0;
5048   return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
5049 }
5050 
5051 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
5052                                                    SelectionDAG &DAG) const {
5053   SDValue Vec = Op.getOperand(0);
5054   SDValue SubVec = Op.getOperand(1);
5055   MVT VecVT = Vec.getSimpleValueType();
5056   MVT SubVecVT = SubVec.getSimpleValueType();
5057 
5058   SDLoc DL(Op);
5059   MVT XLenVT = Subtarget.getXLenVT();
5060   unsigned OrigIdx = Op.getConstantOperandVal(2);
5061   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5062 
5063   // We don't have the ability to slide mask vectors up indexed by their i1
5064   // elements; the smallest we can do is i8. Often we are able to bitcast to
5065   // equivalent i8 vectors. Note that when inserting a fixed-length vector
5066   // into a scalable one, we might not necessarily have enough scalable
5067   // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
5068   if (SubVecVT.getVectorElementType() == MVT::i1 &&
5069       (OrigIdx != 0 || !Vec.isUndef())) {
5070     if (VecVT.getVectorMinNumElements() >= 8 &&
5071         SubVecVT.getVectorMinNumElements() >= 8) {
5072       assert(OrigIdx % 8 == 0 && "Invalid index");
5073       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5074              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5075              "Unexpected mask vector lowering");
5076       OrigIdx /= 8;
5077       SubVecVT =
5078           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5079                            SubVecVT.isScalableVector());
5080       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5081                                VecVT.isScalableVector());
5082       Vec = DAG.getBitcast(VecVT, Vec);
5083       SubVec = DAG.getBitcast(SubVecVT, SubVec);
5084     } else {
5085       // We can't slide this mask vector up indexed by its i1 elements.
5086       // This poses a problem when we wish to insert a scalable vector which
5087       // can't be re-expressed as a larger type. Just choose the slow path and
5088       // extend to a larger type, then truncate back down.
5089       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5090       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5091       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5092       SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
5093       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
5094                         Op.getOperand(2));
5095       SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
5096       return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
5097     }
5098   }
5099 
5100   // If the subvector vector is a fixed-length type, we cannot use subregister
5101   // manipulation to simplify the codegen; we don't know which register of a
5102   // LMUL group contains the specific subvector as we only know the minimum
5103   // register size. Therefore we must slide the vector group up the full
5104   // amount.
5105   if (SubVecVT.isFixedLengthVector()) {
5106     if (OrigIdx == 0 && Vec.isUndef() && !VecVT.isFixedLengthVector())
5107       return Op;
5108     MVT ContainerVT = VecVT;
5109     if (VecVT.isFixedLengthVector()) {
5110       ContainerVT = getContainerForFixedLengthVector(VecVT);
5111       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5112     }
5113     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
5114                          DAG.getUNDEF(ContainerVT), SubVec,
5115                          DAG.getConstant(0, DL, XLenVT));
5116     if (OrigIdx == 0 && Vec.isUndef() && VecVT.isFixedLengthVector()) {
5117       SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
5118       return DAG.getBitcast(Op.getValueType(), SubVec);
5119     }
5120     SDValue Mask =
5121         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5122     // Set the vector length to only the number of elements we care about. Note
5123     // that for slideup this includes the offset.
5124     SDValue VL =
5125         DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
5126     SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5127     SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
5128                                   SubVec, SlideupAmt, Mask, VL);
5129     if (VecVT.isFixedLengthVector())
5130       Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
5131     return DAG.getBitcast(Op.getValueType(), Slideup);
5132   }
5133 
5134   unsigned SubRegIdx, RemIdx;
5135   std::tie(SubRegIdx, RemIdx) =
5136       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5137           VecVT, SubVecVT, OrigIdx, TRI);
5138 
5139   RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
5140   bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
5141                          SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
5142                          SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
5143 
5144   // 1. If the Idx has been completely eliminated and this subvector's size is
5145   // a vector register or a multiple thereof, or the surrounding elements are
5146   // undef, then this is a subvector insert which naturally aligns to a vector
5147   // register. These can easily be handled using subregister manipulation.
5148   // 2. If the subvector is smaller than a vector register, then the insertion
5149   // must preserve the undisturbed elements of the register. We do this by
5150   // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
5151   // (which resolves to a subregister copy), performing a VSLIDEUP to place the
5152   // subvector within the vector register, and an INSERT_SUBVECTOR of that
5153   // LMUL=1 type back into the larger vector (resolving to another subregister
5154   // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
5155   // to avoid allocating a large register group to hold our subvector.
5156   if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
5157     return Op;
5158 
5159   // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
5160   // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
5161   // (in our case undisturbed). This means we can set up a subvector insertion
5162   // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
5163   // size of the subvector.
5164   MVT InterSubVT = VecVT;
5165   SDValue AlignedExtract = Vec;
5166   unsigned AlignedIdx = OrigIdx - RemIdx;
5167   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5168     InterSubVT = getLMUL1VT(VecVT);
5169     // Extract a subvector equal to the nearest full vector register type. This
5170     // should resolve to a EXTRACT_SUBREG instruction.
5171     AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5172                                  DAG.getConstant(AlignedIdx, DL, XLenVT));
5173   }
5174 
5175   SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5176   // For scalable vectors this must be further multiplied by vscale.
5177   SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
5178 
5179   SDValue Mask, VL;
5180   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5181 
5182   // Construct the vector length corresponding to RemIdx + length(SubVecVT).
5183   VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
5184   VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
5185   VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
5186 
5187   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
5188                        DAG.getUNDEF(InterSubVT), SubVec,
5189                        DAG.getConstant(0, DL, XLenVT));
5190 
5191   SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
5192                                 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
5193 
5194   // If required, insert this subvector back into the correct vector register.
5195   // This should resolve to an INSERT_SUBREG instruction.
5196   if (VecVT.bitsGT(InterSubVT))
5197     Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
5198                           DAG.getConstant(AlignedIdx, DL, XLenVT));
5199 
5200   // We might have bitcast from a mask type: cast back to the original type if
5201   // required.
5202   return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
5203 }
5204 
5205 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
5206                                                     SelectionDAG &DAG) const {
5207   SDValue Vec = Op.getOperand(0);
5208   MVT SubVecVT = Op.getSimpleValueType();
5209   MVT VecVT = Vec.getSimpleValueType();
5210 
5211   SDLoc DL(Op);
5212   MVT XLenVT = Subtarget.getXLenVT();
5213   unsigned OrigIdx = Op.getConstantOperandVal(1);
5214   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
5215 
5216   // We don't have the ability to slide mask vectors down indexed by their i1
5217   // elements; the smallest we can do is i8. Often we are able to bitcast to
5218   // equivalent i8 vectors. Note that when extracting a fixed-length vector
5219   // from a scalable one, we might not necessarily have enough scalable
5220   // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
5221   if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
5222     if (VecVT.getVectorMinNumElements() >= 8 &&
5223         SubVecVT.getVectorMinNumElements() >= 8) {
5224       assert(OrigIdx % 8 == 0 && "Invalid index");
5225       assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
5226              SubVecVT.getVectorMinNumElements() % 8 == 0 &&
5227              "Unexpected mask vector lowering");
5228       OrigIdx /= 8;
5229       SubVecVT =
5230           MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
5231                            SubVecVT.isScalableVector());
5232       VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
5233                                VecVT.isScalableVector());
5234       Vec = DAG.getBitcast(VecVT, Vec);
5235     } else {
5236       // We can't slide this mask vector down, indexed by its i1 elements.
5237       // This poses a problem when we wish to extract a scalable vector which
5238       // can't be re-expressed as a larger type. Just choose the slow path and
5239       // extend to a larger type, then truncate back down.
5240       // TODO: We could probably improve this when extracting certain fixed
5241       // from fixed, where we can extract as i8 and shift the correct element
5242       // right to reach the desired subvector?
5243       MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
5244       MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
5245       Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
5246       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
5247                         Op.getOperand(1));
5248       SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
5249       return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
5250     }
5251   }
5252 
5253   // If the subvector vector is a fixed-length type, we cannot use subregister
5254   // manipulation to simplify the codegen; we don't know which register of a
5255   // LMUL group contains the specific subvector as we only know the minimum
5256   // register size. Therefore we must slide the vector group down the full
5257   // amount.
5258   if (SubVecVT.isFixedLengthVector()) {
5259     // With an index of 0 this is a cast-like subvector, which can be performed
5260     // with subregister operations.
5261     if (OrigIdx == 0)
5262       return Op;
5263     MVT ContainerVT = VecVT;
5264     if (VecVT.isFixedLengthVector()) {
5265       ContainerVT = getContainerForFixedLengthVector(VecVT);
5266       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5267     }
5268     SDValue Mask =
5269         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
5270     // Set the vector length to only the number of elements we care about. This
5271     // avoids sliding down elements we're going to discard straight away.
5272     SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
5273     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
5274     SDValue Slidedown =
5275         DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5276                     DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
5277     // Now we can use a cast-like subvector extract to get the result.
5278     Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5279                             DAG.getConstant(0, DL, XLenVT));
5280     return DAG.getBitcast(Op.getValueType(), Slidedown);
5281   }
5282 
5283   unsigned SubRegIdx, RemIdx;
5284   std::tie(SubRegIdx, RemIdx) =
5285       RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
5286           VecVT, SubVecVT, OrigIdx, TRI);
5287 
5288   // If the Idx has been completely eliminated then this is a subvector extract
5289   // which naturally aligns to a vector register. These can easily be handled
5290   // using subregister manipulation.
5291   if (RemIdx == 0)
5292     return Op;
5293 
5294   // Else we must shift our vector register directly to extract the subvector.
5295   // Do this using VSLIDEDOWN.
5296 
5297   // If the vector type is an LMUL-group type, extract a subvector equal to the
5298   // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
5299   // instruction.
5300   MVT InterSubVT = VecVT;
5301   if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
5302     InterSubVT = getLMUL1VT(VecVT);
5303     Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
5304                       DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
5305   }
5306 
5307   // Slide this vector register down by the desired number of elements in order
5308   // to place the desired subvector starting at element 0.
5309   SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
5310   // For scalable vectors this must be further multiplied by vscale.
5311   SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
5312 
5313   SDValue Mask, VL;
5314   std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
5315   SDValue Slidedown =
5316       DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
5317                   DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
5318 
5319   // Now the vector is in the right position, extract our final subvector. This
5320   // should resolve to a COPY.
5321   Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
5322                           DAG.getConstant(0, DL, XLenVT));
5323 
5324   // We might have bitcast from a mask type: cast back to the original type if
5325   // required.
5326   return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
5327 }
5328 
5329 // Lower step_vector to the vid instruction. Any non-identity step value must
5330 // be accounted for my manual expansion.
5331 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
5332                                               SelectionDAG &DAG) const {
5333   SDLoc DL(Op);
5334   MVT VT = Op.getSimpleValueType();
5335   MVT XLenVT = Subtarget.getXLenVT();
5336   SDValue Mask, VL;
5337   std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
5338   SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
5339   uint64_t StepValImm = Op.getConstantOperandVal(0);
5340   if (StepValImm != 1) {
5341     if (isPowerOf2_64(StepValImm)) {
5342       SDValue StepVal =
5343           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
5344                       DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
5345       StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
5346     } else {
5347       SDValue StepVal = lowerScalarSplat(
5348           DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
5349           DL, DAG, Subtarget);
5350       StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
5351     }
5352   }
5353   return StepVec;
5354 }
5355 
5356 // Implement vector_reverse using vrgather.vv with indices determined by
5357 // subtracting the id of each element from (VLMAX-1). This will convert
5358 // the indices like so:
5359 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
5360 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
5361 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
5362                                                  SelectionDAG &DAG) const {
5363   SDLoc DL(Op);
5364   MVT VecVT = Op.getSimpleValueType();
5365   unsigned EltSize = VecVT.getScalarSizeInBits();
5366   unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
5367 
5368   unsigned MaxVLMAX = 0;
5369   unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
5370   if (VectorBitsMax != 0)
5371     MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
5372 
5373   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
5374   MVT IntVT = VecVT.changeVectorElementTypeToInteger();
5375 
5376   // If this is SEW=8 and VLMAX is unknown or more than 256, we need
5377   // to use vrgatherei16.vv.
5378   // TODO: It's also possible to use vrgatherei16.vv for other types to
5379   // decrease register width for the index calculation.
5380   if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
5381     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
5382     // Reverse each half, then reassemble them in reverse order.
5383     // NOTE: It's also possible that after splitting that VLMAX no longer
5384     // requires vrgatherei16.vv.
5385     if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
5386       SDValue Lo, Hi;
5387       std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
5388       EVT LoVT, HiVT;
5389       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
5390       Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
5391       Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
5392       // Reassemble the low and high pieces reversed.
5393       // FIXME: This is a CONCAT_VECTORS.
5394       SDValue Res =
5395           DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
5396                       DAG.getIntPtrConstant(0, DL));
5397       return DAG.getNode(
5398           ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
5399           DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
5400     }
5401 
5402     // Just promote the int type to i16 which will double the LMUL.
5403     IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
5404     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
5405   }
5406 
5407   MVT XLenVT = Subtarget.getXLenVT();
5408   SDValue Mask, VL;
5409   std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
5410 
5411   // Calculate VLMAX-1 for the desired SEW.
5412   unsigned MinElts = VecVT.getVectorMinNumElements();
5413   SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
5414                               DAG.getConstant(MinElts, DL, XLenVT));
5415   SDValue VLMinus1 =
5416       DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
5417 
5418   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
5419   bool IsRV32E64 =
5420       !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
5421   SDValue SplatVL;
5422   if (!IsRV32E64)
5423     SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
5424   else
5425     SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);
5426 
5427   SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
5428   SDValue Indices =
5429       DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
5430 
5431   return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
5432 }
5433 
5434 SDValue
5435 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
5436                                                      SelectionDAG &DAG) const {
5437   SDLoc DL(Op);
5438   auto *Load = cast<LoadSDNode>(Op);
5439 
5440   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5441                                         Load->getMemoryVT(),
5442                                         *Load->getMemOperand()) &&
5443          "Expecting a correctly-aligned load");
5444 
5445   MVT VT = Op.getSimpleValueType();
5446   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5447 
5448   SDValue VL =
5449       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5450 
5451   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5452   SDValue NewLoad = DAG.getMemIntrinsicNode(
5453       RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
5454       Load->getMemoryVT(), Load->getMemOperand());
5455 
5456   SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5457   return DAG.getMergeValues({Result, Load->getChain()}, DL);
5458 }
5459 
5460 SDValue
5461 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
5462                                                       SelectionDAG &DAG) const {
5463   SDLoc DL(Op);
5464   auto *Store = cast<StoreSDNode>(Op);
5465 
5466   assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5467                                         Store->getMemoryVT(),
5468                                         *Store->getMemOperand()) &&
5469          "Expecting a correctly-aligned store");
5470 
5471   SDValue StoreVal = Store->getValue();
5472   MVT VT = StoreVal.getSimpleValueType();
5473 
5474   // If the size less than a byte, we need to pad with zeros to make a byte.
5475   if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
5476     VT = MVT::v8i1;
5477     StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
5478                            DAG.getConstant(0, DL, VT), StoreVal,
5479                            DAG.getIntPtrConstant(0, DL));
5480   }
5481 
5482   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5483 
5484   SDValue VL =
5485       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5486 
5487   SDValue NewValue =
5488       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
5489   return DAG.getMemIntrinsicNode(
5490       RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
5491       {Store->getChain(), NewValue, Store->getBasePtr(), VL},
5492       Store->getMemoryVT(), Store->getMemOperand());
5493 }
5494 
5495 SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
5496                                              SelectionDAG &DAG) const {
5497   SDLoc DL(Op);
5498   MVT VT = Op.getSimpleValueType();
5499 
5500   const auto *MemSD = cast<MemSDNode>(Op);
5501   EVT MemVT = MemSD->getMemoryVT();
5502   MachineMemOperand *MMO = MemSD->getMemOperand();
5503   SDValue Chain = MemSD->getChain();
5504   SDValue BasePtr = MemSD->getBasePtr();
5505 
5506   SDValue Mask, PassThru, VL;
5507   if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
5508     Mask = VPLoad->getMask();
5509     PassThru = DAG.getUNDEF(VT);
5510     VL = VPLoad->getVectorLength();
5511   } else {
5512     const auto *MLoad = cast<MaskedLoadSDNode>(Op);
5513     Mask = MLoad->getMask();
5514     PassThru = MLoad->getPassThru();
5515   }
5516 
5517   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5518 
5519   MVT XLenVT = Subtarget.getXLenVT();
5520 
5521   MVT ContainerVT = VT;
5522   if (VT.isFixedLengthVector()) {
5523     ContainerVT = getContainerForFixedLengthVector(VT);
5524     PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5525     if (!IsUnmasked) {
5526       MVT MaskVT =
5527           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5528       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5529     }
5530   }
5531 
5532   if (!VL)
5533     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5534 
5535   unsigned IntID =
5536       IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
5537   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5538   if (IsUnmasked)
5539     Ops.push_back(DAG.getUNDEF(ContainerVT));
5540   else
5541     Ops.push_back(PassThru);
5542   Ops.push_back(BasePtr);
5543   if (!IsUnmasked)
5544     Ops.push_back(Mask);
5545   Ops.push_back(VL);
5546   if (!IsUnmasked)
5547     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5548 
5549   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5550 
5551   SDValue Result =
5552       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5553   Chain = Result.getValue(1);
5554 
5555   if (VT.isFixedLengthVector())
5556     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5557 
5558   return DAG.getMergeValues({Result, Chain}, DL);
5559 }
5560 
5561 SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
5562                                               SelectionDAG &DAG) const {
5563   SDLoc DL(Op);
5564 
5565   const auto *MemSD = cast<MemSDNode>(Op);
5566   EVT MemVT = MemSD->getMemoryVT();
5567   MachineMemOperand *MMO = MemSD->getMemOperand();
5568   SDValue Chain = MemSD->getChain();
5569   SDValue BasePtr = MemSD->getBasePtr();
5570   SDValue Val, Mask, VL;
5571 
5572   if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
5573     Val = VPStore->getValue();
5574     Mask = VPStore->getMask();
5575     VL = VPStore->getVectorLength();
5576   } else {
5577     const auto *MStore = cast<MaskedStoreSDNode>(Op);
5578     Val = MStore->getValue();
5579     Mask = MStore->getMask();
5580   }
5581 
5582   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5583 
5584   MVT VT = Val.getSimpleValueType();
5585   MVT XLenVT = Subtarget.getXLenVT();
5586 
5587   MVT ContainerVT = VT;
5588   if (VT.isFixedLengthVector()) {
5589     ContainerVT = getContainerForFixedLengthVector(VT);
5590 
5591     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
5592     if (!IsUnmasked) {
5593       MVT MaskVT =
5594           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5595       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5596     }
5597   }
5598 
5599   if (!VL)
5600     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5601 
5602   unsigned IntID =
5603       IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
5604   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5605   Ops.push_back(Val);
5606   Ops.push_back(BasePtr);
5607   if (!IsUnmasked)
5608     Ops.push_back(Mask);
5609   Ops.push_back(VL);
5610 
5611   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
5612                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
5613 }
5614 
5615 SDValue
5616 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
5617                                                       SelectionDAG &DAG) const {
5618   MVT InVT = Op.getOperand(0).getSimpleValueType();
5619   MVT ContainerVT = getContainerForFixedLengthVector(InVT);
5620 
5621   MVT VT = Op.getSimpleValueType();
5622 
5623   SDValue Op1 =
5624       convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
5625   SDValue Op2 =
5626       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5627 
5628   SDLoc DL(Op);
5629   SDValue VL =
5630       DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
5631 
5632   MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5633   SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5634 
5635   SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
5636                             Op.getOperand(2), Mask, VL);
5637 
5638   return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
5639 }
5640 
5641 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
5642     SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
5643   MVT VT = Op.getSimpleValueType();
5644 
5645   if (VT.getVectorElementType() == MVT::i1)
5646     return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
5647 
5648   return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
5649 }
5650 
5651 SDValue
5652 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
5653                                                       SelectionDAG &DAG) const {
5654   unsigned Opc;
5655   switch (Op.getOpcode()) {
5656   default: llvm_unreachable("Unexpected opcode!");
5657   case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
5658   case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
5659   case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
5660   }
5661 
5662   return lowerToScalableOp(Op, DAG, Opc);
5663 }
5664 
5665 // Lower vector ABS to smax(X, sub(0, X)).
5666 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
5667   SDLoc DL(Op);
5668   MVT VT = Op.getSimpleValueType();
5669   SDValue X = Op.getOperand(0);
5670 
5671   assert(VT.isFixedLengthVector() && "Unexpected type");
5672 
5673   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5674   X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5675 
5676   SDValue Mask, VL;
5677   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5678 
5679   SDValue SplatZero =
5680       DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5681                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
5682   SDValue NegX =
5683       DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
5684   SDValue Max =
5685       DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
5686 
5687   return convertFromScalableVector(VT, Max, DAG, Subtarget);
5688 }
5689 
5690 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
5691     SDValue Op, SelectionDAG &DAG) const {
5692   SDLoc DL(Op);
5693   MVT VT = Op.getSimpleValueType();
5694   SDValue Mag = Op.getOperand(0);
5695   SDValue Sign = Op.getOperand(1);
5696   assert(Mag.getValueType() == Sign.getValueType() &&
5697          "Can only handle COPYSIGN with matching types.");
5698 
5699   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5700   Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
5701   Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
5702 
5703   SDValue Mask, VL;
5704   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5705 
5706   SDValue CopySign =
5707       DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
5708 
5709   return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
5710 }
5711 
5712 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
5713     SDValue Op, SelectionDAG &DAG) const {
5714   MVT VT = Op.getSimpleValueType();
5715   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5716 
5717   MVT I1ContainerVT =
5718       MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5719 
5720   SDValue CC =
5721       convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
5722   SDValue Op1 =
5723       convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
5724   SDValue Op2 =
5725       convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
5726 
5727   SDLoc DL(Op);
5728   SDValue Mask, VL;
5729   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5730 
5731   SDValue Select =
5732       DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
5733 
5734   return convertFromScalableVector(VT, Select, DAG, Subtarget);
5735 }
5736 
5737 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
5738                                                unsigned NewOpc,
5739                                                bool HasMask) const {
5740   MVT VT = Op.getSimpleValueType();
5741   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5742 
5743   // Create list of operands by converting existing ones to scalable types.
5744   SmallVector<SDValue, 6> Ops;
5745   for (const SDValue &V : Op->op_values()) {
5746     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5747 
5748     // Pass through non-vector operands.
5749     if (!V.getValueType().isVector()) {
5750       Ops.push_back(V);
5751       continue;
5752     }
5753 
5754     // "cast" fixed length vector to a scalable vector.
5755     assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
5756            "Only fixed length vectors are supported!");
5757     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5758   }
5759 
5760   SDLoc DL(Op);
5761   SDValue Mask, VL;
5762   std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5763   if (HasMask)
5764     Ops.push_back(Mask);
5765   Ops.push_back(VL);
5766 
5767   SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
5768   return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
5769 }
5770 
5771 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
5772 // * Operands of each node are assumed to be in the same order.
5773 // * The EVL operand is promoted from i32 to i64 on RV64.
5774 // * Fixed-length vectors are converted to their scalable-vector container
5775 //   types.
5776 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
5777                                        unsigned RISCVISDOpc) const {
5778   SDLoc DL(Op);
5779   MVT VT = Op.getSimpleValueType();
5780   SmallVector<SDValue, 4> Ops;
5781 
5782   for (const auto &OpIdx : enumerate(Op->ops())) {
5783     SDValue V = OpIdx.value();
5784     assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
5785     // Pass through operands which aren't fixed-length vectors.
5786     if (!V.getValueType().isFixedLengthVector()) {
5787       Ops.push_back(V);
5788       continue;
5789     }
5790     // "cast" fixed length vector to a scalable vector.
5791     MVT OpVT = V.getSimpleValueType();
5792     MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
5793     assert(useRVVForFixedLengthVectorVT(OpVT) &&
5794            "Only fixed length vectors are supported!");
5795     Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
5796   }
5797 
5798   if (!VT.isFixedLengthVector())
5799     return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
5800 
5801   MVT ContainerVT = getContainerForFixedLengthVector(VT);
5802 
5803   SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
5804 
5805   return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
5806 }
5807 
5808 SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG,
5809                                             unsigned MaskOpc,
5810                                             unsigned VecOpc) const {
5811   MVT VT = Op.getSimpleValueType();
5812   if (VT.getVectorElementType() != MVT::i1)
5813     return lowerVPOp(Op, DAG, VecOpc);
5814 
5815   // It is safe to drop mask parameter as masked-off elements are undef.
5816   SDValue Op1 = Op->getOperand(0);
5817   SDValue Op2 = Op->getOperand(1);
5818   SDValue VL = Op->getOperand(3);
5819 
5820   MVT ContainerVT = VT;
5821   const bool IsFixed = VT.isFixedLengthVector();
5822   if (IsFixed) {
5823     ContainerVT = getContainerForFixedLengthVector(VT);
5824     Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5825     Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
5826   }
5827 
5828   SDLoc DL(Op);
5829   SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL);
5830   if (!IsFixed)
5831     return Val;
5832   return convertFromScalableVector(VT, Val, DAG, Subtarget);
5833 }
5834 
5835 // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
5836 // matched to a RVV indexed load. The RVV indexed load instructions only
5837 // support the "unsigned unscaled" addressing mode; indices are implicitly
5838 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5839 // signed or scaled indexing is extended to the XLEN value type and scaled
5840 // accordingly.
5841 SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
5842                                                SelectionDAG &DAG) const {
5843   SDLoc DL(Op);
5844   MVT VT = Op.getSimpleValueType();
5845 
5846   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5847   EVT MemVT = MemSD->getMemoryVT();
5848   MachineMemOperand *MMO = MemSD->getMemOperand();
5849   SDValue Chain = MemSD->getChain();
5850   SDValue BasePtr = MemSD->getBasePtr();
5851 
5852   ISD::LoadExtType LoadExtType;
5853   SDValue Index, Mask, PassThru, VL;
5854 
5855   if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
5856     Index = VPGN->getIndex();
5857     Mask = VPGN->getMask();
5858     PassThru = DAG.getUNDEF(VT);
5859     VL = VPGN->getVectorLength();
5860     // VP doesn't support extending loads.
5861     LoadExtType = ISD::NON_EXTLOAD;
5862   } else {
5863     // Else it must be a MGATHER.
5864     auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
5865     Index = MGN->getIndex();
5866     Mask = MGN->getMask();
5867     PassThru = MGN->getPassThru();
5868     LoadExtType = MGN->getExtensionType();
5869   }
5870 
5871   MVT IndexVT = Index.getSimpleValueType();
5872   MVT XLenVT = Subtarget.getXLenVT();
5873 
5874   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5875          "Unexpected VTs!");
5876   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5877   // Targets have to explicitly opt-in for extending vector loads.
5878   assert(LoadExtType == ISD::NON_EXTLOAD &&
5879          "Unexpected extending MGATHER/VP_GATHER");
5880   (void)LoadExtType;
5881 
5882   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5883   // the selection of the masked intrinsics doesn't do this for us.
5884   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5885 
5886   MVT ContainerVT = VT;
5887   if (VT.isFixedLengthVector()) {
5888     // We need to use the larger of the result and index type to determine the
5889     // scalable type to use so we don't increase LMUL for any operand/result.
5890     if (VT.bitsGE(IndexVT)) {
5891       ContainerVT = getContainerForFixedLengthVector(VT);
5892       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
5893                                  ContainerVT.getVectorElementCount());
5894     } else {
5895       IndexVT = getContainerForFixedLengthVector(IndexVT);
5896       ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
5897                                      IndexVT.getVectorElementCount());
5898     }
5899 
5900     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
5901 
5902     if (!IsUnmasked) {
5903       MVT MaskVT =
5904           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
5905       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
5906       PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
5907     }
5908   }
5909 
5910   if (!VL)
5911     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
5912 
5913   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
5914     IndexVT = IndexVT.changeVectorElementType(XLenVT);
5915     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, Mask.getValueType(),
5916                                    VL);
5917     Index = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, IndexVT, Index,
5918                         TrueMask, VL);
5919   }
5920 
5921   unsigned IntID =
5922       IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
5923   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
5924   if (IsUnmasked)
5925     Ops.push_back(DAG.getUNDEF(ContainerVT));
5926   else
5927     Ops.push_back(PassThru);
5928   Ops.push_back(BasePtr);
5929   Ops.push_back(Index);
5930   if (!IsUnmasked)
5931     Ops.push_back(Mask);
5932   Ops.push_back(VL);
5933   if (!IsUnmasked)
5934     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
5935 
5936   SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5937   SDValue Result =
5938       DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
5939   Chain = Result.getValue(1);
5940 
5941   if (VT.isFixedLengthVector())
5942     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
5943 
5944   return DAG.getMergeValues({Result, Chain}, DL);
5945 }
5946 
5947 // Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
5948 // matched to a RVV indexed store. The RVV indexed store instructions only
5949 // support the "unsigned unscaled" addressing mode; indices are implicitly
5950 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
5951 // signed or scaled indexing is extended to the XLEN value type and scaled
5952 // accordingly.
5953 SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
5954                                                 SelectionDAG &DAG) const {
5955   SDLoc DL(Op);
5956   const auto *MemSD = cast<MemSDNode>(Op.getNode());
5957   EVT MemVT = MemSD->getMemoryVT();
5958   MachineMemOperand *MMO = MemSD->getMemOperand();
5959   SDValue Chain = MemSD->getChain();
5960   SDValue BasePtr = MemSD->getBasePtr();
5961 
5962   bool IsTruncatingStore = false;
5963   SDValue Index, Mask, Val, VL;
5964 
5965   if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
5966     Index = VPSN->getIndex();
5967     Mask = VPSN->getMask();
5968     Val = VPSN->getValue();
5969     VL = VPSN->getVectorLength();
5970     // VP doesn't support truncating stores.
5971     IsTruncatingStore = false;
5972   } else {
5973     // Else it must be a MSCATTER.
5974     auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
5975     Index = MSN->getIndex();
5976     Mask = MSN->getMask();
5977     Val = MSN->getValue();
5978     IsTruncatingStore = MSN->isTruncatingStore();
5979   }
5980 
5981   MVT VT = Val.getSimpleValueType();
5982   MVT IndexVT = Index.getSimpleValueType();
5983   MVT XLenVT = Subtarget.getXLenVT();
5984 
5985   assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
5986          "Unexpected VTs!");
5987   assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
5988   // Targets have to explicitly opt-in for extending vector loads and
5989   // truncating vector stores.
5990   assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
5991   (void)IsTruncatingStore;
5992 
5993   // If the mask is known to be all ones, optimize to an unmasked intrinsic;
5994   // the selection of the masked intrinsics doesn't do this for us.
5995   bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
5996 
5997   MVT ContainerVT = VT;
5998   if (VT.isFixedLengthVector()) {
5999     // We need to use the larger of the value and index type to determine the
6000     // scalable type to use so we don't increase LMUL for any operand/result.
6001     if (VT.bitsGE(IndexVT)) {
6002       ContainerVT = getContainerForFixedLengthVector(VT);
6003       IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
6004                                  ContainerVT.getVectorElementCount());
6005     } else {
6006       IndexVT = getContainerForFixedLengthVector(IndexVT);
6007       ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
6008                                      IndexVT.getVectorElementCount());
6009     }
6010 
6011     Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
6012     Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
6013 
6014     if (!IsUnmasked) {
6015       MVT MaskVT =
6016           MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6017       Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
6018     }
6019   }
6020 
6021   if (!VL)
6022     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6023 
6024   if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
6025     IndexVT = IndexVT.changeVectorElementType(XLenVT);
6026     SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, Mask.getValueType(),
6027                                    VL);
6028     Index = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, IndexVT, Index,
6029                         TrueMask, VL);
6030   }
6031 
6032   unsigned IntID =
6033       IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
6034   SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
6035   Ops.push_back(Val);
6036   Ops.push_back(BasePtr);
6037   Ops.push_back(Index);
6038   if (!IsUnmasked)
6039     Ops.push_back(Mask);
6040   Ops.push_back(VL);
6041 
6042   return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
6043                                  DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
6044 }
6045 
6046 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
6047                                                SelectionDAG &DAG) const {
6048   const MVT XLenVT = Subtarget.getXLenVT();
6049   SDLoc DL(Op);
6050   SDValue Chain = Op->getOperand(0);
6051   SDValue SysRegNo = DAG.getTargetConstant(
6052       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6053   SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
6054   SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
6055 
6056   // Encoding used for rounding mode in RISCV differs from that used in
6057   // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
6058   // table, which consists of a sequence of 4-bit fields, each representing
6059   // corresponding FLT_ROUNDS mode.
6060   static const int Table =
6061       (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
6062       (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
6063       (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
6064       (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
6065       (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
6066 
6067   SDValue Shift =
6068       DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
6069   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6070                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6071   SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6072                                DAG.getConstant(7, DL, XLenVT));
6073 
6074   return DAG.getMergeValues({Masked, Chain}, DL);
6075 }
6076 
6077 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
6078                                                SelectionDAG &DAG) const {
6079   const MVT XLenVT = Subtarget.getXLenVT();
6080   SDLoc DL(Op);
6081   SDValue Chain = Op->getOperand(0);
6082   SDValue RMValue = Op->getOperand(1);
6083   SDValue SysRegNo = DAG.getTargetConstant(
6084       RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
6085 
6086   // Encoding used for rounding mode in RISCV differs from that used in
6087   // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
6088   // a table, which consists of a sequence of 4-bit fields, each representing
6089   // corresponding RISCV mode.
6090   static const unsigned Table =
6091       (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
6092       (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
6093       (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
6094       (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
6095       (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
6096 
6097   SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
6098                               DAG.getConstant(2, DL, XLenVT));
6099   SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
6100                                 DAG.getConstant(Table, DL, XLenVT), Shift);
6101   RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
6102                         DAG.getConstant(0x7, DL, XLenVT));
6103   return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
6104                      RMValue);
6105 }
6106 
6107 static RISCVISD::NodeType getRISCVWOpcodeByIntr(unsigned IntNo) {
6108   switch (IntNo) {
6109   default:
6110     llvm_unreachable("Unexpected Intrinsic");
6111   case Intrinsic::riscv_grev:
6112     return RISCVISD::GREVW;
6113   case Intrinsic::riscv_gorc:
6114     return RISCVISD::GORCW;
6115   case Intrinsic::riscv_bcompress:
6116     return RISCVISD::BCOMPRESSW;
6117   case Intrinsic::riscv_bdecompress:
6118     return RISCVISD::BDECOMPRESSW;
6119   case Intrinsic::riscv_bfp:
6120     return RISCVISD::BFPW;
6121   case Intrinsic::riscv_fsl:
6122     return RISCVISD::FSLW;
6123   case Intrinsic::riscv_fsr:
6124     return RISCVISD::FSRW;
6125   }
6126 }
6127 
6128 // Converts the given intrinsic to a i64 operation with any extension.
6129 static SDValue customLegalizeToWOpByIntr(SDNode *N, SelectionDAG &DAG,
6130                                          unsigned IntNo) {
6131   SDLoc DL(N);
6132   RISCVISD::NodeType WOpcode = getRISCVWOpcodeByIntr(IntNo);
6133   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6134   SDValue NewOp2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6135   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp1, NewOp2);
6136   // ReplaceNodeResults requires we maintain the same type for the return value.
6137   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6138 }
6139 
6140 // Returns the opcode of the target-specific SDNode that implements the 32-bit
6141 // form of the given Opcode.
6142 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
6143   switch (Opcode) {
6144   default:
6145     llvm_unreachable("Unexpected opcode");
6146   case ISD::SHL:
6147     return RISCVISD::SLLW;
6148   case ISD::SRA:
6149     return RISCVISD::SRAW;
6150   case ISD::SRL:
6151     return RISCVISD::SRLW;
6152   case ISD::SDIV:
6153     return RISCVISD::DIVW;
6154   case ISD::UDIV:
6155     return RISCVISD::DIVUW;
6156   case ISD::UREM:
6157     return RISCVISD::REMUW;
6158   case ISD::ROTL:
6159     return RISCVISD::ROLW;
6160   case ISD::ROTR:
6161     return RISCVISD::RORW;
6162   case RISCVISD::GREV:
6163     return RISCVISD::GREVW;
6164   case RISCVISD::GORC:
6165     return RISCVISD::GORCW;
6166   }
6167 }
6168 
6169 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
6170 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
6171 // otherwise be promoted to i64, making it difficult to select the
6172 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
6173 // type i8/i16/i32 is lost.
6174 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
6175                                    unsigned ExtOpc = ISD::ANY_EXTEND) {
6176   SDLoc DL(N);
6177   RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6178   SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
6179   SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
6180   SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6181   // ReplaceNodeResults requires we maintain the same type for the return value.
6182   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
6183 }
6184 
6185 // Converts the given 32-bit operation to a i64 operation with signed extension
6186 // semantic to reduce the signed extension instructions.
6187 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
6188   SDLoc DL(N);
6189   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6190   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6191   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
6192   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6193                                DAG.getValueType(MVT::i32));
6194   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
6195 }
6196 
6197 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
6198                                              SmallVectorImpl<SDValue> &Results,
6199                                              SelectionDAG &DAG) const {
6200   SDLoc DL(N);
6201   switch (N->getOpcode()) {
6202   default:
6203     llvm_unreachable("Don't know how to custom type legalize this operation!");
6204   case ISD::STRICT_FP_TO_SINT:
6205   case ISD::STRICT_FP_TO_UINT:
6206   case ISD::FP_TO_SINT:
6207   case ISD::FP_TO_UINT: {
6208     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6209            "Unexpected custom legalisation");
6210     bool IsStrict = N->isStrictFPOpcode();
6211     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
6212                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
6213     SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
6214     if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
6215         TargetLowering::TypeSoftenFloat) {
6216       if (!isTypeLegal(Op0.getValueType()))
6217         return;
6218       if (IsStrict) {
6219         unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
6220                                 : RISCVISD::STRICT_FCVT_WU_RV64;
6221         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
6222         SDValue Res = DAG.getNode(
6223             Opc, DL, VTs, N->getOperand(0), Op0,
6224             DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6225         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6226         Results.push_back(Res.getValue(1));
6227         return;
6228       }
6229       unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
6230       SDValue Res =
6231           DAG.getNode(Opc, DL, MVT::i64, Op0,
6232                       DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
6233       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6234       return;
6235     }
6236     // If the FP type needs to be softened, emit a library call using the 'si'
6237     // version. If we left it to default legalization we'd end up with 'di'. If
6238     // the FP type doesn't need to be softened just let generic type
6239     // legalization promote the result type.
6240     RTLIB::Libcall LC;
6241     if (IsSigned)
6242       LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
6243     else
6244       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
6245     MakeLibCallOptions CallOptions;
6246     EVT OpVT = Op0.getValueType();
6247     CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
6248     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
6249     SDValue Result;
6250     std::tie(Result, Chain) =
6251         makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
6252     Results.push_back(Result);
6253     if (IsStrict)
6254       Results.push_back(Chain);
6255     break;
6256   }
6257   case ISD::READCYCLECOUNTER: {
6258     assert(!Subtarget.is64Bit() &&
6259            "READCYCLECOUNTER only has custom type legalization on riscv32");
6260 
6261     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
6262     SDValue RCW =
6263         DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
6264 
6265     Results.push_back(
6266         DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
6267     Results.push_back(RCW.getValue(2));
6268     break;
6269   }
6270   case ISD::MUL: {
6271     unsigned Size = N->getSimpleValueType(0).getSizeInBits();
6272     unsigned XLen = Subtarget.getXLen();
6273     // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
6274     if (Size > XLen) {
6275       assert(Size == (XLen * 2) && "Unexpected custom legalisation");
6276       SDValue LHS = N->getOperand(0);
6277       SDValue RHS = N->getOperand(1);
6278       APInt HighMask = APInt::getHighBitsSet(Size, XLen);
6279 
6280       bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
6281       bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
6282       // We need exactly one side to be unsigned.
6283       if (LHSIsU == RHSIsU)
6284         return;
6285 
6286       auto MakeMULPair = [&](SDValue S, SDValue U) {
6287         MVT XLenVT = Subtarget.getXLenVT();
6288         S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
6289         U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
6290         SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
6291         SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
6292         return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
6293       };
6294 
6295       bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
6296       bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
6297 
6298       // The other operand should be signed, but still prefer MULH when
6299       // possible.
6300       if (RHSIsU && LHSIsS && !RHSIsS)
6301         Results.push_back(MakeMULPair(LHS, RHS));
6302       else if (LHSIsU && RHSIsS && !LHSIsS)
6303         Results.push_back(MakeMULPair(RHS, LHS));
6304 
6305       return;
6306     }
6307     LLVM_FALLTHROUGH;
6308   }
6309   case ISD::ADD:
6310   case ISD::SUB:
6311     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6312            "Unexpected custom legalisation");
6313     Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
6314     break;
6315   case ISD::SHL:
6316   case ISD::SRA:
6317   case ISD::SRL:
6318     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6319            "Unexpected custom legalisation");
6320     if (N->getOperand(1).getOpcode() != ISD::Constant) {
6321       Results.push_back(customLegalizeToWOp(N, DAG));
6322       break;
6323     }
6324 
6325     // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
6326     // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
6327     // shift amount.
6328     if (N->getOpcode() == ISD::SHL) {
6329       SDLoc DL(N);
6330       SDValue NewOp0 =
6331           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6332       SDValue NewOp1 =
6333           DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
6334       SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
6335       SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
6336                                    DAG.getValueType(MVT::i32));
6337       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6338     }
6339 
6340     break;
6341   case ISD::ROTL:
6342   case ISD::ROTR:
6343     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6344            "Unexpected custom legalisation");
6345     Results.push_back(customLegalizeToWOp(N, DAG));
6346     break;
6347   case ISD::CTTZ:
6348   case ISD::CTTZ_ZERO_UNDEF:
6349   case ISD::CTLZ:
6350   case ISD::CTLZ_ZERO_UNDEF: {
6351     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6352            "Unexpected custom legalisation");
6353 
6354     SDValue NewOp0 =
6355         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6356     bool IsCTZ =
6357         N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
6358     unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
6359     SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
6360     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6361     return;
6362   }
6363   case ISD::SDIV:
6364   case ISD::UDIV:
6365   case ISD::UREM: {
6366     MVT VT = N->getSimpleValueType(0);
6367     assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
6368            Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
6369            "Unexpected custom legalisation");
6370     // Don't promote division/remainder by constant since we should expand those
6371     // to multiply by magic constant.
6372     // FIXME: What if the expansion is disabled for minsize.
6373     if (N->getOperand(1).getOpcode() == ISD::Constant)
6374       return;
6375 
6376     // If the input is i32, use ANY_EXTEND since the W instructions don't read
6377     // the upper 32 bits. For other types we need to sign or zero extend
6378     // based on the opcode.
6379     unsigned ExtOpc = ISD::ANY_EXTEND;
6380     if (VT != MVT::i32)
6381       ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
6382                                            : ISD::ZERO_EXTEND;
6383 
6384     Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
6385     break;
6386   }
6387   case ISD::UADDO:
6388   case ISD::USUBO: {
6389     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6390            "Unexpected custom legalisation");
6391     bool IsAdd = N->getOpcode() == ISD::UADDO;
6392     // Create an ADDW or SUBW.
6393     SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6394     SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6395     SDValue Res =
6396         DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
6397     Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
6398                       DAG.getValueType(MVT::i32));
6399 
6400     // Sign extend the LHS and perform an unsigned compare with the ADDW result.
6401     // Since the inputs are sign extended from i32, this is equivalent to
6402     // comparing the lower 32 bits.
6403     LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6404     SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
6405                                     IsAdd ? ISD::SETULT : ISD::SETUGT);
6406 
6407     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6408     Results.push_back(Overflow);
6409     return;
6410   }
6411   case ISD::UADDSAT:
6412   case ISD::USUBSAT: {
6413     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6414            "Unexpected custom legalisation");
6415     if (Subtarget.hasStdExtZbb()) {
6416       // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
6417       // sign extend allows overflow of the lower 32 bits to be detected on
6418       // the promoted size.
6419       SDValue LHS =
6420           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
6421       SDValue RHS =
6422           DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
6423       SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
6424       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6425       return;
6426     }
6427 
6428     // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
6429     // promotion for UADDO/USUBO.
6430     Results.push_back(expandAddSubSat(N, DAG));
6431     return;
6432   }
6433   case ISD::BITCAST: {
6434     EVT VT = N->getValueType(0);
6435     assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
6436     SDValue Op0 = N->getOperand(0);
6437     EVT Op0VT = Op0.getValueType();
6438     MVT XLenVT = Subtarget.getXLenVT();
6439     if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
6440       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
6441       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
6442     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
6443                Subtarget.hasStdExtF()) {
6444       SDValue FPConv =
6445           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
6446       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
6447     } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
6448                isTypeLegal(Op0VT)) {
6449       // Custom-legalize bitcasts from fixed-length vector types to illegal
6450       // scalar types in order to improve codegen. Bitcast the vector to a
6451       // one-element vector type whose element type is the same as the result
6452       // type, and extract the first element.
6453       EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6454       if (isTypeLegal(BVT)) {
6455         SDValue BVec = DAG.getBitcast(BVT, Op0);
6456         Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6457                                       DAG.getConstant(0, DL, XLenVT)));
6458       }
6459     }
6460     break;
6461   }
6462   case RISCVISD::GREV:
6463   case RISCVISD::GORC: {
6464     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6465            "Unexpected custom legalisation");
6466     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6467     // This is similar to customLegalizeToWOp, except that we pass the second
6468     // operand (a TargetConstant) straight through: it is already of type
6469     // XLenVT.
6470     RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
6471     SDValue NewOp0 =
6472         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6473     SDValue NewOp1 =
6474         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6475     SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
6476     // ReplaceNodeResults requires we maintain the same type for the return
6477     // value.
6478     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6479     break;
6480   }
6481   case RISCVISD::SHFL: {
6482     // There is no SHFLIW instruction, but we can just promote the operation.
6483     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6484            "Unexpected custom legalisation");
6485     assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
6486     SDValue NewOp0 =
6487         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6488     SDValue NewOp1 =
6489         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6490     SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
6491     // ReplaceNodeResults requires we maintain the same type for the return
6492     // value.
6493     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
6494     break;
6495   }
6496   case ISD::BSWAP:
6497   case ISD::BITREVERSE: {
6498     MVT VT = N->getSimpleValueType(0);
6499     MVT XLenVT = Subtarget.getXLenVT();
6500     assert((VT == MVT::i8 || VT == MVT::i16 ||
6501             (VT == MVT::i32 && Subtarget.is64Bit())) &&
6502            Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
6503     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
6504     unsigned Imm = VT.getSizeInBits() - 1;
6505     // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
6506     if (N->getOpcode() == ISD::BSWAP)
6507       Imm &= ~0x7U;
6508     unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
6509     SDValue GREVI =
6510         DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
6511     // ReplaceNodeResults requires we maintain the same type for the return
6512     // value.
6513     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
6514     break;
6515   }
6516   case ISD::FSHL:
6517   case ISD::FSHR: {
6518     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6519            Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
6520     SDValue NewOp0 =
6521         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
6522     SDValue NewOp1 =
6523         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6524     SDValue NewShAmt =
6525         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6526     // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
6527     // Mask the shift amount to 5 bits to prevent accidentally setting bit 5.
6528     NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt,
6529                            DAG.getConstant(0x1f, DL, MVT::i64));
6530     // fshl and fshr concatenate their operands in the same order. fsrw and fslw
6531     // instruction use different orders. fshl will return its first operand for
6532     // shift of zero, fshr will return its second operand. fsl and fsr both
6533     // return rs1 so the ISD nodes need to have different operand orders.
6534     // Shift amount is in rs2.
6535     unsigned Opc = RISCVISD::FSLW;
6536     if (N->getOpcode() == ISD::FSHR) {
6537       std::swap(NewOp0, NewOp1);
6538       Opc = RISCVISD::FSRW;
6539     }
6540     SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewShAmt);
6541     Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
6542     break;
6543   }
6544   case ISD::EXTRACT_VECTOR_ELT: {
6545     // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
6546     // type is illegal (currently only vXi64 RV32).
6547     // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
6548     // transferred to the destination register. We issue two of these from the
6549     // upper- and lower- halves of the SEW-bit vector element, slid down to the
6550     // first element.
6551     SDValue Vec = N->getOperand(0);
6552     SDValue Idx = N->getOperand(1);
6553 
6554     // The vector type hasn't been legalized yet so we can't issue target
6555     // specific nodes if it needs legalization.
6556     // FIXME: We would manually legalize if it's important.
6557     if (!isTypeLegal(Vec.getValueType()))
6558       return;
6559 
6560     MVT VecVT = Vec.getSimpleValueType();
6561 
6562     assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
6563            VecVT.getVectorElementType() == MVT::i64 &&
6564            "Unexpected EXTRACT_VECTOR_ELT legalization");
6565 
6566     // If this is a fixed vector, we need to convert it to a scalable vector.
6567     MVT ContainerVT = VecVT;
6568     if (VecVT.isFixedLengthVector()) {
6569       ContainerVT = getContainerForFixedLengthVector(VecVT);
6570       Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
6571     }
6572 
6573     MVT XLenVT = Subtarget.getXLenVT();
6574 
6575     // Use a VL of 1 to avoid processing more elements than we need.
6576     MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
6577     SDValue VL = DAG.getConstant(1, DL, XLenVT);
6578     SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6579 
6580     // Unless the index is known to be 0, we must slide the vector down to get
6581     // the desired element into index 0.
6582     if (!isNullConstant(Idx)) {
6583       Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
6584                         DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6585     }
6586 
6587     // Extract the lower XLEN bits of the correct vector element.
6588     SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6589 
6590     // To extract the upper XLEN bits of the vector element, shift the first
6591     // element right by 32 bits and re-extract the lower XLEN bits.
6592     SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
6593                                      DAG.getConstant(32, DL, XLenVT), VL);
6594     SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
6595                                  ThirtyTwoV, Mask, VL);
6596 
6597     SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6598 
6599     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6600     break;
6601   }
6602   case ISD::INTRINSIC_WO_CHAIN: {
6603     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6604     switch (IntNo) {
6605     default:
6606       llvm_unreachable(
6607           "Don't know how to custom type legalize this intrinsic!");
6608     case Intrinsic::riscv_grev:
6609     case Intrinsic::riscv_gorc:
6610     case Intrinsic::riscv_bcompress:
6611     case Intrinsic::riscv_bdecompress:
6612     case Intrinsic::riscv_bfp: {
6613       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6614              "Unexpected custom legalisation");
6615       Results.push_back(customLegalizeToWOpByIntr(N, DAG, IntNo));
6616       break;
6617     }
6618     case Intrinsic::riscv_fsl:
6619     case Intrinsic::riscv_fsr: {
6620       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6621              "Unexpected custom legalisation");
6622       SDValue NewOp1 =
6623           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6624       SDValue NewOp2 =
6625           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6626       SDValue NewOp3 =
6627           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3));
6628       unsigned Opc = getRISCVWOpcodeByIntr(IntNo);
6629       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2, NewOp3);
6630       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6631       break;
6632     }
6633     case Intrinsic::riscv_orc_b: {
6634       // Lower to the GORCI encoding for orc.b with the operand extended.
6635       SDValue NewOp =
6636           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6637       // If Zbp is enabled, use GORCIW which will sign extend the result.
6638       unsigned Opc =
6639           Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
6640       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
6641                                 DAG.getConstant(7, DL, MVT::i64));
6642       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6643       return;
6644     }
6645     case Intrinsic::riscv_shfl:
6646     case Intrinsic::riscv_unshfl: {
6647       assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
6648              "Unexpected custom legalisation");
6649       SDValue NewOp1 =
6650           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
6651       SDValue NewOp2 =
6652           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
6653       unsigned Opc =
6654           IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
6655       // There is no (UN)SHFLIW. If the control word is a constant, we can use
6656       // (UN)SHFLI with bit 4 of the control word cleared. The upper 32 bit half
6657       // will be shuffled the same way as the lower 32 bit half, but the two
6658       // halves won't cross.
6659       if (isa<ConstantSDNode>(NewOp2)) {
6660         NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
6661                              DAG.getConstant(0xf, DL, MVT::i64));
6662         Opc =
6663             IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
6664       }
6665       SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
6666       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
6667       break;
6668     }
6669     case Intrinsic::riscv_vmv_x_s: {
6670       EVT VT = N->getValueType(0);
6671       MVT XLenVT = Subtarget.getXLenVT();
6672       if (VT.bitsLT(XLenVT)) {
6673         // Simple case just extract using vmv.x.s and truncate.
6674         SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
6675                                       Subtarget.getXLenVT(), N->getOperand(1));
6676         Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
6677         return;
6678       }
6679 
6680       assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
6681              "Unexpected custom legalization");
6682 
6683       // We need to do the move in two steps.
6684       SDValue Vec = N->getOperand(1);
6685       MVT VecVT = Vec.getSimpleValueType();
6686 
6687       // First extract the lower XLEN bits of the element.
6688       SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
6689 
6690       // To extract the upper XLEN bits of the vector element, shift the first
6691       // element right by 32 bits and re-extract the lower XLEN bits.
6692       SDValue VL = DAG.getConstant(1, DL, XLenVT);
6693       MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
6694       SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
6695       SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
6696                                        DAG.getConstant(32, DL, XLenVT), VL);
6697       SDValue LShr32 =
6698           DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
6699       SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
6700 
6701       Results.push_back(
6702           DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
6703       break;
6704     }
6705     }
6706     break;
6707   }
6708   case ISD::VECREDUCE_ADD:
6709   case ISD::VECREDUCE_AND:
6710   case ISD::VECREDUCE_OR:
6711   case ISD::VECREDUCE_XOR:
6712   case ISD::VECREDUCE_SMAX:
6713   case ISD::VECREDUCE_UMAX:
6714   case ISD::VECREDUCE_SMIN:
6715   case ISD::VECREDUCE_UMIN:
6716     if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
6717       Results.push_back(V);
6718     break;
6719   case ISD::VP_REDUCE_ADD:
6720   case ISD::VP_REDUCE_AND:
6721   case ISD::VP_REDUCE_OR:
6722   case ISD::VP_REDUCE_XOR:
6723   case ISD::VP_REDUCE_SMAX:
6724   case ISD::VP_REDUCE_UMAX:
6725   case ISD::VP_REDUCE_SMIN:
6726   case ISD::VP_REDUCE_UMIN:
6727     if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
6728       Results.push_back(V);
6729     break;
6730   case ISD::FLT_ROUNDS_: {
6731     SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
6732     SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
6733     Results.push_back(Res.getValue(0));
6734     Results.push_back(Res.getValue(1));
6735     break;
6736   }
6737   }
6738 }
6739 
6740 // A structure to hold one of the bit-manipulation patterns below. Together, a
6741 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
6742 //   (or (and (shl x, 1), 0xAAAAAAAA),
6743 //       (and (srl x, 1), 0x55555555))
6744 struct RISCVBitmanipPat {
6745   SDValue Op;
6746   unsigned ShAmt;
6747   bool IsSHL;
6748 
6749   bool formsPairWith(const RISCVBitmanipPat &Other) const {
6750     return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
6751   }
6752 };
6753 
6754 // Matches patterns of the form
6755 //   (and (shl x, C2), (C1 << C2))
6756 //   (and (srl x, C2), C1)
6757 //   (shl (and x, C1), C2)
6758 //   (srl (and x, (C1 << C2)), C2)
6759 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
6760 // The expected masks for each shift amount are specified in BitmanipMasks where
6761 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
6762 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
6763 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
6764 // XLen is 64.
6765 static Optional<RISCVBitmanipPat>
6766 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
6767   assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
6768          "Unexpected number of masks");
6769   Optional<uint64_t> Mask;
6770   // Optionally consume a mask around the shift operation.
6771   if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
6772     Mask = Op.getConstantOperandVal(1);
6773     Op = Op.getOperand(0);
6774   }
6775   if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
6776     return None;
6777   bool IsSHL = Op.getOpcode() == ISD::SHL;
6778 
6779   if (!isa<ConstantSDNode>(Op.getOperand(1)))
6780     return None;
6781   uint64_t ShAmt = Op.getConstantOperandVal(1);
6782 
6783   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
6784   if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
6785     return None;
6786   // If we don't have enough masks for 64 bit, then we must be trying to
6787   // match SHFL so we're only allowed to shift 1/4 of the width.
6788   if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
6789     return None;
6790 
6791   SDValue Src = Op.getOperand(0);
6792 
6793   // The expected mask is shifted left when the AND is found around SHL
6794   // patterns.
6795   //   ((x >> 1) & 0x55555555)
6796   //   ((x << 1) & 0xAAAAAAAA)
6797   bool SHLExpMask = IsSHL;
6798 
6799   if (!Mask) {
6800     // Sometimes LLVM keeps the mask as an operand of the shift, typically when
6801     // the mask is all ones: consume that now.
6802     if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
6803       Mask = Src.getConstantOperandVal(1);
6804       Src = Src.getOperand(0);
6805       // The expected mask is now in fact shifted left for SRL, so reverse the
6806       // decision.
6807       //   ((x & 0xAAAAAAAA) >> 1)
6808       //   ((x & 0x55555555) << 1)
6809       SHLExpMask = !SHLExpMask;
6810     } else {
6811       // Use a default shifted mask of all-ones if there's no AND, truncated
6812       // down to the expected width. This simplifies the logic later on.
6813       Mask = maskTrailingOnes<uint64_t>(Width);
6814       *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
6815     }
6816   }
6817 
6818   unsigned MaskIdx = Log2_32(ShAmt);
6819   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
6820 
6821   if (SHLExpMask)
6822     ExpMask <<= ShAmt;
6823 
6824   if (Mask != ExpMask)
6825     return None;
6826 
6827   return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
6828 }
6829 
6830 // Matches any of the following bit-manipulation patterns:
6831 //   (and (shl x, 1), (0x55555555 << 1))
6832 //   (and (srl x, 1), 0x55555555)
6833 //   (shl (and x, 0x55555555), 1)
6834 //   (srl (and x, (0x55555555 << 1)), 1)
6835 // where the shift amount and mask may vary thus:
6836 //   [1]  = 0x55555555 / 0xAAAAAAAA
6837 //   [2]  = 0x33333333 / 0xCCCCCCCC
6838 //   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
6839 //   [8]  = 0x00FF00FF / 0xFF00FF00
6840 //   [16] = 0x0000FFFF / 0xFFFFFFFF
6841 //   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
6842 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
6843   // These are the unshifted masks which we use to match bit-manipulation
6844   // patterns. They may be shifted left in certain circumstances.
6845   static const uint64_t BitmanipMasks[] = {
6846       0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
6847       0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
6848 
6849   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6850 }
6851 
6852 // Match the following pattern as a GREVI(W) operation
6853 //   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
6854 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
6855                                const RISCVSubtarget &Subtarget) {
6856   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6857   EVT VT = Op.getValueType();
6858 
6859   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6860     auto LHS = matchGREVIPat(Op.getOperand(0));
6861     auto RHS = matchGREVIPat(Op.getOperand(1));
6862     if (LHS && RHS && LHS->formsPairWith(*RHS)) {
6863       SDLoc DL(Op);
6864       return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
6865                          DAG.getConstant(LHS->ShAmt, DL, VT));
6866     }
6867   }
6868   return SDValue();
6869 }
6870 
6871 // Matches any the following pattern as a GORCI(W) operation
6872 // 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
6873 // 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
6874 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
6875 // Note that with the variant of 3.,
6876 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
6877 // the inner pattern will first be matched as GREVI and then the outer
6878 // pattern will be matched to GORC via the first rule above.
6879 // 4.  (or (rotl/rotr x, bitwidth/2), x)
6880 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
6881                                const RISCVSubtarget &Subtarget) {
6882   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6883   EVT VT = Op.getValueType();
6884 
6885   if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
6886     SDLoc DL(Op);
6887     SDValue Op0 = Op.getOperand(0);
6888     SDValue Op1 = Op.getOperand(1);
6889 
6890     auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
6891       if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
6892           isa<ConstantSDNode>(Reverse.getOperand(1)) &&
6893           isPowerOf2_32(Reverse.getConstantOperandVal(1)))
6894         return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
6895       // We can also form GORCI from ROTL/ROTR by half the bitwidth.
6896       if ((Reverse.getOpcode() == ISD::ROTL ||
6897            Reverse.getOpcode() == ISD::ROTR) &&
6898           Reverse.getOperand(0) == X &&
6899           isa<ConstantSDNode>(Reverse.getOperand(1))) {
6900         uint64_t RotAmt = Reverse.getConstantOperandVal(1);
6901         if (RotAmt == (VT.getSizeInBits() / 2))
6902           return DAG.getNode(RISCVISD::GORC, DL, VT, X,
6903                              DAG.getConstant(RotAmt, DL, VT));
6904       }
6905       return SDValue();
6906     };
6907 
6908     // Check for either commutable permutation of (or (GREVI x, shamt), x)
6909     if (SDValue V = MatchOROfReverse(Op0, Op1))
6910       return V;
6911     if (SDValue V = MatchOROfReverse(Op1, Op0))
6912       return V;
6913 
6914     // OR is commutable so canonicalize its OR operand to the left
6915     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
6916       std::swap(Op0, Op1);
6917     if (Op0.getOpcode() != ISD::OR)
6918       return SDValue();
6919     SDValue OrOp0 = Op0.getOperand(0);
6920     SDValue OrOp1 = Op0.getOperand(1);
6921     auto LHS = matchGREVIPat(OrOp0);
6922     // OR is commutable so swap the operands and try again: x might have been
6923     // on the left
6924     if (!LHS) {
6925       std::swap(OrOp0, OrOp1);
6926       LHS = matchGREVIPat(OrOp0);
6927     }
6928     auto RHS = matchGREVIPat(Op1);
6929     if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
6930       return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
6931                          DAG.getConstant(LHS->ShAmt, DL, VT));
6932     }
6933   }
6934   return SDValue();
6935 }
6936 
6937 // Matches any of the following bit-manipulation patterns:
6938 //   (and (shl x, 1), (0x22222222 << 1))
6939 //   (and (srl x, 1), 0x22222222)
6940 //   (shl (and x, 0x22222222), 1)
6941 //   (srl (and x, (0x22222222 << 1)), 1)
6942 // where the shift amount and mask may vary thus:
6943 //   [1]  = 0x22222222 / 0x44444444
6944 //   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
6945 //   [4]  = 0x00F000F0 / 0x0F000F00
6946 //   [8]  = 0x0000FF00 / 0x00FF0000
6947 //   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
6948 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
6949   // These are the unshifted masks which we use to match bit-manipulation
6950   // patterns. They may be shifted left in certain circumstances.
6951   static const uint64_t BitmanipMasks[] = {
6952       0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
6953       0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
6954 
6955   return matchRISCVBitmanipPat(Op, BitmanipMasks);
6956 }
6957 
6958 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
6959 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
6960                                const RISCVSubtarget &Subtarget) {
6961   assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
6962   EVT VT = Op.getValueType();
6963 
6964   if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
6965     return SDValue();
6966 
6967   SDValue Op0 = Op.getOperand(0);
6968   SDValue Op1 = Op.getOperand(1);
6969 
6970   // Or is commutable so canonicalize the second OR to the LHS.
6971   if (Op0.getOpcode() != ISD::OR)
6972     std::swap(Op0, Op1);
6973   if (Op0.getOpcode() != ISD::OR)
6974     return SDValue();
6975 
6976   // We found an inner OR, so our operands are the operands of the inner OR
6977   // and the other operand of the outer OR.
6978   SDValue A = Op0.getOperand(0);
6979   SDValue B = Op0.getOperand(1);
6980   SDValue C = Op1;
6981 
6982   auto Match1 = matchSHFLPat(A);
6983   auto Match2 = matchSHFLPat(B);
6984 
6985   // If neither matched, we failed.
6986   if (!Match1 && !Match2)
6987     return SDValue();
6988 
6989   // We had at least one match. if one failed, try the remaining C operand.
6990   if (!Match1) {
6991     std::swap(A, C);
6992     Match1 = matchSHFLPat(A);
6993     if (!Match1)
6994       return SDValue();
6995   } else if (!Match2) {
6996     std::swap(B, C);
6997     Match2 = matchSHFLPat(B);
6998     if (!Match2)
6999       return SDValue();
7000   }
7001   assert(Match1 && Match2);
7002 
7003   // Make sure our matches pair up.
7004   if (!Match1->formsPairWith(*Match2))
7005     return SDValue();
7006 
7007   // All the remains is to make sure C is an AND with the same input, that masks
7008   // out the bits that are being shuffled.
7009   if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
7010       C.getOperand(0) != Match1->Op)
7011     return SDValue();
7012 
7013   uint64_t Mask = C.getConstantOperandVal(1);
7014 
7015   static const uint64_t BitmanipMasks[] = {
7016       0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
7017       0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
7018   };
7019 
7020   unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
7021   unsigned MaskIdx = Log2_32(Match1->ShAmt);
7022   uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
7023 
7024   if (Mask != ExpMask)
7025     return SDValue();
7026 
7027   SDLoc DL(Op);
7028   return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
7029                      DAG.getConstant(Match1->ShAmt, DL, VT));
7030 }
7031 
7032 // Optimize (add (shl x, c0), (shl y, c1)) ->
7033 //          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
7034 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
7035                                   const RISCVSubtarget &Subtarget) {
7036   // Perform this optimization only in the zba extension.
7037   if (!Subtarget.hasStdExtZba())
7038     return SDValue();
7039 
7040   // Skip for vector types and larger types.
7041   EVT VT = N->getValueType(0);
7042   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7043     return SDValue();
7044 
7045   // The two operand nodes must be SHL and have no other use.
7046   SDValue N0 = N->getOperand(0);
7047   SDValue N1 = N->getOperand(1);
7048   if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
7049       !N0->hasOneUse() || !N1->hasOneUse())
7050     return SDValue();
7051 
7052   // Check c0 and c1.
7053   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7054   auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
7055   if (!N0C || !N1C)
7056     return SDValue();
7057   int64_t C0 = N0C->getSExtValue();
7058   int64_t C1 = N1C->getSExtValue();
7059   if (C0 <= 0 || C1 <= 0)
7060     return SDValue();
7061 
7062   // Skip if SH1ADD/SH2ADD/SH3ADD are not applicable.
7063   int64_t Bits = std::min(C0, C1);
7064   int64_t Diff = std::abs(C0 - C1);
7065   if (Diff != 1 && Diff != 2 && Diff != 3)
7066     return SDValue();
7067 
7068   // Build nodes.
7069   SDLoc DL(N);
7070   SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
7071   SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
7072   SDValue NA0 =
7073       DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT));
7074   SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS);
7075   return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT));
7076 }
7077 
7078 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
7079 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
7080 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
7081 // not undo itself, but they are redundant.
7082 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
7083   SDValue Src = N->getOperand(0);
7084 
7085   if (Src.getOpcode() != N->getOpcode())
7086     return SDValue();
7087 
7088   if (!isa<ConstantSDNode>(N->getOperand(1)) ||
7089       !isa<ConstantSDNode>(Src.getOperand(1)))
7090     return SDValue();
7091 
7092   unsigned ShAmt1 = N->getConstantOperandVal(1);
7093   unsigned ShAmt2 = Src.getConstantOperandVal(1);
7094   Src = Src.getOperand(0);
7095 
7096   unsigned CombinedShAmt;
7097   if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
7098     CombinedShAmt = ShAmt1 | ShAmt2;
7099   else
7100     CombinedShAmt = ShAmt1 ^ ShAmt2;
7101 
7102   if (CombinedShAmt == 0)
7103     return Src;
7104 
7105   SDLoc DL(N);
7106   return DAG.getNode(
7107       N->getOpcode(), DL, N->getValueType(0), Src,
7108       DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
7109 }
7110 
7111 // Combine a constant select operand into its use:
7112 //
7113 // (and (select cond, -1, c), x)
7114 //   -> (select cond, x, (and x, c))  [AllOnes=1]
7115 // (or  (select cond, 0, c), x)
7116 //   -> (select cond, x, (or x, c))  [AllOnes=0]
7117 // (xor (select cond, 0, c), x)
7118 //   -> (select cond, x, (xor x, c))  [AllOnes=0]
7119 // (add (select cond, 0, c), x)
7120 //   -> (select cond, x, (add x, c))  [AllOnes=0]
7121 // (sub x, (select cond, 0, c))
7122 //   -> (select cond, x, (sub x, c))  [AllOnes=0]
7123 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
7124                                    SelectionDAG &DAG, bool AllOnes) {
7125   EVT VT = N->getValueType(0);
7126 
7127   // Skip vectors.
7128   if (VT.isVector())
7129     return SDValue();
7130 
7131   if ((Slct.getOpcode() != ISD::SELECT &&
7132        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
7133       !Slct.hasOneUse())
7134     return SDValue();
7135 
7136   auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
7137     return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
7138   };
7139 
7140   bool SwapSelectOps;
7141   unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
7142   SDValue TrueVal = Slct.getOperand(1 + OpOffset);
7143   SDValue FalseVal = Slct.getOperand(2 + OpOffset);
7144   SDValue NonConstantVal;
7145   if (isZeroOrAllOnes(TrueVal, AllOnes)) {
7146     SwapSelectOps = false;
7147     NonConstantVal = FalseVal;
7148   } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
7149     SwapSelectOps = true;
7150     NonConstantVal = TrueVal;
7151   } else
7152     return SDValue();
7153 
7154   // Slct is now know to be the desired identity constant when CC is true.
7155   TrueVal = OtherOp;
7156   FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
7157   // Unless SwapSelectOps says the condition should be false.
7158   if (SwapSelectOps)
7159     std::swap(TrueVal, FalseVal);
7160 
7161   if (Slct.getOpcode() == RISCVISD::SELECT_CC)
7162     return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
7163                        {Slct.getOperand(0), Slct.getOperand(1),
7164                         Slct.getOperand(2), TrueVal, FalseVal});
7165 
7166   return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
7167                      {Slct.getOperand(0), TrueVal, FalseVal});
7168 }
7169 
7170 // Attempt combineSelectAndUse on each operand of a commutative operator N.
7171 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
7172                                               bool AllOnes) {
7173   SDValue N0 = N->getOperand(0);
7174   SDValue N1 = N->getOperand(1);
7175   if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
7176     return Result;
7177   if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
7178     return Result;
7179   return SDValue();
7180 }
7181 
7182 // Transform (add (mul x, c0), c1) ->
7183 //           (add (mul (add x, c1/c0), c0), c1%c0).
7184 // if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
7185 // that should be excluded is when c0*(c1/c0) is simm12, which will lead
7186 // to an infinite loop in DAGCombine if transformed.
7187 // Or transform (add (mul x, c0), c1) ->
7188 //              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
7189 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
7190 // case that should be excluded is when c0*(c1/c0+1) is simm12, which will
7191 // lead to an infinite loop in DAGCombine if transformed.
7192 // Or transform (add (mul x, c0), c1) ->
7193 //              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
7194 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
7195 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
7196 // lead to an infinite loop in DAGCombine if transformed.
7197 // Or transform (add (mul x, c0), c1) ->
7198 //              (mul (add x, c1/c0), c0).
7199 // if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
7200 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
7201                                      const RISCVSubtarget &Subtarget) {
7202   // Skip for vector types and larger types.
7203   EVT VT = N->getValueType(0);
7204   if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
7205     return SDValue();
7206   // The first operand node must be a MUL and has no other use.
7207   SDValue N0 = N->getOperand(0);
7208   if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
7209     return SDValue();
7210   // Check if c0 and c1 match above conditions.
7211   auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
7212   auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
7213   if (!N0C || !N1C)
7214     return SDValue();
7215   // If N0C has multiple uses it's possible one of the cases in
7216   // DAGCombiner::isMulAddWithConstProfitable will be true, which would result
7217   // in an infinite loop.
7218   if (!N0C->hasOneUse())
7219     return SDValue();
7220   int64_t C0 = N0C->getSExtValue();
7221   int64_t C1 = N1C->getSExtValue();
7222   int64_t CA, CB;
7223   if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
7224     return SDValue();
7225   // Search for proper CA (non-zero) and CB that both are simm12.
7226   if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
7227       !isInt<12>(C0 * (C1 / C0))) {
7228     CA = C1 / C0;
7229     CB = C1 % C0;
7230   } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
7231              isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
7232     CA = C1 / C0 + 1;
7233     CB = C1 % C0 - C0;
7234   } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
7235              isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
7236     CA = C1 / C0 - 1;
7237     CB = C1 % C0 + C0;
7238   } else
7239     return SDValue();
7240   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
7241   SDLoc DL(N);
7242   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
7243                              DAG.getConstant(CA, DL, VT));
7244   SDValue New1 =
7245       DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
7246   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
7247 }
7248 
7249 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
7250                                  const RISCVSubtarget &Subtarget) {
7251   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
7252     return V;
7253   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
7254     return V;
7255   // fold (add (select lhs, rhs, cc, 0, y), x) ->
7256   //      (select lhs, rhs, cc, x, (add x, y))
7257   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7258 }
7259 
7260 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
7261   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
7262   //      (select lhs, rhs, cc, x, (sub x, y))
7263   SDValue N0 = N->getOperand(0);
7264   SDValue N1 = N->getOperand(1);
7265   return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
7266 }
7267 
7268 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
7269   // fold (and (select lhs, rhs, cc, -1, y), x) ->
7270   //      (select lhs, rhs, cc, x, (and x, y))
7271   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
7272 }
7273 
7274 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
7275                                 const RISCVSubtarget &Subtarget) {
7276   if (Subtarget.hasStdExtZbp()) {
7277     if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
7278       return GREV;
7279     if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
7280       return GORC;
7281     if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
7282       return SHFL;
7283   }
7284 
7285   // fold (or (select cond, 0, y), x) ->
7286   //      (select cond, x, (or x, y))
7287   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7288 }
7289 
7290 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
7291   // fold (xor (select cond, 0, y), x) ->
7292   //      (select cond, x, (xor x, y))
7293   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
7294 }
7295 
7296 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
7297 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
7298 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
7299 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
7300 // ADDW/SUBW/MULW.
7301 static SDValue performANY_EXTENDCombine(SDNode *N,
7302                                         TargetLowering::DAGCombinerInfo &DCI,
7303                                         const RISCVSubtarget &Subtarget) {
7304   if (!Subtarget.is64Bit())
7305     return SDValue();
7306 
7307   SelectionDAG &DAG = DCI.DAG;
7308 
7309   SDValue Src = N->getOperand(0);
7310   EVT VT = N->getValueType(0);
7311   if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
7312     return SDValue();
7313 
7314   // The opcode must be one that can implicitly sign_extend.
7315   // FIXME: Additional opcodes.
7316   switch (Src.getOpcode()) {
7317   default:
7318     return SDValue();
7319   case ISD::MUL:
7320     if (!Subtarget.hasStdExtM())
7321       return SDValue();
7322     LLVM_FALLTHROUGH;
7323   case ISD::ADD:
7324   case ISD::SUB:
7325     break;
7326   }
7327 
7328   // Only handle cases where the result is used by a CopyToReg. That likely
7329   // means the value is a liveout of the basic block. This helps prevent
7330   // infinite combine loops like PR51206.
7331   if (none_of(N->uses(),
7332               [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
7333     return SDValue();
7334 
7335   SmallVector<SDNode *, 4> SetCCs;
7336   for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
7337                             UE = Src.getNode()->use_end();
7338        UI != UE; ++UI) {
7339     SDNode *User = *UI;
7340     if (User == N)
7341       continue;
7342     if (UI.getUse().getResNo() != Src.getResNo())
7343       continue;
7344     // All i32 setccs are legalized by sign extending operands.
7345     if (User->getOpcode() == ISD::SETCC) {
7346       SetCCs.push_back(User);
7347       continue;
7348     }
7349     // We don't know if we can extend this user.
7350     break;
7351   }
7352 
7353   // If we don't have any SetCCs, this isn't worthwhile.
7354   if (SetCCs.empty())
7355     return SDValue();
7356 
7357   SDLoc DL(N);
7358   SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
7359   DCI.CombineTo(N, SExt);
7360 
7361   // Promote all the setccs.
7362   for (SDNode *SetCC : SetCCs) {
7363     SmallVector<SDValue, 4> Ops;
7364 
7365     for (unsigned j = 0; j != 2; ++j) {
7366       SDValue SOp = SetCC->getOperand(j);
7367       if (SOp == Src)
7368         Ops.push_back(SExt);
7369       else
7370         Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
7371     }
7372 
7373     Ops.push_back(SetCC->getOperand(2));
7374     DCI.CombineTo(SetCC,
7375                   DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7376   }
7377   return SDValue(N, 0);
7378 }
7379 
7380 // Try to form VWMUL, VWMULU or VWMULSU.
7381 // TODO: Support VWMULSU.vx with a sign extend Op and a splat of scalar Op.
7382 static SDValue combineMUL_VLToVWMUL_VL(SDNode *N, SelectionDAG &DAG,
7383                                        bool Commute) {
7384   assert(N->getOpcode() == RISCVISD::MUL_VL && "Unexpected opcode");
7385   SDValue Op0 = N->getOperand(0);
7386   SDValue Op1 = N->getOperand(1);
7387   if (Commute)
7388     std::swap(Op0, Op1);
7389 
7390   bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
7391   bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
7392   bool IsVWMULSU = IsSignExt && Op1.getOpcode() == RISCVISD::VZEXT_VL;
7393   if ((!IsSignExt && !IsZeroExt) || !Op0.hasOneUse())
7394     return SDValue();
7395 
7396   SDValue Mask = N->getOperand(2);
7397   SDValue VL = N->getOperand(3);
7398 
7399   // Make sure the mask and VL match.
7400   if (Op0.getOperand(1) != Mask || Op0.getOperand(2) != VL)
7401     return SDValue();
7402 
7403   MVT VT = N->getSimpleValueType(0);
7404 
7405   // Determine the narrow size for a widening multiply.
7406   unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
7407   MVT NarrowVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowSize),
7408                                   VT.getVectorElementCount());
7409 
7410   SDLoc DL(N);
7411 
7412   // See if the other operand is the same opcode.
7413   if (IsVWMULSU || Op0.getOpcode() == Op1.getOpcode()) {
7414     if (!Op1.hasOneUse())
7415       return SDValue();
7416 
7417     // Make sure the mask and VL match.
7418     if (Op1.getOperand(1) != Mask || Op1.getOperand(2) != VL)
7419       return SDValue();
7420 
7421     Op1 = Op1.getOperand(0);
7422   } else if (Op1.getOpcode() == RISCVISD::VMV_V_X_VL) {
7423     // The operand is a splat of a scalar.
7424 
7425     // The VL must be the same.
7426     if (Op1.getOperand(1) != VL)
7427       return SDValue();
7428 
7429     // Get the scalar value.
7430     Op1 = Op1.getOperand(0);
7431 
7432     // See if have enough sign bits or zero bits in the scalar to use a
7433     // widening multiply by splatting to smaller element size.
7434     unsigned EltBits = VT.getScalarSizeInBits();
7435     unsigned ScalarBits = Op1.getValueSizeInBits();
7436     // Make sure we're getting all element bits from the scalar register.
7437     // FIXME: Support implicit sign extension of vmv.v.x?
7438     if (ScalarBits < EltBits)
7439       return SDValue();
7440 
7441     if (IsSignExt) {
7442       if (DAG.ComputeNumSignBits(Op1) <= (ScalarBits - NarrowSize))
7443         return SDValue();
7444     } else {
7445       APInt Mask = APInt::getBitsSetFrom(ScalarBits, NarrowSize);
7446       if (!DAG.MaskedValueIsZero(Op1, Mask))
7447         return SDValue();
7448     }
7449 
7450     Op1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT, Op1, VL);
7451   } else
7452     return SDValue();
7453 
7454   Op0 = Op0.getOperand(0);
7455 
7456   // Re-introduce narrower extends if needed.
7457   unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
7458   if (Op0.getValueType() != NarrowVT)
7459     Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
7460   // vwmulsu requires second operand to be zero extended.
7461   ExtOpc = IsVWMULSU ? RISCVISD::VZEXT_VL : ExtOpc;
7462   if (Op1.getValueType() != NarrowVT)
7463     Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
7464 
7465   unsigned WMulOpc = RISCVISD::VWMULSU_VL;
7466   if (!IsVWMULSU)
7467     WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
7468   return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
7469 }
7470 
7471 static RISCVFPRndMode::RoundingMode matchRoundingOp(SDValue Op) {
7472   switch (Op.getOpcode()) {
7473   case ISD::FROUNDEVEN: return RISCVFPRndMode::RNE;
7474   case ISD::FTRUNC:     return RISCVFPRndMode::RTZ;
7475   case ISD::FFLOOR:     return RISCVFPRndMode::RDN;
7476   case ISD::FCEIL:      return RISCVFPRndMode::RUP;
7477   case ISD::FROUND:     return RISCVFPRndMode::RMM;
7478   }
7479 
7480   return RISCVFPRndMode::Invalid;
7481 }
7482 
7483 // Fold
7484 //   (fp_to_int (froundeven X)) -> fcvt X, rne
7485 //   (fp_to_int (ftrunc X))     -> fcvt X, rtz
7486 //   (fp_to_int (ffloor X))     -> fcvt X, rdn
7487 //   (fp_to_int (fceil X))      -> fcvt X, rup
7488 //   (fp_to_int (fround X))     -> fcvt X, rmm
7489 static SDValue performFP_TO_INTCombine(SDNode *N,
7490                                        TargetLowering::DAGCombinerInfo &DCI,
7491                                        const RISCVSubtarget &Subtarget) {
7492   SelectionDAG &DAG = DCI.DAG;
7493   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7494   MVT XLenVT = Subtarget.getXLenVT();
7495 
7496   // Only handle XLen or i32 types. Other types narrower than XLen will
7497   // eventually be legalized to XLenVT.
7498   EVT VT = N->getValueType(0);
7499   if (VT != MVT::i32 && VT != XLenVT)
7500     return SDValue();
7501 
7502   SDValue Src = N->getOperand(0);
7503 
7504   // Ensure the FP type is also legal.
7505   if (!TLI.isTypeLegal(Src.getValueType()))
7506     return SDValue();
7507 
7508   // Don't do this for f16 with Zfhmin and not Zfh.
7509   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7510     return SDValue();
7511 
7512   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7513   if (FRM == RISCVFPRndMode::Invalid)
7514     return SDValue();
7515 
7516   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
7517 
7518   unsigned Opc;
7519   if (VT == XLenVT)
7520     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7521   else
7522     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7523 
7524   SDLoc DL(N);
7525   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
7526                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7527   return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
7528 }
7529 
7530 // Fold
7531 //   (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
7532 //   (fp_to_int_sat (ftrunc X))     -> (select X == nan, 0, (fcvt X, rtz))
7533 //   (fp_to_int_sat (ffloor X))     -> (select X == nan, 0, (fcvt X, rdn))
7534 //   (fp_to_int_sat (fceil X))      -> (select X == nan, 0, (fcvt X, rup))
7535 //   (fp_to_int_sat (fround X))     -> (select X == nan, 0, (fcvt X, rmm))
7536 static SDValue performFP_TO_INT_SATCombine(SDNode *N,
7537                                        TargetLowering::DAGCombinerInfo &DCI,
7538                                        const RISCVSubtarget &Subtarget) {
7539   SelectionDAG &DAG = DCI.DAG;
7540   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7541   MVT XLenVT = Subtarget.getXLenVT();
7542 
7543   // Only handle XLen types. Other types narrower than XLen will eventually be
7544   // legalized to XLenVT.
7545   EVT DstVT = N->getValueType(0);
7546   if (DstVT != XLenVT)
7547     return SDValue();
7548 
7549   SDValue Src = N->getOperand(0);
7550 
7551   // Ensure the FP type is also legal.
7552   if (!TLI.isTypeLegal(Src.getValueType()))
7553     return SDValue();
7554 
7555   // Don't do this for f16 with Zfhmin and not Zfh.
7556   if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
7557     return SDValue();
7558 
7559   EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
7560 
7561   RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src);
7562   if (FRM == RISCVFPRndMode::Invalid)
7563     return SDValue();
7564 
7565   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
7566 
7567   unsigned Opc;
7568   if (SatVT == DstVT)
7569     Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
7570   else if (DstVT == MVT::i64 && SatVT == MVT::i32)
7571     Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
7572   else
7573     return SDValue();
7574   // FIXME: Support other SatVTs by clamping before or after the conversion.
7575 
7576   Src = Src.getOperand(0);
7577 
7578   SDLoc DL(N);
7579   SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
7580                                 DAG.getTargetConstant(FRM, DL, XLenVT));
7581 
7582   // RISCV FP-to-int conversions saturate to the destination register size, but
7583   // don't produce 0 for nan.
7584   SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
7585   return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
7586 }
7587 
7588 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
7589                                                DAGCombinerInfo &DCI) const {
7590   SelectionDAG &DAG = DCI.DAG;
7591 
7592   // Helper to call SimplifyDemandedBits on an operand of N where only some low
7593   // bits are demanded. N will be added to the Worklist if it was not deleted.
7594   // Caller should return SDValue(N, 0) if this returns true.
7595   auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
7596     SDValue Op = N->getOperand(OpNo);
7597     APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
7598     if (!SimplifyDemandedBits(Op, Mask, DCI))
7599       return false;
7600 
7601     if (N->getOpcode() != ISD::DELETED_NODE)
7602       DCI.AddToWorklist(N);
7603     return true;
7604   };
7605 
7606   switch (N->getOpcode()) {
7607   default:
7608     break;
7609   case RISCVISD::SplitF64: {
7610     SDValue Op0 = N->getOperand(0);
7611     // If the input to SplitF64 is just BuildPairF64 then the operation is
7612     // redundant. Instead, use BuildPairF64's operands directly.
7613     if (Op0->getOpcode() == RISCVISD::BuildPairF64)
7614       return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
7615 
7616     SDLoc DL(N);
7617 
7618     // It's cheaper to materialise two 32-bit integers than to load a double
7619     // from the constant pool and transfer it to integer registers through the
7620     // stack.
7621     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
7622       APInt V = C->getValueAPF().bitcastToAPInt();
7623       SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
7624       SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
7625       return DCI.CombineTo(N, Lo, Hi);
7626     }
7627 
7628     // This is a target-specific version of a DAGCombine performed in
7629     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7630     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7631     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7632     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7633         !Op0.getNode()->hasOneUse())
7634       break;
7635     SDValue NewSplitF64 =
7636         DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
7637                     Op0.getOperand(0));
7638     SDValue Lo = NewSplitF64.getValue(0);
7639     SDValue Hi = NewSplitF64.getValue(1);
7640     APInt SignBit = APInt::getSignMask(32);
7641     if (Op0.getOpcode() == ISD::FNEG) {
7642       SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
7643                                   DAG.getConstant(SignBit, DL, MVT::i32));
7644       return DCI.CombineTo(N, Lo, NewHi);
7645     }
7646     assert(Op0.getOpcode() == ISD::FABS);
7647     SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
7648                                 DAG.getConstant(~SignBit, DL, MVT::i32));
7649     return DCI.CombineTo(N, Lo, NewHi);
7650   }
7651   case RISCVISD::SLLW:
7652   case RISCVISD::SRAW:
7653   case RISCVISD::SRLW:
7654   case RISCVISD::ROLW:
7655   case RISCVISD::RORW: {
7656     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7657     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7658         SimplifyDemandedLowBitsHelper(1, 5))
7659       return SDValue(N, 0);
7660     break;
7661   }
7662   case RISCVISD::CLZW:
7663   case RISCVISD::CTZW: {
7664     // Only the lower 32 bits of the first operand are read
7665     if (SimplifyDemandedLowBitsHelper(0, 32))
7666       return SDValue(N, 0);
7667     break;
7668   }
7669   case RISCVISD::GREV:
7670   case RISCVISD::GORC: {
7671     // Only the lower log2(Bitwidth) bits of the the shift amount are read.
7672     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7673     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7674     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
7675       return SDValue(N, 0);
7676 
7677     return combineGREVI_GORCI(N, DAG);
7678   }
7679   case RISCVISD::GREVW:
7680   case RISCVISD::GORCW: {
7681     // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
7682     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7683         SimplifyDemandedLowBitsHelper(1, 5))
7684       return SDValue(N, 0);
7685 
7686     return combineGREVI_GORCI(N, DAG);
7687   }
7688   case RISCVISD::SHFL:
7689   case RISCVISD::UNSHFL: {
7690     // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
7691     unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
7692     assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
7693     if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
7694       return SDValue(N, 0);
7695 
7696     break;
7697   }
7698   case RISCVISD::SHFLW:
7699   case RISCVISD::UNSHFLW: {
7700     // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
7701     SDValue LHS = N->getOperand(0);
7702     SDValue RHS = N->getOperand(1);
7703     APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
7704     APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 4);
7705     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7706         SimplifyDemandedLowBitsHelper(1, 4))
7707       return SDValue(N, 0);
7708 
7709     break;
7710   }
7711   case RISCVISD::BCOMPRESSW:
7712   case RISCVISD::BDECOMPRESSW: {
7713     // Only the lower 32 bits of LHS and RHS are read.
7714     if (SimplifyDemandedLowBitsHelper(0, 32) ||
7715         SimplifyDemandedLowBitsHelper(1, 32))
7716       return SDValue(N, 0);
7717 
7718     break;
7719   }
7720   case RISCVISD::FMV_X_ANYEXTH:
7721   case RISCVISD::FMV_X_ANYEXTW_RV64: {
7722     SDLoc DL(N);
7723     SDValue Op0 = N->getOperand(0);
7724     MVT VT = N->getSimpleValueType(0);
7725     // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
7726     // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
7727     // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
7728     if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
7729          Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
7730         (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
7731          Op0->getOpcode() == RISCVISD::FMV_H_X)) {
7732       assert(Op0.getOperand(0).getValueType() == VT &&
7733              "Unexpected value type!");
7734       return Op0.getOperand(0);
7735     }
7736 
7737     // This is a target-specific version of a DAGCombine performed in
7738     // DAGCombiner::visitBITCAST. It performs the equivalent of:
7739     // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
7740     // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
7741     if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
7742         !Op0.getNode()->hasOneUse())
7743       break;
7744     SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
7745     unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
7746     APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
7747     if (Op0.getOpcode() == ISD::FNEG)
7748       return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
7749                          DAG.getConstant(SignBit, DL, VT));
7750 
7751     assert(Op0.getOpcode() == ISD::FABS);
7752     return DAG.getNode(ISD::AND, DL, VT, NewFMV,
7753                        DAG.getConstant(~SignBit, DL, VT));
7754   }
7755   case ISD::ADD:
7756     return performADDCombine(N, DAG, Subtarget);
7757   case ISD::SUB:
7758     return performSUBCombine(N, DAG);
7759   case ISD::AND:
7760     return performANDCombine(N, DAG);
7761   case ISD::OR:
7762     return performORCombine(N, DAG, Subtarget);
7763   case ISD::XOR:
7764     return performXORCombine(N, DAG);
7765   case ISD::ANY_EXTEND:
7766     return performANY_EXTENDCombine(N, DCI, Subtarget);
7767   case ISD::ZERO_EXTEND:
7768     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
7769     // type legalization. This is safe because fp_to_uint produces poison if
7770     // it overflows.
7771     if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
7772       SDValue Src = N->getOperand(0);
7773       if (Src.getOpcode() == ISD::FP_TO_UINT &&
7774           isTypeLegal(Src.getOperand(0).getValueType()))
7775         return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
7776                            Src.getOperand(0));
7777       if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
7778           isTypeLegal(Src.getOperand(1).getValueType())) {
7779         SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
7780         SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
7781                                   Src.getOperand(0), Src.getOperand(1));
7782         DCI.CombineTo(N, Res);
7783         DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
7784         DCI.recursivelyDeleteUnusedNodes(Src.getNode());
7785         return SDValue(N, 0); // Return N so it doesn't get rechecked.
7786       }
7787     }
7788     return SDValue();
7789   case RISCVISD::SELECT_CC: {
7790     // Transform
7791     SDValue LHS = N->getOperand(0);
7792     SDValue RHS = N->getOperand(1);
7793     SDValue TrueV = N->getOperand(3);
7794     SDValue FalseV = N->getOperand(4);
7795 
7796     // If the True and False values are the same, we don't need a select_cc.
7797     if (TrueV == FalseV)
7798       return TrueV;
7799 
7800     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
7801     if (!ISD::isIntEqualitySetCC(CCVal))
7802       break;
7803 
7804     // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
7805     //      (select_cc X, Y, lt, trueV, falseV)
7806     // Sometimes the setcc is introduced after select_cc has been formed.
7807     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7808         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7809       // If we're looking for eq 0 instead of ne 0, we need to invert the
7810       // condition.
7811       bool Invert = CCVal == ISD::SETEQ;
7812       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7813       if (Invert)
7814         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7815 
7816       SDLoc DL(N);
7817       RHS = LHS.getOperand(1);
7818       LHS = LHS.getOperand(0);
7819       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7820 
7821       SDValue TargetCC = DAG.getCondCode(CCVal);
7822       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7823                          {LHS, RHS, TargetCC, TrueV, FalseV});
7824     }
7825 
7826     // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
7827     //      (select_cc X, Y, eq/ne, trueV, falseV)
7828     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7829       return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
7830                          {LHS.getOperand(0), LHS.getOperand(1),
7831                           N->getOperand(2), TrueV, FalseV});
7832     // (select_cc X, 1, setne, trueV, falseV) ->
7833     // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
7834     // This can occur when legalizing some floating point comparisons.
7835     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7836     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7837       SDLoc DL(N);
7838       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7839       SDValue TargetCC = DAG.getCondCode(CCVal);
7840       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7841       return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
7842                          {LHS, RHS, TargetCC, TrueV, FalseV});
7843     }
7844 
7845     break;
7846   }
7847   case RISCVISD::BR_CC: {
7848     SDValue LHS = N->getOperand(1);
7849     SDValue RHS = N->getOperand(2);
7850     ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
7851     if (!ISD::isIntEqualitySetCC(CCVal))
7852       break;
7853 
7854     // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
7855     //      (br_cc X, Y, lt, dest)
7856     // Sometimes the setcc is introduced after br_cc has been formed.
7857     if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
7858         LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
7859       // If we're looking for eq 0 instead of ne 0, we need to invert the
7860       // condition.
7861       bool Invert = CCVal == ISD::SETEQ;
7862       CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
7863       if (Invert)
7864         CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7865 
7866       SDLoc DL(N);
7867       RHS = LHS.getOperand(1);
7868       LHS = LHS.getOperand(0);
7869       translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7870 
7871       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7872                          N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
7873                          N->getOperand(4));
7874     }
7875 
7876     // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
7877     //      (br_cc X, Y, eq/ne, trueV, falseV)
7878     if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
7879       return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
7880                          N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
7881                          N->getOperand(3), N->getOperand(4));
7882 
7883     // (br_cc X, 1, setne, br_cc) ->
7884     // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
7885     // This can occur when legalizing some floating point comparisons.
7886     APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
7887     if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
7888       SDLoc DL(N);
7889       CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
7890       SDValue TargetCC = DAG.getCondCode(CCVal);
7891       RHS = DAG.getConstant(0, DL, LHS.getValueType());
7892       return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
7893                          N->getOperand(0), LHS, RHS, TargetCC,
7894                          N->getOperand(4));
7895     }
7896     break;
7897   }
7898   case ISD::FP_TO_SINT:
7899   case ISD::FP_TO_UINT:
7900     return performFP_TO_INTCombine(N, DCI, Subtarget);
7901   case ISD::FP_TO_SINT_SAT:
7902   case ISD::FP_TO_UINT_SAT:
7903     return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
7904   case ISD::FCOPYSIGN: {
7905     EVT VT = N->getValueType(0);
7906     if (!VT.isVector())
7907       break;
7908     // There is a form of VFSGNJ which injects the negated sign of its second
7909     // operand. Try and bubble any FNEG up after the extend/round to produce
7910     // this optimized pattern. Avoid modifying cases where FP_ROUND and
7911     // TRUNC=1.
7912     SDValue In2 = N->getOperand(1);
7913     // Avoid cases where the extend/round has multiple uses, as duplicating
7914     // those is typically more expensive than removing a fneg.
7915     if (!In2.hasOneUse())
7916       break;
7917     if (In2.getOpcode() != ISD::FP_EXTEND &&
7918         (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
7919       break;
7920     In2 = In2.getOperand(0);
7921     if (In2.getOpcode() != ISD::FNEG)
7922       break;
7923     SDLoc DL(N);
7924     SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
7925     return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
7926                        DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
7927   }
7928   case ISD::MGATHER:
7929   case ISD::MSCATTER:
7930   case ISD::VP_GATHER:
7931   case ISD::VP_SCATTER: {
7932     if (!DCI.isBeforeLegalize())
7933       break;
7934     SDValue Index, ScaleOp;
7935     bool IsIndexScaled = false;
7936     bool IsIndexSigned = false;
7937     if (const auto *VPGSN = dyn_cast<VPGatherScatterSDNode>(N)) {
7938       Index = VPGSN->getIndex();
7939       ScaleOp = VPGSN->getScale();
7940       IsIndexScaled = VPGSN->isIndexScaled();
7941       IsIndexSigned = VPGSN->isIndexSigned();
7942     } else {
7943       const auto *MGSN = cast<MaskedGatherScatterSDNode>(N);
7944       Index = MGSN->getIndex();
7945       ScaleOp = MGSN->getScale();
7946       IsIndexScaled = MGSN->isIndexScaled();
7947       IsIndexSigned = MGSN->isIndexSigned();
7948     }
7949     EVT IndexVT = Index.getValueType();
7950     MVT XLenVT = Subtarget.getXLenVT();
7951     // RISCV indexed loads only support the "unsigned unscaled" addressing
7952     // mode, so anything else must be manually legalized.
7953     bool NeedsIdxLegalization =
7954         IsIndexScaled ||
7955         (IsIndexSigned && IndexVT.getVectorElementType().bitsLT(XLenVT));
7956     if (!NeedsIdxLegalization)
7957       break;
7958 
7959     SDLoc DL(N);
7960 
7961     // Any index legalization should first promote to XLenVT, so we don't lose
7962     // bits when scaling. This may create an illegal index type so we let
7963     // LLVM's legalization take care of the splitting.
7964     // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
7965     if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
7966       IndexVT = IndexVT.changeVectorElementType(XLenVT);
7967       Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
7968                           DL, IndexVT, Index);
7969     }
7970 
7971     unsigned Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue();
7972     if (IsIndexScaled && Scale != 1) {
7973       // Manually scale the indices by the element size.
7974       // TODO: Sanitize the scale operand here?
7975       // TODO: For VP nodes, should we use VP_SHL here?
7976       assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
7977       SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
7978       Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
7979     }
7980 
7981     ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
7982     if (const auto *VPGN = dyn_cast<VPGatherSDNode>(N))
7983       return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
7984                              {VPGN->getChain(), VPGN->getBasePtr(), Index,
7985                               VPGN->getScale(), VPGN->getMask(),
7986                               VPGN->getVectorLength()},
7987                              VPGN->getMemOperand(), NewIndexTy);
7988     if (const auto *VPSN = dyn_cast<VPScatterSDNode>(N))
7989       return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
7990                               {VPSN->getChain(), VPSN->getValue(),
7991                                VPSN->getBasePtr(), Index, VPSN->getScale(),
7992                                VPSN->getMask(), VPSN->getVectorLength()},
7993                               VPSN->getMemOperand(), NewIndexTy);
7994     if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N))
7995       return DAG.getMaskedGather(
7996           N->getVTList(), MGN->getMemoryVT(), DL,
7997           {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
7998            MGN->getBasePtr(), Index, MGN->getScale()},
7999           MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
8000     const auto *MSN = cast<MaskedScatterSDNode>(N);
8001     return DAG.getMaskedScatter(
8002         N->getVTList(), MSN->getMemoryVT(), DL,
8003         {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
8004          Index, MSN->getScale()},
8005         MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
8006   }
8007   case RISCVISD::SRA_VL:
8008   case RISCVISD::SRL_VL:
8009   case RISCVISD::SHL_VL: {
8010     SDValue ShAmt = N->getOperand(1);
8011     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
8012       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
8013       SDLoc DL(N);
8014       SDValue VL = N->getOperand(3);
8015       EVT VT = N->getValueType(0);
8016       ShAmt =
8017           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
8018       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
8019                          N->getOperand(2), N->getOperand(3));
8020     }
8021     break;
8022   }
8023   case ISD::SRA:
8024   case ISD::SRL:
8025   case ISD::SHL: {
8026     SDValue ShAmt = N->getOperand(1);
8027     if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
8028       // We don't need the upper 32 bits of a 64-bit element for a shift amount.
8029       SDLoc DL(N);
8030       EVT VT = N->getValueType(0);
8031       ShAmt =
8032           DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT, ShAmt.getOperand(0));
8033       return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
8034     }
8035     break;
8036   }
8037   case RISCVISD::MUL_VL:
8038     if (SDValue V = combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ false))
8039       return V;
8040     // Mul is commutative.
8041     return combineMUL_VLToVWMUL_VL(N, DAG, /*Commute*/ true);
8042   case ISD::STORE: {
8043     auto *Store = cast<StoreSDNode>(N);
8044     SDValue Val = Store->getValue();
8045     // Combine store of vmv.x.s to vse with VL of 1.
8046     // FIXME: Support FP.
8047     if (Val.getOpcode() == RISCVISD::VMV_X_S) {
8048       SDValue Src = Val.getOperand(0);
8049       EVT VecVT = Src.getValueType();
8050       EVT MemVT = Store->getMemoryVT();
8051       // The memory VT and the element type must match.
8052       if (VecVT.getVectorElementType() == MemVT) {
8053         SDLoc DL(N);
8054         MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
8055         return DAG.getStoreVP(
8056             Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
8057             DAG.getConstant(1, DL, MaskVT),
8058             DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
8059             Store->getMemOperand(), Store->getAddressingMode(),
8060             Store->isTruncatingStore(), /*IsCompress*/ false);
8061       }
8062     }
8063 
8064     break;
8065   }
8066   }
8067 
8068   return SDValue();
8069 }
8070 
8071 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
8072     const SDNode *N, CombineLevel Level) const {
8073   // The following folds are only desirable if `(OP _, c1 << c2)` can be
8074   // materialised in fewer instructions than `(OP _, c1)`:
8075   //
8076   //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
8077   //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
8078   SDValue N0 = N->getOperand(0);
8079   EVT Ty = N0.getValueType();
8080   if (Ty.isScalarInteger() &&
8081       (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
8082     auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
8083     auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
8084     if (C1 && C2) {
8085       const APInt &C1Int = C1->getAPIntValue();
8086       APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
8087 
8088       // We can materialise `c1 << c2` into an add immediate, so it's "free",
8089       // and the combine should happen, to potentially allow further combines
8090       // later.
8091       if (ShiftedC1Int.getMinSignedBits() <= 64 &&
8092           isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
8093         return true;
8094 
8095       // We can materialise `c1` in an add immediate, so it's "free", and the
8096       // combine should be prevented.
8097       if (C1Int.getMinSignedBits() <= 64 &&
8098           isLegalAddImmediate(C1Int.getSExtValue()))
8099         return false;
8100 
8101       // Neither constant will fit into an immediate, so find materialisation
8102       // costs.
8103       int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
8104                                               Subtarget.getFeatureBits(),
8105                                               /*CompressionCost*/true);
8106       int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
8107           ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
8108           /*CompressionCost*/true);
8109 
8110       // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
8111       // combine should be prevented.
8112       if (C1Cost < ShiftedC1Cost)
8113         return false;
8114     }
8115   }
8116   return true;
8117 }
8118 
8119 bool RISCVTargetLowering::targetShrinkDemandedConstant(
8120     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
8121     TargetLoweringOpt &TLO) const {
8122   // Delay this optimization as late as possible.
8123   if (!TLO.LegalOps)
8124     return false;
8125 
8126   EVT VT = Op.getValueType();
8127   if (VT.isVector())
8128     return false;
8129 
8130   // Only handle AND for now.
8131   if (Op.getOpcode() != ISD::AND)
8132     return false;
8133 
8134   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
8135   if (!C)
8136     return false;
8137 
8138   const APInt &Mask = C->getAPIntValue();
8139 
8140   // Clear all non-demanded bits initially.
8141   APInt ShrunkMask = Mask & DemandedBits;
8142 
8143   // Try to make a smaller immediate by setting undemanded bits.
8144 
8145   APInt ExpandedMask = Mask | ~DemandedBits;
8146 
8147   auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
8148     return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
8149   };
8150   auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
8151     if (NewMask == Mask)
8152       return true;
8153     SDLoc DL(Op);
8154     SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
8155     SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
8156     return TLO.CombineTo(Op, NewOp);
8157   };
8158 
8159   // If the shrunk mask fits in sign extended 12 bits, let the target
8160   // independent code apply it.
8161   if (ShrunkMask.isSignedIntN(12))
8162     return false;
8163 
8164   // Preserve (and X, 0xffff) when zext.h is supported.
8165   if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
8166     APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
8167     if (IsLegalMask(NewMask))
8168       return UseMask(NewMask);
8169   }
8170 
8171   // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
8172   if (VT == MVT::i64) {
8173     APInt NewMask = APInt(64, 0xffffffff);
8174     if (IsLegalMask(NewMask))
8175       return UseMask(NewMask);
8176   }
8177 
8178   // For the remaining optimizations, we need to be able to make a negative
8179   // number through a combination of mask and undemanded bits.
8180   if (!ExpandedMask.isNegative())
8181     return false;
8182 
8183   // What is the fewest number of bits we need to represent the negative number.
8184   unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
8185 
8186   // Try to make a 12 bit negative immediate. If that fails try to make a 32
8187   // bit negative immediate unless the shrunk immediate already fits in 32 bits.
8188   APInt NewMask = ShrunkMask;
8189   if (MinSignedBits <= 12)
8190     NewMask.setBitsFrom(11);
8191   else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
8192     NewMask.setBitsFrom(31);
8193   else
8194     return false;
8195 
8196   // Check that our new mask is a subset of the demanded mask.
8197   assert(IsLegalMask(NewMask));
8198   return UseMask(NewMask);
8199 }
8200 
8201 static void computeGREV(APInt &Src, unsigned ShAmt) {
8202   ShAmt &= Src.getBitWidth() - 1;
8203   uint64_t x = Src.getZExtValue();
8204   if (ShAmt & 1)
8205     x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
8206   if (ShAmt & 2)
8207     x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
8208   if (ShAmt & 4)
8209     x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
8210   if (ShAmt & 8)
8211     x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
8212   if (ShAmt & 16)
8213     x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
8214   if (ShAmt & 32)
8215     x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
8216   Src = x;
8217 }
8218 
8219 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
8220                                                         KnownBits &Known,
8221                                                         const APInt &DemandedElts,
8222                                                         const SelectionDAG &DAG,
8223                                                         unsigned Depth) const {
8224   unsigned BitWidth = Known.getBitWidth();
8225   unsigned Opc = Op.getOpcode();
8226   assert((Opc >= ISD::BUILTIN_OP_END ||
8227           Opc == ISD::INTRINSIC_WO_CHAIN ||
8228           Opc == ISD::INTRINSIC_W_CHAIN ||
8229           Opc == ISD::INTRINSIC_VOID) &&
8230          "Should use MaskedValueIsZero if you don't know whether Op"
8231          " is a target node!");
8232 
8233   Known.resetAll();
8234   switch (Opc) {
8235   default: break;
8236   case RISCVISD::SELECT_CC: {
8237     Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
8238     // If we don't know any bits, early out.
8239     if (Known.isUnknown())
8240       break;
8241     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
8242 
8243     // Only known if known in both the LHS and RHS.
8244     Known = KnownBits::commonBits(Known, Known2);
8245     break;
8246   }
8247   case RISCVISD::REMUW: {
8248     KnownBits Known2;
8249     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8250     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8251     // We only care about the lower 32 bits.
8252     Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
8253     // Restore the original width by sign extending.
8254     Known = Known.sext(BitWidth);
8255     break;
8256   }
8257   case RISCVISD::DIVUW: {
8258     KnownBits Known2;
8259     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
8260     Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
8261     // We only care about the lower 32 bits.
8262     Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
8263     // Restore the original width by sign extending.
8264     Known = Known.sext(BitWidth);
8265     break;
8266   }
8267   case RISCVISD::CTZW: {
8268     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8269     unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
8270     unsigned LowBits = Log2_32(PossibleTZ) + 1;
8271     Known.Zero.setBitsFrom(LowBits);
8272     break;
8273   }
8274   case RISCVISD::CLZW: {
8275     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8276     unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
8277     unsigned LowBits = Log2_32(PossibleLZ) + 1;
8278     Known.Zero.setBitsFrom(LowBits);
8279     break;
8280   }
8281   case RISCVISD::GREV:
8282   case RISCVISD::GREVW: {
8283     if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8284       Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
8285       if (Opc == RISCVISD::GREVW)
8286         Known = Known.trunc(32);
8287       unsigned ShAmt = C->getZExtValue();
8288       computeGREV(Known.Zero, ShAmt);
8289       computeGREV(Known.One, ShAmt);
8290       if (Opc == RISCVISD::GREVW)
8291         Known = Known.sext(BitWidth);
8292     }
8293     break;
8294   }
8295   case RISCVISD::READ_VLENB: {
8296     // If we know the minimum VLen from Zvl extensions, we can use that to
8297     // determine the trailing zeros of VLENB.
8298     // FIXME: Limit to 128 bit vectors until we have more testing.
8299     unsigned MinVLenB = std::min(128U, Subtarget.getMinVLen()) / 8;
8300     if (MinVLenB > 0)
8301       Known.Zero.setLowBits(Log2_32(MinVLenB));
8302     // We assume VLENB is no more than 65536 / 8 bytes.
8303     Known.Zero.setBitsFrom(14);
8304     break;
8305   }
8306   case ISD::INTRINSIC_W_CHAIN:
8307   case ISD::INTRINSIC_WO_CHAIN: {
8308     unsigned IntNo =
8309         Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
8310     switch (IntNo) {
8311     default:
8312       // We can't do anything for most intrinsics.
8313       break;
8314     case Intrinsic::riscv_vsetvli:
8315     case Intrinsic::riscv_vsetvlimax:
8316     case Intrinsic::riscv_vsetvli_opt:
8317     case Intrinsic::riscv_vsetvlimax_opt:
8318       // Assume that VL output is positive and would fit in an int32_t.
8319       // TODO: VLEN might be capped at 16 bits in a future V spec update.
8320       if (BitWidth >= 32)
8321         Known.Zero.setBitsFrom(31);
8322       break;
8323     }
8324     break;
8325   }
8326   }
8327 }
8328 
8329 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
8330     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
8331     unsigned Depth) const {
8332   switch (Op.getOpcode()) {
8333   default:
8334     break;
8335   case RISCVISD::SELECT_CC: {
8336     unsigned Tmp =
8337         DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
8338     if (Tmp == 1) return 1;  // Early out.
8339     unsigned Tmp2 =
8340         DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
8341     return std::min(Tmp, Tmp2);
8342   }
8343   case RISCVISD::SLLW:
8344   case RISCVISD::SRAW:
8345   case RISCVISD::SRLW:
8346   case RISCVISD::DIVW:
8347   case RISCVISD::DIVUW:
8348   case RISCVISD::REMUW:
8349   case RISCVISD::ROLW:
8350   case RISCVISD::RORW:
8351   case RISCVISD::GREVW:
8352   case RISCVISD::GORCW:
8353   case RISCVISD::FSLW:
8354   case RISCVISD::FSRW:
8355   case RISCVISD::SHFLW:
8356   case RISCVISD::UNSHFLW:
8357   case RISCVISD::BCOMPRESSW:
8358   case RISCVISD::BDECOMPRESSW:
8359   case RISCVISD::BFPW:
8360   case RISCVISD::FCVT_W_RV64:
8361   case RISCVISD::FCVT_WU_RV64:
8362   case RISCVISD::STRICT_FCVT_W_RV64:
8363   case RISCVISD::STRICT_FCVT_WU_RV64:
8364     // TODO: As the result is sign-extended, this is conservatively correct. A
8365     // more precise answer could be calculated for SRAW depending on known
8366     // bits in the shift amount.
8367     return 33;
8368   case RISCVISD::SHFL:
8369   case RISCVISD::UNSHFL: {
8370     // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
8371     // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
8372     // will stay within the upper 32 bits. If there were more than 32 sign bits
8373     // before there will be at least 33 sign bits after.
8374     if (Op.getValueType() == MVT::i64 &&
8375         isa<ConstantSDNode>(Op.getOperand(1)) &&
8376         (Op.getConstantOperandVal(1) & 0x10) == 0) {
8377       unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
8378       if (Tmp > 32)
8379         return 33;
8380     }
8381     break;
8382   }
8383   case RISCVISD::VMV_X_S: {
8384     // The number of sign bits of the scalar result is computed by obtaining the
8385     // element type of the input vector operand, subtracting its width from the
8386     // XLEN, and then adding one (sign bit within the element type). If the
8387     // element type is wider than XLen, the least-significant XLEN bits are
8388     // taken.
8389     unsigned XLen = Subtarget.getXLen();
8390     unsigned EltBits = Op.getOperand(0).getScalarValueSizeInBits();
8391     if (EltBits <= XLen)
8392       return XLen - EltBits + 1;
8393     break;
8394   }
8395   }
8396 
8397   return 1;
8398 }
8399 
8400 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
8401                                                   MachineBasicBlock *BB) {
8402   assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
8403 
8404   // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
8405   // Should the count have wrapped while it was being read, we need to try
8406   // again.
8407   // ...
8408   // read:
8409   // rdcycleh x3 # load high word of cycle
8410   // rdcycle  x2 # load low word of cycle
8411   // rdcycleh x4 # load high word of cycle
8412   // bne x3, x4, read # check if high word reads match, otherwise try again
8413   // ...
8414 
8415   MachineFunction &MF = *BB->getParent();
8416   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8417   MachineFunction::iterator It = ++BB->getIterator();
8418 
8419   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8420   MF.insert(It, LoopMBB);
8421 
8422   MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
8423   MF.insert(It, DoneMBB);
8424 
8425   // Transfer the remainder of BB and its successor edges to DoneMBB.
8426   DoneMBB->splice(DoneMBB->begin(), BB,
8427                   std::next(MachineBasicBlock::iterator(MI)), BB->end());
8428   DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
8429 
8430   BB->addSuccessor(LoopMBB);
8431 
8432   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8433   Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
8434   Register LoReg = MI.getOperand(0).getReg();
8435   Register HiReg = MI.getOperand(1).getReg();
8436   DebugLoc DL = MI.getDebugLoc();
8437 
8438   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8439   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
8440       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8441       .addReg(RISCV::X0);
8442   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
8443       .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
8444       .addReg(RISCV::X0);
8445   BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
8446       .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
8447       .addReg(RISCV::X0);
8448 
8449   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
8450       .addReg(HiReg)
8451       .addReg(ReadAgainReg)
8452       .addMBB(LoopMBB);
8453 
8454   LoopMBB->addSuccessor(LoopMBB);
8455   LoopMBB->addSuccessor(DoneMBB);
8456 
8457   MI.eraseFromParent();
8458 
8459   return DoneMBB;
8460 }
8461 
8462 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
8463                                              MachineBasicBlock *BB) {
8464   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
8465 
8466   MachineFunction &MF = *BB->getParent();
8467   DebugLoc DL = MI.getDebugLoc();
8468   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8469   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8470   Register LoReg = MI.getOperand(0).getReg();
8471   Register HiReg = MI.getOperand(1).getReg();
8472   Register SrcReg = MI.getOperand(2).getReg();
8473   const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
8474   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8475 
8476   TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
8477                           RI);
8478   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8479   MachineMemOperand *MMOLo =
8480       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
8481   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8482       MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
8483   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
8484       .addFrameIndex(FI)
8485       .addImm(0)
8486       .addMemOperand(MMOLo);
8487   BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
8488       .addFrameIndex(FI)
8489       .addImm(4)
8490       .addMemOperand(MMOHi);
8491   MI.eraseFromParent(); // The pseudo instruction is gone now.
8492   return BB;
8493 }
8494 
8495 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
8496                                                  MachineBasicBlock *BB) {
8497   assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
8498          "Unexpected instruction");
8499 
8500   MachineFunction &MF = *BB->getParent();
8501   DebugLoc DL = MI.getDebugLoc();
8502   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
8503   const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
8504   Register DstReg = MI.getOperand(0).getReg();
8505   Register LoReg = MI.getOperand(1).getReg();
8506   Register HiReg = MI.getOperand(2).getReg();
8507   const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
8508   int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
8509 
8510   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
8511   MachineMemOperand *MMOLo =
8512       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
8513   MachineMemOperand *MMOHi = MF.getMachineMemOperand(
8514       MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
8515   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8516       .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
8517       .addFrameIndex(FI)
8518       .addImm(0)
8519       .addMemOperand(MMOLo);
8520   BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
8521       .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
8522       .addFrameIndex(FI)
8523       .addImm(4)
8524       .addMemOperand(MMOHi);
8525   TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
8526   MI.eraseFromParent(); // The pseudo instruction is gone now.
8527   return BB;
8528 }
8529 
8530 static bool isSelectPseudo(MachineInstr &MI) {
8531   switch (MI.getOpcode()) {
8532   default:
8533     return false;
8534   case RISCV::Select_GPR_Using_CC_GPR:
8535   case RISCV::Select_FPR16_Using_CC_GPR:
8536   case RISCV::Select_FPR32_Using_CC_GPR:
8537   case RISCV::Select_FPR64_Using_CC_GPR:
8538     return true;
8539   }
8540 }
8541 
8542 static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
8543                                         unsigned RelOpcode, unsigned EqOpcode,
8544                                         const RISCVSubtarget &Subtarget) {
8545   DebugLoc DL = MI.getDebugLoc();
8546   Register DstReg = MI.getOperand(0).getReg();
8547   Register Src1Reg = MI.getOperand(1).getReg();
8548   Register Src2Reg = MI.getOperand(2).getReg();
8549   MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
8550   Register SavedFFlags = MRI.createVirtualRegister(&RISCV::GPRRegClass);
8551   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
8552 
8553   // Save the current FFLAGS.
8554   BuildMI(*BB, MI, DL, TII.get(RISCV::ReadFFLAGS), SavedFFlags);
8555 
8556   auto MIB = BuildMI(*BB, MI, DL, TII.get(RelOpcode), DstReg)
8557                  .addReg(Src1Reg)
8558                  .addReg(Src2Reg);
8559   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8560     MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
8561 
8562   // Restore the FFLAGS.
8563   BuildMI(*BB, MI, DL, TII.get(RISCV::WriteFFLAGS))
8564       .addReg(SavedFFlags, RegState::Kill);
8565 
8566   // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
8567   auto MIB2 = BuildMI(*BB, MI, DL, TII.get(EqOpcode), RISCV::X0)
8568                   .addReg(Src1Reg, getKillRegState(MI.getOperand(1).isKill()))
8569                   .addReg(Src2Reg, getKillRegState(MI.getOperand(2).isKill()));
8570   if (MI.getFlag(MachineInstr::MIFlag::NoFPExcept))
8571     MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
8572 
8573   // Erase the pseudoinstruction.
8574   MI.eraseFromParent();
8575   return BB;
8576 }
8577 
8578 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
8579                                            MachineBasicBlock *BB,
8580                                            const RISCVSubtarget &Subtarget) {
8581   // To "insert" Select_* instructions, we actually have to insert the triangle
8582   // control-flow pattern.  The incoming instructions know the destination vreg
8583   // to set, the condition code register to branch on, the true/false values to
8584   // select between, and the condcode to use to select the appropriate branch.
8585   //
8586   // We produce the following control flow:
8587   //     HeadMBB
8588   //     |  \
8589   //     |  IfFalseMBB
8590   //     | /
8591   //    TailMBB
8592   //
8593   // When we find a sequence of selects we attempt to optimize their emission
8594   // by sharing the control flow. Currently we only handle cases where we have
8595   // multiple selects with the exact same condition (same LHS, RHS and CC).
8596   // The selects may be interleaved with other instructions if the other
8597   // instructions meet some requirements we deem safe:
8598   // - They are debug instructions. Otherwise,
8599   // - They do not have side-effects, do not access memory and their inputs do
8600   //   not depend on the results of the select pseudo-instructions.
8601   // The TrueV/FalseV operands of the selects cannot depend on the result of
8602   // previous selects in the sequence.
8603   // These conditions could be further relaxed. See the X86 target for a
8604   // related approach and more information.
8605   Register LHS = MI.getOperand(1).getReg();
8606   Register RHS = MI.getOperand(2).getReg();
8607   auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
8608 
8609   SmallVector<MachineInstr *, 4> SelectDebugValues;
8610   SmallSet<Register, 4> SelectDests;
8611   SelectDests.insert(MI.getOperand(0).getReg());
8612 
8613   MachineInstr *LastSelectPseudo = &MI;
8614 
8615   for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
8616        SequenceMBBI != E; ++SequenceMBBI) {
8617     if (SequenceMBBI->isDebugInstr())
8618       continue;
8619     else if (isSelectPseudo(*SequenceMBBI)) {
8620       if (SequenceMBBI->getOperand(1).getReg() != LHS ||
8621           SequenceMBBI->getOperand(2).getReg() != RHS ||
8622           SequenceMBBI->getOperand(3).getImm() != CC ||
8623           SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
8624           SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
8625         break;
8626       LastSelectPseudo = &*SequenceMBBI;
8627       SequenceMBBI->collectDebugValues(SelectDebugValues);
8628       SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
8629     } else {
8630       if (SequenceMBBI->hasUnmodeledSideEffects() ||
8631           SequenceMBBI->mayLoadOrStore())
8632         break;
8633       if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
8634             return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
8635           }))
8636         break;
8637     }
8638   }
8639 
8640   const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
8641   const BasicBlock *LLVM_BB = BB->getBasicBlock();
8642   DebugLoc DL = MI.getDebugLoc();
8643   MachineFunction::iterator I = ++BB->getIterator();
8644 
8645   MachineBasicBlock *HeadMBB = BB;
8646   MachineFunction *F = BB->getParent();
8647   MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
8648   MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
8649 
8650   F->insert(I, IfFalseMBB);
8651   F->insert(I, TailMBB);
8652 
8653   // Transfer debug instructions associated with the selects to TailMBB.
8654   for (MachineInstr *DebugInstr : SelectDebugValues) {
8655     TailMBB->push_back(DebugInstr->removeFromParent());
8656   }
8657 
8658   // Move all instructions after the sequence to TailMBB.
8659   TailMBB->splice(TailMBB->end(), HeadMBB,
8660                   std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
8661   // Update machine-CFG edges by transferring all successors of the current
8662   // block to the new block which will contain the Phi nodes for the selects.
8663   TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
8664   // Set the successors for HeadMBB.
8665   HeadMBB->addSuccessor(IfFalseMBB);
8666   HeadMBB->addSuccessor(TailMBB);
8667 
8668   // Insert appropriate branch.
8669   BuildMI(HeadMBB, DL, TII.getBrCond(CC))
8670     .addReg(LHS)
8671     .addReg(RHS)
8672     .addMBB(TailMBB);
8673 
8674   // IfFalseMBB just falls through to TailMBB.
8675   IfFalseMBB->addSuccessor(TailMBB);
8676 
8677   // Create PHIs for all of the select pseudo-instructions.
8678   auto SelectMBBI = MI.getIterator();
8679   auto SelectEnd = std::next(LastSelectPseudo->getIterator());
8680   auto InsertionPoint = TailMBB->begin();
8681   while (SelectMBBI != SelectEnd) {
8682     auto Next = std::next(SelectMBBI);
8683     if (isSelectPseudo(*SelectMBBI)) {
8684       // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
8685       BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
8686               TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
8687           .addReg(SelectMBBI->getOperand(4).getReg())
8688           .addMBB(HeadMBB)
8689           .addReg(SelectMBBI->getOperand(5).getReg())
8690           .addMBB(IfFalseMBB);
8691       SelectMBBI->eraseFromParent();
8692     }
8693     SelectMBBI = Next;
8694   }
8695 
8696   F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
8697   return TailMBB;
8698 }
8699 
8700 MachineBasicBlock *
8701 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
8702                                                  MachineBasicBlock *BB) const {
8703   switch (MI.getOpcode()) {
8704   default:
8705     llvm_unreachable("Unexpected instr type to insert");
8706   case RISCV::ReadCycleWide:
8707     assert(!Subtarget.is64Bit() &&
8708            "ReadCycleWrite is only to be used on riscv32");
8709     return emitReadCycleWidePseudo(MI, BB);
8710   case RISCV::Select_GPR_Using_CC_GPR:
8711   case RISCV::Select_FPR16_Using_CC_GPR:
8712   case RISCV::Select_FPR32_Using_CC_GPR:
8713   case RISCV::Select_FPR64_Using_CC_GPR:
8714     return emitSelectPseudo(MI, BB, Subtarget);
8715   case RISCV::BuildPairF64Pseudo:
8716     return emitBuildPairF64Pseudo(MI, BB);
8717   case RISCV::SplitF64Pseudo:
8718     return emitSplitF64Pseudo(MI, BB);
8719   case RISCV::PseudoQuietFLE_H:
8720     return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget);
8721   case RISCV::PseudoQuietFLT_H:
8722     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
8723   case RISCV::PseudoQuietFLE_S:
8724     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
8725   case RISCV::PseudoQuietFLT_S:
8726     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
8727   case RISCV::PseudoQuietFLE_D:
8728     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
8729   case RISCV::PseudoQuietFLT_D:
8730     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
8731   }
8732 }
8733 
8734 void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
8735                                                         SDNode *Node) const {
8736   // Add FRM dependency to any instructions with dynamic rounding mode.
8737   unsigned Opc = MI.getOpcode();
8738   auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm);
8739   if (Idx < 0)
8740     return;
8741   if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN)
8742     return;
8743   // If the instruction already reads FRM, don't add another read.
8744   if (MI.readsRegister(RISCV::FRM))
8745     return;
8746   MI.addOperand(
8747       MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
8748 }
8749 
8750 // Calling Convention Implementation.
8751 // The expectations for frontend ABI lowering vary from target to target.
8752 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
8753 // details, but this is a longer term goal. For now, we simply try to keep the
8754 // role of the frontend as simple and well-defined as possible. The rules can
8755 // be summarised as:
8756 // * Never split up large scalar arguments. We handle them here.
8757 // * If a hardfloat calling convention is being used, and the struct may be
8758 // passed in a pair of registers (fp+fp, int+fp), and both registers are
8759 // available, then pass as two separate arguments. If either the GPRs or FPRs
8760 // are exhausted, then pass according to the rule below.
8761 // * If a struct could never be passed in registers or directly in a stack
8762 // slot (as it is larger than 2*XLEN and the floating point rules don't
8763 // apply), then pass it using a pointer with the byval attribute.
8764 // * If a struct is less than 2*XLEN, then coerce to either a two-element
8765 // word-sized array or a 2*XLEN scalar (depending on alignment).
8766 // * The frontend can determine whether a struct is returned by reference or
8767 // not based on its size and fields. If it will be returned by reference, the
8768 // frontend must modify the prototype so a pointer with the sret annotation is
8769 // passed as the first argument. This is not necessary for large scalar
8770 // returns.
8771 // * Struct return values and varargs should be coerced to structs containing
8772 // register-size fields in the same situations they would be for fixed
8773 // arguments.
8774 
8775 static const MCPhysReg ArgGPRs[] = {
8776   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
8777   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
8778 };
8779 static const MCPhysReg ArgFPR16s[] = {
8780   RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
8781   RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
8782 };
8783 static const MCPhysReg ArgFPR32s[] = {
8784   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
8785   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
8786 };
8787 static const MCPhysReg ArgFPR64s[] = {
8788   RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
8789   RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
8790 };
8791 // This is an interim calling convention and it may be changed in the future.
8792 static const MCPhysReg ArgVRs[] = {
8793     RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
8794     RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
8795     RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
8796 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
8797                                      RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
8798                                      RISCV::V20M2, RISCV::V22M2};
8799 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
8800                                      RISCV::V20M4};
8801 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
8802 
8803 // Pass a 2*XLEN argument that has been split into two XLEN values through
8804 // registers or the stack as necessary.
8805 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
8806                                 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
8807                                 MVT ValVT2, MVT LocVT2,
8808                                 ISD::ArgFlagsTy ArgFlags2) {
8809   unsigned XLenInBytes = XLen / 8;
8810   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8811     // At least one half can be passed via register.
8812     State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
8813                                      VA1.getLocVT(), CCValAssign::Full));
8814   } else {
8815     // Both halves must be passed on the stack, with proper alignment.
8816     Align StackAlign =
8817         std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
8818     State.addLoc(
8819         CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
8820                             State.AllocateStack(XLenInBytes, StackAlign),
8821                             VA1.getLocVT(), CCValAssign::Full));
8822     State.addLoc(CCValAssign::getMem(
8823         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8824         LocVT2, CCValAssign::Full));
8825     return false;
8826   }
8827 
8828   if (Register Reg = State.AllocateReg(ArgGPRs)) {
8829     // The second half can also be passed via register.
8830     State.addLoc(
8831         CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
8832   } else {
8833     // The second half is passed via the stack, without additional alignment.
8834     State.addLoc(CCValAssign::getMem(
8835         ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
8836         LocVT2, CCValAssign::Full));
8837   }
8838 
8839   return false;
8840 }
8841 
8842 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
8843                                Optional<unsigned> FirstMaskArgument,
8844                                CCState &State, const RISCVTargetLowering &TLI) {
8845   const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
8846   if (RC == &RISCV::VRRegClass) {
8847     // Assign the first mask argument to V0.
8848     // This is an interim calling convention and it may be changed in the
8849     // future.
8850     if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
8851       return State.AllocateReg(RISCV::V0);
8852     return State.AllocateReg(ArgVRs);
8853   }
8854   if (RC == &RISCV::VRM2RegClass)
8855     return State.AllocateReg(ArgVRM2s);
8856   if (RC == &RISCV::VRM4RegClass)
8857     return State.AllocateReg(ArgVRM4s);
8858   if (RC == &RISCV::VRM8RegClass)
8859     return State.AllocateReg(ArgVRM8s);
8860   llvm_unreachable("Unhandled register class for ValueType");
8861 }
8862 
8863 // Implements the RISC-V calling convention. Returns true upon failure.
8864 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
8865                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
8866                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
8867                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
8868                      Optional<unsigned> FirstMaskArgument) {
8869   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
8870   assert(XLen == 32 || XLen == 64);
8871   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
8872 
8873   // Any return value split in to more than two values can't be returned
8874   // directly. Vectors are returned via the available vector registers.
8875   if (!LocVT.isVector() && IsRet && ValNo > 1)
8876     return true;
8877 
8878   // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
8879   // variadic argument, or if no F16/F32 argument registers are available.
8880   bool UseGPRForF16_F32 = true;
8881   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
8882   // variadic argument, or if no F64 argument registers are available.
8883   bool UseGPRForF64 = true;
8884 
8885   switch (ABI) {
8886   default:
8887     llvm_unreachable("Unexpected ABI");
8888   case RISCVABI::ABI_ILP32:
8889   case RISCVABI::ABI_LP64:
8890     break;
8891   case RISCVABI::ABI_ILP32F:
8892   case RISCVABI::ABI_LP64F:
8893     UseGPRForF16_F32 = !IsFixed;
8894     break;
8895   case RISCVABI::ABI_ILP32D:
8896   case RISCVABI::ABI_LP64D:
8897     UseGPRForF16_F32 = !IsFixed;
8898     UseGPRForF64 = !IsFixed;
8899     break;
8900   }
8901 
8902   // FPR16, FPR32, and FPR64 alias each other.
8903   if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
8904     UseGPRForF16_F32 = true;
8905     UseGPRForF64 = true;
8906   }
8907 
8908   // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
8909   // similar local variables rather than directly checking against the target
8910   // ABI.
8911 
8912   if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
8913     LocVT = XLenVT;
8914     LocInfo = CCValAssign::BCvt;
8915   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
8916     LocVT = MVT::i64;
8917     LocInfo = CCValAssign::BCvt;
8918   }
8919 
8920   // If this is a variadic argument, the RISC-V calling convention requires
8921   // that it is assigned an 'even' or 'aligned' register if it has 8-byte
8922   // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
8923   // be used regardless of whether the original argument was split during
8924   // legalisation or not. The argument will not be passed by registers if the
8925   // original type is larger than 2*XLEN, so the register alignment rule does
8926   // not apply.
8927   unsigned TwoXLenInBytes = (2 * XLen) / 8;
8928   if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
8929       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
8930     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
8931     // Skip 'odd' register if necessary.
8932     if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
8933       State.AllocateReg(ArgGPRs);
8934   }
8935 
8936   SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
8937   SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
8938       State.getPendingArgFlags();
8939 
8940   assert(PendingLocs.size() == PendingArgFlags.size() &&
8941          "PendingLocs and PendingArgFlags out of sync");
8942 
8943   // Handle passing f64 on RV32D with a soft float ABI or when floating point
8944   // registers are exhausted.
8945   if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
8946     assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
8947            "Can't lower f64 if it is split");
8948     // Depending on available argument GPRS, f64 may be passed in a pair of
8949     // GPRs, split between a GPR and the stack, or passed completely on the
8950     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
8951     // cases.
8952     Register Reg = State.AllocateReg(ArgGPRs);
8953     LocVT = MVT::i32;
8954     if (!Reg) {
8955       unsigned StackOffset = State.AllocateStack(8, Align(8));
8956       State.addLoc(
8957           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
8958       return false;
8959     }
8960     if (!State.AllocateReg(ArgGPRs))
8961       State.AllocateStack(4, Align(4));
8962     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
8963     return false;
8964   }
8965 
8966   // Fixed-length vectors are located in the corresponding scalable-vector
8967   // container types.
8968   if (ValVT.isFixedLengthVector())
8969     LocVT = TLI.getContainerForFixedLengthVector(LocVT);
8970 
8971   // Split arguments might be passed indirectly, so keep track of the pending
8972   // values. Split vectors are passed via a mix of registers and indirectly, so
8973   // treat them as we would any other argument.
8974   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
8975     LocVT = XLenVT;
8976     LocInfo = CCValAssign::Indirect;
8977     PendingLocs.push_back(
8978         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
8979     PendingArgFlags.push_back(ArgFlags);
8980     if (!ArgFlags.isSplitEnd()) {
8981       return false;
8982     }
8983   }
8984 
8985   // If the split argument only had two elements, it should be passed directly
8986   // in registers or on the stack.
8987   if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
8988       PendingLocs.size() <= 2) {
8989     assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
8990     // Apply the normal calling convention rules to the first half of the
8991     // split argument.
8992     CCValAssign VA = PendingLocs[0];
8993     ISD::ArgFlagsTy AF = PendingArgFlags[0];
8994     PendingLocs.clear();
8995     PendingArgFlags.clear();
8996     return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
8997                                ArgFlags);
8998   }
8999 
9000   // Allocate to a register if possible, or else a stack slot.
9001   Register Reg;
9002   unsigned StoreSizeBytes = XLen / 8;
9003   Align StackAlign = Align(XLen / 8);
9004 
9005   if (ValVT == MVT::f16 && !UseGPRForF16_F32)
9006     Reg = State.AllocateReg(ArgFPR16s);
9007   else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
9008     Reg = State.AllocateReg(ArgFPR32s);
9009   else if (ValVT == MVT::f64 && !UseGPRForF64)
9010     Reg = State.AllocateReg(ArgFPR64s);
9011   else if (ValVT.isVector()) {
9012     Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
9013     if (!Reg) {
9014       // For return values, the vector must be passed fully via registers or
9015       // via the stack.
9016       // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
9017       // but we're using all of them.
9018       if (IsRet)
9019         return true;
9020       // Try using a GPR to pass the address
9021       if ((Reg = State.AllocateReg(ArgGPRs))) {
9022         LocVT = XLenVT;
9023         LocInfo = CCValAssign::Indirect;
9024       } else if (ValVT.isScalableVector()) {
9025         LocVT = XLenVT;
9026         LocInfo = CCValAssign::Indirect;
9027       } else {
9028         // Pass fixed-length vectors on the stack.
9029         LocVT = ValVT;
9030         StoreSizeBytes = ValVT.getStoreSize();
9031         // Align vectors to their element sizes, being careful for vXi1
9032         // vectors.
9033         StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9034       }
9035     }
9036   } else {
9037     Reg = State.AllocateReg(ArgGPRs);
9038   }
9039 
9040   unsigned StackOffset =
9041       Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
9042 
9043   // If we reach this point and PendingLocs is non-empty, we must be at the
9044   // end of a split argument that must be passed indirectly.
9045   if (!PendingLocs.empty()) {
9046     assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
9047     assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
9048 
9049     for (auto &It : PendingLocs) {
9050       if (Reg)
9051         It.convertToReg(Reg);
9052       else
9053         It.convertToMem(StackOffset);
9054       State.addLoc(It);
9055     }
9056     PendingLocs.clear();
9057     PendingArgFlags.clear();
9058     return false;
9059   }
9060 
9061   assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
9062           (TLI.getSubtarget().hasVInstructions() && ValVT.isVector())) &&
9063          "Expected an XLenVT or vector types at this stage");
9064 
9065   if (Reg) {
9066     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9067     return false;
9068   }
9069 
9070   // When a floating-point value is passed on the stack, no bit-conversion is
9071   // needed.
9072   if (ValVT.isFloatingPoint()) {
9073     LocVT = ValVT;
9074     LocInfo = CCValAssign::Full;
9075   }
9076   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9077   return false;
9078 }
9079 
9080 template <typename ArgTy>
9081 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
9082   for (const auto &ArgIdx : enumerate(Args)) {
9083     MVT ArgVT = ArgIdx.value().VT;
9084     if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
9085       return ArgIdx.index();
9086   }
9087   return None;
9088 }
9089 
9090 void RISCVTargetLowering::analyzeInputArgs(
9091     MachineFunction &MF, CCState &CCInfo,
9092     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
9093     RISCVCCAssignFn Fn) const {
9094   unsigned NumArgs = Ins.size();
9095   FunctionType *FType = MF.getFunction().getFunctionType();
9096 
9097   Optional<unsigned> FirstMaskArgument;
9098   if (Subtarget.hasVInstructions())
9099     FirstMaskArgument = preAssignMask(Ins);
9100 
9101   for (unsigned i = 0; i != NumArgs; ++i) {
9102     MVT ArgVT = Ins[i].VT;
9103     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
9104 
9105     Type *ArgTy = nullptr;
9106     if (IsRet)
9107       ArgTy = FType->getReturnType();
9108     else if (Ins[i].isOrigArg())
9109       ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
9110 
9111     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9112     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9113            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
9114            FirstMaskArgument)) {
9115       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
9116                         << EVT(ArgVT).getEVTString() << '\n');
9117       llvm_unreachable(nullptr);
9118     }
9119   }
9120 }
9121 
9122 void RISCVTargetLowering::analyzeOutputArgs(
9123     MachineFunction &MF, CCState &CCInfo,
9124     const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
9125     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
9126   unsigned NumArgs = Outs.size();
9127 
9128   Optional<unsigned> FirstMaskArgument;
9129   if (Subtarget.hasVInstructions())
9130     FirstMaskArgument = preAssignMask(Outs);
9131 
9132   for (unsigned i = 0; i != NumArgs; i++) {
9133     MVT ArgVT = Outs[i].VT;
9134     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9135     Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
9136 
9137     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9138     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
9139            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
9140            FirstMaskArgument)) {
9141       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
9142                         << EVT(ArgVT).getEVTString() << "\n");
9143       llvm_unreachable(nullptr);
9144     }
9145   }
9146 }
9147 
9148 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
9149 // values.
9150 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
9151                                    const CCValAssign &VA, const SDLoc &DL,
9152                                    const RISCVSubtarget &Subtarget) {
9153   switch (VA.getLocInfo()) {
9154   default:
9155     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9156   case CCValAssign::Full:
9157     if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
9158       Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
9159     break;
9160   case CCValAssign::BCvt:
9161     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9162       Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
9163     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9164       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
9165     else
9166       Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
9167     break;
9168   }
9169   return Val;
9170 }
9171 
9172 // The caller is responsible for loading the full value if the argument is
9173 // passed with CCValAssign::Indirect.
9174 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
9175                                 const CCValAssign &VA, const SDLoc &DL,
9176                                 const RISCVTargetLowering &TLI) {
9177   MachineFunction &MF = DAG.getMachineFunction();
9178   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9179   EVT LocVT = VA.getLocVT();
9180   SDValue Val;
9181   const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
9182   Register VReg = RegInfo.createVirtualRegister(RC);
9183   RegInfo.addLiveIn(VA.getLocReg(), VReg);
9184   Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
9185 
9186   if (VA.getLocInfo() == CCValAssign::Indirect)
9187     return Val;
9188 
9189   return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
9190 }
9191 
9192 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
9193                                    const CCValAssign &VA, const SDLoc &DL,
9194                                    const RISCVSubtarget &Subtarget) {
9195   EVT LocVT = VA.getLocVT();
9196 
9197   switch (VA.getLocInfo()) {
9198   default:
9199     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9200   case CCValAssign::Full:
9201     if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
9202       Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
9203     break;
9204   case CCValAssign::BCvt:
9205     if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
9206       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
9207     else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
9208       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
9209     else
9210       Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
9211     break;
9212   }
9213   return Val;
9214 }
9215 
9216 // The caller is responsible for loading the full value if the argument is
9217 // passed with CCValAssign::Indirect.
9218 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
9219                                 const CCValAssign &VA, const SDLoc &DL) {
9220   MachineFunction &MF = DAG.getMachineFunction();
9221   MachineFrameInfo &MFI = MF.getFrameInfo();
9222   EVT LocVT = VA.getLocVT();
9223   EVT ValVT = VA.getValVT();
9224   EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
9225   if (ValVT.isScalableVector()) {
9226     // When the value is a scalable vector, we save the pointer which points to
9227     // the scalable vector value in the stack. The ValVT will be the pointer
9228     // type, instead of the scalable vector type.
9229     ValVT = LocVT;
9230   }
9231   int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
9232                                  /*IsImmutable=*/true);
9233   SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
9234   SDValue Val;
9235 
9236   ISD::LoadExtType ExtType;
9237   switch (VA.getLocInfo()) {
9238   default:
9239     llvm_unreachable("Unexpected CCValAssign::LocInfo");
9240   case CCValAssign::Full:
9241   case CCValAssign::Indirect:
9242   case CCValAssign::BCvt:
9243     ExtType = ISD::NON_EXTLOAD;
9244     break;
9245   }
9246   Val = DAG.getExtLoad(
9247       ExtType, DL, LocVT, Chain, FIN,
9248       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
9249   return Val;
9250 }
9251 
9252 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
9253                                        const CCValAssign &VA, const SDLoc &DL) {
9254   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
9255          "Unexpected VA");
9256   MachineFunction &MF = DAG.getMachineFunction();
9257   MachineFrameInfo &MFI = MF.getFrameInfo();
9258   MachineRegisterInfo &RegInfo = MF.getRegInfo();
9259 
9260   if (VA.isMemLoc()) {
9261     // f64 is passed on the stack.
9262     int FI =
9263         MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
9264     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9265     return DAG.getLoad(MVT::f64, DL, Chain, FIN,
9266                        MachinePointerInfo::getFixedStack(MF, FI));
9267   }
9268 
9269   assert(VA.isRegLoc() && "Expected register VA assignment");
9270 
9271   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9272   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
9273   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
9274   SDValue Hi;
9275   if (VA.getLocReg() == RISCV::X17) {
9276     // Second half of f64 is passed on the stack.
9277     int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
9278     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
9279     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
9280                      MachinePointerInfo::getFixedStack(MF, FI));
9281   } else {
9282     // Second half of f64 is passed in another GPR.
9283     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
9284     RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
9285     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
9286   }
9287   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
9288 }
9289 
9290 // FastCC has less than 1% performance improvement for some particular
9291 // benchmark. But theoretically, it may has benenfit for some cases.
9292 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
9293                             unsigned ValNo, MVT ValVT, MVT LocVT,
9294                             CCValAssign::LocInfo LocInfo,
9295                             ISD::ArgFlagsTy ArgFlags, CCState &State,
9296                             bool IsFixed, bool IsRet, Type *OrigTy,
9297                             const RISCVTargetLowering &TLI,
9298                             Optional<unsigned> FirstMaskArgument) {
9299 
9300   // X5 and X6 might be used for save-restore libcall.
9301   static const MCPhysReg GPRList[] = {
9302       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
9303       RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
9304       RISCV::X29, RISCV::X30, RISCV::X31};
9305 
9306   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9307     if (unsigned Reg = State.AllocateReg(GPRList)) {
9308       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9309       return false;
9310     }
9311   }
9312 
9313   if (LocVT == MVT::f16) {
9314     static const MCPhysReg FPR16List[] = {
9315         RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
9316         RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
9317         RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
9318         RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
9319     if (unsigned Reg = State.AllocateReg(FPR16List)) {
9320       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9321       return false;
9322     }
9323   }
9324 
9325   if (LocVT == MVT::f32) {
9326     static const MCPhysReg FPR32List[] = {
9327         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
9328         RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
9329         RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
9330         RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
9331     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9332       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9333       return false;
9334     }
9335   }
9336 
9337   if (LocVT == MVT::f64) {
9338     static const MCPhysReg FPR64List[] = {
9339         RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
9340         RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
9341         RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
9342         RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
9343     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9344       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9345       return false;
9346     }
9347   }
9348 
9349   if (LocVT == MVT::i32 || LocVT == MVT::f32) {
9350     unsigned Offset4 = State.AllocateStack(4, Align(4));
9351     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
9352     return false;
9353   }
9354 
9355   if (LocVT == MVT::i64 || LocVT == MVT::f64) {
9356     unsigned Offset5 = State.AllocateStack(8, Align(8));
9357     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
9358     return false;
9359   }
9360 
9361   if (LocVT.isVector()) {
9362     if (unsigned Reg =
9363             allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
9364       // Fixed-length vectors are located in the corresponding scalable-vector
9365       // container types.
9366       if (ValVT.isFixedLengthVector())
9367         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
9368       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9369     } else {
9370       // Try and pass the address via a "fast" GPR.
9371       if (unsigned GPRReg = State.AllocateReg(GPRList)) {
9372         LocInfo = CCValAssign::Indirect;
9373         LocVT = TLI.getSubtarget().getXLenVT();
9374         State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
9375       } else if (ValVT.isFixedLengthVector()) {
9376         auto StackAlign =
9377             MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
9378         unsigned StackOffset =
9379             State.AllocateStack(ValVT.getStoreSize(), StackAlign);
9380         State.addLoc(
9381             CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
9382       } else {
9383         // Can't pass scalable vectors on the stack.
9384         return true;
9385       }
9386     }
9387 
9388     return false;
9389   }
9390 
9391   return true; // CC didn't match.
9392 }
9393 
9394 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
9395                          CCValAssign::LocInfo LocInfo,
9396                          ISD::ArgFlagsTy ArgFlags, CCState &State) {
9397 
9398   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9399     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
9400     //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
9401     static const MCPhysReg GPRList[] = {
9402         RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
9403         RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
9404     if (unsigned Reg = State.AllocateReg(GPRList)) {
9405       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9406       return false;
9407     }
9408   }
9409 
9410   if (LocVT == MVT::f32) {
9411     // Pass in STG registers: F1, ..., F6
9412     //                        fs0 ... fs5
9413     static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
9414                                           RISCV::F18_F, RISCV::F19_F,
9415                                           RISCV::F20_F, RISCV::F21_F};
9416     if (unsigned Reg = State.AllocateReg(FPR32List)) {
9417       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9418       return false;
9419     }
9420   }
9421 
9422   if (LocVT == MVT::f64) {
9423     // Pass in STG registers: D1, ..., D6
9424     //                        fs6 ... fs11
9425     static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
9426                                           RISCV::F24_D, RISCV::F25_D,
9427                                           RISCV::F26_D, RISCV::F27_D};
9428     if (unsigned Reg = State.AllocateReg(FPR64List)) {
9429       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
9430       return false;
9431     }
9432   }
9433 
9434   report_fatal_error("No registers left in GHC calling convention");
9435   return true;
9436 }
9437 
9438 // Transform physical registers into virtual registers.
9439 SDValue RISCVTargetLowering::LowerFormalArguments(
9440     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
9441     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
9442     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
9443 
9444   MachineFunction &MF = DAG.getMachineFunction();
9445 
9446   switch (CallConv) {
9447   default:
9448     report_fatal_error("Unsupported calling convention");
9449   case CallingConv::C:
9450   case CallingConv::Fast:
9451     break;
9452   case CallingConv::GHC:
9453     if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
9454         !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
9455       report_fatal_error(
9456         "GHC calling convention requires the F and D instruction set extensions");
9457   }
9458 
9459   const Function &Func = MF.getFunction();
9460   if (Func.hasFnAttribute("interrupt")) {
9461     if (!Func.arg_empty())
9462       report_fatal_error(
9463         "Functions with the interrupt attribute cannot have arguments!");
9464 
9465     StringRef Kind =
9466       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
9467 
9468     if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
9469       report_fatal_error(
9470         "Function interrupt attribute argument not supported!");
9471   }
9472 
9473   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9474   MVT XLenVT = Subtarget.getXLenVT();
9475   unsigned XLenInBytes = Subtarget.getXLen() / 8;
9476   // Used with vargs to acumulate store chains.
9477   std::vector<SDValue> OutChains;
9478 
9479   // Assign locations to all of the incoming arguments.
9480   SmallVector<CCValAssign, 16> ArgLocs;
9481   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9482 
9483   if (CallConv == CallingConv::GHC)
9484     CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
9485   else
9486     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
9487                      CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9488                                                    : CC_RISCV);
9489 
9490   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
9491     CCValAssign &VA = ArgLocs[i];
9492     SDValue ArgValue;
9493     // Passing f64 on RV32D with a soft float ABI must be handled as a special
9494     // case.
9495     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
9496       ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
9497     else if (VA.isRegLoc())
9498       ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
9499     else
9500       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
9501 
9502     if (VA.getLocInfo() == CCValAssign::Indirect) {
9503       // If the original argument was split and passed by reference (e.g. i128
9504       // on RV32), we need to load all parts of it here (using the same
9505       // address). Vectors may be partly split to registers and partly to the
9506       // stack, in which case the base address is partly offset and subsequent
9507       // stores are relative to that.
9508       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
9509                                    MachinePointerInfo()));
9510       unsigned ArgIndex = Ins[i].OrigArgIndex;
9511       unsigned ArgPartOffset = Ins[i].PartOffset;
9512       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9513       while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
9514         CCValAssign &PartVA = ArgLocs[i + 1];
9515         unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
9516         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9517         if (PartVA.getValVT().isScalableVector())
9518           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9519         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
9520         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
9521                                      MachinePointerInfo()));
9522         ++i;
9523       }
9524       continue;
9525     }
9526     InVals.push_back(ArgValue);
9527   }
9528 
9529   if (IsVarArg) {
9530     ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
9531     unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
9532     const TargetRegisterClass *RC = &RISCV::GPRRegClass;
9533     MachineFrameInfo &MFI = MF.getFrameInfo();
9534     MachineRegisterInfo &RegInfo = MF.getRegInfo();
9535     RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
9536 
9537     // Offset of the first variable argument from stack pointer, and size of
9538     // the vararg save area. For now, the varargs save area is either zero or
9539     // large enough to hold a0-a7.
9540     int VaArgOffset, VarArgsSaveSize;
9541 
9542     // If all registers are allocated, then all varargs must be passed on the
9543     // stack and we don't need to save any argregs.
9544     if (ArgRegs.size() == Idx) {
9545       VaArgOffset = CCInfo.getNextStackOffset();
9546       VarArgsSaveSize = 0;
9547     } else {
9548       VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
9549       VaArgOffset = -VarArgsSaveSize;
9550     }
9551 
9552     // Record the frame index of the first variable argument
9553     // which is a value necessary to VASTART.
9554     int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9555     RVFI->setVarArgsFrameIndex(FI);
9556 
9557     // If saving an odd number of registers then create an extra stack slot to
9558     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
9559     // offsets to even-numbered registered remain 2*XLEN-aligned.
9560     if (Idx % 2) {
9561       MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
9562       VarArgsSaveSize += XLenInBytes;
9563     }
9564 
9565     // Copy the integer registers that may have been used for passing varargs
9566     // to the vararg save area.
9567     for (unsigned I = Idx; I < ArgRegs.size();
9568          ++I, VaArgOffset += XLenInBytes) {
9569       const Register Reg = RegInfo.createVirtualRegister(RC);
9570       RegInfo.addLiveIn(ArgRegs[I], Reg);
9571       SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
9572       FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
9573       SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9574       SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
9575                                    MachinePointerInfo::getFixedStack(MF, FI));
9576       cast<StoreSDNode>(Store.getNode())
9577           ->getMemOperand()
9578           ->setValue((Value *)nullptr);
9579       OutChains.push_back(Store);
9580     }
9581     RVFI->setVarArgsSaveSize(VarArgsSaveSize);
9582   }
9583 
9584   // All stores are grouped in one node to allow the matching between
9585   // the size of Ins and InVals. This only happens for vararg functions.
9586   if (!OutChains.empty()) {
9587     OutChains.push_back(Chain);
9588     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
9589   }
9590 
9591   return Chain;
9592 }
9593 
9594 /// isEligibleForTailCallOptimization - Check whether the call is eligible
9595 /// for tail call optimization.
9596 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
9597 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
9598     CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
9599     const SmallVector<CCValAssign, 16> &ArgLocs) const {
9600 
9601   auto &Callee = CLI.Callee;
9602   auto CalleeCC = CLI.CallConv;
9603   auto &Outs = CLI.Outs;
9604   auto &Caller = MF.getFunction();
9605   auto CallerCC = Caller.getCallingConv();
9606 
9607   // Exception-handling functions need a special set of instructions to
9608   // indicate a return to the hardware. Tail-calling another function would
9609   // probably break this.
9610   // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
9611   // should be expanded as new function attributes are introduced.
9612   if (Caller.hasFnAttribute("interrupt"))
9613     return false;
9614 
9615   // Do not tail call opt if the stack is used to pass parameters.
9616   if (CCInfo.getNextStackOffset() != 0)
9617     return false;
9618 
9619   // Do not tail call opt if any parameters need to be passed indirectly.
9620   // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
9621   // passed indirectly. So the address of the value will be passed in a
9622   // register, or if not available, then the address is put on the stack. In
9623   // order to pass indirectly, space on the stack often needs to be allocated
9624   // in order to store the value. In this case the CCInfo.getNextStackOffset()
9625   // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
9626   // are passed CCValAssign::Indirect.
9627   for (auto &VA : ArgLocs)
9628     if (VA.getLocInfo() == CCValAssign::Indirect)
9629       return false;
9630 
9631   // Do not tail call opt if either caller or callee uses struct return
9632   // semantics.
9633   auto IsCallerStructRet = Caller.hasStructRetAttr();
9634   auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
9635   if (IsCallerStructRet || IsCalleeStructRet)
9636     return false;
9637 
9638   // Externally-defined functions with weak linkage should not be
9639   // tail-called. The behaviour of branch instructions in this situation (as
9640   // used for tail calls) is implementation-defined, so we cannot rely on the
9641   // linker replacing the tail call with a return.
9642   if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
9643     const GlobalValue *GV = G->getGlobal();
9644     if (GV->hasExternalWeakLinkage())
9645       return false;
9646   }
9647 
9648   // The callee has to preserve all registers the caller needs to preserve.
9649   const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
9650   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
9651   if (CalleeCC != CallerCC) {
9652     const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
9653     if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9654       return false;
9655   }
9656 
9657   // Byval parameters hand the function a pointer directly into the stack area
9658   // we want to reuse during a tail call. Working around this *is* possible
9659   // but less efficient and uglier in LowerCall.
9660   for (auto &Arg : Outs)
9661     if (Arg.Flags.isByVal())
9662       return false;
9663 
9664   return true;
9665 }
9666 
9667 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
9668   return DAG.getDataLayout().getPrefTypeAlign(
9669       VT.getTypeForEVT(*DAG.getContext()));
9670 }
9671 
9672 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
9673 // and output parameter nodes.
9674 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
9675                                        SmallVectorImpl<SDValue> &InVals) const {
9676   SelectionDAG &DAG = CLI.DAG;
9677   SDLoc &DL = CLI.DL;
9678   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
9679   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
9680   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
9681   SDValue Chain = CLI.Chain;
9682   SDValue Callee = CLI.Callee;
9683   bool &IsTailCall = CLI.IsTailCall;
9684   CallingConv::ID CallConv = CLI.CallConv;
9685   bool IsVarArg = CLI.IsVarArg;
9686   EVT PtrVT = getPointerTy(DAG.getDataLayout());
9687   MVT XLenVT = Subtarget.getXLenVT();
9688 
9689   MachineFunction &MF = DAG.getMachineFunction();
9690 
9691   // Analyze the operands of the call, assigning locations to each operand.
9692   SmallVector<CCValAssign, 16> ArgLocs;
9693   CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
9694 
9695   if (CallConv == CallingConv::GHC)
9696     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
9697   else
9698     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
9699                       CallConv == CallingConv::Fast ? CC_RISCV_FastCC
9700                                                     : CC_RISCV);
9701 
9702   // Check if it's really possible to do a tail call.
9703   if (IsTailCall)
9704     IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
9705 
9706   if (IsTailCall)
9707     ++NumTailCalls;
9708   else if (CLI.CB && CLI.CB->isMustTailCall())
9709     report_fatal_error("failed to perform tail call elimination on a call "
9710                        "site marked musttail");
9711 
9712   // Get a count of how many bytes are to be pushed on the stack.
9713   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
9714 
9715   // Create local copies for byval args
9716   SmallVector<SDValue, 8> ByValArgs;
9717   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9718     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9719     if (!Flags.isByVal())
9720       continue;
9721 
9722     SDValue Arg = OutVals[i];
9723     unsigned Size = Flags.getByValSize();
9724     Align Alignment = Flags.getNonZeroByValAlign();
9725 
9726     int FI =
9727         MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
9728     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
9729     SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
9730 
9731     Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
9732                           /*IsVolatile=*/false,
9733                           /*AlwaysInline=*/false, IsTailCall,
9734                           MachinePointerInfo(), MachinePointerInfo());
9735     ByValArgs.push_back(FIPtr);
9736   }
9737 
9738   if (!IsTailCall)
9739     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
9740 
9741   // Copy argument values to their designated locations.
9742   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
9743   SmallVector<SDValue, 8> MemOpChains;
9744   SDValue StackPtr;
9745   for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
9746     CCValAssign &VA = ArgLocs[i];
9747     SDValue ArgValue = OutVals[i];
9748     ISD::ArgFlagsTy Flags = Outs[i].Flags;
9749 
9750     // Handle passing f64 on RV32D with a soft float ABI as a special case.
9751     bool IsF64OnRV32DSoftABI =
9752         VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
9753     if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
9754       SDValue SplitF64 = DAG.getNode(
9755           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
9756       SDValue Lo = SplitF64.getValue(0);
9757       SDValue Hi = SplitF64.getValue(1);
9758 
9759       Register RegLo = VA.getLocReg();
9760       RegsToPass.push_back(std::make_pair(RegLo, Lo));
9761 
9762       if (RegLo == RISCV::X17) {
9763         // Second half of f64 is passed on the stack.
9764         // Work out the address of the stack slot.
9765         if (!StackPtr.getNode())
9766           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9767         // Emit the store.
9768         MemOpChains.push_back(
9769             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
9770       } else {
9771         // Second half of f64 is passed in another GPR.
9772         assert(RegLo < RISCV::X31 && "Invalid register pair");
9773         Register RegHigh = RegLo + 1;
9774         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
9775       }
9776       continue;
9777     }
9778 
9779     // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
9780     // as any other MemLoc.
9781 
9782     // Promote the value if needed.
9783     // For now, only handle fully promoted and indirect arguments.
9784     if (VA.getLocInfo() == CCValAssign::Indirect) {
9785       // Store the argument in a stack slot and pass its address.
9786       Align StackAlign =
9787           std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
9788                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
9789       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
9790       // If the original argument was split (e.g. i128), we need
9791       // to store the required parts of it here (and pass just one address).
9792       // Vectors may be partly split to registers and partly to the stack, in
9793       // which case the base address is partly offset and subsequent stores are
9794       // relative to that.
9795       unsigned ArgIndex = Outs[i].OrigArgIndex;
9796       unsigned ArgPartOffset = Outs[i].PartOffset;
9797       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
9798       // Calculate the total size to store. We don't have access to what we're
9799       // actually storing other than performing the loop and collecting the
9800       // info.
9801       SmallVector<std::pair<SDValue, SDValue>> Parts;
9802       while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
9803         SDValue PartValue = OutVals[i + 1];
9804         unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
9805         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
9806         EVT PartVT = PartValue.getValueType();
9807         if (PartVT.isScalableVector())
9808           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
9809         StoredSize += PartVT.getStoreSize();
9810         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
9811         Parts.push_back(std::make_pair(PartValue, Offset));
9812         ++i;
9813       }
9814       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
9815       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
9816       MemOpChains.push_back(
9817           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
9818                        MachinePointerInfo::getFixedStack(MF, FI)));
9819       for (const auto &Part : Parts) {
9820         SDValue PartValue = Part.first;
9821         SDValue PartOffset = Part.second;
9822         SDValue Address =
9823             DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
9824         MemOpChains.push_back(
9825             DAG.getStore(Chain, DL, PartValue, Address,
9826                          MachinePointerInfo::getFixedStack(MF, FI)));
9827       }
9828       ArgValue = SpillSlot;
9829     } else {
9830       ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
9831     }
9832 
9833     // Use local copy if it is a byval arg.
9834     if (Flags.isByVal())
9835       ArgValue = ByValArgs[j++];
9836 
9837     if (VA.isRegLoc()) {
9838       // Queue up the argument copies and emit them at the end.
9839       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
9840     } else {
9841       assert(VA.isMemLoc() && "Argument not register or memory");
9842       assert(!IsTailCall && "Tail call not allowed if stack is used "
9843                             "for passing parameters");
9844 
9845       // Work out the address of the stack slot.
9846       if (!StackPtr.getNode())
9847         StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
9848       SDValue Address =
9849           DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
9850                       DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
9851 
9852       // Emit the store.
9853       MemOpChains.push_back(
9854           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
9855     }
9856   }
9857 
9858   // Join the stores, which are independent of one another.
9859   if (!MemOpChains.empty())
9860     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
9861 
9862   SDValue Glue;
9863 
9864   // Build a sequence of copy-to-reg nodes, chained and glued together.
9865   for (auto &Reg : RegsToPass) {
9866     Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
9867     Glue = Chain.getValue(1);
9868   }
9869 
9870   // Validate that none of the argument registers have been marked as
9871   // reserved, if so report an error. Do the same for the return address if this
9872   // is not a tailcall.
9873   validateCCReservedRegs(RegsToPass, MF);
9874   if (!IsTailCall &&
9875       MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
9876     MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
9877         MF.getFunction(),
9878         "Return address register required, but has been reserved."});
9879 
9880   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
9881   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
9882   // split it and then direct call can be matched by PseudoCALL.
9883   if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
9884     const GlobalValue *GV = S->getGlobal();
9885 
9886     unsigned OpFlags = RISCVII::MO_CALL;
9887     if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
9888       OpFlags = RISCVII::MO_PLT;
9889 
9890     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
9891   } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
9892     unsigned OpFlags = RISCVII::MO_CALL;
9893 
9894     if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
9895                                                  nullptr))
9896       OpFlags = RISCVII::MO_PLT;
9897 
9898     Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
9899   }
9900 
9901   // The first call operand is the chain and the second is the target address.
9902   SmallVector<SDValue, 8> Ops;
9903   Ops.push_back(Chain);
9904   Ops.push_back(Callee);
9905 
9906   // Add argument registers to the end of the list so that they are
9907   // known live into the call.
9908   for (auto &Reg : RegsToPass)
9909     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
9910 
9911   if (!IsTailCall) {
9912     // Add a register mask operand representing the call-preserved registers.
9913     const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
9914     const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
9915     assert(Mask && "Missing call preserved mask for calling convention");
9916     Ops.push_back(DAG.getRegisterMask(Mask));
9917   }
9918 
9919   // Glue the call to the argument copies, if any.
9920   if (Glue.getNode())
9921     Ops.push_back(Glue);
9922 
9923   // Emit the call.
9924   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9925 
9926   if (IsTailCall) {
9927     MF.getFrameInfo().setHasTailCall();
9928     return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
9929   }
9930 
9931   Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
9932   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
9933   Glue = Chain.getValue(1);
9934 
9935   // Mark the end of the call, which is glued to the call itself.
9936   Chain = DAG.getCALLSEQ_END(Chain,
9937                              DAG.getConstant(NumBytes, DL, PtrVT, true),
9938                              DAG.getConstant(0, DL, PtrVT, true),
9939                              Glue, DL);
9940   Glue = Chain.getValue(1);
9941 
9942   // Assign locations to each value returned by this call.
9943   SmallVector<CCValAssign, 16> RVLocs;
9944   CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
9945   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
9946 
9947   // Copy all of the result registers out of their specified physreg.
9948   for (auto &VA : RVLocs) {
9949     // Copy the value out
9950     SDValue RetValue =
9951         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
9952     // Glue the RetValue to the end of the call sequence
9953     Chain = RetValue.getValue(1);
9954     Glue = RetValue.getValue(2);
9955 
9956     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
9957       assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
9958       SDValue RetValue2 =
9959           DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
9960       Chain = RetValue2.getValue(1);
9961       Glue = RetValue2.getValue(2);
9962       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
9963                              RetValue2);
9964     }
9965 
9966     RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
9967 
9968     InVals.push_back(RetValue);
9969   }
9970 
9971   return Chain;
9972 }
9973 
9974 bool RISCVTargetLowering::CanLowerReturn(
9975     CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
9976     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
9977   SmallVector<CCValAssign, 16> RVLocs;
9978   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
9979 
9980   Optional<unsigned> FirstMaskArgument;
9981   if (Subtarget.hasVInstructions())
9982     FirstMaskArgument = preAssignMask(Outs);
9983 
9984   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
9985     MVT VT = Outs[i].VT;
9986     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
9987     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
9988     if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
9989                  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
9990                  *this, FirstMaskArgument))
9991       return false;
9992   }
9993   return true;
9994 }
9995 
9996 SDValue
9997 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
9998                                  bool IsVarArg,
9999                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
10000                                  const SmallVectorImpl<SDValue> &OutVals,
10001                                  const SDLoc &DL, SelectionDAG &DAG) const {
10002   const MachineFunction &MF = DAG.getMachineFunction();
10003   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
10004 
10005   // Stores the assignment of the return value to a location.
10006   SmallVector<CCValAssign, 16> RVLocs;
10007 
10008   // Info about the registers and stack slot.
10009   CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
10010                  *DAG.getContext());
10011 
10012   analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
10013                     nullptr, CC_RISCV);
10014 
10015   if (CallConv == CallingConv::GHC && !RVLocs.empty())
10016     report_fatal_error("GHC functions return void only");
10017 
10018   SDValue Glue;
10019   SmallVector<SDValue, 4> RetOps(1, Chain);
10020 
10021   // Copy the result values into the output registers.
10022   for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
10023     SDValue Val = OutVals[i];
10024     CCValAssign &VA = RVLocs[i];
10025     assert(VA.isRegLoc() && "Can only return in registers!");
10026 
10027     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10028       // Handle returning f64 on RV32D with a soft float ABI.
10029       assert(VA.isRegLoc() && "Expected return via registers");
10030       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
10031                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
10032       SDValue Lo = SplitF64.getValue(0);
10033       SDValue Hi = SplitF64.getValue(1);
10034       Register RegLo = VA.getLocReg();
10035       assert(RegLo < RISCV::X31 && "Invalid register pair");
10036       Register RegHi = RegLo + 1;
10037 
10038       if (STI.isRegisterReservedByUser(RegLo) ||
10039           STI.isRegisterReservedByUser(RegHi))
10040         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
10041             MF.getFunction(),
10042             "Return value register required, but has been reserved."});
10043 
10044       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
10045       Glue = Chain.getValue(1);
10046       RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
10047       Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
10048       Glue = Chain.getValue(1);
10049       RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
10050     } else {
10051       // Handle a 'normal' return.
10052       Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
10053       Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
10054 
10055       if (STI.isRegisterReservedByUser(VA.getLocReg()))
10056         MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
10057             MF.getFunction(),
10058             "Return value register required, but has been reserved."});
10059 
10060       // Guarantee that all emitted copies are stuck together.
10061       Glue = Chain.getValue(1);
10062       RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
10063     }
10064   }
10065 
10066   RetOps[0] = Chain; // Update chain.
10067 
10068   // Add the glue node if we have it.
10069   if (Glue.getNode()) {
10070     RetOps.push_back(Glue);
10071   }
10072 
10073   unsigned RetOpc = RISCVISD::RET_FLAG;
10074   // Interrupt service routines use different return instructions.
10075   const Function &Func = DAG.getMachineFunction().getFunction();
10076   if (Func.hasFnAttribute("interrupt")) {
10077     if (!Func.getReturnType()->isVoidTy())
10078       report_fatal_error(
10079           "Functions with the interrupt attribute must have void return type!");
10080 
10081     MachineFunction &MF = DAG.getMachineFunction();
10082     StringRef Kind =
10083       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
10084 
10085     if (Kind == "user")
10086       RetOpc = RISCVISD::URET_FLAG;
10087     else if (Kind == "supervisor")
10088       RetOpc = RISCVISD::SRET_FLAG;
10089     else
10090       RetOpc = RISCVISD::MRET_FLAG;
10091   }
10092 
10093   return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
10094 }
10095 
10096 void RISCVTargetLowering::validateCCReservedRegs(
10097     const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
10098     MachineFunction &MF) const {
10099   const Function &F = MF.getFunction();
10100   const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
10101 
10102   if (llvm::any_of(Regs, [&STI](auto Reg) {
10103         return STI.isRegisterReservedByUser(Reg.first);
10104       }))
10105     F.getContext().diagnose(DiagnosticInfoUnsupported{
10106         F, "Argument register required, but has been reserved."});
10107 }
10108 
10109 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
10110   return CI->isTailCall();
10111 }
10112 
10113 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
10114 #define NODE_NAME_CASE(NODE)                                                   \
10115   case RISCVISD::NODE:                                                         \
10116     return "RISCVISD::" #NODE;
10117   // clang-format off
10118   switch ((RISCVISD::NodeType)Opcode) {
10119   case RISCVISD::FIRST_NUMBER:
10120     break;
10121   NODE_NAME_CASE(RET_FLAG)
10122   NODE_NAME_CASE(URET_FLAG)
10123   NODE_NAME_CASE(SRET_FLAG)
10124   NODE_NAME_CASE(MRET_FLAG)
10125   NODE_NAME_CASE(CALL)
10126   NODE_NAME_CASE(SELECT_CC)
10127   NODE_NAME_CASE(BR_CC)
10128   NODE_NAME_CASE(BuildPairF64)
10129   NODE_NAME_CASE(SplitF64)
10130   NODE_NAME_CASE(TAIL)
10131   NODE_NAME_CASE(MULHSU)
10132   NODE_NAME_CASE(SLLW)
10133   NODE_NAME_CASE(SRAW)
10134   NODE_NAME_CASE(SRLW)
10135   NODE_NAME_CASE(DIVW)
10136   NODE_NAME_CASE(DIVUW)
10137   NODE_NAME_CASE(REMUW)
10138   NODE_NAME_CASE(ROLW)
10139   NODE_NAME_CASE(RORW)
10140   NODE_NAME_CASE(CLZW)
10141   NODE_NAME_CASE(CTZW)
10142   NODE_NAME_CASE(FSLW)
10143   NODE_NAME_CASE(FSRW)
10144   NODE_NAME_CASE(FSL)
10145   NODE_NAME_CASE(FSR)
10146   NODE_NAME_CASE(FMV_H_X)
10147   NODE_NAME_CASE(FMV_X_ANYEXTH)
10148   NODE_NAME_CASE(FMV_W_X_RV64)
10149   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
10150   NODE_NAME_CASE(FCVT_X)
10151   NODE_NAME_CASE(FCVT_XU)
10152   NODE_NAME_CASE(FCVT_W_RV64)
10153   NODE_NAME_CASE(FCVT_WU_RV64)
10154   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
10155   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
10156   NODE_NAME_CASE(READ_CYCLE_WIDE)
10157   NODE_NAME_CASE(GREV)
10158   NODE_NAME_CASE(GREVW)
10159   NODE_NAME_CASE(GORC)
10160   NODE_NAME_CASE(GORCW)
10161   NODE_NAME_CASE(SHFL)
10162   NODE_NAME_CASE(SHFLW)
10163   NODE_NAME_CASE(UNSHFL)
10164   NODE_NAME_CASE(UNSHFLW)
10165   NODE_NAME_CASE(BFP)
10166   NODE_NAME_CASE(BFPW)
10167   NODE_NAME_CASE(BCOMPRESS)
10168   NODE_NAME_CASE(BCOMPRESSW)
10169   NODE_NAME_CASE(BDECOMPRESS)
10170   NODE_NAME_CASE(BDECOMPRESSW)
10171   NODE_NAME_CASE(VMV_V_X_VL)
10172   NODE_NAME_CASE(VFMV_V_F_VL)
10173   NODE_NAME_CASE(VMV_X_S)
10174   NODE_NAME_CASE(VMV_S_X_VL)
10175   NODE_NAME_CASE(VFMV_S_F_VL)
10176   NODE_NAME_CASE(SPLAT_VECTOR_I64)
10177   NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
10178   NODE_NAME_CASE(READ_VLENB)
10179   NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
10180   NODE_NAME_CASE(VSLIDEUP_VL)
10181   NODE_NAME_CASE(VSLIDE1UP_VL)
10182   NODE_NAME_CASE(VSLIDEDOWN_VL)
10183   NODE_NAME_CASE(VSLIDE1DOWN_VL)
10184   NODE_NAME_CASE(VID_VL)
10185   NODE_NAME_CASE(VFNCVT_ROD_VL)
10186   NODE_NAME_CASE(VECREDUCE_ADD_VL)
10187   NODE_NAME_CASE(VECREDUCE_UMAX_VL)
10188   NODE_NAME_CASE(VECREDUCE_SMAX_VL)
10189   NODE_NAME_CASE(VECREDUCE_UMIN_VL)
10190   NODE_NAME_CASE(VECREDUCE_SMIN_VL)
10191   NODE_NAME_CASE(VECREDUCE_AND_VL)
10192   NODE_NAME_CASE(VECREDUCE_OR_VL)
10193   NODE_NAME_CASE(VECREDUCE_XOR_VL)
10194   NODE_NAME_CASE(VECREDUCE_FADD_VL)
10195   NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
10196   NODE_NAME_CASE(VECREDUCE_FMIN_VL)
10197   NODE_NAME_CASE(VECREDUCE_FMAX_VL)
10198   NODE_NAME_CASE(ADD_VL)
10199   NODE_NAME_CASE(AND_VL)
10200   NODE_NAME_CASE(MUL_VL)
10201   NODE_NAME_CASE(OR_VL)
10202   NODE_NAME_CASE(SDIV_VL)
10203   NODE_NAME_CASE(SHL_VL)
10204   NODE_NAME_CASE(SREM_VL)
10205   NODE_NAME_CASE(SRA_VL)
10206   NODE_NAME_CASE(SRL_VL)
10207   NODE_NAME_CASE(SUB_VL)
10208   NODE_NAME_CASE(UDIV_VL)
10209   NODE_NAME_CASE(UREM_VL)
10210   NODE_NAME_CASE(XOR_VL)
10211   NODE_NAME_CASE(SADDSAT_VL)
10212   NODE_NAME_CASE(UADDSAT_VL)
10213   NODE_NAME_CASE(SSUBSAT_VL)
10214   NODE_NAME_CASE(USUBSAT_VL)
10215   NODE_NAME_CASE(FADD_VL)
10216   NODE_NAME_CASE(FSUB_VL)
10217   NODE_NAME_CASE(FMUL_VL)
10218   NODE_NAME_CASE(FDIV_VL)
10219   NODE_NAME_CASE(FNEG_VL)
10220   NODE_NAME_CASE(FABS_VL)
10221   NODE_NAME_CASE(FSQRT_VL)
10222   NODE_NAME_CASE(FMA_VL)
10223   NODE_NAME_CASE(FCOPYSIGN_VL)
10224   NODE_NAME_CASE(SMIN_VL)
10225   NODE_NAME_CASE(SMAX_VL)
10226   NODE_NAME_CASE(UMIN_VL)
10227   NODE_NAME_CASE(UMAX_VL)
10228   NODE_NAME_CASE(FMINNUM_VL)
10229   NODE_NAME_CASE(FMAXNUM_VL)
10230   NODE_NAME_CASE(MULHS_VL)
10231   NODE_NAME_CASE(MULHU_VL)
10232   NODE_NAME_CASE(FP_TO_SINT_VL)
10233   NODE_NAME_CASE(FP_TO_UINT_VL)
10234   NODE_NAME_CASE(SINT_TO_FP_VL)
10235   NODE_NAME_CASE(UINT_TO_FP_VL)
10236   NODE_NAME_CASE(FP_EXTEND_VL)
10237   NODE_NAME_CASE(FP_ROUND_VL)
10238   NODE_NAME_CASE(VWMUL_VL)
10239   NODE_NAME_CASE(VWMULU_VL)
10240   NODE_NAME_CASE(VWMULSU_VL)
10241   NODE_NAME_CASE(VWADDU_VL)
10242   NODE_NAME_CASE(SETCC_VL)
10243   NODE_NAME_CASE(VSELECT_VL)
10244   NODE_NAME_CASE(VP_MERGE_VL)
10245   NODE_NAME_CASE(VMAND_VL)
10246   NODE_NAME_CASE(VMOR_VL)
10247   NODE_NAME_CASE(VMXOR_VL)
10248   NODE_NAME_CASE(VMCLR_VL)
10249   NODE_NAME_CASE(VMSET_VL)
10250   NODE_NAME_CASE(VRGATHER_VX_VL)
10251   NODE_NAME_CASE(VRGATHER_VV_VL)
10252   NODE_NAME_CASE(VRGATHEREI16_VV_VL)
10253   NODE_NAME_CASE(VSEXT_VL)
10254   NODE_NAME_CASE(VZEXT_VL)
10255   NODE_NAME_CASE(VCPOP_VL)
10256   NODE_NAME_CASE(VLE_VL)
10257   NODE_NAME_CASE(VSE_VL)
10258   NODE_NAME_CASE(READ_CSR)
10259   NODE_NAME_CASE(WRITE_CSR)
10260   NODE_NAME_CASE(SWAP_CSR)
10261   }
10262   // clang-format on
10263   return nullptr;
10264 #undef NODE_NAME_CASE
10265 }
10266 
10267 /// getConstraintType - Given a constraint letter, return the type of
10268 /// constraint it is for this target.
10269 RISCVTargetLowering::ConstraintType
10270 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
10271   if (Constraint.size() == 1) {
10272     switch (Constraint[0]) {
10273     default:
10274       break;
10275     case 'f':
10276       return C_RegisterClass;
10277     case 'I':
10278     case 'J':
10279     case 'K':
10280       return C_Immediate;
10281     case 'A':
10282       return C_Memory;
10283     case 'S': // A symbolic address
10284       return C_Other;
10285     }
10286   } else {
10287     if (Constraint == "vr" || Constraint == "vm")
10288       return C_RegisterClass;
10289   }
10290   return TargetLowering::getConstraintType(Constraint);
10291 }
10292 
10293 std::pair<unsigned, const TargetRegisterClass *>
10294 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
10295                                                   StringRef Constraint,
10296                                                   MVT VT) const {
10297   // First, see if this is a constraint that directly corresponds to a
10298   // RISCV register class.
10299   if (Constraint.size() == 1) {
10300     switch (Constraint[0]) {
10301     case 'r':
10302       // TODO: Support fixed vectors up to XLen for P extension?
10303       if (VT.isVector())
10304         break;
10305       return std::make_pair(0U, &RISCV::GPRRegClass);
10306     case 'f':
10307       if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
10308         return std::make_pair(0U, &RISCV::FPR16RegClass);
10309       if (Subtarget.hasStdExtF() && VT == MVT::f32)
10310         return std::make_pair(0U, &RISCV::FPR32RegClass);
10311       if (Subtarget.hasStdExtD() && VT == MVT::f64)
10312         return std::make_pair(0U, &RISCV::FPR64RegClass);
10313       break;
10314     default:
10315       break;
10316     }
10317   } else if (Constraint == "vr") {
10318     for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
10319                            &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10320       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
10321         return std::make_pair(0U, RC);
10322     }
10323   } else if (Constraint == "vm") {
10324     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
10325       return std::make_pair(0U, &RISCV::VMV0RegClass);
10326   }
10327 
10328   // Clang will correctly decode the usage of register name aliases into their
10329   // official names. However, other frontends like `rustc` do not. This allows
10330   // users of these frontends to use the ABI names for registers in LLVM-style
10331   // register constraints.
10332   unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
10333                                .Case("{zero}", RISCV::X0)
10334                                .Case("{ra}", RISCV::X1)
10335                                .Case("{sp}", RISCV::X2)
10336                                .Case("{gp}", RISCV::X3)
10337                                .Case("{tp}", RISCV::X4)
10338                                .Case("{t0}", RISCV::X5)
10339                                .Case("{t1}", RISCV::X6)
10340                                .Case("{t2}", RISCV::X7)
10341                                .Cases("{s0}", "{fp}", RISCV::X8)
10342                                .Case("{s1}", RISCV::X9)
10343                                .Case("{a0}", RISCV::X10)
10344                                .Case("{a1}", RISCV::X11)
10345                                .Case("{a2}", RISCV::X12)
10346                                .Case("{a3}", RISCV::X13)
10347                                .Case("{a4}", RISCV::X14)
10348                                .Case("{a5}", RISCV::X15)
10349                                .Case("{a6}", RISCV::X16)
10350                                .Case("{a7}", RISCV::X17)
10351                                .Case("{s2}", RISCV::X18)
10352                                .Case("{s3}", RISCV::X19)
10353                                .Case("{s4}", RISCV::X20)
10354                                .Case("{s5}", RISCV::X21)
10355                                .Case("{s6}", RISCV::X22)
10356                                .Case("{s7}", RISCV::X23)
10357                                .Case("{s8}", RISCV::X24)
10358                                .Case("{s9}", RISCV::X25)
10359                                .Case("{s10}", RISCV::X26)
10360                                .Case("{s11}", RISCV::X27)
10361                                .Case("{t3}", RISCV::X28)
10362                                .Case("{t4}", RISCV::X29)
10363                                .Case("{t5}", RISCV::X30)
10364                                .Case("{t6}", RISCV::X31)
10365                                .Default(RISCV::NoRegister);
10366   if (XRegFromAlias != RISCV::NoRegister)
10367     return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
10368 
10369   // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
10370   // TableGen record rather than the AsmName to choose registers for InlineAsm
10371   // constraints, plus we want to match those names to the widest floating point
10372   // register type available, manually select floating point registers here.
10373   //
10374   // The second case is the ABI name of the register, so that frontends can also
10375   // use the ABI names in register constraint lists.
10376   if (Subtarget.hasStdExtF()) {
10377     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
10378                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
10379                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
10380                         .Cases("{f2}", "{ft2}", RISCV::F2_F)
10381                         .Cases("{f3}", "{ft3}", RISCV::F3_F)
10382                         .Cases("{f4}", "{ft4}", RISCV::F4_F)
10383                         .Cases("{f5}", "{ft5}", RISCV::F5_F)
10384                         .Cases("{f6}", "{ft6}", RISCV::F6_F)
10385                         .Cases("{f7}", "{ft7}", RISCV::F7_F)
10386                         .Cases("{f8}", "{fs0}", RISCV::F8_F)
10387                         .Cases("{f9}", "{fs1}", RISCV::F9_F)
10388                         .Cases("{f10}", "{fa0}", RISCV::F10_F)
10389                         .Cases("{f11}", "{fa1}", RISCV::F11_F)
10390                         .Cases("{f12}", "{fa2}", RISCV::F12_F)
10391                         .Cases("{f13}", "{fa3}", RISCV::F13_F)
10392                         .Cases("{f14}", "{fa4}", RISCV::F14_F)
10393                         .Cases("{f15}", "{fa5}", RISCV::F15_F)
10394                         .Cases("{f16}", "{fa6}", RISCV::F16_F)
10395                         .Cases("{f17}", "{fa7}", RISCV::F17_F)
10396                         .Cases("{f18}", "{fs2}", RISCV::F18_F)
10397                         .Cases("{f19}", "{fs3}", RISCV::F19_F)
10398                         .Cases("{f20}", "{fs4}", RISCV::F20_F)
10399                         .Cases("{f21}", "{fs5}", RISCV::F21_F)
10400                         .Cases("{f22}", "{fs6}", RISCV::F22_F)
10401                         .Cases("{f23}", "{fs7}", RISCV::F23_F)
10402                         .Cases("{f24}", "{fs8}", RISCV::F24_F)
10403                         .Cases("{f25}", "{fs9}", RISCV::F25_F)
10404                         .Cases("{f26}", "{fs10}", RISCV::F26_F)
10405                         .Cases("{f27}", "{fs11}", RISCV::F27_F)
10406                         .Cases("{f28}", "{ft8}", RISCV::F28_F)
10407                         .Cases("{f29}", "{ft9}", RISCV::F29_F)
10408                         .Cases("{f30}", "{ft10}", RISCV::F30_F)
10409                         .Cases("{f31}", "{ft11}", RISCV::F31_F)
10410                         .Default(RISCV::NoRegister);
10411     if (FReg != RISCV::NoRegister) {
10412       assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
10413       if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
10414         unsigned RegNo = FReg - RISCV::F0_F;
10415         unsigned DReg = RISCV::F0_D + RegNo;
10416         return std::make_pair(DReg, &RISCV::FPR64RegClass);
10417       }
10418       if (VT == MVT::f32 || VT == MVT::Other)
10419         return std::make_pair(FReg, &RISCV::FPR32RegClass);
10420       if (Subtarget.hasStdExtZfh() && VT == MVT::f16) {
10421         unsigned RegNo = FReg - RISCV::F0_F;
10422         unsigned HReg = RISCV::F0_H + RegNo;
10423         return std::make_pair(HReg, &RISCV::FPR16RegClass);
10424       }
10425     }
10426   }
10427 
10428   if (Subtarget.hasVInstructions()) {
10429     Register VReg = StringSwitch<Register>(Constraint.lower())
10430                         .Case("{v0}", RISCV::V0)
10431                         .Case("{v1}", RISCV::V1)
10432                         .Case("{v2}", RISCV::V2)
10433                         .Case("{v3}", RISCV::V3)
10434                         .Case("{v4}", RISCV::V4)
10435                         .Case("{v5}", RISCV::V5)
10436                         .Case("{v6}", RISCV::V6)
10437                         .Case("{v7}", RISCV::V7)
10438                         .Case("{v8}", RISCV::V8)
10439                         .Case("{v9}", RISCV::V9)
10440                         .Case("{v10}", RISCV::V10)
10441                         .Case("{v11}", RISCV::V11)
10442                         .Case("{v12}", RISCV::V12)
10443                         .Case("{v13}", RISCV::V13)
10444                         .Case("{v14}", RISCV::V14)
10445                         .Case("{v15}", RISCV::V15)
10446                         .Case("{v16}", RISCV::V16)
10447                         .Case("{v17}", RISCV::V17)
10448                         .Case("{v18}", RISCV::V18)
10449                         .Case("{v19}", RISCV::V19)
10450                         .Case("{v20}", RISCV::V20)
10451                         .Case("{v21}", RISCV::V21)
10452                         .Case("{v22}", RISCV::V22)
10453                         .Case("{v23}", RISCV::V23)
10454                         .Case("{v24}", RISCV::V24)
10455                         .Case("{v25}", RISCV::V25)
10456                         .Case("{v26}", RISCV::V26)
10457                         .Case("{v27}", RISCV::V27)
10458                         .Case("{v28}", RISCV::V28)
10459                         .Case("{v29}", RISCV::V29)
10460                         .Case("{v30}", RISCV::V30)
10461                         .Case("{v31}", RISCV::V31)
10462                         .Default(RISCV::NoRegister);
10463     if (VReg != RISCV::NoRegister) {
10464       if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
10465         return std::make_pair(VReg, &RISCV::VMRegClass);
10466       if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
10467         return std::make_pair(VReg, &RISCV::VRRegClass);
10468       for (const auto *RC :
10469            {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
10470         if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
10471           VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
10472           return std::make_pair(VReg, RC);
10473         }
10474       }
10475     }
10476   }
10477 
10478   std::pair<Register, const TargetRegisterClass *> Res =
10479       TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
10480 
10481   // If we picked one of the Zfinx register classes, remap it to the GPR class.
10482   // FIXME: When Zfinx is supported in CodeGen this will need to take the
10483   // Subtarget into account.
10484   if (Res.second == &RISCV::GPRF16RegClass ||
10485       Res.second == &RISCV::GPRF32RegClass ||
10486       Res.second == &RISCV::GPRF64RegClass)
10487     return std::make_pair(Res.first, &RISCV::GPRRegClass);
10488 
10489   return Res;
10490 }
10491 
10492 unsigned
10493 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
10494   // Currently only support length 1 constraints.
10495   if (ConstraintCode.size() == 1) {
10496     switch (ConstraintCode[0]) {
10497     case 'A':
10498       return InlineAsm::Constraint_A;
10499     default:
10500       break;
10501     }
10502   }
10503 
10504   return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
10505 }
10506 
10507 void RISCVTargetLowering::LowerAsmOperandForConstraint(
10508     SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
10509     SelectionDAG &DAG) const {
10510   // Currently only support length 1 constraints.
10511   if (Constraint.length() == 1) {
10512     switch (Constraint[0]) {
10513     case 'I':
10514       // Validate & create a 12-bit signed immediate operand.
10515       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10516         uint64_t CVal = C->getSExtValue();
10517         if (isInt<12>(CVal))
10518           Ops.push_back(
10519               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10520       }
10521       return;
10522     case 'J':
10523       // Validate & create an integer zero operand.
10524       if (auto *C = dyn_cast<ConstantSDNode>(Op))
10525         if (C->getZExtValue() == 0)
10526           Ops.push_back(
10527               DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
10528       return;
10529     case 'K':
10530       // Validate & create a 5-bit unsigned immediate operand.
10531       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
10532         uint64_t CVal = C->getZExtValue();
10533         if (isUInt<5>(CVal))
10534           Ops.push_back(
10535               DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
10536       }
10537       return;
10538     case 'S':
10539       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10540         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
10541                                                  GA->getValueType(0)));
10542       } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
10543         Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
10544                                                 BA->getValueType(0)));
10545       }
10546       return;
10547     default:
10548       break;
10549     }
10550   }
10551   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
10552 }
10553 
10554 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
10555                                                    Instruction *Inst,
10556                                                    AtomicOrdering Ord) const {
10557   if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
10558     return Builder.CreateFence(Ord);
10559   if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
10560     return Builder.CreateFence(AtomicOrdering::Release);
10561   return nullptr;
10562 }
10563 
10564 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
10565                                                     Instruction *Inst,
10566                                                     AtomicOrdering Ord) const {
10567   if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
10568     return Builder.CreateFence(AtomicOrdering::Acquire);
10569   return nullptr;
10570 }
10571 
10572 TargetLowering::AtomicExpansionKind
10573 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
10574   // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
10575   // point operations can't be used in an lr/sc sequence without breaking the
10576   // forward-progress guarantee.
10577   if (AI->isFloatingPointOperation())
10578     return AtomicExpansionKind::CmpXChg;
10579 
10580   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
10581   if (Size == 8 || Size == 16)
10582     return AtomicExpansionKind::MaskedIntrinsic;
10583   return AtomicExpansionKind::None;
10584 }
10585 
10586 static Intrinsic::ID
10587 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
10588   if (XLen == 32) {
10589     switch (BinOp) {
10590     default:
10591       llvm_unreachable("Unexpected AtomicRMW BinOp");
10592     case AtomicRMWInst::Xchg:
10593       return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
10594     case AtomicRMWInst::Add:
10595       return Intrinsic::riscv_masked_atomicrmw_add_i32;
10596     case AtomicRMWInst::Sub:
10597       return Intrinsic::riscv_masked_atomicrmw_sub_i32;
10598     case AtomicRMWInst::Nand:
10599       return Intrinsic::riscv_masked_atomicrmw_nand_i32;
10600     case AtomicRMWInst::Max:
10601       return Intrinsic::riscv_masked_atomicrmw_max_i32;
10602     case AtomicRMWInst::Min:
10603       return Intrinsic::riscv_masked_atomicrmw_min_i32;
10604     case AtomicRMWInst::UMax:
10605       return Intrinsic::riscv_masked_atomicrmw_umax_i32;
10606     case AtomicRMWInst::UMin:
10607       return Intrinsic::riscv_masked_atomicrmw_umin_i32;
10608     }
10609   }
10610 
10611   if (XLen == 64) {
10612     switch (BinOp) {
10613     default:
10614       llvm_unreachable("Unexpected AtomicRMW BinOp");
10615     case AtomicRMWInst::Xchg:
10616       return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
10617     case AtomicRMWInst::Add:
10618       return Intrinsic::riscv_masked_atomicrmw_add_i64;
10619     case AtomicRMWInst::Sub:
10620       return Intrinsic::riscv_masked_atomicrmw_sub_i64;
10621     case AtomicRMWInst::Nand:
10622       return Intrinsic::riscv_masked_atomicrmw_nand_i64;
10623     case AtomicRMWInst::Max:
10624       return Intrinsic::riscv_masked_atomicrmw_max_i64;
10625     case AtomicRMWInst::Min:
10626       return Intrinsic::riscv_masked_atomicrmw_min_i64;
10627     case AtomicRMWInst::UMax:
10628       return Intrinsic::riscv_masked_atomicrmw_umax_i64;
10629     case AtomicRMWInst::UMin:
10630       return Intrinsic::riscv_masked_atomicrmw_umin_i64;
10631     }
10632   }
10633 
10634   llvm_unreachable("Unexpected XLen\n");
10635 }
10636 
10637 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
10638     IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
10639     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
10640   unsigned XLen = Subtarget.getXLen();
10641   Value *Ordering =
10642       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
10643   Type *Tys[] = {AlignedAddr->getType()};
10644   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
10645       AI->getModule(),
10646       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
10647 
10648   if (XLen == 64) {
10649     Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10650     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10651     ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10652   }
10653 
10654   Value *Result;
10655 
10656   // Must pass the shift amount needed to sign extend the loaded value prior
10657   // to performing a signed comparison for min/max. ShiftAmt is the number of
10658   // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
10659   // is the number of bits to left+right shift the value in order to
10660   // sign-extend.
10661   if (AI->getOperation() == AtomicRMWInst::Min ||
10662       AI->getOperation() == AtomicRMWInst::Max) {
10663     const DataLayout &DL = AI->getModule()->getDataLayout();
10664     unsigned ValWidth =
10665         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
10666     Value *SextShamt =
10667         Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
10668     Result = Builder.CreateCall(LrwOpScwLoop,
10669                                 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10670   } else {
10671     Result =
10672         Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
10673   }
10674 
10675   if (XLen == 64)
10676     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10677   return Result;
10678 }
10679 
10680 TargetLowering::AtomicExpansionKind
10681 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
10682     AtomicCmpXchgInst *CI) const {
10683   unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
10684   if (Size == 8 || Size == 16)
10685     return AtomicExpansionKind::MaskedIntrinsic;
10686   return AtomicExpansionKind::None;
10687 }
10688 
10689 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
10690     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
10691     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
10692   unsigned XLen = Subtarget.getXLen();
10693   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
10694   Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
10695   if (XLen == 64) {
10696     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
10697     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
10698     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10699     CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
10700   }
10701   Type *Tys[] = {AlignedAddr->getType()};
10702   Function *MaskedCmpXchg =
10703       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
10704   Value *Result = Builder.CreateCall(
10705       MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
10706   if (XLen == 64)
10707     Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10708   return Result;
10709 }
10710 
10711 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
10712   return false;
10713 }
10714 
10715 bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
10716                                                EVT VT) const {
10717   if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
10718     return false;
10719 
10720   switch (FPVT.getSimpleVT().SimpleTy) {
10721   case MVT::f16:
10722     return Subtarget.hasStdExtZfh();
10723   case MVT::f32:
10724     return Subtarget.hasStdExtF();
10725   case MVT::f64:
10726     return Subtarget.hasStdExtD();
10727   default:
10728     return false;
10729   }
10730 }
10731 
10732 unsigned RISCVTargetLowering::getJumpTableEncoding() const {
10733   // If we are using the small code model, we can reduce size of jump table
10734   // entry to 4 bytes.
10735   if (Subtarget.is64Bit() && !isPositionIndependent() &&
10736       getTargetMachine().getCodeModel() == CodeModel::Small) {
10737     return MachineJumpTableInfo::EK_Custom32;
10738   }
10739   return TargetLowering::getJumpTableEncoding();
10740 }
10741 
10742 const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
10743     const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
10744     unsigned uid, MCContext &Ctx) const {
10745   assert(Subtarget.is64Bit() && !isPositionIndependent() &&
10746          getTargetMachine().getCodeModel() == CodeModel::Small);
10747   return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
10748 }
10749 
10750 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
10751                                                      EVT VT) const {
10752   VT = VT.getScalarType();
10753 
10754   if (!VT.isSimple())
10755     return false;
10756 
10757   switch (VT.getSimpleVT().SimpleTy) {
10758   case MVT::f16:
10759     return Subtarget.hasStdExtZfh();
10760   case MVT::f32:
10761     return Subtarget.hasStdExtF();
10762   case MVT::f64:
10763     return Subtarget.hasStdExtD();
10764   default:
10765     break;
10766   }
10767 
10768   return false;
10769 }
10770 
10771 Register RISCVTargetLowering::getExceptionPointerRegister(
10772     const Constant *PersonalityFn) const {
10773   return RISCV::X10;
10774 }
10775 
10776 Register RISCVTargetLowering::getExceptionSelectorRegister(
10777     const Constant *PersonalityFn) const {
10778   return RISCV::X11;
10779 }
10780 
10781 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
10782   // Return false to suppress the unnecessary extensions if the LibCall
10783   // arguments or return value is f32 type for LP64 ABI.
10784   RISCVABI::ABI ABI = Subtarget.getTargetABI();
10785   if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
10786     return false;
10787 
10788   return true;
10789 }
10790 
10791 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
10792   if (Subtarget.is64Bit() && Type == MVT::i32)
10793     return true;
10794 
10795   return IsSigned;
10796 }
10797 
10798 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
10799                                                  SDValue C) const {
10800   // Check integral scalar types.
10801   if (VT.isScalarInteger()) {
10802     // Omit the optimization if the sub target has the M extension and the data
10803     // size exceeds XLen.
10804     if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
10805       return false;
10806     if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
10807       // Break the MUL to a SLLI and an ADD/SUB.
10808       const APInt &Imm = ConstNode->getAPIntValue();
10809       if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
10810           (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
10811         return true;
10812       // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
10813       if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
10814           ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
10815            (Imm - 8).isPowerOf2()))
10816         return true;
10817       // Omit the following optimization if the sub target has the M extension
10818       // and the data size >= XLen.
10819       if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
10820         return false;
10821       // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
10822       // a pair of LUI/ADDI.
10823       if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
10824         APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
10825         if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
10826             (1 - ImmS).isPowerOf2())
10827         return true;
10828       }
10829     }
10830   }
10831 
10832   return false;
10833 }
10834 
10835 bool RISCVTargetLowering::isMulAddWithConstProfitable(
10836     const SDValue &AddNode, const SDValue &ConstNode) const {
10837   // Let the DAGCombiner decide for vectors.
10838   EVT VT = AddNode.getValueType();
10839   if (VT.isVector())
10840     return true;
10841 
10842   // Let the DAGCombiner decide for larger types.
10843   if (VT.getScalarSizeInBits() > Subtarget.getXLen())
10844     return true;
10845 
10846   // It is worse if c1 is simm12 while c1*c2 is not.
10847   ConstantSDNode *C1Node = cast<ConstantSDNode>(AddNode.getOperand(1));
10848   ConstantSDNode *C2Node = cast<ConstantSDNode>(ConstNode);
10849   const APInt &C1 = C1Node->getAPIntValue();
10850   const APInt &C2 = C2Node->getAPIntValue();
10851   if (C1.isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
10852     return false;
10853 
10854   // Default to true and let the DAGCombiner decide.
10855   return true;
10856 }
10857 
10858 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
10859     EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10860     bool *Fast) const {
10861   if (!VT.isVector())
10862     return false;
10863 
10864   EVT ElemVT = VT.getVectorElementType();
10865   if (Alignment >= ElemVT.getStoreSize()) {
10866     if (Fast)
10867       *Fast = true;
10868     return true;
10869   }
10870 
10871   return false;
10872 }
10873 
10874 bool RISCVTargetLowering::splitValueIntoRegisterParts(
10875     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
10876     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
10877   bool IsABIRegCopy = CC.hasValue();
10878   EVT ValueVT = Val.getValueType();
10879   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10880     // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
10881     // and cast to f32.
10882     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
10883     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
10884     Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
10885                       DAG.getConstant(0xFFFF0000, DL, MVT::i32));
10886     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
10887     Parts[0] = Val;
10888     return true;
10889   }
10890 
10891   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10892     LLVMContext &Context = *DAG.getContext();
10893     EVT ValueEltVT = ValueVT.getVectorElementType();
10894     EVT PartEltVT = PartVT.getVectorElementType();
10895     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10896     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10897     if (PartVTBitSize % ValueVTBitSize == 0) {
10898       assert(PartVTBitSize >= ValueVTBitSize);
10899       // If the element types are different, bitcast to the same element type of
10900       // PartVT first.
10901       // Give an example here, we want copy a <vscale x 1 x i8> value to
10902       // <vscale x 4 x i16>.
10903       // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
10904       // subvector, then we can bitcast to <vscale x 4 x i16>.
10905       if (ValueEltVT != PartEltVT) {
10906         if (PartVTBitSize > ValueVTBitSize) {
10907           unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10908           assert(Count != 0 && "The number of element should not be zero.");
10909           EVT SameEltTypeVT =
10910               EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10911           Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
10912                             DAG.getUNDEF(SameEltTypeVT), Val,
10913                             DAG.getVectorIdxConstant(0, DL));
10914         }
10915         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
10916       } else {
10917         Val =
10918             DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
10919                         Val, DAG.getVectorIdxConstant(0, DL));
10920       }
10921       Parts[0] = Val;
10922       return true;
10923     }
10924   }
10925   return false;
10926 }
10927 
10928 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
10929     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
10930     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
10931   bool IsABIRegCopy = CC.hasValue();
10932   if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
10933     SDValue Val = Parts[0];
10934 
10935     // Cast the f32 to i32, truncate to i16, and cast back to f16.
10936     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
10937     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
10938     Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
10939     return Val;
10940   }
10941 
10942   if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
10943     LLVMContext &Context = *DAG.getContext();
10944     SDValue Val = Parts[0];
10945     EVT ValueEltVT = ValueVT.getVectorElementType();
10946     EVT PartEltVT = PartVT.getVectorElementType();
10947     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
10948     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
10949     if (PartVTBitSize % ValueVTBitSize == 0) {
10950       assert(PartVTBitSize >= ValueVTBitSize);
10951       EVT SameEltTypeVT = ValueVT;
10952       // If the element types are different, convert it to the same element type
10953       // of PartVT.
10954       // Give an example here, we want copy a <vscale x 1 x i8> value from
10955       // <vscale x 4 x i16>.
10956       // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
10957       // then we can extract <vscale x 1 x i8>.
10958       if (ValueEltVT != PartEltVT) {
10959         unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
10960         assert(Count != 0 && "The number of element should not be zero.");
10961         SameEltTypeVT =
10962             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
10963         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
10964       }
10965       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
10966                         DAG.getVectorIdxConstant(0, DL));
10967       return Val;
10968     }
10969   }
10970   return SDValue();
10971 }
10972 
10973 SDValue
10974 RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
10975                                    SelectionDAG &DAG,
10976                                    SmallVectorImpl<SDNode *> &Created) const {
10977   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
10978   if (isIntDivCheap(N->getValueType(0), Attr))
10979     return SDValue(N, 0); // Lower SDIV as SDIV
10980 
10981   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
10982          "Unexpected divisor!");
10983 
10984   // Conditional move is needed, so do the transformation iff Zbt is enabled.
10985   if (!Subtarget.hasStdExtZbt())
10986     return SDValue();
10987 
10988   // When |Divisor| >= 2 ^ 12, it isn't profitable to do such transformation.
10989   // Besides, more critical path instructions will be generated when dividing
10990   // by 2. So we keep using the original DAGs for these cases.
10991   unsigned Lg2 = Divisor.countTrailingZeros();
10992   if (Lg2 == 1 || Lg2 >= 12)
10993     return SDValue();
10994 
10995   // fold (sdiv X, pow2)
10996   EVT VT = N->getValueType(0);
10997   if (VT != MVT::i32 && !(Subtarget.is64Bit() && VT == MVT::i64))
10998     return SDValue();
10999 
11000   SDLoc DL(N);
11001   SDValue N0 = N->getOperand(0);
11002   SDValue Zero = DAG.getConstant(0, DL, VT);
11003   SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
11004 
11005   // Add (N0 < 0) ? Pow2 - 1 : 0;
11006   SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT);
11007   SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
11008   SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
11009 
11010   Created.push_back(Cmp.getNode());
11011   Created.push_back(Add.getNode());
11012   Created.push_back(Sel.getNode());
11013 
11014   // Divide by pow2.
11015   SDValue SRA =
11016       DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT));
11017 
11018   // If we're dividing by a positive value, we're done.  Otherwise, we must
11019   // negate the result.
11020   if (Divisor.isNonNegative())
11021     return SRA;
11022 
11023   Created.push_back(SRA.getNode());
11024   return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
11025 }
11026 
11027 #define GET_REGISTER_MATCHER
11028 #include "RISCVGenAsmMatcher.inc"
11029 
11030 Register
11031 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
11032                                        const MachineFunction &MF) const {
11033   Register Reg = MatchRegisterAltName(RegName);
11034   if (Reg == RISCV::NoRegister)
11035     Reg = MatchRegisterName(RegName);
11036   if (Reg == RISCV::NoRegister)
11037     report_fatal_error(
11038         Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
11039   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
11040   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
11041     report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
11042                              StringRef(RegName) + "\"."));
11043   return Reg;
11044 }
11045 
11046 namespace llvm {
11047 namespace RISCVVIntrinsicsTable {
11048 
11049 #define GET_RISCVVIntrinsicsTable_IMPL
11050 #include "RISCVGenSearchableTables.inc"
11051 
11052 } // namespace RISCVVIntrinsicsTable
11053 
11054 } // namespace llvm
11055