xref: /freebsd/contrib/llvm-project/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp (revision 0d8fe2373503aeac48492f28073049a8bfa4feb5)
1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "llvm/CodeGen/MachineFrameInfo.h"
17 #include "llvm/IR/IntrinsicsRISCV.h"
18 #include "llvm/Support/Alignment.h"
19 #include "llvm/Support/Debug.h"
20 #include "llvm/Support/MathExtras.h"
21 #include "llvm/Support/raw_ostream.h"
22 
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "riscv-isel"
26 
27 void RISCVDAGToDAGISel::PostprocessISelDAG() {
28   doPeepholeLoadStoreADDI();
29 }
30 
31 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
32                          MVT XLenVT) {
33   RISCVMatInt::InstSeq Seq;
34   RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
35 
36   SDNode *Result = nullptr;
37   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
38   for (RISCVMatInt::Inst &Inst : Seq) {
39     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
40     if (Inst.Opc == RISCV::LUI)
41       Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
42     else
43       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
44 
45     // Only the first instruction has X0 as its source.
46     SrcReg = SDValue(Result, 0);
47   }
48 
49   return Result;
50 }
51 
52 static RISCVVLMUL getLMUL(EVT VT) {
53   switch (VT.getSizeInBits().getKnownMinValue() / 8) {
54   default:
55     llvm_unreachable("Invalid LMUL.");
56   case 1:
57     return RISCVVLMUL::LMUL_F8;
58   case 2:
59     return RISCVVLMUL::LMUL_F4;
60   case 4:
61     return RISCVVLMUL::LMUL_F2;
62   case 8:
63     return RISCVVLMUL::LMUL_1;
64   case 16:
65     return RISCVVLMUL::LMUL_2;
66   case 32:
67     return RISCVVLMUL::LMUL_4;
68   case 64:
69     return RISCVVLMUL::LMUL_8;
70   }
71 }
72 
73 static unsigned getSubregIndexByEVT(EVT VT, unsigned Index) {
74   RISCVVLMUL LMUL = getLMUL(VT);
75   if (LMUL == RISCVVLMUL::LMUL_F8 || LMUL == RISCVVLMUL::LMUL_F4 ||
76       LMUL == RISCVVLMUL::LMUL_F2 || LMUL == RISCVVLMUL::LMUL_1) {
77     static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
78                   "Unexpected subreg numbering");
79     return RISCV::sub_vrm1_0 + Index;
80   } else if (LMUL == RISCVVLMUL::LMUL_2) {
81     static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
82                   "Unexpected subreg numbering");
83     return RISCV::sub_vrm2_0 + Index;
84   } else if (LMUL == RISCVVLMUL::LMUL_4) {
85     static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
86                   "Unexpected subreg numbering");
87     return RISCV::sub_vrm4_0 + Index;
88   }
89   llvm_unreachable("Invalid vector type.");
90 }
91 
92 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
93                                unsigned RegClassID, unsigned SubReg0) {
94   assert(Regs.size() >= 2 && Regs.size() <= 8);
95 
96   SDLoc DL(Regs[0]);
97   SmallVector<SDValue, 8> Ops;
98 
99   Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
100 
101   for (unsigned I = 0; I < Regs.size(); ++I) {
102     Ops.push_back(Regs[I]);
103     Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
104   }
105   SDNode *N =
106       CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
107   return SDValue(N, 0);
108 }
109 
110 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
111                              unsigned NF) {
112   static const unsigned RegClassIDs[] = {
113       RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
114       RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
115       RISCV::VRN8M1RegClassID};
116 
117   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
118 }
119 
120 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
121                              unsigned NF) {
122   static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
123                                          RISCV::VRN3M2RegClassID,
124                                          RISCV::VRN4M2RegClassID};
125 
126   return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
127 }
128 
129 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
130                              unsigned NF) {
131   return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
132                          RISCV::sub_vrm4_0);
133 }
134 
135 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
136                            unsigned NF, RISCVVLMUL LMUL) {
137   switch (LMUL) {
138   default:
139     llvm_unreachable("Invalid LMUL.");
140   case RISCVVLMUL::LMUL_F8:
141   case RISCVVLMUL::LMUL_F4:
142   case RISCVVLMUL::LMUL_F2:
143   case RISCVVLMUL::LMUL_1:
144     return createM1Tuple(CurDAG, Regs, NF);
145   case RISCVVLMUL::LMUL_2:
146     return createM2Tuple(CurDAG, Regs, NF);
147   case RISCVVLMUL::LMUL_4:
148     return createM4Tuple(CurDAG, Regs, NF);
149   }
150 }
151 
152 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned IntNo,
153                                     bool IsStrided) {
154   SDLoc DL(Node);
155   unsigned NF = Node->getNumValues() - 1;
156   EVT VT = Node->getValueType(0);
157   unsigned ScalarSize = VT.getScalarSizeInBits();
158   MVT XLenVT = Subtarget->getXLenVT();
159   RISCVVLMUL LMUL = getLMUL(VT);
160   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
161   SmallVector<SDValue, 5> Operands;
162   Operands.push_back(Node->getOperand(2)); // Base pointer.
163   if (IsStrided) {
164     Operands.push_back(Node->getOperand(3)); // Stride.
165     Operands.push_back(Node->getOperand(4)); // VL.
166   } else {
167     Operands.push_back(Node->getOperand(3)); // VL.
168   }
169   Operands.push_back(SEW);
170   Operands.push_back(Node->getOperand(0)); // Chain.
171   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
172       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
173       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
174   SDNode *Load =
175       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
176   SDValue SuperReg = SDValue(Load, 0);
177   for (unsigned I = 0; I < NF; ++I)
178     ReplaceUses(SDValue(Node, I),
179                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
180                                                VT, SuperReg));
181 
182   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
183   CurDAG->RemoveDeadNode(Node);
184 }
185 
186 void RISCVDAGToDAGISel::selectVLSEGMask(SDNode *Node, unsigned IntNo,
187                                         bool IsStrided) {
188   SDLoc DL(Node);
189   unsigned NF = Node->getNumValues() - 1;
190   EVT VT = Node->getValueType(0);
191   unsigned ScalarSize = VT.getScalarSizeInBits();
192   MVT XLenVT = Subtarget->getXLenVT();
193   RISCVVLMUL LMUL = getLMUL(VT);
194   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
195   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
196   SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
197   SmallVector<SDValue, 7> Operands;
198   Operands.push_back(MaskedOff);
199   Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
200   if (IsStrided) {
201     Operands.push_back(Node->getOperand(NF + 3)); // Stride.
202     Operands.push_back(Node->getOperand(NF + 4)); // Mask.
203     Operands.push_back(Node->getOperand(NF + 5)); // VL.
204   } else {
205     Operands.push_back(Node->getOperand(NF + 3)); // Mask.
206     Operands.push_back(Node->getOperand(NF + 4)); // VL.
207   }
208   Operands.push_back(SEW);
209   Operands.push_back(Node->getOperand(0)); /// Chain.
210   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
211       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
212       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
213   SDNode *Load =
214       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
215   SDValue SuperReg = SDValue(Load, 0);
216   for (unsigned I = 0; I < NF; ++I)
217     ReplaceUses(SDValue(Node, I),
218                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
219                                                VT, SuperReg));
220 
221   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
222   CurDAG->RemoveDeadNode(Node);
223 }
224 
225 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
226   SDLoc DL(Node);
227   unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
228   unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
229   EVT VT = Node->getValueType(0);
230   unsigned ScalarSize = VT.getScalarSizeInBits();
231   MVT XLenVT = Subtarget->getXLenVT();
232   RISCVVLMUL LMUL = getLMUL(VT);
233   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
234   SmallVector<SDValue, 5> Operands;
235   Operands.push_back(Node->getOperand(2)); // Base pointer.
236   Operands.push_back(Node->getOperand(3)); // VL.
237   Operands.push_back(SEW);
238   Operands.push_back(Node->getOperand(0)); // Chain.
239   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
240       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
241       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
242   SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
243                                         MVT::Glue, Operands);
244   SDValue SuperReg = SDValue(Load, 0);
245   for (unsigned I = 0; I < NF; ++I)
246     ReplaceUses(SDValue(Node, I),
247                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
248                                                VT, SuperReg));
249 
250   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));     // Chain.
251   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
252   CurDAG->RemoveDeadNode(Node);
253 }
254 
255 void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
256   SDLoc DL(Node);
257   unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
258   unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
259   EVT VT = Node->getValueType(0);
260   unsigned ScalarSize = VT.getScalarSizeInBits();
261   MVT XLenVT = Subtarget->getXLenVT();
262   RISCVVLMUL LMUL = getLMUL(VT);
263   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
264   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
265   SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
266   SmallVector<SDValue, 7> Operands;
267   Operands.push_back(MaskedOff);
268   Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
269   Operands.push_back(Node->getOperand(NF + 3)); // Mask.
270   Operands.push_back(Node->getOperand(NF + 4)); // VL.
271   Operands.push_back(SEW);
272   Operands.push_back(Node->getOperand(0)); /// Chain.
273   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
274       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
275       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
276   SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
277                                         MVT::Glue, Operands);
278   SDValue SuperReg = SDValue(Load, 0);
279   for (unsigned I = 0; I < NF; ++I)
280     ReplaceUses(SDValue(Node, I),
281                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
282                                                VT, SuperReg));
283 
284   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));     // Chain.
285   ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
286   CurDAG->RemoveDeadNode(Node);
287 }
288 
289 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo) {
290   SDLoc DL(Node);
291   unsigned NF = Node->getNumValues() - 1;
292   EVT VT = Node->getValueType(0);
293   unsigned ScalarSize = VT.getScalarSizeInBits();
294   MVT XLenVT = Subtarget->getXLenVT();
295   RISCVVLMUL LMUL = getLMUL(VT);
296   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
297   SDValue Operands[] = {
298       Node->getOperand(2),     // Base pointer.
299       Node->getOperand(3),     // Index.
300       Node->getOperand(4),     // VL.
301       SEW, Node->getOperand(0) // Chain.
302   };
303 
304   EVT IndexVT = Node->getOperand(3)->getValueType(0);
305   RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
306   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
307   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
308       IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
309       static_cast<unsigned>(IndexLMUL));
310   SDNode *Load =
311       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
312   SDValue SuperReg = SDValue(Load, 0);
313   for (unsigned I = 0; I < NF; ++I)
314     ReplaceUses(SDValue(Node, I),
315                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
316                                                VT, SuperReg));
317 
318   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
319   CurDAG->RemoveDeadNode(Node);
320 }
321 
322 void RISCVDAGToDAGISel::selectVLXSEGMask(SDNode *Node, unsigned IntNo) {
323   SDLoc DL(Node);
324   unsigned NF = Node->getNumValues() - 1;
325   EVT VT = Node->getValueType(0);
326   unsigned ScalarSize = VT.getScalarSizeInBits();
327   MVT XLenVT = Subtarget->getXLenVT();
328   RISCVVLMUL LMUL = getLMUL(VT);
329   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
330   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
331   SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
332   SDValue Operands[] = {
333       MaskedOff,
334       Node->getOperand(NF + 2), // Base pointer.
335       Node->getOperand(NF + 3), // Index.
336       Node->getOperand(NF + 4), // Mask.
337       Node->getOperand(NF + 5), // VL.
338       SEW,
339       Node->getOperand(0) // Chain.
340   };
341 
342   EVT IndexVT = Node->getOperand(NF + 3)->getValueType(0);
343   RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
344   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
345   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
346       IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
347       static_cast<unsigned>(IndexLMUL));
348   SDNode *Load =
349       CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
350   SDValue SuperReg = SDValue(Load, 0);
351   for (unsigned I = 0; I < NF; ++I)
352     ReplaceUses(SDValue(Node, I),
353                 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
354                                                VT, SuperReg));
355 
356   ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
357   CurDAG->RemoveDeadNode(Node);
358 }
359 
360 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned IntNo,
361                                     bool IsStrided) {
362   SDLoc DL(Node);
363   unsigned NF = Node->getNumOperands() - 4;
364   if (IsStrided)
365     NF--;
366   EVT VT = Node->getOperand(2)->getValueType(0);
367   unsigned ScalarSize = VT.getScalarSizeInBits();
368   MVT XLenVT = Subtarget->getXLenVT();
369   RISCVVLMUL LMUL = getLMUL(VT);
370   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
371   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
372   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
373   SmallVector<SDValue, 6> Operands;
374   Operands.push_back(StoreVal);
375   Operands.push_back(Node->getOperand(2 + NF)); // Base pointer.
376   if (IsStrided) {
377     Operands.push_back(Node->getOperand(3 + NF)); // Stride.
378     Operands.push_back(Node->getOperand(4 + NF)); // VL.
379   } else {
380     Operands.push_back(Node->getOperand(3 + NF)); // VL.
381   }
382   Operands.push_back(SEW);
383   Operands.push_back(Node->getOperand(0)); // Chain.
384   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
385       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
386       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
387   SDNode *Store =
388       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
389   ReplaceNode(Node, Store);
390 }
391 
392 void RISCVDAGToDAGISel::selectVSSEGMask(SDNode *Node, unsigned IntNo,
393                                         bool IsStrided) {
394   SDLoc DL(Node);
395   unsigned NF = Node->getNumOperands() - 5;
396   if (IsStrided)
397     NF--;
398   EVT VT = Node->getOperand(2)->getValueType(0);
399   unsigned ScalarSize = VT.getScalarSizeInBits();
400   MVT XLenVT = Subtarget->getXLenVT();
401   RISCVVLMUL LMUL = getLMUL(VT);
402   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
403   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
404   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
405   SmallVector<SDValue, 7> Operands;
406   Operands.push_back(StoreVal);
407   Operands.push_back(Node->getOperand(2 + NF)); // Base pointer.
408   if (IsStrided) {
409     Operands.push_back(Node->getOperand(3 + NF)); // Stride.
410     Operands.push_back(Node->getOperand(4 + NF)); // Mask.
411     Operands.push_back(Node->getOperand(5 + NF)); // VL.
412   } else {
413     Operands.push_back(Node->getOperand(3 + NF)); // Mask.
414     Operands.push_back(Node->getOperand(4 + NF)); // VL.
415   }
416   Operands.push_back(SEW);
417   Operands.push_back(Node->getOperand(0)); // Chain.
418   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
419       IntNo, ScalarSize, static_cast<unsigned>(LMUL),
420       static_cast<unsigned>(RISCVVLMUL::LMUL_1));
421   SDNode *Store =
422       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
423   ReplaceNode(Node, Store);
424 }
425 
426 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned IntNo) {
427   SDLoc DL(Node);
428   unsigned NF = Node->getNumOperands() - 5;
429   EVT VT = Node->getOperand(2)->getValueType(0);
430   unsigned ScalarSize = VT.getScalarSizeInBits();
431   MVT XLenVT = Subtarget->getXLenVT();
432   RISCVVLMUL LMUL = getLMUL(VT);
433   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
434   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
435   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
436   SDValue Operands[] = {
437       StoreVal,
438       Node->getOperand(2 + NF), // Base pointer.
439       Node->getOperand(3 + NF), // Index.
440       Node->getOperand(4 + NF), // VL.
441       SEW,
442       Node->getOperand(0) // Chain.
443   };
444 
445   EVT IndexVT = Node->getOperand(3 + NF)->getValueType(0);
446   RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
447   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
448   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
449       IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
450       static_cast<unsigned>(IndexLMUL));
451   SDNode *Store =
452       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
453   ReplaceNode(Node, Store);
454 }
455 
456 void RISCVDAGToDAGISel::selectVSXSEGMask(SDNode *Node, unsigned IntNo) {
457   SDLoc DL(Node);
458   unsigned NF = Node->getNumOperands() - 6;
459   EVT VT = Node->getOperand(2)->getValueType(0);
460   unsigned ScalarSize = VT.getScalarSizeInBits();
461   MVT XLenVT = Subtarget->getXLenVT();
462   RISCVVLMUL LMUL = getLMUL(VT);
463   SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
464   SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
465   SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
466   SDValue Operands[] = {
467       StoreVal,
468       Node->getOperand(2 + NF), // Base pointer.
469       Node->getOperand(3 + NF), // Index.
470       Node->getOperand(4 + NF), // Mask.
471       Node->getOperand(5 + NF), // VL.
472       SEW,
473       Node->getOperand(0) // Chain.
474   };
475 
476   EVT IndexVT = Node->getOperand(3 + NF)->getValueType(0);
477   RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
478   unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
479   const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
480       IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
481       static_cast<unsigned>(IndexLMUL));
482   SDNode *Store =
483       CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
484   ReplaceNode(Node, Store);
485 }
486 
487 void RISCVDAGToDAGISel::Select(SDNode *Node) {
488   // If we have a custom node, we have already selected.
489   if (Node->isMachineOpcode()) {
490     LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
491     Node->setNodeId(-1);
492     return;
493   }
494 
495   // Instruction Selection not handled by the auto-generated tablegen selection
496   // should be handled here.
497   unsigned Opcode = Node->getOpcode();
498   MVT XLenVT = Subtarget->getXLenVT();
499   SDLoc DL(Node);
500   EVT VT = Node->getValueType(0);
501 
502   switch (Opcode) {
503   case ISD::ADD: {
504     // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
505     // immediate must be in specific ranges and have a single use.
506     if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
507       if (!(ConstOp->hasOneUse()))
508         break;
509       // The imm must be in range [-4096,-2049] or [2048,4094].
510       int64_t Imm = ConstOp->getSExtValue();
511       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
512         break;
513       // Break the imm to imm0+imm1.
514       EVT VT = Node->getValueType(0);
515       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
516       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
517       auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
518                                                Node->getOperand(0), ImmOp0);
519       auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
520                                                SDValue(NodeAddi0, 0), ImmOp1);
521       ReplaceNode(Node, NodeAddi1);
522       return;
523     }
524     break;
525   }
526   case ISD::Constant: {
527     auto ConstNode = cast<ConstantSDNode>(Node);
528     if (VT == XLenVT && ConstNode->isNullValue()) {
529       SDValue New =
530           CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
531       ReplaceNode(Node, New.getNode());
532       return;
533     }
534     int64_t Imm = ConstNode->getSExtValue();
535     if (XLenVT == MVT::i64) {
536       ReplaceNode(Node, selectImm(CurDAG, DL, Imm, XLenVT));
537       return;
538     }
539     break;
540   }
541   case ISD::FrameIndex: {
542     SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
543     int FI = cast<FrameIndexSDNode>(Node)->getIndex();
544     SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
545     ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
546     return;
547   }
548   case ISD::INTRINSIC_W_CHAIN: {
549     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
550     switch (IntNo) {
551       // By default we do not custom select any intrinsic.
552     default:
553       break;
554 
555     case Intrinsic::riscv_vsetvli: {
556       if (!Subtarget->hasStdExtV())
557         break;
558 
559       assert(Node->getNumOperands() == 5);
560 
561       RISCVVSEW VSEW =
562           static_cast<RISCVVSEW>(Node->getConstantOperandVal(3) & 0x7);
563       RISCVVLMUL VLMul =
564           static_cast<RISCVVLMUL>(Node->getConstantOperandVal(4) & 0x7);
565 
566       unsigned VTypeI = RISCVVType::encodeVTYPE(
567           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
568       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
569 
570       SDValue VLOperand = Node->getOperand(2);
571       if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
572         uint64_t AVL = C->getZExtValue();
573         if (isUInt<5>(AVL)) {
574           SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
575           ReplaceNode(Node,
576                       CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
577                                              MVT::Other, VLImm, VTypeIOp,
578                                              /* Chain */ Node->getOperand(0)));
579           return;
580         }
581       }
582 
583       ReplaceNode(Node,
584                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
585                                          MVT::Other, VLOperand, VTypeIOp,
586                                          /* Chain */ Node->getOperand(0)));
587       return;
588     }
589     case Intrinsic::riscv_vsetvlimax: {
590       if (!Subtarget->hasStdExtV())
591         break;
592 
593       assert(Node->getNumOperands() == 4);
594 
595       RISCVVSEW VSEW =
596           static_cast<RISCVVSEW>(Node->getConstantOperandVal(2) & 0x7);
597       RISCVVLMUL VLMul =
598           static_cast<RISCVVLMUL>(Node->getConstantOperandVal(3) & 0x7);
599 
600       unsigned VTypeI = RISCVVType::encodeVTYPE(
601           VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
602       SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
603 
604       SDValue VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
605       ReplaceNode(Node,
606                   CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
607                                          MVT::Other, VLOperand, VTypeIOp,
608                                          /* Chain */ Node->getOperand(0)));
609       return;
610     }
611     case Intrinsic::riscv_vlseg2:
612     case Intrinsic::riscv_vlseg3:
613     case Intrinsic::riscv_vlseg4:
614     case Intrinsic::riscv_vlseg5:
615     case Intrinsic::riscv_vlseg6:
616     case Intrinsic::riscv_vlseg7:
617     case Intrinsic::riscv_vlseg8: {
618       selectVLSEG(Node, IntNo, /*IsStrided=*/false);
619       return;
620     }
621     case Intrinsic::riscv_vlseg2_mask:
622     case Intrinsic::riscv_vlseg3_mask:
623     case Intrinsic::riscv_vlseg4_mask:
624     case Intrinsic::riscv_vlseg5_mask:
625     case Intrinsic::riscv_vlseg6_mask:
626     case Intrinsic::riscv_vlseg7_mask:
627     case Intrinsic::riscv_vlseg8_mask: {
628       selectVLSEGMask(Node, IntNo, /*IsStrided=*/false);
629       return;
630     }
631     case Intrinsic::riscv_vlsseg2:
632     case Intrinsic::riscv_vlsseg3:
633     case Intrinsic::riscv_vlsseg4:
634     case Intrinsic::riscv_vlsseg5:
635     case Intrinsic::riscv_vlsseg6:
636     case Intrinsic::riscv_vlsseg7:
637     case Intrinsic::riscv_vlsseg8: {
638       selectVLSEG(Node, IntNo, /*IsStrided=*/true);
639       return;
640     }
641     case Intrinsic::riscv_vlsseg2_mask:
642     case Intrinsic::riscv_vlsseg3_mask:
643     case Intrinsic::riscv_vlsseg4_mask:
644     case Intrinsic::riscv_vlsseg5_mask:
645     case Intrinsic::riscv_vlsseg6_mask:
646     case Intrinsic::riscv_vlsseg7_mask:
647     case Intrinsic::riscv_vlsseg8_mask: {
648       selectVLSEGMask(Node, IntNo, /*IsStrided=*/true);
649       return;
650     }
651     case Intrinsic::riscv_vloxseg2:
652     case Intrinsic::riscv_vloxseg3:
653     case Intrinsic::riscv_vloxseg4:
654     case Intrinsic::riscv_vloxseg5:
655     case Intrinsic::riscv_vloxseg6:
656     case Intrinsic::riscv_vloxseg7:
657     case Intrinsic::riscv_vloxseg8:
658     case Intrinsic::riscv_vluxseg2:
659     case Intrinsic::riscv_vluxseg3:
660     case Intrinsic::riscv_vluxseg4:
661     case Intrinsic::riscv_vluxseg5:
662     case Intrinsic::riscv_vluxseg6:
663     case Intrinsic::riscv_vluxseg7:
664     case Intrinsic::riscv_vluxseg8: {
665       selectVLXSEG(Node, IntNo);
666       return;
667     }
668     case Intrinsic::riscv_vloxseg2_mask:
669     case Intrinsic::riscv_vloxseg3_mask:
670     case Intrinsic::riscv_vloxseg4_mask:
671     case Intrinsic::riscv_vloxseg5_mask:
672     case Intrinsic::riscv_vloxseg6_mask:
673     case Intrinsic::riscv_vloxseg7_mask:
674     case Intrinsic::riscv_vloxseg8_mask:
675     case Intrinsic::riscv_vluxseg2_mask:
676     case Intrinsic::riscv_vluxseg3_mask:
677     case Intrinsic::riscv_vluxseg4_mask:
678     case Intrinsic::riscv_vluxseg5_mask:
679     case Intrinsic::riscv_vluxseg6_mask:
680     case Intrinsic::riscv_vluxseg7_mask:
681     case Intrinsic::riscv_vluxseg8_mask: {
682       selectVLXSEGMask(Node, IntNo);
683       return;
684     }
685     }
686     break;
687   }
688   case ISD::INTRINSIC_VOID: {
689     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
690     switch (IntNo) {
691     case Intrinsic::riscv_vsseg2:
692     case Intrinsic::riscv_vsseg3:
693     case Intrinsic::riscv_vsseg4:
694     case Intrinsic::riscv_vsseg5:
695     case Intrinsic::riscv_vsseg6:
696     case Intrinsic::riscv_vsseg7:
697     case Intrinsic::riscv_vsseg8: {
698       selectVSSEG(Node, IntNo, /*IsStrided=*/false);
699       return;
700     }
701     case Intrinsic::riscv_vsseg2_mask:
702     case Intrinsic::riscv_vsseg3_mask:
703     case Intrinsic::riscv_vsseg4_mask:
704     case Intrinsic::riscv_vsseg5_mask:
705     case Intrinsic::riscv_vsseg6_mask:
706     case Intrinsic::riscv_vsseg7_mask:
707     case Intrinsic::riscv_vsseg8_mask: {
708       selectVSSEGMask(Node, IntNo, /*IsStrided=*/false);
709       return;
710     }
711     case Intrinsic::riscv_vssseg2:
712     case Intrinsic::riscv_vssseg3:
713     case Intrinsic::riscv_vssseg4:
714     case Intrinsic::riscv_vssseg5:
715     case Intrinsic::riscv_vssseg6:
716     case Intrinsic::riscv_vssseg7:
717     case Intrinsic::riscv_vssseg8: {
718       selectVSSEG(Node, IntNo, /*IsStrided=*/true);
719       return;
720     }
721     case Intrinsic::riscv_vssseg2_mask:
722     case Intrinsic::riscv_vssseg3_mask:
723     case Intrinsic::riscv_vssseg4_mask:
724     case Intrinsic::riscv_vssseg5_mask:
725     case Intrinsic::riscv_vssseg6_mask:
726     case Intrinsic::riscv_vssseg7_mask:
727     case Intrinsic::riscv_vssseg8_mask: {
728       selectVSSEGMask(Node, IntNo, /*IsStrided=*/true);
729       return;
730     }
731     case Intrinsic::riscv_vsoxseg2:
732     case Intrinsic::riscv_vsoxseg3:
733     case Intrinsic::riscv_vsoxseg4:
734     case Intrinsic::riscv_vsoxseg5:
735     case Intrinsic::riscv_vsoxseg6:
736     case Intrinsic::riscv_vsoxseg7:
737     case Intrinsic::riscv_vsoxseg8:
738     case Intrinsic::riscv_vsuxseg2:
739     case Intrinsic::riscv_vsuxseg3:
740     case Intrinsic::riscv_vsuxseg4:
741     case Intrinsic::riscv_vsuxseg5:
742     case Intrinsic::riscv_vsuxseg6:
743     case Intrinsic::riscv_vsuxseg7:
744     case Intrinsic::riscv_vsuxseg8: {
745       selectVSXSEG(Node, IntNo);
746       return;
747     }
748     case Intrinsic::riscv_vsoxseg2_mask:
749     case Intrinsic::riscv_vsoxseg3_mask:
750     case Intrinsic::riscv_vsoxseg4_mask:
751     case Intrinsic::riscv_vsoxseg5_mask:
752     case Intrinsic::riscv_vsoxseg6_mask:
753     case Intrinsic::riscv_vsoxseg7_mask:
754     case Intrinsic::riscv_vsoxseg8_mask:
755     case Intrinsic::riscv_vsuxseg2_mask:
756     case Intrinsic::riscv_vsuxseg3_mask:
757     case Intrinsic::riscv_vsuxseg4_mask:
758     case Intrinsic::riscv_vsuxseg5_mask:
759     case Intrinsic::riscv_vsuxseg6_mask:
760     case Intrinsic::riscv_vsuxseg7_mask:
761     case Intrinsic::riscv_vsuxseg8_mask: {
762       selectVSXSEGMask(Node, IntNo);
763       return;
764     }
765     }
766     break;
767   }
768   case RISCVISD::VLSEGFF: {
769     selectVLSEGFF(Node);
770     return;
771   }
772   case RISCVISD::VLSEGFF_MASK: {
773     selectVLSEGFFMask(Node);
774     return;
775   }
776   }
777 
778   // Select the default instruction.
779   SelectCode(Node);
780 }
781 
782 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
783     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
784   switch (ConstraintID) {
785   case InlineAsm::Constraint_m:
786     // We just support simple memory operands that have a single address
787     // operand and need no special handling.
788     OutOps.push_back(Op);
789     return false;
790   case InlineAsm::Constraint_A:
791     OutOps.push_back(Op);
792     return false;
793   default:
794     break;
795   }
796 
797   return true;
798 }
799 
800 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
801   if (auto FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
802     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
803     return true;
804   }
805   return false;
806 }
807 
808 // Match (srl (and val, mask), imm) where the result would be a
809 // zero-extended 32-bit integer. i.e. the mask is 0xffffffff or the result
810 // is equivalent to this (SimplifyDemandedBits may have removed lower bits
811 // from the mask that aren't necessary due to the right-shifting).
812 bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const {
813   assert(N->getOpcode() == ISD::SRL);
814   assert(N->getOperand(0).getOpcode() == ISD::AND);
815   assert(isa<ConstantSDNode>(N->getOperand(1)));
816   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
817 
818   // The IsRV64 predicate is checked after PatFrag predicates so we can get
819   // here even on RV32.
820   if (!Subtarget->is64Bit())
821     return false;
822 
823   SDValue And = N->getOperand(0);
824   uint64_t ShAmt = N->getConstantOperandVal(1);
825   uint64_t Mask = And.getConstantOperandVal(1);
826   return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff;
827 }
828 
829 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
830 // on RV64).
831 // SLLIUW is the same as SLLI except for the fact that it clears the bits
832 // XLEN-1:32 of the input RS1 before shifting.
833 // A PatFrag has already checked that it has the right structure:
834 //
835 //  (AND (SHL RS1, VC2), VC1)
836 //
837 // We check that VC2, the shamt is less than 32, otherwise the pattern is
838 // exactly the same as SLLI and we give priority to that.
839 // Eventually we check that VC1, the mask used to clear the upper 32 bits
840 // of RS1, is correct:
841 //
842 //  VC1 == (0xFFFFFFFF << VC2)
843 //
844 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
845   assert(N->getOpcode() == ISD::AND);
846   assert(N->getOperand(0).getOpcode() == ISD::SHL);
847   assert(isa<ConstantSDNode>(N->getOperand(1)));
848   assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
849 
850   // The IsRV64 predicate is checked after PatFrag predicates so we can get
851   // here even on RV32.
852   if (!Subtarget->is64Bit())
853     return false;
854 
855   SDValue Shl = N->getOperand(0);
856   uint64_t VC1 = N->getConstantOperandVal(1);
857   uint64_t VC2 = Shl.getConstantOperandVal(1);
858 
859   // Immediate range should be enforced by uimm5 predicate.
860   assert(VC2 < 32 && "Unexpected immediate");
861   return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
862 }
863 
864 // X0 has special meaning for vsetvl/vsetvli.
865 //  rd | rs1 |   AVL value | Effect on vl
866 //--------------------------------------------------------------
867 // !X0 |  X0 |       VLMAX | Set vl to VLMAX
868 //  X0 |  X0 | Value in vl | Keep current vl, just change vtype.
869 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
870   // If the VL value is a constant 0, manually select it to an ADDI with 0
871   // immediate to prevent the default selection path from matching it to X0.
872   auto *C = dyn_cast<ConstantSDNode>(N);
873   if (C && C->isNullValue())
874     VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
875   else
876     VL = N;
877 
878   return true;
879 }
880 
881 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
882   if (N.getOpcode() != ISD::SPLAT_VECTOR &&
883       N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64)
884     return false;
885   SplatVal = N.getOperand(0);
886   return true;
887 }
888 
889 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
890   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
891        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64) ||
892       !isa<ConstantSDNode>(N.getOperand(0)))
893     return false;
894 
895   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
896 
897   // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
898   // the operand type is wider than the resulting vector element type: an
899   // implicit truncation first takes place. Therefore, perform a manual
900   // truncation/sign-extension in order to ignore any truncated bits and catch
901   // any zero-extended immediate.
902   // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
903   // sign-extending to (XLenVT -1).
904   auto XLenVT = Subtarget->getXLenVT();
905   assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
906          "Unexpected splat operand type");
907   auto EltVT = N.getValueType().getVectorElementType();
908   if (EltVT.bitsLT(XLenVT)) {
909     SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
910   }
911 
912   if (!isInt<5>(SplatImm))
913     return false;
914 
915   SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
916   return true;
917 }
918 
919 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
920   if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
921        N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64) ||
922       !isa<ConstantSDNode>(N.getOperand(0)))
923     return false;
924 
925   int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
926 
927   if (!isUInt<5>(SplatImm))
928     return false;
929 
930   SplatVal =
931       CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
932 
933   return true;
934 }
935 
936 // Merge an ADDI into the offset of a load/store instruction where possible.
937 // (load (addi base, off1), off2) -> (load base, off1+off2)
938 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
939 // This is possible when off1+off2 fits a 12-bit immediate.
940 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
941   SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
942   ++Position;
943 
944   while (Position != CurDAG->allnodes_begin()) {
945     SDNode *N = &*--Position;
946     // Skip dead nodes and any non-machine opcodes.
947     if (N->use_empty() || !N->isMachineOpcode())
948       continue;
949 
950     int OffsetOpIdx;
951     int BaseOpIdx;
952 
953     // Only attempt this optimisation for I-type loads and S-type stores.
954     switch (N->getMachineOpcode()) {
955     default:
956       continue;
957     case RISCV::LB:
958     case RISCV::LH:
959     case RISCV::LW:
960     case RISCV::LBU:
961     case RISCV::LHU:
962     case RISCV::LWU:
963     case RISCV::LD:
964     case RISCV::FLH:
965     case RISCV::FLW:
966     case RISCV::FLD:
967       BaseOpIdx = 0;
968       OffsetOpIdx = 1;
969       break;
970     case RISCV::SB:
971     case RISCV::SH:
972     case RISCV::SW:
973     case RISCV::SD:
974     case RISCV::FSH:
975     case RISCV::FSW:
976     case RISCV::FSD:
977       BaseOpIdx = 1;
978       OffsetOpIdx = 2;
979       break;
980     }
981 
982     if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
983       continue;
984 
985     SDValue Base = N->getOperand(BaseOpIdx);
986 
987     // If the base is an ADDI, we can merge it in to the load/store.
988     if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
989       continue;
990 
991     SDValue ImmOperand = Base.getOperand(1);
992     uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
993 
994     if (auto Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
995       int64_t Offset1 = Const->getSExtValue();
996       int64_t CombinedOffset = Offset1 + Offset2;
997       if (!isInt<12>(CombinedOffset))
998         continue;
999       ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1000                                              ImmOperand.getValueType());
1001     } else if (auto GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1002       // If the off1 in (addi base, off1) is a global variable's address (its
1003       // low part, really), then we can rely on the alignment of that variable
1004       // to provide a margin of safety before off1 can overflow the 12 bits.
1005       // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1006       const DataLayout &DL = CurDAG->getDataLayout();
1007       Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1008       if (Offset2 != 0 && Alignment <= Offset2)
1009         continue;
1010       int64_t Offset1 = GA->getOffset();
1011       int64_t CombinedOffset = Offset1 + Offset2;
1012       ImmOperand = CurDAG->getTargetGlobalAddress(
1013           GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1014           CombinedOffset, GA->getTargetFlags());
1015     } else if (auto CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1016       // Ditto.
1017       Align Alignment = CP->getAlign();
1018       if (Offset2 != 0 && Alignment <= Offset2)
1019         continue;
1020       int64_t Offset1 = CP->getOffset();
1021       int64_t CombinedOffset = Offset1 + Offset2;
1022       ImmOperand = CurDAG->getTargetConstantPool(
1023           CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1024           CombinedOffset, CP->getTargetFlags());
1025     } else {
1026       continue;
1027     }
1028 
1029     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
1030     LLVM_DEBUG(Base->dump(CurDAG));
1031     LLVM_DEBUG(dbgs() << "\nN: ");
1032     LLVM_DEBUG(N->dump(CurDAG));
1033     LLVM_DEBUG(dbgs() << "\n");
1034 
1035     // Modify the offset operand of the load/store.
1036     if (BaseOpIdx == 0) // Load
1037       CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1038                                  N->getOperand(2));
1039     else // Store
1040       CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1041                                  ImmOperand, N->getOperand(3));
1042 
1043     // The add-immediate may now be dead, in which case remove it.
1044     if (Base.getNode()->use_empty())
1045       CurDAG->RemoveDeadNode(Base.getNode());
1046   }
1047 }
1048 
1049 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1050 // for instruction scheduling.
1051 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1052   return new RISCVDAGToDAGISel(TM);
1053 }
1054