xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp (revision a90b9d0159070121c221b966469c3e36d912bf82)
1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPUISelDAGToDAG.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUInstrInfo.h"
17 #include "AMDGPUSubtarget.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "MCTargetDesc/R600MCTargetDesc.h"
21 #include "R600RegisterInfo.h"
22 #include "SIISelLowering.h"
23 #include "SIMachineFunctionInfo.h"
24 #include "llvm/Analysis/UniformityAnalysis.h"
25 #include "llvm/Analysis/ValueTracking.h"
26 #include "llvm/CodeGen/FunctionLoweringInfo.h"
27 #include "llvm/CodeGen/SelectionDAG.h"
28 #include "llvm/CodeGen/SelectionDAGISel.h"
29 #include "llvm/CodeGen/SelectionDAGNodes.h"
30 #include "llvm/IR/IntrinsicsAMDGPU.h"
31 #include "llvm/InitializePasses.h"
32 #include "llvm/Support/ErrorHandling.h"
33 
34 #ifdef EXPENSIVE_CHECKS
35 #include "llvm/Analysis/LoopInfo.h"
36 #include "llvm/IR/Dominators.h"
37 #endif
38 
39 #define DEBUG_TYPE "amdgpu-isel"
40 
41 using namespace llvm;
42 
43 //===----------------------------------------------------------------------===//
44 // Instruction Selector Implementation
45 //===----------------------------------------------------------------------===//
46 
47 namespace {
48 static SDValue stripBitcast(SDValue Val) {
49   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
50 }
51 
52 // Figure out if this is really an extract of the high 16-bits of a dword.
53 static bool isExtractHiElt(SDValue In, SDValue &Out) {
54   In = stripBitcast(In);
55 
56   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
57     if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
58       if (!Idx->isOne())
59         return false;
60       Out = In.getOperand(0);
61       return true;
62     }
63   }
64 
65   if (In.getOpcode() != ISD::TRUNCATE)
66     return false;
67 
68   SDValue Srl = In.getOperand(0);
69   if (Srl.getOpcode() == ISD::SRL) {
70     if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
71       if (ShiftAmt->getZExtValue() == 16) {
72         Out = stripBitcast(Srl.getOperand(0));
73         return true;
74       }
75     }
76   }
77 
78   return false;
79 }
80 
81 // Look through operations that obscure just looking at the low 16-bits of the
82 // same register.
83 static SDValue stripExtractLoElt(SDValue In) {
84   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
85     SDValue Idx = In.getOperand(1);
86     if (isNullConstant(Idx) && In.getValueSizeInBits() <= 32)
87       return In.getOperand(0);
88   }
89 
90   if (In.getOpcode() == ISD::TRUNCATE) {
91     SDValue Src = In.getOperand(0);
92     if (Src.getValueType().getSizeInBits() == 32)
93       return stripBitcast(Src);
94   }
95 
96   return In;
97 }
98 
99 } // end anonymous namespace
100 
101 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
102                       "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
103 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
104 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
105 INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
106 #ifdef EXPENSIVE_CHECKS
107 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
108 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
109 #endif
110 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
111                     "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
112 
113 /// This pass converts a legalized DAG into a AMDGPU-specific
114 // DAG, ready for instruction scheduling.
115 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM,
116                                         CodeGenOptLevel OptLevel) {
117   return new AMDGPUDAGToDAGISel(TM, OptLevel);
118 }
119 
120 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM,
121                                        CodeGenOptLevel OptLevel)
122     : SelectionDAGISel(ID, TM, OptLevel) {
123   EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
124 }
125 
126 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
127 #ifdef EXPENSIVE_CHECKS
128   DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
129   LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
130   for (auto &L : LI->getLoopsInPreorder()) {
131     assert(L->isLCSSAForm(DT));
132   }
133 #endif
134   Subtarget = &MF.getSubtarget<GCNSubtarget>();
135   Mode = SIModeRegisterDefaults(MF.getFunction(), *Subtarget);
136   return SelectionDAGISel::runOnMachineFunction(MF);
137 }
138 
139 bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(unsigned Opc) const {
140   // XXX - only need to list legal operations.
141   switch (Opc) {
142   case ISD::FADD:
143   case ISD::FSUB:
144   case ISD::FMUL:
145   case ISD::FDIV:
146   case ISD::FREM:
147   case ISD::FCANONICALIZE:
148   case ISD::UINT_TO_FP:
149   case ISD::SINT_TO_FP:
150   case ISD::FABS:
151     // Fabs is lowered to a bit operation, but it's an and which will clear the
152     // high bits anyway.
153   case ISD::FSQRT:
154   case ISD::FSIN:
155   case ISD::FCOS:
156   case ISD::FPOWI:
157   case ISD::FPOW:
158   case ISD::FLOG:
159   case ISD::FLOG2:
160   case ISD::FLOG10:
161   case ISD::FEXP:
162   case ISD::FEXP2:
163   case ISD::FCEIL:
164   case ISD::FTRUNC:
165   case ISD::FRINT:
166   case ISD::FNEARBYINT:
167   case ISD::FROUNDEVEN:
168   case ISD::FROUND:
169   case ISD::FFLOOR:
170   case ISD::FMINNUM:
171   case ISD::FMAXNUM:
172   case ISD::FLDEXP:
173   case AMDGPUISD::FRACT:
174   case AMDGPUISD::CLAMP:
175   case AMDGPUISD::COS_HW:
176   case AMDGPUISD::SIN_HW:
177   case AMDGPUISD::FMIN3:
178   case AMDGPUISD::FMAX3:
179   case AMDGPUISD::FMED3:
180   case AMDGPUISD::FMAD_FTZ:
181   case AMDGPUISD::RCP:
182   case AMDGPUISD::RSQ:
183   case AMDGPUISD::RCP_IFLAG:
184     // On gfx10, all 16-bit instructions preserve the high bits.
185     return Subtarget->getGeneration() <= AMDGPUSubtarget::GFX9;
186   case ISD::FP_ROUND:
187     // We may select fptrunc (fma/mad) to mad_mixlo, which does not zero the
188     // high bits on gfx9.
189     // TODO: If we had the source node we could see if the source was fma/mad
190     return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
191   case ISD::FMA:
192   case ISD::FMAD:
193   case AMDGPUISD::DIV_FIXUP:
194     return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
195   default:
196     // fcopysign, select and others may be lowered to 32-bit bit operations
197     // which don't zero the high bits.
198     return false;
199   }
200 }
201 
202 void AMDGPUDAGToDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
203   AU.addRequired<AMDGPUArgumentUsageInfo>();
204   AU.addRequired<UniformityInfoWrapperPass>();
205 #ifdef EXPENSIVE_CHECKS
206   AU.addRequired<DominatorTreeWrapperPass>();
207   AU.addRequired<LoopInfoWrapperPass>();
208 #endif
209   SelectionDAGISel::getAnalysisUsage(AU);
210 }
211 
212 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
213   assert(Subtarget->d16PreservesUnusedBits());
214   MVT VT = N->getValueType(0).getSimpleVT();
215   if (VT != MVT::v2i16 && VT != MVT::v2f16)
216     return false;
217 
218   SDValue Lo = N->getOperand(0);
219   SDValue Hi = N->getOperand(1);
220 
221   LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
222 
223   // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
224   // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
225   // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
226 
227   // Need to check for possible indirect dependencies on the other half of the
228   // vector to avoid introducing a cycle.
229   if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
230     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
231 
232     SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
233     SDValue Ops[] = {
234       LdHi->getChain(), LdHi->getBasePtr(), TiedIn
235     };
236 
237     unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
238     if (LdHi->getMemoryVT() == MVT::i8) {
239       LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
240         AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
241     } else {
242       assert(LdHi->getMemoryVT() == MVT::i16);
243     }
244 
245     SDValue NewLoadHi =
246       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
247                                   Ops, LdHi->getMemoryVT(),
248                                   LdHi->getMemOperand());
249 
250     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
251     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
252     return true;
253   }
254 
255   // build_vector (load ptr), hi -> load_d16_lo ptr, hi
256   // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
257   // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
258   LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
259   if (LdLo && Lo.hasOneUse()) {
260     SDValue TiedIn = getHi16Elt(Hi);
261     if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
262       return false;
263 
264     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
265     unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
266     if (LdLo->getMemoryVT() == MVT::i8) {
267       LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
268         AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
269     } else {
270       assert(LdLo->getMemoryVT() == MVT::i16);
271     }
272 
273     TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
274 
275     SDValue Ops[] = {
276       LdLo->getChain(), LdLo->getBasePtr(), TiedIn
277     };
278 
279     SDValue NewLoadLo =
280       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
281                                   Ops, LdLo->getMemoryVT(),
282                                   LdLo->getMemOperand());
283 
284     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
285     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
286     return true;
287   }
288 
289   return false;
290 }
291 
292 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
293   if (!Subtarget->d16PreservesUnusedBits())
294     return;
295 
296   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
297 
298   bool MadeChange = false;
299   while (Position != CurDAG->allnodes_begin()) {
300     SDNode *N = &*--Position;
301     if (N->use_empty())
302       continue;
303 
304     switch (N->getOpcode()) {
305     case ISD::BUILD_VECTOR:
306       // TODO: Match load d16 from shl (extload:i16), 16
307       MadeChange |= matchLoadD16FromBuildVector(N);
308       break;
309     default:
310       break;
311     }
312   }
313 
314   if (MadeChange) {
315     CurDAG->RemoveDeadNodes();
316     LLVM_DEBUG(dbgs() << "After PreProcess:\n";
317                CurDAG->dump(););
318   }
319 }
320 
321 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N) const {
322   if (N->isUndef())
323     return true;
324 
325   const SIInstrInfo *TII = Subtarget->getInstrInfo();
326   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
327     return TII->isInlineConstant(C->getAPIntValue());
328 
329   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
330     return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
331 
332   return false;
333 }
334 
335 /// Determine the register class for \p OpNo
336 /// \returns The register class of the virtual register that will be used for
337 /// the given operand number \OpNo or NULL if the register class cannot be
338 /// determined.
339 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
340                                                           unsigned OpNo) const {
341   if (!N->isMachineOpcode()) {
342     if (N->getOpcode() == ISD::CopyToReg) {
343       Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
344       if (Reg.isVirtual()) {
345         MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
346         return MRI.getRegClass(Reg);
347       }
348 
349       const SIRegisterInfo *TRI
350         = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
351       return TRI->getPhysRegBaseClass(Reg);
352     }
353 
354     return nullptr;
355   }
356 
357   switch (N->getMachineOpcode()) {
358   default: {
359     const MCInstrDesc &Desc =
360         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
361     unsigned OpIdx = Desc.getNumDefs() + OpNo;
362     if (OpIdx >= Desc.getNumOperands())
363       return nullptr;
364     int RegClass = Desc.operands()[OpIdx].RegClass;
365     if (RegClass == -1)
366       return nullptr;
367 
368     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
369   }
370   case AMDGPU::REG_SEQUENCE: {
371     unsigned RCID = N->getConstantOperandVal(0);
372     const TargetRegisterClass *SuperRC =
373         Subtarget->getRegisterInfo()->getRegClass(RCID);
374 
375     SDValue SubRegOp = N->getOperand(OpNo + 1);
376     unsigned SubRegIdx = SubRegOp->getAsZExtVal();
377     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
378                                                               SubRegIdx);
379   }
380   }
381 }
382 
383 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
384                                          SDValue Glue) const {
385   SmallVector <SDValue, 8> Ops;
386   Ops.push_back(NewChain); // Replace the chain.
387   for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
388     Ops.push_back(N->getOperand(i));
389 
390   Ops.push_back(Glue);
391   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
392 }
393 
394 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
395   const SITargetLowering& Lowering =
396     *static_cast<const SITargetLowering*>(getTargetLowering());
397 
398   assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
399 
400   SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
401   return glueCopyToOp(N, M0, M0.getValue(1));
402 }
403 
404 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
405   unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
406   if (AS == AMDGPUAS::LOCAL_ADDRESS) {
407     if (Subtarget->ldsRequiresM0Init())
408       return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
409   } else if (AS == AMDGPUAS::REGION_ADDRESS) {
410     MachineFunction &MF = CurDAG->getMachineFunction();
411     unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
412     return
413         glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
414   }
415   return N;
416 }
417 
418 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
419                                                   EVT VT) const {
420   SDNode *Lo = CurDAG->getMachineNode(
421       AMDGPU::S_MOV_B32, DL, MVT::i32,
422       CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
423   SDNode *Hi =
424       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
425                              CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
426   const SDValue Ops[] = {
427       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
428       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
429       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
430 
431   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
432 }
433 
434 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
435   EVT VT = N->getValueType(0);
436   unsigned NumVectorElts = VT.getVectorNumElements();
437   EVT EltVT = VT.getVectorElementType();
438   SDLoc DL(N);
439   SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
440 
441   if (NumVectorElts == 1) {
442     CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
443                          RegClass);
444     return;
445   }
446 
447   assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
448                                   "supported yet");
449   // 32 = Max Num Vector Elements
450   // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
451   // 1 = Vector Register Class
452   SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
453 
454   bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
455                Triple::amdgcn;
456   RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
457   bool IsRegSeq = true;
458   unsigned NOps = N->getNumOperands();
459   for (unsigned i = 0; i < NOps; i++) {
460     // XXX: Why is this here?
461     if (isa<RegisterSDNode>(N->getOperand(i))) {
462       IsRegSeq = false;
463       break;
464     }
465     unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
466                          : R600RegisterInfo::getSubRegFromChannel(i);
467     RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
468     RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
469   }
470   if (NOps != NumVectorElts) {
471     // Fill in the missing undef elements if this was a scalar_to_vector.
472     assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
473     MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
474                                                    DL, EltVT);
475     for (unsigned i = NOps; i < NumVectorElts; ++i) {
476       unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
477                            : R600RegisterInfo::getSubRegFromChannel(i);
478       RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
479       RegSeqArgs[1 + (2 * i) + 1] =
480           CurDAG->getTargetConstant(Sub, DL, MVT::i32);
481     }
482   }
483 
484   if (!IsRegSeq)
485     SelectCode(N);
486   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
487 }
488 
489 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
490   unsigned int Opc = N->getOpcode();
491   if (N->isMachineOpcode()) {
492     N->setNodeId(-1);
493     return;   // Already selected.
494   }
495 
496   // isa<MemSDNode> almost works but is slightly too permissive for some DS
497   // intrinsics.
498   if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
499       Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
500       Opc == AMDGPUISD::ATOMIC_LOAD_FMAX) {
501     N = glueCopyToM0LDSInit(N);
502     SelectCode(N);
503     return;
504   }
505 
506   switch (Opc) {
507   default:
508     break;
509   // We are selecting i64 ADD here instead of custom lower it during
510   // DAG legalization, so we can fold some i64 ADDs used for address
511   // calculation into the LOAD and STORE instructions.
512   case ISD::ADDC:
513   case ISD::ADDE:
514   case ISD::SUBC:
515   case ISD::SUBE: {
516     if (N->getValueType(0) != MVT::i64)
517       break;
518 
519     SelectADD_SUB_I64(N);
520     return;
521   }
522   case ISD::UADDO_CARRY:
523   case ISD::USUBO_CARRY:
524     if (N->getValueType(0) != MVT::i32)
525       break;
526 
527     SelectAddcSubb(N);
528     return;
529   case ISD::UADDO:
530   case ISD::USUBO: {
531     SelectUADDO_USUBO(N);
532     return;
533   }
534   case AMDGPUISD::FMUL_W_CHAIN: {
535     SelectFMUL_W_CHAIN(N);
536     return;
537   }
538   case AMDGPUISD::FMA_W_CHAIN: {
539     SelectFMA_W_CHAIN(N);
540     return;
541   }
542 
543   case ISD::SCALAR_TO_VECTOR:
544   case ISD::BUILD_VECTOR: {
545     EVT VT = N->getValueType(0);
546     unsigned NumVectorElts = VT.getVectorNumElements();
547     if (VT.getScalarSizeInBits() == 16) {
548       if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
549         if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
550           ReplaceNode(N, Packed);
551           return;
552         }
553       }
554 
555       break;
556     }
557 
558     assert(VT.getVectorElementType().bitsEq(MVT::i32));
559     unsigned RegClassID =
560         SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
561     SelectBuildVector(N, RegClassID);
562     return;
563   }
564   case ISD::BUILD_PAIR: {
565     SDValue RC, SubReg0, SubReg1;
566     SDLoc DL(N);
567     if (N->getValueType(0) == MVT::i128) {
568       RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
569       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
570       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
571     } else if (N->getValueType(0) == MVT::i64) {
572       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
573       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
574       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
575     } else {
576       llvm_unreachable("Unhandled value type for BUILD_PAIR");
577     }
578     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
579                             N->getOperand(1), SubReg1 };
580     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
581                                           N->getValueType(0), Ops));
582     return;
583   }
584 
585   case ISD::Constant:
586   case ISD::ConstantFP: {
587     if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
588       break;
589 
590     uint64_t Imm;
591     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) {
592       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
593       if (AMDGPU::isValid32BitLiteral(Imm, true))
594         break;
595     } else {
596       ConstantSDNode *C = cast<ConstantSDNode>(N);
597       Imm = C->getZExtValue();
598       if (AMDGPU::isValid32BitLiteral(Imm, false))
599         break;
600     }
601 
602     SDLoc DL(N);
603     ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
604     return;
605   }
606   case AMDGPUISD::BFE_I32:
607   case AMDGPUISD::BFE_U32: {
608     // There is a scalar version available, but unlike the vector version which
609     // has a separate operand for the offset and width, the scalar version packs
610     // the width and offset into a single operand. Try to move to the scalar
611     // version if the offsets are constant, so that we can try to keep extended
612     // loads of kernel arguments in SGPRs.
613 
614     // TODO: Technically we could try to pattern match scalar bitshifts of
615     // dynamic values, but it's probably not useful.
616     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
617     if (!Offset)
618       break;
619 
620     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
621     if (!Width)
622       break;
623 
624     bool Signed = Opc == AMDGPUISD::BFE_I32;
625 
626     uint32_t OffsetVal = Offset->getZExtValue();
627     uint32_t WidthVal = Width->getZExtValue();
628 
629     ReplaceNode(N, getBFE32(Signed, SDLoc(N), N->getOperand(0), OffsetVal,
630                             WidthVal));
631     return;
632   }
633   case AMDGPUISD::DIV_SCALE: {
634     SelectDIV_SCALE(N);
635     return;
636   }
637   case AMDGPUISD::MAD_I64_I32:
638   case AMDGPUISD::MAD_U64_U32: {
639     SelectMAD_64_32(N);
640     return;
641   }
642   case ISD::SMUL_LOHI:
643   case ISD::UMUL_LOHI:
644     return SelectMUL_LOHI(N);
645   case ISD::CopyToReg: {
646     const SITargetLowering& Lowering =
647       *static_cast<const SITargetLowering*>(getTargetLowering());
648     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
649     break;
650   }
651   case ISD::AND:
652   case ISD::SRL:
653   case ISD::SRA:
654   case ISD::SIGN_EXTEND_INREG:
655     if (N->getValueType(0) != MVT::i32)
656       break;
657 
658     SelectS_BFE(N);
659     return;
660   case ISD::BRCOND:
661     SelectBRCOND(N);
662     return;
663   case ISD::FP_EXTEND:
664     SelectFP_EXTEND(N);
665     return;
666   case AMDGPUISD::CVT_PKRTZ_F16_F32:
667   case AMDGPUISD::CVT_PKNORM_I16_F32:
668   case AMDGPUISD::CVT_PKNORM_U16_F32:
669   case AMDGPUISD::CVT_PK_U16_U32:
670   case AMDGPUISD::CVT_PK_I16_I32: {
671     // Hack around using a legal type if f16 is illegal.
672     if (N->getValueType(0) == MVT::i32) {
673       MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
674       N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
675                               { N->getOperand(0), N->getOperand(1) });
676       SelectCode(N);
677       return;
678     }
679 
680     break;
681   }
682   case ISD::INTRINSIC_W_CHAIN: {
683     SelectINTRINSIC_W_CHAIN(N);
684     return;
685   }
686   case ISD::INTRINSIC_WO_CHAIN: {
687     SelectINTRINSIC_WO_CHAIN(N);
688     return;
689   }
690   case ISD::INTRINSIC_VOID: {
691     SelectINTRINSIC_VOID(N);
692     return;
693   }
694   case AMDGPUISD::WAVE_ADDRESS: {
695     SelectWAVE_ADDRESS(N);
696     return;
697   }
698   case ISD::STACKRESTORE: {
699     SelectSTACKRESTORE(N);
700     return;
701   }
702   }
703 
704   SelectCode(N);
705 }
706 
707 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
708   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
709   const Instruction *Term = BB->getTerminator();
710   return Term->getMetadata("amdgpu.uniform") ||
711          Term->getMetadata("structurizecfg.uniform");
712 }
713 
714 bool AMDGPUDAGToDAGISel::isUnneededShiftMask(const SDNode *N,
715                                              unsigned ShAmtBits) const {
716   assert(N->getOpcode() == ISD::AND);
717 
718   const APInt &RHS = N->getConstantOperandAPInt(1);
719   if (RHS.countr_one() >= ShAmtBits)
720     return true;
721 
722   const APInt &LHSKnownZeros = CurDAG->computeKnownBits(N->getOperand(0)).Zero;
723   return (LHSKnownZeros | RHS).countr_one() >= ShAmtBits;
724 }
725 
726 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
727                                           SDValue &N0, SDValue &N1) {
728   if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
729       Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
730     // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
731     // (i64 (bitcast (v2i32 (build_vector
732     //                        (or (extract_vector_elt V, 0), OFFSET),
733     //                        (extract_vector_elt V, 1)))))
734     SDValue Lo = Addr.getOperand(0).getOperand(0);
735     if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
736       SDValue BaseLo = Lo.getOperand(0);
737       SDValue BaseHi = Addr.getOperand(0).getOperand(1);
738       // Check that split base (Lo and Hi) are extracted from the same one.
739       if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
740           BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
741           BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
742           // Lo is statically extracted from index 0.
743           isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
744           BaseLo.getConstantOperandVal(1) == 0 &&
745           // Hi is statically extracted from index 0.
746           isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
747           BaseHi.getConstantOperandVal(1) == 1) {
748         N0 = BaseLo.getOperand(0).getOperand(0);
749         N1 = Lo.getOperand(1);
750         return true;
751       }
752     }
753   }
754   return false;
755 }
756 
757 bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
758                                                     SDValue &RHS) const {
759   if (CurDAG->isBaseWithConstantOffset(Addr)) {
760     LHS = Addr.getOperand(0);
761     RHS = Addr.getOperand(1);
762     return true;
763   }
764 
765   if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, LHS, RHS)) {
766     assert(LHS && RHS && isa<ConstantSDNode>(RHS));
767     return true;
768   }
769 
770   return false;
771 }
772 
773 StringRef AMDGPUDAGToDAGISel::getPassName() const {
774   return "AMDGPU DAG->DAG Pattern Instruction Selection";
775 }
776 
777 //===----------------------------------------------------------------------===//
778 // Complex Patterns
779 //===----------------------------------------------------------------------===//
780 
781 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
782                                             SDValue &Offset) {
783   return false;
784 }
785 
786 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
787                                             SDValue &Offset) {
788   ConstantSDNode *C;
789   SDLoc DL(Addr);
790 
791   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
792     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
793     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
794   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
795              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
796     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
797     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
798   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
799             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
800     Base = Addr.getOperand(0);
801     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
802   } else {
803     Base = Addr;
804     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
805   }
806 
807   return true;
808 }
809 
810 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
811                                                        const SDLoc &DL) const {
812   SDNode *Mov = CurDAG->getMachineNode(
813     AMDGPU::S_MOV_B32, DL, MVT::i32,
814     CurDAG->getTargetConstant(Val, DL, MVT::i32));
815   return SDValue(Mov, 0);
816 }
817 
818 // FIXME: Should only handle uaddo_carry/usubo_carry
819 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
820   SDLoc DL(N);
821   SDValue LHS = N->getOperand(0);
822   SDValue RHS = N->getOperand(1);
823 
824   unsigned Opcode = N->getOpcode();
825   bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
826   bool ProduceCarry =
827       ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
828   bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
829 
830   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
831   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
832 
833   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
834                                        DL, MVT::i32, LHS, Sub0);
835   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
836                                        DL, MVT::i32, LHS, Sub1);
837 
838   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
839                                        DL, MVT::i32, RHS, Sub0);
840   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
841                                        DL, MVT::i32, RHS, Sub1);
842 
843   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
844 
845   static const unsigned OpcMap[2][2][2] = {
846       {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
847        {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
848       {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
849        {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
850 
851   unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
852   unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
853 
854   SDNode *AddLo;
855   if (!ConsumeCarry) {
856     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
857     AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
858   } else {
859     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
860     AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
861   }
862   SDValue AddHiArgs[] = {
863     SDValue(Hi0, 0),
864     SDValue(Hi1, 0),
865     SDValue(AddLo, 1)
866   };
867   SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
868 
869   SDValue RegSequenceArgs[] = {
870     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
871     SDValue(AddLo,0),
872     Sub0,
873     SDValue(AddHi,0),
874     Sub1,
875   };
876   SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
877                                                MVT::i64, RegSequenceArgs);
878 
879   if (ProduceCarry) {
880     // Replace the carry-use
881     ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
882   }
883 
884   // Replace the remaining uses.
885   ReplaceNode(N, RegSequence);
886 }
887 
888 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
889   SDLoc DL(N);
890   SDValue LHS = N->getOperand(0);
891   SDValue RHS = N->getOperand(1);
892   SDValue CI = N->getOperand(2);
893 
894   if (N->isDivergent()) {
895     unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::V_ADDC_U32_e64
896                                                       : AMDGPU::V_SUBB_U32_e64;
897     CurDAG->SelectNodeTo(
898         N, Opc, N->getVTList(),
899         {LHS, RHS, CI,
900          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
901   } else {
902     unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::S_ADD_CO_PSEUDO
903                                                       : AMDGPU::S_SUB_CO_PSEUDO;
904     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
905   }
906 }
907 
908 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
909   // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
910   // carry out despite the _i32 name. These were renamed in VI to _U32.
911   // FIXME: We should probably rename the opcodes here.
912   bool IsAdd = N->getOpcode() == ISD::UADDO;
913   bool IsVALU = N->isDivergent();
914 
915   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
916        ++UI)
917     if (UI.getUse().getResNo() == 1) {
918       if ((IsAdd && (UI->getOpcode() != ISD::UADDO_CARRY)) ||
919           (!IsAdd && (UI->getOpcode() != ISD::USUBO_CARRY))) {
920         IsVALU = true;
921         break;
922       }
923     }
924 
925   if (IsVALU) {
926     unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
927 
928     CurDAG->SelectNodeTo(
929         N, Opc, N->getVTList(),
930         {N->getOperand(0), N->getOperand(1),
931          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
932   } else {
933     unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
934                                                 : AMDGPU::S_USUBO_PSEUDO;
935 
936     CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
937                          {N->getOperand(0), N->getOperand(1)});
938   }
939 }
940 
941 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
942   SDLoc SL(N);
943   //  src0_modifiers, src0,  src1_modifiers, src1, src2_modifiers, src2, clamp, omod
944   SDValue Ops[10];
945 
946   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
947   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
948   SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
949   Ops[8] = N->getOperand(0);
950   Ops[9] = N->getOperand(4);
951 
952   // If there are no source modifiers, prefer fmac over fma because it can use
953   // the smaller VOP2 encoding.
954   bool UseFMAC = Subtarget->hasDLInsts() &&
955                  cast<ConstantSDNode>(Ops[0])->isZero() &&
956                  cast<ConstantSDNode>(Ops[2])->isZero() &&
957                  cast<ConstantSDNode>(Ops[4])->isZero();
958   unsigned Opcode = UseFMAC ? AMDGPU::V_FMAC_F32_e64 : AMDGPU::V_FMA_F32_e64;
959   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), Ops);
960 }
961 
962 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
963   SDLoc SL(N);
964   //    src0_modifiers, src0,  src1_modifiers, src1, clamp, omod
965   SDValue Ops[8];
966 
967   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
968   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
969   Ops[6] = N->getOperand(0);
970   Ops[7] = N->getOperand(3);
971 
972   CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
973 }
974 
975 // We need to handle this here because tablegen doesn't support matching
976 // instructions with multiple outputs.
977 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
978   SDLoc SL(N);
979   EVT VT = N->getValueType(0);
980 
981   assert(VT == MVT::f32 || VT == MVT::f64);
982 
983   unsigned Opc
984     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
985 
986   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
987   // omod
988   SDValue Ops[8];
989   SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
990   SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
991   SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
992   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
993 }
994 
995 // We need to handle this here because tablegen doesn't support matching
996 // instructions with multiple outputs.
997 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
998   SDLoc SL(N);
999   bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1000   unsigned Opc;
1001   if (Subtarget->hasMADIntraFwdBug())
1002     Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1003                  : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1004   else
1005     Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1006 
1007   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1008   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1009                     Clamp };
1010   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1011 }
1012 
1013 // We need to handle this here because tablegen doesn't support matching
1014 // instructions with multiple outputs.
1015 void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
1016   SDLoc SL(N);
1017   bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
1018   unsigned Opc;
1019   if (Subtarget->hasMADIntraFwdBug())
1020     Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1021                  : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1022   else
1023     Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1024 
1025   SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64);
1026   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1027   SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp};
1028   SDNode *Mad = CurDAG->getMachineNode(Opc, SL, N->getVTList(), Ops);
1029   if (!SDValue(N, 0).use_empty()) {
1030     SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
1031     SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1032                                         MVT::i32, SDValue(Mad, 0), Sub0);
1033     ReplaceUses(SDValue(N, 0), SDValue(Lo, 0));
1034   }
1035   if (!SDValue(N, 1).use_empty()) {
1036     SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32);
1037     SDNode *Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1038                                         MVT::i32, SDValue(Mad, 0), Sub1);
1039     ReplaceUses(SDValue(N, 1), SDValue(Hi, 0));
1040   }
1041   CurDAG->RemoveDeadNode(N);
1042 }
1043 
1044 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1045   if (!isUInt<16>(Offset))
1046     return false;
1047 
1048   if (!Base || Subtarget->hasUsableDSOffset() ||
1049       Subtarget->unsafeDSOffsetFoldingEnabled())
1050     return true;
1051 
1052   // On Southern Islands instruction with a negative base value and an offset
1053   // don't seem to work.
1054   return CurDAG->SignBitIsZero(Base);
1055 }
1056 
1057 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1058                                               SDValue &Offset) const {
1059   SDLoc DL(Addr);
1060   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1061     SDValue N0 = Addr.getOperand(0);
1062     SDValue N1 = Addr.getOperand(1);
1063     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1064     if (isDSOffsetLegal(N0, C1->getSExtValue())) {
1065       // (add n0, c0)
1066       Base = N0;
1067       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1068       return true;
1069     }
1070   } else if (Addr.getOpcode() == ISD::SUB) {
1071     // sub C, x -> add (sub 0, x), C
1072     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1073       int64_t ByteOffset = C->getSExtValue();
1074       if (isDSOffsetLegal(SDValue(), ByteOffset)) {
1075         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1076 
1077         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1078         // the known bits in isDSOffsetLegal. We need to emit the selected node
1079         // here, so this is thrown away.
1080         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1081                                       Zero, Addr.getOperand(1));
1082 
1083         if (isDSOffsetLegal(Sub, ByteOffset)) {
1084           SmallVector<SDValue, 3> Opnds;
1085           Opnds.push_back(Zero);
1086           Opnds.push_back(Addr.getOperand(1));
1087 
1088           // FIXME: Select to VOP3 version for with-carry.
1089           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1090           if (Subtarget->hasAddNoCarry()) {
1091             SubOp = AMDGPU::V_SUB_U32_e64;
1092             Opnds.push_back(
1093                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1094           }
1095 
1096           MachineSDNode *MachineSub =
1097               CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1098 
1099           Base = SDValue(MachineSub, 0);
1100           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1101           return true;
1102         }
1103       }
1104     }
1105   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1106     // If we have a constant address, prefer to put the constant into the
1107     // offset. This can save moves to load the constant address since multiple
1108     // operations can share the zero base address register, and enables merging
1109     // into read2 / write2 instructions.
1110 
1111     SDLoc DL(Addr);
1112 
1113     if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) {
1114       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1115       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1116                                  DL, MVT::i32, Zero);
1117       Base = SDValue(MovZero, 0);
1118       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1119       return true;
1120     }
1121   }
1122 
1123   // default case
1124   Base = Addr;
1125   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1126   return true;
1127 }
1128 
1129 bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1130                                           unsigned Offset1,
1131                                           unsigned Size) const {
1132   if (Offset0 % Size != 0 || Offset1 % Size != 0)
1133     return false;
1134   if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
1135     return false;
1136 
1137   if (!Base || Subtarget->hasUsableDSOffset() ||
1138       Subtarget->unsafeDSOffsetFoldingEnabled())
1139     return true;
1140 
1141   // On Southern Islands instruction with a negative base value and an offset
1142   // don't seem to work.
1143   return CurDAG->SignBitIsZero(Base);
1144 }
1145 
1146 // Return whether the operation has NoUnsignedWrap property.
1147 static bool isNoUnsignedWrap(SDValue Addr) {
1148   return (Addr.getOpcode() == ISD::ADD &&
1149           Addr->getFlags().hasNoUnsignedWrap()) ||
1150          Addr->getOpcode() == ISD::OR;
1151 }
1152 
1153 // Check that the base address of flat scratch load/store in the form of `base +
1154 // offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
1155 // requirement). We always treat the first operand as the base address here.
1156 bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegal(SDValue Addr) const {
1157   if (isNoUnsignedWrap(Addr))
1158     return true;
1159 
1160   // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1161   // values.
1162   if (Subtarget->hasSignedScratchOffsets())
1163     return true;
1164 
1165   auto LHS = Addr.getOperand(0);
1166   auto RHS = Addr.getOperand(1);
1167 
1168   // If the immediate offset is negative and within certain range, the base
1169   // address cannot also be negative. If the base is also negative, the sum
1170   // would be either negative or much larger than the valid range of scratch
1171   // memory a thread can access.
1172   ConstantSDNode *ImmOp = nullptr;
1173   if (Addr.getOpcode() == ISD::ADD && (ImmOp = dyn_cast<ConstantSDNode>(RHS))) {
1174     if (ImmOp->getSExtValue() < 0 && ImmOp->getSExtValue() > -0x40000000)
1175       return true;
1176   }
1177 
1178   return CurDAG->SignBitIsZero(LHS);
1179 }
1180 
1181 // Check address value in SGPR/VGPR are legal for flat scratch in the form
1182 // of: SGPR + VGPR.
1183 bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSV(SDValue Addr) const {
1184   if (isNoUnsignedWrap(Addr))
1185     return true;
1186 
1187   // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1188   // values.
1189   if (Subtarget->hasSignedScratchOffsets())
1190     return true;
1191 
1192   auto LHS = Addr.getOperand(0);
1193   auto RHS = Addr.getOperand(1);
1194   return CurDAG->SignBitIsZero(RHS) && CurDAG->SignBitIsZero(LHS);
1195 }
1196 
1197 // Check address value in SGPR/VGPR are legal for flat scratch in the form
1198 // of: SGPR + VGPR + Imm.
1199 bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSVImm(SDValue Addr) const {
1200   // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1201   // values.
1202   if (AMDGPU::isGFX12Plus(*Subtarget))
1203     return true;
1204 
1205   auto Base = Addr.getOperand(0);
1206   auto *RHSImm = cast<ConstantSDNode>(Addr.getOperand(1));
1207   // If the immediate offset is negative and within certain range, the base
1208   // address cannot also be negative. If the base is also negative, the sum
1209   // would be either negative or much larger than the valid range of scratch
1210   // memory a thread can access.
1211   if (isNoUnsignedWrap(Base) &&
1212       (isNoUnsignedWrap(Addr) ||
1213        (RHSImm->getSExtValue() < 0 && RHSImm->getSExtValue() > -0x40000000)))
1214     return true;
1215 
1216   auto LHS = Base.getOperand(0);
1217   auto RHS = Base.getOperand(1);
1218   return CurDAG->SignBitIsZero(RHS) && CurDAG->SignBitIsZero(LHS);
1219 }
1220 
1221 // TODO: If offset is too big, put low 16-bit into offset.
1222 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1223                                                    SDValue &Offset0,
1224                                                    SDValue &Offset1) const {
1225   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4);
1226 }
1227 
1228 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1229                                                     SDValue &Offset0,
1230                                                     SDValue &Offset1) const {
1231   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8);
1232 }
1233 
1234 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1235                                             SDValue &Offset0, SDValue &Offset1,
1236                                             unsigned Size) const {
1237   SDLoc DL(Addr);
1238 
1239   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1240     SDValue N0 = Addr.getOperand(0);
1241     SDValue N1 = Addr.getOperand(1);
1242     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1243     unsigned OffsetValue0 = C1->getZExtValue();
1244     unsigned OffsetValue1 = OffsetValue0 + Size;
1245 
1246     // (add n0, c0)
1247     if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) {
1248       Base = N0;
1249       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1250       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1251       return true;
1252     }
1253   } else if (Addr.getOpcode() == ISD::SUB) {
1254     // sub C, x -> add (sub 0, x), C
1255     if (const ConstantSDNode *C =
1256             dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1257       unsigned OffsetValue0 = C->getZExtValue();
1258       unsigned OffsetValue1 = OffsetValue0 + Size;
1259 
1260       if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1261         SDLoc DL(Addr);
1262         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1263 
1264         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1265         // the known bits in isDSOffsetLegal. We need to emit the selected node
1266         // here, so this is thrown away.
1267         SDValue Sub =
1268             CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1269 
1270         if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) {
1271           SmallVector<SDValue, 3> Opnds;
1272           Opnds.push_back(Zero);
1273           Opnds.push_back(Addr.getOperand(1));
1274           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1275           if (Subtarget->hasAddNoCarry()) {
1276             SubOp = AMDGPU::V_SUB_U32_e64;
1277             Opnds.push_back(
1278                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1279           }
1280 
1281           MachineSDNode *MachineSub = CurDAG->getMachineNode(
1282               SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds);
1283 
1284           Base = SDValue(MachineSub, 0);
1285           Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1286           Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1287           return true;
1288         }
1289       }
1290     }
1291   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1292     unsigned OffsetValue0 = CAddr->getZExtValue();
1293     unsigned OffsetValue1 = OffsetValue0 + Size;
1294 
1295     if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1296       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1297       MachineSDNode *MovZero =
1298           CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1299       Base = SDValue(MovZero, 0);
1300       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1301       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1302       return true;
1303     }
1304   }
1305 
1306   // default case
1307 
1308   Base = Addr;
1309   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1310   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1311   return true;
1312 }
1313 
1314 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
1315                                      SDValue &SOffset, SDValue &Offset,
1316                                      SDValue &Offen, SDValue &Idxen,
1317                                      SDValue &Addr64) const {
1318   // Subtarget prefers to use flat instruction
1319   // FIXME: This should be a pattern predicate and not reach here
1320   if (Subtarget->useFlatForGlobal())
1321     return false;
1322 
1323   SDLoc DL(Addr);
1324 
1325   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1326   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1327   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1328   SOffset = Subtarget->hasRestrictedSOffset()
1329                 ? CurDAG->getRegister(AMDGPU::SGPR_NULL, MVT::i32)
1330                 : CurDAG->getTargetConstant(0, DL, MVT::i32);
1331 
1332   ConstantSDNode *C1 = nullptr;
1333   SDValue N0 = Addr;
1334   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1335     C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1336     if (isUInt<32>(C1->getZExtValue()))
1337       N0 = Addr.getOperand(0);
1338     else
1339       C1 = nullptr;
1340   }
1341 
1342   if (N0.getOpcode() == ISD::ADD) {
1343     // (add N2, N3) -> addr64, or
1344     // (add (add N2, N3), C1) -> addr64
1345     SDValue N2 = N0.getOperand(0);
1346     SDValue N3 = N0.getOperand(1);
1347     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1348 
1349     if (N2->isDivergent()) {
1350       if (N3->isDivergent()) {
1351         // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1352         // addr64, and construct the resource from a 0 address.
1353         Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1354         VAddr = N0;
1355       } else {
1356         // N2 is divergent, N3 is not.
1357         Ptr = N3;
1358         VAddr = N2;
1359       }
1360     } else {
1361       // N2 is not divergent.
1362       Ptr = N2;
1363       VAddr = N3;
1364     }
1365     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1366   } else if (N0->isDivergent()) {
1367     // N0 is divergent. Use it as the addr64, and construct the resource from a
1368     // 0 address.
1369     Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1370     VAddr = N0;
1371     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1372   } else {
1373     // N0 -> offset, or
1374     // (N0 + C1) -> offset
1375     VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1376     Ptr = N0;
1377   }
1378 
1379   if (!C1) {
1380     // No offset.
1381     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1382     return true;
1383   }
1384 
1385   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1386   if (TII->isLegalMUBUFImmOffset(C1->getZExtValue())) {
1387     // Legal offset for instruction.
1388     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1389     return true;
1390   }
1391 
1392   // Illegal offset, store it in soffset.
1393   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1394   SOffset =
1395       SDValue(CurDAG->getMachineNode(
1396                   AMDGPU::S_MOV_B32, DL, MVT::i32,
1397                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1398               0);
1399   return true;
1400 }
1401 
1402 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1403                                            SDValue &VAddr, SDValue &SOffset,
1404                                            SDValue &Offset) const {
1405   SDValue Ptr, Offen, Idxen, Addr64;
1406 
1407   // addr64 bit was removed for volcanic islands.
1408   // FIXME: This should be a pattern predicate and not reach here
1409   if (!Subtarget->hasAddr64())
1410     return false;
1411 
1412   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1413     return false;
1414 
1415   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1416   if (C->getSExtValue()) {
1417     SDLoc DL(Addr);
1418 
1419     const SITargetLowering& Lowering =
1420       *static_cast<const SITargetLowering*>(getTargetLowering());
1421 
1422     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1423     return true;
1424   }
1425 
1426   return false;
1427 }
1428 
1429 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1430   SDLoc DL(N);
1431 
1432   auto *FI = dyn_cast<FrameIndexSDNode>(N);
1433   SDValue TFI =
1434       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1435 
1436   // We rebase the base address into an absolute stack address and hence
1437   // use constant 0 for soffset. This value must be retained until
1438   // frame elimination and eliminateFrameIndex will choose the appropriate
1439   // frame register if need be.
1440   return std::pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1441 }
1442 
1443 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1444                                                  SDValue Addr, SDValue &Rsrc,
1445                                                  SDValue &VAddr, SDValue &SOffset,
1446                                                  SDValue &ImmOffset) const {
1447 
1448   SDLoc DL(Addr);
1449   MachineFunction &MF = CurDAG->getMachineFunction();
1450   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1451 
1452   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1453 
1454   if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1455     int64_t Imm = CAddr->getSExtValue();
1456     const int64_t NullPtr =
1457         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1458     // Don't fold null pointer.
1459     if (Imm != NullPtr) {
1460       const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(*Subtarget);
1461       SDValue HighBits =
1462           CurDAG->getTargetConstant(Imm & ~MaxOffset, DL, MVT::i32);
1463       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1464         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1465       VAddr = SDValue(MovHighBits, 0);
1466 
1467       SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1468       ImmOffset = CurDAG->getTargetConstant(Imm & MaxOffset, DL, MVT::i32);
1469       return true;
1470     }
1471   }
1472 
1473   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1474     // (add n0, c1)
1475 
1476     SDValue N0 = Addr.getOperand(0);
1477     SDValue N1 = Addr.getOperand(1);
1478 
1479     // Offsets in vaddr must be positive if range checking is enabled.
1480     //
1481     // The total computation of vaddr + soffset + offset must not overflow.  If
1482     // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1483     // overflowing.
1484     //
1485     // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1486     // always perform a range check. If a negative vaddr base index was used,
1487     // this would fail the range check. The overall address computation would
1488     // compute a valid address, but this doesn't happen due to the range
1489     // check. For out-of-bounds MUBUF loads, a 0 is returned.
1490     //
1491     // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1492     // MUBUF vaddr, but not on older subtargets which can only do this if the
1493     // sign bit is known 0.
1494     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1495     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1496     if (TII->isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1497         (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1498          CurDAG->SignBitIsZero(N0))) {
1499       std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1500       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1501       return true;
1502     }
1503   }
1504 
1505   // (node)
1506   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1507   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1508   return true;
1509 }
1510 
1511 static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
1512   if (Val.getOpcode() != ISD::CopyFromReg)
1513     return false;
1514   auto Reg = cast<RegisterSDNode>(Val.getOperand(1))->getReg();
1515   if (!Reg.isPhysical())
1516     return false;
1517   auto RC = TRI.getPhysRegBaseClass(Reg);
1518   return RC && TRI.isSGPRClass(RC);
1519 }
1520 
1521 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1522                                                   SDValue Addr,
1523                                                   SDValue &SRsrc,
1524                                                   SDValue &SOffset,
1525                                                   SDValue &Offset) const {
1526   const SIRegisterInfo *TRI =
1527       static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
1528   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1529   MachineFunction &MF = CurDAG->getMachineFunction();
1530   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1531   SDLoc DL(Addr);
1532 
1533   // CopyFromReg <sgpr>
1534   if (IsCopyFromSGPR(*TRI, Addr)) {
1535     SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1536     SOffset = Addr;
1537     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1538     return true;
1539   }
1540 
1541   ConstantSDNode *CAddr;
1542   if (Addr.getOpcode() == ISD::ADD) {
1543     // Add (CopyFromReg <sgpr>) <constant>
1544     CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
1545     if (!CAddr || !TII->isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1546       return false;
1547     if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0)))
1548       return false;
1549 
1550     SOffset = Addr.getOperand(0);
1551   } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) &&
1552              TII->isLegalMUBUFImmOffset(CAddr->getZExtValue())) {
1553     // <constant>
1554     SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1555   } else {
1556     return false;
1557   }
1558 
1559   SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1560 
1561   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i32);
1562   return true;
1563 }
1564 
1565 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1566                                            SDValue &SOffset, SDValue &Offset
1567                                            ) const {
1568   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1569   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1570 
1571   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1572     return false;
1573 
1574   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1575       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1576       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1577     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1578                     APInt::getAllOnes(32).getZExtValue(); // Size
1579     SDLoc DL(Addr);
1580 
1581     const SITargetLowering& Lowering =
1582       *static_cast<const SITargetLowering*>(getTargetLowering());
1583 
1584     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1585     return true;
1586   }
1587   return false;
1588 }
1589 
1590 bool AMDGPUDAGToDAGISel::SelectBUFSOffset(SDValue ByteOffsetNode,
1591                                           SDValue &SOffset) const {
1592   if (Subtarget->hasRestrictedSOffset() && isNullConstant(ByteOffsetNode)) {
1593     SOffset = CurDAG->getRegister(AMDGPU::SGPR_NULL, MVT::i32);
1594     return true;
1595   }
1596 
1597   SOffset = ByteOffsetNode;
1598   return true;
1599 }
1600 
1601 // Find a load or store from corresponding pattern root.
1602 // Roots may be build_vector, bitconvert or their combinations.
1603 static MemSDNode* findMemSDNode(SDNode *N) {
1604   N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1605   if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1606     return MN;
1607   assert(isa<BuildVectorSDNode>(N));
1608   for (SDValue V : N->op_values())
1609     if (MemSDNode *MN =
1610           dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1611       return MN;
1612   llvm_unreachable("cannot find MemSDNode in the pattern!");
1613 }
1614 
1615 bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(SDNode *N, SDValue Addr,
1616                                               SDValue &VAddr, SDValue &Offset,
1617                                               uint64_t FlatVariant) const {
1618   int64_t OffsetVal = 0;
1619 
1620   unsigned AS = findMemSDNode(N)->getAddressSpace();
1621 
1622   bool CanHaveFlatSegmentOffsetBug =
1623       Subtarget->hasFlatSegmentOffsetBug() &&
1624       FlatVariant == SIInstrFlags::FLAT &&
1625       (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS);
1626 
1627   if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1628     SDValue N0, N1;
1629     if (isBaseWithConstantOffset64(Addr, N0, N1) &&
1630         (FlatVariant != SIInstrFlags::FlatScratch ||
1631          isFlatScratchBaseLegal(Addr))) {
1632       int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1633 
1634       const SIInstrInfo *TII = Subtarget->getInstrInfo();
1635       if (TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) {
1636         Addr = N0;
1637         OffsetVal = COffsetVal;
1638       } else {
1639         // If the offset doesn't fit, put the low bits into the offset field and
1640         // add the rest.
1641         //
1642         // For a FLAT instruction the hardware decides whether to access
1643         // global/scratch/shared memory based on the high bits of vaddr,
1644         // ignoring the offset field, so we have to ensure that when we add
1645         // remainder to vaddr it still points into the same underlying object.
1646         // The easiest way to do that is to make sure that we split the offset
1647         // into two pieces that are both >= 0 or both <= 0.
1648 
1649         SDLoc DL(N);
1650         uint64_t RemainderOffset;
1651 
1652         std::tie(OffsetVal, RemainderOffset) =
1653             TII->splitFlatOffset(COffsetVal, AS, FlatVariant);
1654 
1655         SDValue AddOffsetLo =
1656             getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1657         SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1658 
1659         if (Addr.getValueType().getSizeInBits() == 32) {
1660           SmallVector<SDValue, 3> Opnds;
1661           Opnds.push_back(N0);
1662           Opnds.push_back(AddOffsetLo);
1663           unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1664           if (Subtarget->hasAddNoCarry()) {
1665             AddOp = AMDGPU::V_ADD_U32_e64;
1666             Opnds.push_back(Clamp);
1667           }
1668           Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1669         } else {
1670           // TODO: Should this try to use a scalar add pseudo if the base address
1671           // is uniform and saddr is usable?
1672           SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1673           SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1674 
1675           SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1676                                                 DL, MVT::i32, N0, Sub0);
1677           SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1678                                                 DL, MVT::i32, N0, Sub1);
1679 
1680           SDValue AddOffsetHi =
1681               getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1682 
1683           SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1684 
1685           SDNode *Add =
1686               CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1687                                      {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1688 
1689           SDNode *Addc = CurDAG->getMachineNode(
1690               AMDGPU::V_ADDC_U32_e64, DL, VTs,
1691               {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1692 
1693           SDValue RegSequenceArgs[] = {
1694               CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1695               SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1696 
1697           Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1698                                                 MVT::i64, RegSequenceArgs),
1699                          0);
1700         }
1701       }
1702     }
1703   }
1704 
1705   VAddr = Addr;
1706   Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
1707   return true;
1708 }
1709 
1710 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr,
1711                                           SDValue &VAddr,
1712                                           SDValue &Offset) const {
1713   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FLAT);
1714 }
1715 
1716 bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr,
1717                                             SDValue &VAddr,
1718                                             SDValue &Offset) const {
1719   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FlatGlobal);
1720 }
1721 
1722 bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr,
1723                                              SDValue &VAddr,
1724                                              SDValue &Offset) const {
1725   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
1726                               SIInstrFlags::FlatScratch);
1727 }
1728 
1729 // If this matches zero_extend i32:x, return x
1730 static SDValue matchZExtFromI32(SDValue Op) {
1731   if (Op.getOpcode() != ISD::ZERO_EXTEND)
1732     return SDValue();
1733 
1734   SDValue ExtSrc = Op.getOperand(0);
1735   return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1736 }
1737 
1738 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1739 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1740                                            SDValue Addr,
1741                                            SDValue &SAddr,
1742                                            SDValue &VOffset,
1743                                            SDValue &Offset) const {
1744   int64_t ImmOffset = 0;
1745 
1746   // Match the immediate offset first, which canonically is moved as low as
1747   // possible.
1748 
1749   SDValue LHS, RHS;
1750   if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1751     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1752     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1753 
1754     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS,
1755                                SIInstrFlags::FlatGlobal)) {
1756       Addr = LHS;
1757       ImmOffset = COffsetVal;
1758     } else if (!LHS->isDivergent()) {
1759       if (COffsetVal > 0) {
1760         SDLoc SL(N);
1761         // saddr + large_offset -> saddr +
1762         //                         (voffset = large_offset & ~MaxOffset) +
1763         //                         (large_offset & MaxOffset);
1764         int64_t SplitImmOffset, RemainderOffset;
1765         std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1766             COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, SIInstrFlags::FlatGlobal);
1767 
1768         if (isUInt<32>(RemainderOffset)) {
1769           SDNode *VMov = CurDAG->getMachineNode(
1770               AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1771               CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1772           VOffset = SDValue(VMov, 0);
1773           SAddr = LHS;
1774           Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i32);
1775           return true;
1776         }
1777       }
1778 
1779       // We are adding a 64 bit SGPR and a constant. If constant bus limit
1780       // is 1 we would need to perform 1 or 2 extra moves for each half of
1781       // the constant and it is better to do a scalar add and then issue a
1782       // single VALU instruction to materialize zero. Otherwise it is less
1783       // instructions to perform VALU adds with immediates or inline literals.
1784       unsigned NumLiterals =
1785           !TII->isInlineConstant(APInt(32, COffsetVal & 0xffffffff)) +
1786           !TII->isInlineConstant(APInt(32, COffsetVal >> 32));
1787       if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
1788         return false;
1789     }
1790   }
1791 
1792   // Match the variable offset.
1793   if (Addr.getOpcode() == ISD::ADD) {
1794     LHS = Addr.getOperand(0);
1795     RHS = Addr.getOperand(1);
1796 
1797     if (!LHS->isDivergent()) {
1798       // add (i64 sgpr), (zero_extend (i32 vgpr))
1799       if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1800         SAddr = LHS;
1801         VOffset = ZextRHS;
1802       }
1803     }
1804 
1805     if (!SAddr && !RHS->isDivergent()) {
1806       // add (zero_extend (i32 vgpr)), (i64 sgpr)
1807       if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1808         SAddr = RHS;
1809         VOffset = ZextLHS;
1810       }
1811     }
1812 
1813     if (SAddr) {
1814       Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i32);
1815       return true;
1816     }
1817   }
1818 
1819   if (Addr->isDivergent() || Addr.getOpcode() == ISD::UNDEF ||
1820       isa<ConstantSDNode>(Addr))
1821     return false;
1822 
1823   // It's cheaper to materialize a single 32-bit zero for vaddr than the two
1824   // moves required to copy a 64-bit SGPR to VGPR.
1825   SAddr = Addr;
1826   SDNode *VMov =
1827       CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32,
1828                              CurDAG->getTargetConstant(0, SDLoc(), MVT::i32));
1829   VOffset = SDValue(VMov, 0);
1830   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i32);
1831   return true;
1832 }
1833 
1834 static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) {
1835   if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1836     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1837   } else if (SAddr.getOpcode() == ISD::ADD &&
1838              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1839     // Materialize this into a scalar move for scalar address to avoid
1840     // readfirstlane.
1841     auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1842     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1843                                               FI->getValueType(0));
1844     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, SDLoc(SAddr),
1845                                            MVT::i32, TFI, SAddr.getOperand(1)),
1846                     0);
1847   }
1848 
1849   return SAddr;
1850 }
1851 
1852 // Match (32-bit SGPR base) + sext(imm offset)
1853 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
1854                                             SDValue &SAddr,
1855                                             SDValue &Offset) const {
1856   if (Addr->isDivergent())
1857     return false;
1858 
1859   SDLoc DL(Addr);
1860 
1861   int64_t COffsetVal = 0;
1862 
1863   if (CurDAG->isBaseWithConstantOffset(Addr) && isFlatScratchBaseLegal(Addr)) {
1864     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1865     SAddr = Addr.getOperand(0);
1866   } else {
1867     SAddr = Addr;
1868   }
1869 
1870   SAddr = SelectSAddrFI(CurDAG, SAddr);
1871 
1872   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1873 
1874   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
1875                               SIInstrFlags::FlatScratch)) {
1876     int64_t SplitImmOffset, RemainderOffset;
1877     std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1878         COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch);
1879 
1880     COffsetVal = SplitImmOffset;
1881 
1882     SDValue AddOffset =
1883         SAddr.getOpcode() == ISD::TargetFrameIndex
1884             ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL)
1885             : CurDAG->getTargetConstant(RemainderOffset, DL, MVT::i32);
1886     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, DL, MVT::i32,
1887                                            SAddr, AddOffset),
1888                     0);
1889   }
1890 
1891   Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16);
1892 
1893   return true;
1894 }
1895 
1896 // Check whether the flat scratch SVS swizzle bug affects this access.
1897 bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug(
1898     SDValue VAddr, SDValue SAddr, uint64_t ImmOffset) const {
1899   if (!Subtarget->hasFlatScratchSVSSwizzleBug())
1900     return false;
1901 
1902   // The bug affects the swizzling of SVS accesses if there is any carry out
1903   // from the two low order bits (i.e. from bit 1 into bit 2) when adding
1904   // voffset to (soffset + inst_offset).
1905   KnownBits VKnown = CurDAG->computeKnownBits(VAddr);
1906   KnownBits SKnown = KnownBits::computeForAddSub(
1907       true, false, CurDAG->computeKnownBits(SAddr),
1908       KnownBits::makeConstant(APInt(32, ImmOffset)));
1909   uint64_t VMax = VKnown.getMaxValue().getZExtValue();
1910   uint64_t SMax = SKnown.getMaxValue().getZExtValue();
1911   return (VMax & 3) + (SMax & 3) >= 4;
1912 }
1913 
1914 bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
1915                                              SDValue &VAddr, SDValue &SAddr,
1916                                              SDValue &Offset) const  {
1917   int64_t ImmOffset = 0;
1918 
1919   SDValue LHS, RHS;
1920   SDValue OrigAddr = Addr;
1921   if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1922     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1923     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1924 
1925     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
1926       Addr = LHS;
1927       ImmOffset = COffsetVal;
1928     } else if (!LHS->isDivergent() && COffsetVal > 0) {
1929       SDLoc SL(N);
1930       // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
1931       //                         (large_offset & MaxOffset);
1932       int64_t SplitImmOffset, RemainderOffset;
1933       std::tie(SplitImmOffset, RemainderOffset)
1934         = TII->splitFlatOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true);
1935 
1936       if (isUInt<32>(RemainderOffset)) {
1937         SDNode *VMov = CurDAG->getMachineNode(
1938           AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1939           CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1940         VAddr = SDValue(VMov, 0);
1941         SAddr = LHS;
1942         if (!isFlatScratchBaseLegal(Addr))
1943           return false;
1944         if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, SplitImmOffset))
1945           return false;
1946         Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
1947         return true;
1948       }
1949     }
1950   }
1951 
1952   if (Addr.getOpcode() != ISD::ADD)
1953     return false;
1954 
1955   LHS = Addr.getOperand(0);
1956   RHS = Addr.getOperand(1);
1957 
1958   if (!LHS->isDivergent() && RHS->isDivergent()) {
1959     SAddr = LHS;
1960     VAddr = RHS;
1961   } else if (!RHS->isDivergent() && LHS->isDivergent()) {
1962     SAddr = RHS;
1963     VAddr = LHS;
1964   } else {
1965     return false;
1966   }
1967 
1968   if (OrigAddr != Addr) {
1969     if (!isFlatScratchBaseLegalSVImm(OrigAddr))
1970       return false;
1971   } else {
1972     if (!isFlatScratchBaseLegalSV(OrigAddr))
1973       return false;
1974   }
1975 
1976   if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset))
1977     return false;
1978   SAddr = SelectSAddrFI(CurDAG, SAddr);
1979   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1980   return true;
1981 }
1982 
1983 // Match an immediate (if Offset is not null) or an SGPR (if SOffset is
1984 // not null) offset. If Imm32Only is true, match only 32-bit immediate
1985 // offsets available on CI.
1986 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1987                                           SDValue *SOffset, SDValue *Offset,
1988                                           bool Imm32Only, bool IsBuffer) const {
1989   assert((!SOffset || !Offset) &&
1990          "Cannot match both soffset and offset at the same time!");
1991 
1992   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1993   if (!C) {
1994     if (!SOffset)
1995       return false;
1996     if (ByteOffsetNode.getValueType().isScalarInteger() &&
1997         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1998       *SOffset = ByteOffsetNode;
1999       return true;
2000     }
2001     if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
2002       if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
2003         *SOffset = ByteOffsetNode.getOperand(0);
2004         return true;
2005       }
2006     }
2007     return false;
2008   }
2009 
2010   SDLoc SL(ByteOffsetNode);
2011 
2012   // GFX9 and GFX10 have signed byte immediate offsets. The immediate
2013   // offset for S_BUFFER instructions is unsigned.
2014   int64_t ByteOffset = IsBuffer ? C->getZExtValue() : C->getSExtValue();
2015   std::optional<int64_t> EncodedOffset =
2016       AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, IsBuffer);
2017   if (EncodedOffset && Offset && !Imm32Only) {
2018     *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
2019     return true;
2020   }
2021 
2022   // SGPR and literal offsets are unsigned.
2023   if (ByteOffset < 0)
2024     return false;
2025 
2026   EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
2027   if (EncodedOffset && Offset && Imm32Only) {
2028     *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
2029     return true;
2030   }
2031 
2032   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
2033     return false;
2034 
2035   if (SOffset) {
2036     SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
2037     *SOffset = SDValue(
2038         CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
2039     return true;
2040   }
2041 
2042   return false;
2043 }
2044 
2045 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
2046   if (Addr.getValueType() != MVT::i32)
2047     return Addr;
2048 
2049   // Zero-extend a 32-bit address.
2050   SDLoc SL(Addr);
2051 
2052   const MachineFunction &MF = CurDAG->getMachineFunction();
2053   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2054   unsigned AddrHiVal = Info->get32BitAddressHighBits();
2055   SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
2056 
2057   const SDValue Ops[] = {
2058     CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
2059     Addr,
2060     CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2061     SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
2062             0),
2063     CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
2064   };
2065 
2066   return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
2067                                         Ops), 0);
2068 }
2069 
2070 // Match a base and an immediate (if Offset is not null) or an SGPR (if
2071 // SOffset is not null) or an immediate+SGPR offset. If Imm32Only is
2072 // true, match only 32-bit immediate offsets available on CI.
2073 bool AMDGPUDAGToDAGISel::SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase,
2074                                               SDValue *SOffset, SDValue *Offset,
2075                                               bool Imm32Only,
2076                                               bool IsBuffer) const {
2077   if (SOffset && Offset) {
2078     assert(!Imm32Only && !IsBuffer);
2079     SDValue B;
2080     return SelectSMRDBaseOffset(Addr, B, nullptr, Offset) &&
2081            SelectSMRDBaseOffset(B, SBase, SOffset, nullptr);
2082   }
2083 
2084   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
2085   // wraparound, because s_load instructions perform the addition in 64 bits.
2086   if (Addr.getValueType() == MVT::i32 && Addr.getOpcode() == ISD::ADD &&
2087       !Addr->getFlags().hasNoUnsignedWrap())
2088     return false;
2089 
2090   SDValue N0, N1;
2091   // Extract the base and offset if possible.
2092   if (CurDAG->isBaseWithConstantOffset(Addr) || Addr.getOpcode() == ISD::ADD) {
2093     N0 = Addr.getOperand(0);
2094     N1 = Addr.getOperand(1);
2095   } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
2096     assert(N0 && N1 && isa<ConstantSDNode>(N1));
2097   }
2098   if (!N0 || !N1)
2099     return false;
2100   if (SelectSMRDOffset(N1, SOffset, Offset, Imm32Only, IsBuffer)) {
2101     SBase = N0;
2102     return true;
2103   }
2104   if (SelectSMRDOffset(N0, SOffset, Offset, Imm32Only, IsBuffer)) {
2105     SBase = N1;
2106     return true;
2107   }
2108   return false;
2109 }
2110 
2111 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
2112                                     SDValue *SOffset, SDValue *Offset,
2113                                     bool Imm32Only) const {
2114   if (SelectSMRDBaseOffset(Addr, SBase, SOffset, Offset, Imm32Only)) {
2115     SBase = Expand32BitAddress(SBase);
2116     return true;
2117   }
2118 
2119   if (Addr.getValueType() == MVT::i32 && Offset && !SOffset) {
2120     SBase = Expand32BitAddress(Addr);
2121     *Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2122     return true;
2123   }
2124 
2125   return false;
2126 }
2127 
2128 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2129                                        SDValue &Offset) const {
2130   return SelectSMRD(Addr, SBase, /* SOffset */ nullptr, &Offset);
2131 }
2132 
2133 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2134                                          SDValue &Offset) const {
2135   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2136   return SelectSMRD(Addr, SBase, /* SOffset */ nullptr, &Offset,
2137                     /* Imm32Only */ true);
2138 }
2139 
2140 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2141                                         SDValue &SOffset) const {
2142   return SelectSMRD(Addr, SBase, &SOffset, /* Offset */ nullptr);
2143 }
2144 
2145 bool AMDGPUDAGToDAGISel::SelectSMRDSgprImm(SDValue Addr, SDValue &SBase,
2146                                            SDValue &SOffset,
2147                                            SDValue &Offset) const {
2148   return SelectSMRD(Addr, SBase, &SOffset, &Offset);
2149 }
2150 
2151 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue N, SDValue &Offset) const {
2152   return SelectSMRDOffset(N, /* SOffset */ nullptr, &Offset,
2153                           /* Imm32Only */ false, /* IsBuffer */ true);
2154 }
2155 
2156 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue N,
2157                                                SDValue &Offset) const {
2158   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2159   return SelectSMRDOffset(N, /* SOffset */ nullptr, &Offset,
2160                           /* Imm32Only */ true, /* IsBuffer */ true);
2161 }
2162 
2163 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
2164                                                  SDValue &Offset) const {
2165   // Match the (soffset + offset) pair as a 32-bit register base and
2166   // an immediate offset.
2167   return N.getValueType() == MVT::i32 &&
2168          SelectSMRDBaseOffset(N, /* SBase */ SOffset, /* SOffset*/ nullptr,
2169                               &Offset, /* Imm32Only */ false,
2170                               /* IsBuffer */ true);
2171 }
2172 
2173 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2174                                             SDValue &Base,
2175                                             SDValue &Offset) const {
2176   SDLoc DL(Index);
2177 
2178   if (CurDAG->isBaseWithConstantOffset(Index)) {
2179     SDValue N0 = Index.getOperand(0);
2180     SDValue N1 = Index.getOperand(1);
2181     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2182 
2183     // (add n0, c0)
2184     // Don't peel off the offset (c0) if doing so could possibly lead
2185     // the base (n0) to be negative.
2186     // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2187     if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2188         (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2189       Base = N0;
2190       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2191       return true;
2192     }
2193   }
2194 
2195   if (isa<ConstantSDNode>(Index))
2196     return false;
2197 
2198   Base = Index;
2199   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2200   return true;
2201 }
2202 
2203 SDNode *AMDGPUDAGToDAGISel::getBFE32(bool IsSigned, const SDLoc &DL,
2204                                      SDValue Val, uint32_t Offset,
2205                                      uint32_t Width) {
2206   if (Val->isDivergent()) {
2207     unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2208     SDValue Off = CurDAG->getTargetConstant(Offset, DL, MVT::i32);
2209     SDValue W = CurDAG->getTargetConstant(Width, DL, MVT::i32);
2210 
2211     return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, Off, W);
2212   }
2213   unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2214   // Transformation function, pack the offset and width of a BFE into
2215   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2216   // source, bits [5:0] contain the offset and bits [22:16] the width.
2217   uint32_t PackedVal = Offset | (Width << 16);
2218   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2219 
2220   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2221 }
2222 
2223 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2224   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2225   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2226   // Predicate: 0 < b <= c < 32
2227 
2228   const SDValue &Shl = N->getOperand(0);
2229   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2230   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2231 
2232   if (B && C) {
2233     uint32_t BVal = B->getZExtValue();
2234     uint32_t CVal = C->getZExtValue();
2235 
2236     if (0 < BVal && BVal <= CVal && CVal < 32) {
2237       bool Signed = N->getOpcode() == ISD::SRA;
2238       ReplaceNode(N, getBFE32(Signed, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2239                   32 - CVal));
2240       return;
2241     }
2242   }
2243   SelectCode(N);
2244 }
2245 
2246 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2247   switch (N->getOpcode()) {
2248   case ISD::AND:
2249     if (N->getOperand(0).getOpcode() == ISD::SRL) {
2250       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2251       // Predicate: isMask(mask)
2252       const SDValue &Srl = N->getOperand(0);
2253       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2254       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2255 
2256       if (Shift && Mask) {
2257         uint32_t ShiftVal = Shift->getZExtValue();
2258         uint32_t MaskVal = Mask->getZExtValue();
2259 
2260         if (isMask_32(MaskVal)) {
2261           uint32_t WidthVal = llvm::popcount(MaskVal);
2262           ReplaceNode(N, getBFE32(false, SDLoc(N), Srl.getOperand(0), ShiftVal,
2263                                   WidthVal));
2264           return;
2265         }
2266       }
2267     }
2268     break;
2269   case ISD::SRL:
2270     if (N->getOperand(0).getOpcode() == ISD::AND) {
2271       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2272       // Predicate: isMask(mask >> b)
2273       const SDValue &And = N->getOperand(0);
2274       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2275       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2276 
2277       if (Shift && Mask) {
2278         uint32_t ShiftVal = Shift->getZExtValue();
2279         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2280 
2281         if (isMask_32(MaskVal)) {
2282           uint32_t WidthVal = llvm::popcount(MaskVal);
2283           ReplaceNode(N, getBFE32(false, SDLoc(N), And.getOperand(0), ShiftVal,
2284                       WidthVal));
2285           return;
2286         }
2287       }
2288     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2289       SelectS_BFEFromShifts(N);
2290       return;
2291     }
2292     break;
2293   case ISD::SRA:
2294     if (N->getOperand(0).getOpcode() == ISD::SHL) {
2295       SelectS_BFEFromShifts(N);
2296       return;
2297     }
2298     break;
2299 
2300   case ISD::SIGN_EXTEND_INREG: {
2301     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2302     SDValue Src = N->getOperand(0);
2303     if (Src.getOpcode() != ISD::SRL)
2304       break;
2305 
2306     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2307     if (!Amt)
2308       break;
2309 
2310     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2311     ReplaceNode(N, getBFE32(true, SDLoc(N), Src.getOperand(0),
2312                             Amt->getZExtValue(), Width));
2313     return;
2314   }
2315   }
2316 
2317   SelectCode(N);
2318 }
2319 
2320 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2321   assert(N->getOpcode() == ISD::BRCOND);
2322   if (!N->hasOneUse())
2323     return false;
2324 
2325   SDValue Cond = N->getOperand(1);
2326   if (Cond.getOpcode() == ISD::CopyToReg)
2327     Cond = Cond.getOperand(2);
2328 
2329   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2330     return false;
2331 
2332   MVT VT = Cond.getOperand(0).getSimpleValueType();
2333   if (VT == MVT::i32)
2334     return true;
2335 
2336   if (VT == MVT::i64) {
2337     auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2338 
2339     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2340     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2341   }
2342 
2343   return false;
2344 }
2345 
2346 static SDValue combineBallotPattern(SDValue VCMP, bool &Negate) {
2347   assert(VCMP->getOpcode() == AMDGPUISD::SETCC);
2348   // Special case for amdgcn.ballot:
2349   // %Cond = i1 (and/or combination of i1 ISD::SETCCs)
2350   // %VCMP = i(WaveSize) AMDGPUISD::SETCC (ext %Cond), 0, setne/seteq
2351   // =>
2352   // Use i1 %Cond value instead of i(WaveSize) %VCMP.
2353   // This is possible because divergent ISD::SETCC is selected as V_CMP and
2354   // Cond becomes a i(WaveSize) full mask value.
2355   // Note that ballot doesn't use SETEQ condition but its easy to support it
2356   // here for completeness, so in this case Negate is set true on return.
2357   auto VCMP_CC = cast<CondCodeSDNode>(VCMP.getOperand(2))->get();
2358   if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) &&
2359       isNullConstant(VCMP.getOperand(1))) {
2360 
2361     auto Cond = VCMP.getOperand(0);
2362     if (ISD::isExtOpcode(Cond->getOpcode())) // Skip extension.
2363       Cond = Cond.getOperand(0);
2364 
2365     if (isBoolSGPR(Cond)) {
2366       Negate = VCMP_CC == ISD::SETEQ;
2367       return Cond;
2368     }
2369   }
2370   return SDValue();
2371 }
2372 
2373 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2374   SDValue Cond = N->getOperand(1);
2375 
2376   if (Cond.isUndef()) {
2377     CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2378                          N->getOperand(2), N->getOperand(0));
2379     return;
2380   }
2381 
2382   const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2383   const SIRegisterInfo *TRI = ST->getRegisterInfo();
2384 
2385   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2386   bool AndExec = !UseSCCBr;
2387   bool Negate = false;
2388 
2389   if (Cond.getOpcode() == ISD::SETCC &&
2390       Cond->getOperand(0)->getOpcode() == AMDGPUISD::SETCC) {
2391     SDValue VCMP = Cond->getOperand(0);
2392     auto CC = cast<CondCodeSDNode>(Cond->getOperand(2))->get();
2393     if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
2394         isNullConstant(Cond->getOperand(1)) &&
2395         // TODO: make condition below an assert after fixing ballot bitwidth.
2396         VCMP.getValueType().getSizeInBits() == ST->getWavefrontSize()) {
2397       // %VCMP = i(WaveSize) AMDGPUISD::SETCC ...
2398       // %C = i1 ISD::SETCC %VCMP, 0, setne/seteq
2399       // BRCOND i1 %C, %BB
2400       // =>
2401       // %VCMP = i(WaveSize) AMDGPUISD::SETCC ...
2402       // VCC = COPY i(WaveSize) %VCMP
2403       // S_CBRANCH_VCCNZ/VCCZ %BB
2404       Negate = CC == ISD::SETEQ;
2405       bool NegatedBallot = false;
2406       if (auto BallotCond = combineBallotPattern(VCMP, NegatedBallot)) {
2407         Cond = BallotCond;
2408         UseSCCBr = !BallotCond->isDivergent();
2409         Negate = Negate ^ NegatedBallot;
2410       } else {
2411         // TODO: don't use SCC here assuming that AMDGPUISD::SETCC is always
2412         // selected as V_CMP, but this may change for uniform condition.
2413         Cond = VCMP;
2414         UseSCCBr = false;
2415       }
2416     }
2417     // Cond is either V_CMP resulted from AMDGPUISD::SETCC or a combination of
2418     // V_CMPs resulted from ballot or ballot has uniform condition and SCC is
2419     // used.
2420     AndExec = false;
2421   }
2422 
2423   unsigned BrOp =
2424       UseSCCBr ? (Negate ? AMDGPU::S_CBRANCH_SCC0 : AMDGPU::S_CBRANCH_SCC1)
2425                : (Negate ? AMDGPU::S_CBRANCH_VCCZ : AMDGPU::S_CBRANCH_VCCNZ);
2426   Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2427   SDLoc SL(N);
2428 
2429   if (AndExec) {
2430     // This is the case that we are selecting to S_CBRANCH_VCCNZ.  We have not
2431     // analyzed what generates the vcc value, so we do not know whether vcc
2432     // bits for disabled lanes are 0.  Thus we need to mask out bits for
2433     // disabled lanes.
2434     //
2435     // For the case that we select S_CBRANCH_SCC1 and it gets
2436     // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2437     // SIInstrInfo::moveToVALU which inserts the S_AND).
2438     //
2439     // We could add an analysis of what generates the vcc value here and omit
2440     // the S_AND when is unnecessary. But it would be better to add a separate
2441     // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2442     // catches both cases.
2443     Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2444                                                          : AMDGPU::S_AND_B64,
2445                      SL, MVT::i1,
2446                      CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2447                                                         : AMDGPU::EXEC,
2448                                          MVT::i1),
2449                     Cond),
2450                    0);
2451   }
2452 
2453   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2454   CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2455                        N->getOperand(2), // Basic Block
2456                        VCC.getValue(0));
2457 }
2458 
2459 void AMDGPUDAGToDAGISel::SelectFP_EXTEND(SDNode *N) {
2460   if (Subtarget->hasSALUFloatInsts() && N->getValueType(0) == MVT::f32 &&
2461       !N->isDivergent()) {
2462     SDValue Src = N->getOperand(0);
2463     if (Src.getValueType() == MVT::f16) {
2464       if (isExtractHiElt(Src, Src)) {
2465         CurDAG->SelectNodeTo(N, AMDGPU::S_CVT_HI_F32_F16, N->getVTList(),
2466                              {Src});
2467         return;
2468       }
2469     }
2470   }
2471 
2472   SelectCode(N);
2473 }
2474 
2475 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2476   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2477   // be copied to an SGPR with readfirstlane.
2478   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2479     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2480 
2481   SDValue Chain = N->getOperand(0);
2482   SDValue Ptr = N->getOperand(2);
2483   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2484   MachineMemOperand *MMO = M->getMemOperand();
2485   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2486 
2487   SDValue Offset;
2488   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2489     SDValue PtrBase = Ptr.getOperand(0);
2490     SDValue PtrOffset = Ptr.getOperand(1);
2491 
2492     const APInt &OffsetVal = PtrOffset->getAsAPIntVal();
2493     if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) {
2494       N = glueCopyToM0(N, PtrBase);
2495       Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2496     }
2497   }
2498 
2499   if (!Offset) {
2500     N = glueCopyToM0(N, Ptr);
2501     Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2502   }
2503 
2504   SDValue Ops[] = {
2505     Offset,
2506     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2507     Chain,
2508     N->getOperand(N->getNumOperands() - 1) // New glue
2509   };
2510 
2511   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2512   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2513 }
2514 
2515 // We need to handle this here because tablegen doesn't support matching
2516 // instructions with multiple outputs.
2517 void AMDGPUDAGToDAGISel::SelectDSBvhStackIntrinsic(SDNode *N) {
2518   unsigned Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2519   SDValue Ops[] = {N->getOperand(2), N->getOperand(3), N->getOperand(4),
2520                    N->getOperand(5), N->getOperand(0)};
2521 
2522   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2523   MachineMemOperand *MMO = M->getMemOperand();
2524   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2525   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2526 }
2527 
2528 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2529   switch (IntrID) {
2530   case Intrinsic::amdgcn_ds_gws_init:
2531     return AMDGPU::DS_GWS_INIT;
2532   case Intrinsic::amdgcn_ds_gws_barrier:
2533     return AMDGPU::DS_GWS_BARRIER;
2534   case Intrinsic::amdgcn_ds_gws_sema_v:
2535     return AMDGPU::DS_GWS_SEMA_V;
2536   case Intrinsic::amdgcn_ds_gws_sema_br:
2537     return AMDGPU::DS_GWS_SEMA_BR;
2538   case Intrinsic::amdgcn_ds_gws_sema_p:
2539     return AMDGPU::DS_GWS_SEMA_P;
2540   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2541     return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2542   default:
2543     llvm_unreachable("not a gws intrinsic");
2544   }
2545 }
2546 
2547 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2548   if (!Subtarget->hasGWS() ||
2549       (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2550        !Subtarget->hasGWSSemaReleaseAll())) {
2551     // Let this error.
2552     SelectCode(N);
2553     return;
2554   }
2555 
2556   // Chain, intrinsic ID, vsrc, offset
2557   const bool HasVSrc = N->getNumOperands() == 4;
2558   assert(HasVSrc || N->getNumOperands() == 3);
2559 
2560   SDLoc SL(N);
2561   SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2562   int ImmOffset = 0;
2563   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2564   MachineMemOperand *MMO = M->getMemOperand();
2565 
2566   // Don't worry if the offset ends up in a VGPR. Only one lane will have
2567   // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2568 
2569   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2570   // offset field) % 64. Some versions of the programming guide omit the m0
2571   // part, or claim it's from offset 0.
2572   if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2573     // If we have a constant offset, try to use the 0 in m0 as the base.
2574     // TODO: Look into changing the default m0 initialization value. If the
2575     // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2576     // the immediate offset.
2577     glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2578     ImmOffset = ConstOffset->getZExtValue();
2579   } else {
2580     if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2581       ImmOffset = BaseOffset.getConstantOperandVal(1);
2582       BaseOffset = BaseOffset.getOperand(0);
2583     }
2584 
2585     // Prefer to do the shift in an SGPR since it should be possible to use m0
2586     // as the result directly. If it's already an SGPR, it will be eliminated
2587     // later.
2588     SDNode *SGPROffset
2589       = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2590                                BaseOffset);
2591     // Shift to offset in m0
2592     SDNode *M0Base
2593       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2594                                SDValue(SGPROffset, 0),
2595                                CurDAG->getTargetConstant(16, SL, MVT::i32));
2596     glueCopyToM0(N, SDValue(M0Base, 0));
2597   }
2598 
2599   SDValue Chain = N->getOperand(0);
2600   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2601 
2602   const unsigned Opc = gwsIntrinToOpcode(IntrID);
2603   SmallVector<SDValue, 5> Ops;
2604   if (HasVSrc)
2605     Ops.push_back(N->getOperand(2));
2606   Ops.push_back(OffsetField);
2607   Ops.push_back(Chain);
2608 
2609   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2610   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2611 }
2612 
2613 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2614   if (Subtarget->getLDSBankCount() != 16) {
2615     // This is a single instruction with a pattern.
2616     SelectCode(N);
2617     return;
2618   }
2619 
2620   SDLoc DL(N);
2621 
2622   // This requires 2 instructions. It is possible to write a pattern to support
2623   // this, but the generated isel emitter doesn't correctly deal with multiple
2624   // output instructions using the same physical register input. The copy to m0
2625   // is incorrectly placed before the second instruction.
2626   //
2627   // TODO: Match source modifiers.
2628   //
2629   // def : Pat <
2630   //   (int_amdgcn_interp_p1_f16
2631   //    (VOP3Mods f32:$src0, i32:$src0_modifiers),
2632   //                             (i32 timm:$attrchan), (i32 timm:$attr),
2633   //                             (i1 timm:$high), M0),
2634   //   (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2635   //       timm:$attrchan, 0,
2636   //       (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2637   //   let Predicates = [has16BankLDS];
2638   // }
2639 
2640   // 16 bank LDS
2641   SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2642                                       N->getOperand(5), SDValue());
2643 
2644   SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2645 
2646   SDNode *InterpMov =
2647     CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2648         CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2649         N->getOperand(3),  // Attr
2650         N->getOperand(2),  // Attrchan
2651         ToM0.getValue(1) // In glue
2652   });
2653 
2654   SDNode *InterpP1LV =
2655     CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2656         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2657         N->getOperand(1), // Src0
2658         N->getOperand(3), // Attr
2659         N->getOperand(2), // Attrchan
2660         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2661         SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2662         N->getOperand(4), // high
2663         CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2664         CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2665         SDValue(InterpMov, 1)
2666   });
2667 
2668   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2669 }
2670 
2671 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2672   unsigned IntrID = N->getConstantOperandVal(1);
2673   switch (IntrID) {
2674   case Intrinsic::amdgcn_ds_append:
2675   case Intrinsic::amdgcn_ds_consume: {
2676     if (N->getValueType(0) != MVT::i32)
2677       break;
2678     SelectDSAppendConsume(N, IntrID);
2679     return;
2680   }
2681   case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2682     SelectDSBvhStackIntrinsic(N);
2683     return;
2684   }
2685 
2686   SelectCode(N);
2687 }
2688 
2689 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2690   unsigned IntrID = N->getConstantOperandVal(0);
2691   unsigned Opcode;
2692   switch (IntrID) {
2693   case Intrinsic::amdgcn_wqm:
2694     Opcode = AMDGPU::WQM;
2695     break;
2696   case Intrinsic::amdgcn_softwqm:
2697     Opcode = AMDGPU::SOFT_WQM;
2698     break;
2699   case Intrinsic::amdgcn_wwm:
2700   case Intrinsic::amdgcn_strict_wwm:
2701     Opcode = AMDGPU::STRICT_WWM;
2702     break;
2703   case Intrinsic::amdgcn_strict_wqm:
2704     Opcode = AMDGPU::STRICT_WQM;
2705     break;
2706   case Intrinsic::amdgcn_interp_p1_f16:
2707     SelectInterpP1F16(N);
2708     return;
2709   case Intrinsic::amdgcn_inverse_ballot:
2710     switch (N->getOperand(1).getValueSizeInBits()) {
2711     case 32:
2712       Opcode = AMDGPU::S_INVERSE_BALLOT_U32;
2713       break;
2714     case 64:
2715       Opcode = AMDGPU::S_INVERSE_BALLOT_U64;
2716       break;
2717     default:
2718       llvm_unreachable("Unsupported size for inverse ballot mask.");
2719     }
2720     break;
2721   default:
2722     SelectCode(N);
2723     return;
2724   }
2725 
2726   SDValue Src = N->getOperand(1);
2727   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2728 }
2729 
2730 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2731   unsigned IntrID = N->getConstantOperandVal(1);
2732   switch (IntrID) {
2733   case Intrinsic::amdgcn_ds_gws_init:
2734   case Intrinsic::amdgcn_ds_gws_barrier:
2735   case Intrinsic::amdgcn_ds_gws_sema_v:
2736   case Intrinsic::amdgcn_ds_gws_sema_br:
2737   case Intrinsic::amdgcn_ds_gws_sema_p:
2738   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2739     SelectDS_GWS(N, IntrID);
2740     return;
2741   default:
2742     break;
2743   }
2744 
2745   SelectCode(N);
2746 }
2747 
2748 void AMDGPUDAGToDAGISel::SelectWAVE_ADDRESS(SDNode *N) {
2749   SDValue Log2WaveSize =
2750     CurDAG->getTargetConstant(Subtarget->getWavefrontSizeLog2(), SDLoc(N), MVT::i32);
2751   CurDAG->SelectNodeTo(N, AMDGPU::S_LSHR_B32, N->getVTList(),
2752                        {N->getOperand(0), Log2WaveSize});
2753 }
2754 
2755 void AMDGPUDAGToDAGISel::SelectSTACKRESTORE(SDNode *N) {
2756   SDValue SrcVal = N->getOperand(1);
2757   if (SrcVal.getValueType() != MVT::i32) {
2758     SelectCode(N); // Emit default error
2759     return;
2760   }
2761 
2762   SDValue CopyVal;
2763   Register SP = TLI->getStackPointerRegisterToSaveRestore();
2764   SDLoc SL(N);
2765 
2766   if (SrcVal.getOpcode() == AMDGPUISD::WAVE_ADDRESS) {
2767     CopyVal = SrcVal.getOperand(0);
2768   } else {
2769     SDValue Log2WaveSize = CurDAG->getTargetConstant(
2770         Subtarget->getWavefrontSizeLog2(), SL, MVT::i32);
2771 
2772     if (N->isDivergent()) {
2773       SrcVal = SDValue(CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL,
2774                                               MVT::i32, SrcVal),
2775                        0);
2776     }
2777 
2778     CopyVal = SDValue(CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2779                                              {SrcVal, Log2WaveSize}),
2780                       0);
2781   }
2782 
2783   SDValue CopyToSP = CurDAG->getCopyToReg(N->getOperand(0), SL, SP, CopyVal);
2784   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), CopyToSP);
2785 }
2786 
2787 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2788                                             unsigned &Mods,
2789                                             bool IsCanonicalizing,
2790                                             bool AllowAbs) const {
2791   Mods = SISrcMods::NONE;
2792   Src = In;
2793 
2794   if (Src.getOpcode() == ISD::FNEG) {
2795     Mods |= SISrcMods::NEG;
2796     Src = Src.getOperand(0);
2797   } else if (Src.getOpcode() == ISD::FSUB && IsCanonicalizing) {
2798     // Fold fsub [+-]0 into fneg. This may not have folded depending on the
2799     // denormal mode, but we're implicitly canonicalizing in a source operand.
2800     auto *LHS = dyn_cast<ConstantFPSDNode>(Src.getOperand(0));
2801     if (LHS && LHS->isZero()) {
2802       Mods |= SISrcMods::NEG;
2803       Src = Src.getOperand(1);
2804     }
2805   }
2806 
2807   if (AllowAbs && Src.getOpcode() == ISD::FABS) {
2808     Mods |= SISrcMods::ABS;
2809     Src = Src.getOperand(0);
2810   }
2811 
2812   return true;
2813 }
2814 
2815 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2816                                         SDValue &SrcMods) const {
2817   unsigned Mods;
2818   if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/true,
2819                          /*AllowAbs=*/true)) {
2820     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2821     return true;
2822   }
2823 
2824   return false;
2825 }
2826 
2827 bool AMDGPUDAGToDAGISel::SelectVOP3ModsNonCanonicalizing(
2828     SDValue In, SDValue &Src, SDValue &SrcMods) const {
2829   unsigned Mods;
2830   if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/false,
2831                          /*AllowAbs=*/true)) {
2832     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2833     return true;
2834   }
2835 
2836   return false;
2837 }
2838 
2839 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
2840                                          SDValue &SrcMods) const {
2841   unsigned Mods;
2842   if (SelectVOP3ModsImpl(In, Src, Mods,
2843                          /*IsCanonicalizing=*/true,
2844                          /*AllowAbs=*/false)) {
2845     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2846     return true;
2847   }
2848 
2849   return false;
2850 }
2851 
2852 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2853   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2854     return false;
2855 
2856   Src = In;
2857   return true;
2858 }
2859 
2860 bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
2861                                                SDValue &SrcMods,
2862                                                bool OpSel) const {
2863   unsigned Mods;
2864   if (SelectVOP3ModsImpl(In, Src, Mods,
2865                          /*IsCanonicalizing=*/true,
2866                          /*AllowAbs=*/false)) {
2867     if (OpSel)
2868       Mods |= SISrcMods::OP_SEL_0;
2869     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2870     return true;
2871   }
2872 
2873   return false;
2874 }
2875 
2876 bool AMDGPUDAGToDAGISel::SelectVINTERPMods(SDValue In, SDValue &Src,
2877                                            SDValue &SrcMods) const {
2878   return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ false);
2879 }
2880 
2881 bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(SDValue In, SDValue &Src,
2882                                              SDValue &SrcMods) const {
2883   return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ true);
2884 }
2885 
2886 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2887                                          SDValue &SrcMods, SDValue &Clamp,
2888                                          SDValue &Omod) const {
2889   SDLoc DL(In);
2890   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2891   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2892 
2893   return SelectVOP3Mods(In, Src, SrcMods);
2894 }
2895 
2896 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
2897                                           SDValue &SrcMods, SDValue &Clamp,
2898                                           SDValue &Omod) const {
2899   SDLoc DL(In);
2900   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2901   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2902 
2903   return SelectVOP3BMods(In, Src, SrcMods);
2904 }
2905 
2906 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2907                                          SDValue &Clamp, SDValue &Omod) const {
2908   Src = In;
2909 
2910   SDLoc DL(In);
2911   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2912   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2913 
2914   return true;
2915 }
2916 
2917 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2918                                          SDValue &SrcMods, bool IsDOT) const {
2919   unsigned Mods = SISrcMods::NONE;
2920   Src = In;
2921 
2922   // TODO: Handle G_FSUB 0 as fneg
2923   if (Src.getOpcode() == ISD::FNEG) {
2924     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2925     Src = Src.getOperand(0);
2926   }
2927 
2928   if (Src.getOpcode() == ISD::BUILD_VECTOR && Src.getNumOperands() == 2 &&
2929       (!IsDOT || !Subtarget->hasDOTOpSelHazard())) {
2930     unsigned VecMods = Mods;
2931 
2932     SDValue Lo = stripBitcast(Src.getOperand(0));
2933     SDValue Hi = stripBitcast(Src.getOperand(1));
2934 
2935     if (Lo.getOpcode() == ISD::FNEG) {
2936       Lo = stripBitcast(Lo.getOperand(0));
2937       Mods ^= SISrcMods::NEG;
2938     }
2939 
2940     if (Hi.getOpcode() == ISD::FNEG) {
2941       Hi = stripBitcast(Hi.getOperand(0));
2942       Mods ^= SISrcMods::NEG_HI;
2943     }
2944 
2945     if (isExtractHiElt(Lo, Lo))
2946       Mods |= SISrcMods::OP_SEL_0;
2947 
2948     if (isExtractHiElt(Hi, Hi))
2949       Mods |= SISrcMods::OP_SEL_1;
2950 
2951     unsigned VecSize = Src.getValueSizeInBits();
2952     Lo = stripExtractLoElt(Lo);
2953     Hi = stripExtractLoElt(Hi);
2954 
2955     if (Lo.getValueSizeInBits() > VecSize) {
2956       Lo = CurDAG->getTargetExtractSubreg(
2957         (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2958         MVT::getIntegerVT(VecSize), Lo);
2959     }
2960 
2961     if (Hi.getValueSizeInBits() > VecSize) {
2962       Hi = CurDAG->getTargetExtractSubreg(
2963         (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2964         MVT::getIntegerVT(VecSize), Hi);
2965     }
2966 
2967     assert(Lo.getValueSizeInBits() <= VecSize &&
2968            Hi.getValueSizeInBits() <= VecSize);
2969 
2970     if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2971       // Really a scalar input. Just select from the low half of the register to
2972       // avoid packing.
2973 
2974       if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) {
2975         Src = Lo;
2976       } else {
2977         assert(Lo.getValueSizeInBits() == 32 && VecSize == 64);
2978 
2979         SDLoc SL(In);
2980         SDValue Undef = SDValue(
2981           CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
2982                                  Lo.getValueType()), 0);
2983         auto RC = Lo->isDivergent() ? AMDGPU::VReg_64RegClassID
2984                                     : AMDGPU::SReg_64RegClassID;
2985         const SDValue Ops[] = {
2986           CurDAG->getTargetConstant(RC, SL, MVT::i32),
2987           Lo, CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2988           Undef, CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32) };
2989 
2990         Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
2991                                              Src.getValueType(), Ops), 0);
2992       }
2993       SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2994       return true;
2995     }
2996 
2997     if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) {
2998       uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF()
2999                       .bitcastToAPInt().getZExtValue();
3000       if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) {
3001         Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);
3002         SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3003         return true;
3004       }
3005     }
3006 
3007     Mods = VecMods;
3008   }
3009 
3010   // Packed instructions do not have abs modifiers.
3011   Mods |= SISrcMods::OP_SEL_1;
3012 
3013   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3014   return true;
3015 }
3016 
3017 bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(SDValue In, SDValue &Src,
3018                                             SDValue &SrcMods) const {
3019   return SelectVOP3PMods(In, Src, SrcMods, true);
3020 }
3021 
3022 bool AMDGPUDAGToDAGISel::SelectVOP3PModsNeg(SDValue In, SDValue &Src) const {
3023   const ConstantSDNode *C = cast<ConstantSDNode>(In);
3024   // Literal i1 value set in intrinsic, represents SrcMods for the next operand.
3025   // 1 promotes packed values to signed, 0 treats them as unsigned.
3026   assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
3027 
3028   unsigned Mods = SISrcMods::OP_SEL_1;
3029   unsigned SrcSign = C->getZExtValue();
3030   if (SrcSign == 1)
3031     Mods ^= SISrcMods::NEG;
3032 
3033   Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3034   return true;
3035 }
3036 
3037 bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(SDValue In,
3038                                                   SDValue &Src) const {
3039   const ConstantSDNode *C = cast<ConstantSDNode>(In);
3040   assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
3041 
3042   unsigned Mods = SISrcMods::OP_SEL_1;
3043   unsigned SrcVal = C->getZExtValue();
3044   if (SrcVal == 1)
3045     Mods |= SISrcMods::OP_SEL_0;
3046 
3047   Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3048   return true;
3049 }
3050 
3051 static MachineSDNode *buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
3052                                          llvm::SelectionDAG *CurDAG,
3053                                          const SDLoc &DL) {
3054   unsigned DstRegClass;
3055   EVT DstTy;
3056   switch (Elts.size()) {
3057   case 8:
3058     DstRegClass = AMDGPU::VReg_256RegClassID;
3059     DstTy = MVT::v8i32;
3060     break;
3061   case 4:
3062     DstRegClass = AMDGPU::VReg_128RegClassID;
3063     DstTy = MVT::v4i32;
3064     break;
3065   case 2:
3066     DstRegClass = AMDGPU::VReg_64RegClassID;
3067     DstTy = MVT::v2i32;
3068     break;
3069   default:
3070     llvm_unreachable("unhandled Reg sequence size");
3071   }
3072 
3073   SmallVector<SDValue, 17> Ops;
3074   Ops.push_back(CurDAG->getTargetConstant(DstRegClass, DL, MVT::i32));
3075   for (unsigned i = 0; i < Elts.size(); ++i) {
3076     Ops.push_back(Elts[i]);
3077     Ops.push_back(CurDAG->getTargetConstant(
3078         SIRegisterInfo::getSubRegFromChannel(i), DL, MVT::i32));
3079   }
3080   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, DstTy, Ops);
3081 }
3082 
3083 static MachineSDNode *buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
3084                                          llvm::SelectionDAG *CurDAG,
3085                                          const SDLoc &DL) {
3086   SmallVector<SDValue, 8> PackedElts;
3087   assert("unhandled Reg sequence size" &&
3088          (Elts.size() == 8 || Elts.size() == 16));
3089 
3090   // Pack 16-bit elements in pairs into 32-bit register. If both elements are
3091   // unpacked from 32-bit source use it, otherwise pack them using v_perm.
3092   for (unsigned i = 0; i < Elts.size(); i += 2) {
3093     SDValue LoSrc = stripExtractLoElt(stripBitcast(Elts[i]));
3094     SDValue HiSrc;
3095     if (isExtractHiElt(Elts[i + 1], HiSrc) && LoSrc == HiSrc) {
3096       PackedElts.push_back(HiSrc);
3097     } else {
3098       SDValue PackLoLo = CurDAG->getTargetConstant(0x05040100, DL, MVT::i32);
3099       MachineSDNode *Packed =
3100           CurDAG->getMachineNode(AMDGPU::V_PERM_B32_e64, DL, MVT::i32,
3101                                  {Elts[i + 1], Elts[i], PackLoLo});
3102       PackedElts.push_back(SDValue(Packed, 0));
3103     }
3104   }
3105 
3106   return buildRegSequence32(PackedElts, CurDAG, DL);
3107 }
3108 
3109 static MachineSDNode *buildRegSequence(SmallVectorImpl<SDValue> &Elts,
3110                                        llvm::SelectionDAG *CurDAG,
3111                                        const SDLoc &DL, unsigned ElementSize) {
3112   if (ElementSize == 16)
3113     return buildRegSequence16(Elts, CurDAG, DL);
3114   if (ElementSize == 32)
3115     return buildRegSequence32(Elts, CurDAG, DL);
3116   llvm_unreachable("Unhandled element size");
3117 }
3118 
3119 static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
3120                                  SmallVectorImpl<SDValue> &Elts, SDValue &Src,
3121                                  llvm::SelectionDAG *CurDAG, const SDLoc &DL,
3122                                  unsigned ElementSize) {
3123   if (ModOpcode == ISD::FNEG) {
3124     Mods |= SISrcMods::NEG;
3125     // Check if all elements also have abs modifier
3126     SmallVector<SDValue, 8> NegAbsElts;
3127     for (auto El : Elts) {
3128       if (El.getOpcode() != ISD::FABS)
3129         break;
3130       NegAbsElts.push_back(El->getOperand(0));
3131     }
3132     if (Elts.size() != NegAbsElts.size()) {
3133       // Neg
3134       Src = SDValue(buildRegSequence(Elts, CurDAG, DL, ElementSize), 0);
3135     } else {
3136       // Neg and Abs
3137       Mods |= SISrcMods::NEG_HI;
3138       Src = SDValue(buildRegSequence(NegAbsElts, CurDAG, DL, ElementSize), 0);
3139     }
3140   } else {
3141     assert(ModOpcode == ISD::FABS);
3142     // Abs
3143     Mods |= SISrcMods::NEG_HI;
3144     Src = SDValue(buildRegSequence(Elts, CurDAG, DL, ElementSize), 0);
3145   }
3146 }
3147 
3148 // Check all f16 elements for modifiers while looking through b32 and v2b16
3149 // build vector, stop if element does not satisfy ModifierCheck.
3150 static void
3151 checkWMMAElementsModifiersF16(BuildVectorSDNode *BV,
3152                               std::function<bool(SDValue)> ModifierCheck) {
3153   for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
3154     if (auto *F16Pair =
3155             dyn_cast<BuildVectorSDNode>(stripBitcast(BV->getOperand(i)))) {
3156       for (unsigned i = 0; i < F16Pair->getNumOperands(); ++i) {
3157         SDValue ElF16 = stripBitcast(F16Pair->getOperand(i));
3158         if (!ModifierCheck(ElF16))
3159           break;
3160       }
3161     }
3162   }
3163 }
3164 
3165 bool AMDGPUDAGToDAGISel::SelectWMMAModsF16Neg(SDValue In, SDValue &Src,
3166                                               SDValue &SrcMods) const {
3167   Src = In;
3168   unsigned Mods = SISrcMods::OP_SEL_1;
3169 
3170   // mods are on f16 elements
3171   if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
3172     SmallVector<SDValue, 8> EltsF16;
3173 
3174     checkWMMAElementsModifiersF16(BV, [&](SDValue Element) -> bool {
3175       if (Element.getOpcode() != ISD::FNEG)
3176         return false;
3177       EltsF16.push_back(Element.getOperand(0));
3178       return true;
3179     });
3180 
3181     // All elements have neg modifier
3182     if (BV->getNumOperands() * 2 == EltsF16.size()) {
3183       Src = SDValue(buildRegSequence16(EltsF16, CurDAG, SDLoc(In)), 0);
3184       Mods |= SISrcMods::NEG;
3185       Mods |= SISrcMods::NEG_HI;
3186     }
3187   }
3188 
3189   // mods are on v2f16 elements
3190   if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
3191     SmallVector<SDValue, 8> EltsV2F16;
3192     for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
3193       SDValue ElV2f16 = stripBitcast(BV->getOperand(i));
3194       // Based on first element decide which mod we match, neg or abs
3195       if (ElV2f16.getOpcode() != ISD::FNEG)
3196         break;
3197       EltsV2F16.push_back(ElV2f16.getOperand(0));
3198     }
3199 
3200     // All pairs of elements have neg modifier
3201     if (BV->getNumOperands() == EltsV2F16.size()) {
3202       Src = SDValue(buildRegSequence32(EltsV2F16, CurDAG, SDLoc(In)), 0);
3203       Mods |= SISrcMods::NEG;
3204       Mods |= SISrcMods::NEG_HI;
3205     }
3206   }
3207 
3208   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3209   return true;
3210 }
3211 
3212 bool AMDGPUDAGToDAGISel::SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
3213                                                  SDValue &SrcMods) const {
3214   Src = In;
3215   unsigned Mods = SISrcMods::OP_SEL_1;
3216   unsigned ModOpcode;
3217 
3218   // mods are on f16 elements
3219   if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
3220     SmallVector<SDValue, 8> EltsF16;
3221     checkWMMAElementsModifiersF16(BV, [&](SDValue ElF16) -> bool {
3222       // Based on first element decide which mod we match, neg or abs
3223       if (EltsF16.empty())
3224         ModOpcode = (ElF16.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
3225       if (ElF16.getOpcode() != ModOpcode)
3226         return false;
3227       EltsF16.push_back(ElF16.getOperand(0));
3228       return true;
3229     });
3230 
3231     // All elements have ModOpcode modifier
3232     if (BV->getNumOperands() * 2 == EltsF16.size())
3233       selectWMMAModsNegAbs(ModOpcode, Mods, EltsF16, Src, CurDAG, SDLoc(In),
3234                            16);
3235   }
3236 
3237   // mods are on v2f16 elements
3238   if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
3239     SmallVector<SDValue, 8> EltsV2F16;
3240 
3241     for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
3242       SDValue ElV2f16 = stripBitcast(BV->getOperand(i));
3243       // Based on first element decide which mod we match, neg or abs
3244       if (EltsV2F16.empty())
3245         ModOpcode = (ElV2f16.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
3246       if (ElV2f16->getOpcode() != ModOpcode)
3247         break;
3248       EltsV2F16.push_back(ElV2f16->getOperand(0));
3249     }
3250 
3251     // All elements have ModOpcode modifier
3252     if (BV->getNumOperands() == EltsV2F16.size())
3253       selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, CurDAG, SDLoc(In),
3254                            32);
3255   }
3256 
3257   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3258   return true;
3259 }
3260 
3261 bool AMDGPUDAGToDAGISel::SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
3262                                                  SDValue &SrcMods) const {
3263   Src = In;
3264   unsigned Mods = SISrcMods::OP_SEL_1;
3265   unsigned ModOpcode;
3266   SmallVector<SDValue, 8> EltsF32;
3267 
3268   if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
3269     for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
3270       SDValue ElF32 = stripBitcast(BV->getOperand(i));
3271       // Based on first element decide which mod we match, neg or abs
3272       if (EltsF32.empty())
3273         ModOpcode = (ElF32.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
3274       if (ElF32.getOpcode() != ModOpcode)
3275         break;
3276       EltsF32.push_back(ElF32.getOperand(0));
3277     }
3278 
3279     // All elements had ModOpcode modifier
3280     if (BV->getNumOperands() == EltsF32.size())
3281       selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, CurDAG, SDLoc(In),
3282                            32);
3283   }
3284 
3285   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3286   return true;
3287 }
3288 
3289 bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const {
3290   if (auto *BV = dyn_cast<BuildVectorSDNode>(In)) {
3291     BitVector UndefElements;
3292     if (SDValue Splat = BV->getSplatValue(&UndefElements))
3293       if (isInlineImmediate(Splat.getNode())) {
3294         if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Splat)) {
3295           unsigned Imm = C->getAPIntValue().getSExtValue();
3296           Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
3297           return true;
3298         }
3299         if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Splat)) {
3300           unsigned Imm = C->getValueAPF().bitcastToAPInt().getSExtValue();
3301           Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
3302           return true;
3303         }
3304         llvm_unreachable("unhandled Constant node");
3305       }
3306   }
3307 
3308   // 16 bit splat
3309   SDValue SplatSrc32 = stripBitcast(In);
3310   if (auto *SplatSrc32BV = dyn_cast<BuildVectorSDNode>(SplatSrc32)) {
3311     if (SDValue Splat32 = SplatSrc32BV->getSplatValue()) {
3312       SDValue SplatSrc16 = stripBitcast(Splat32);
3313       if (auto *SplatSrc16BV = dyn_cast<BuildVectorSDNode>(SplatSrc16)) {
3314         if (SDValue Splat = SplatSrc16BV->getSplatValue()) {
3315 
3316           // f16
3317           if (isInlineImmediate(Splat.getNode())) {
3318             const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Splat);
3319             int64_t Imm = C->getValueAPF().bitcastToAPInt().getSExtValue();
3320             Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i16);
3321             return true;
3322           }
3323 
3324           // bf16
3325           if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Splat)) {
3326             const SIInstrInfo *TII = Subtarget->getInstrInfo();
3327             APInt BF16Value = C->getAPIntValue();
3328             APInt F32Value = BF16Value.zext(32).shl(16);
3329             if (TII->isInlineConstant(F32Value)) {
3330               int64_t Imm = F32Value.getSExtValue();
3331               Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
3332               return true;
3333             }
3334           }
3335         }
3336       }
3337     }
3338   }
3339 
3340   return false;
3341 }
3342 
3343 bool AMDGPUDAGToDAGISel::SelectSWMMACIndex8(SDValue In, SDValue &Src,
3344                                             SDValue &IndexKey) const {
3345   unsigned Key = 0;
3346   Src = In;
3347 
3348   if (In.getOpcode() == ISD::SRL) {
3349     const llvm::SDValue &ShiftSrc = In.getOperand(0);
3350     ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(In.getOperand(1));
3351     if (ShiftSrc.getValueType().getSizeInBits() == 32 && ShiftAmt &&
3352         ShiftAmt->getZExtValue() % 8 == 0) {
3353       Key = ShiftAmt->getZExtValue() / 8;
3354       Src = ShiftSrc;
3355     }
3356   }
3357 
3358   IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32);
3359   return true;
3360 }
3361 
3362 bool AMDGPUDAGToDAGISel::SelectSWMMACIndex16(SDValue In, SDValue &Src,
3363                                              SDValue &IndexKey) const {
3364   unsigned Key = 0;
3365   Src = In;
3366 
3367   if (In.getOpcode() == ISD::SRL) {
3368     const llvm::SDValue &ShiftSrc = In.getOperand(0);
3369     ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(In.getOperand(1));
3370     if (ShiftSrc.getValueType().getSizeInBits() == 32 && ShiftAmt &&
3371         ShiftAmt->getZExtValue() == 16) {
3372       Key = 1;
3373       Src = ShiftSrc;
3374     }
3375   }
3376 
3377   IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32);
3378   return true;
3379 }
3380 
3381 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
3382                                          SDValue &SrcMods) const {
3383   Src = In;
3384   // FIXME: Handle op_sel
3385   SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
3386   return true;
3387 }
3388 
3389 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
3390                                              SDValue &SrcMods) const {
3391   // FIXME: Handle op_sel
3392   return SelectVOP3Mods(In, Src, SrcMods);
3393 }
3394 
3395 // The return value is not whether the match is possible (which it always is),
3396 // but whether or not it a conversion is really used.
3397 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
3398                                                    unsigned &Mods) const {
3399   Mods = 0;
3400   SelectVOP3ModsImpl(In, Src, Mods);
3401 
3402   if (Src.getOpcode() == ISD::FP_EXTEND) {
3403     Src = Src.getOperand(0);
3404     assert(Src.getValueType() == MVT::f16);
3405     Src = stripBitcast(Src);
3406 
3407     // Be careful about folding modifiers if we already have an abs. fneg is
3408     // applied last, so we don't want to apply an earlier fneg.
3409     if ((Mods & SISrcMods::ABS) == 0) {
3410       unsigned ModsTmp;
3411       SelectVOP3ModsImpl(Src, Src, ModsTmp);
3412 
3413       if ((ModsTmp & SISrcMods::NEG) != 0)
3414         Mods ^= SISrcMods::NEG;
3415 
3416       if ((ModsTmp & SISrcMods::ABS) != 0)
3417         Mods |= SISrcMods::ABS;
3418     }
3419 
3420     // op_sel/op_sel_hi decide the source type and source.
3421     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
3422     // If the sources's op_sel is set, it picks the high half of the source
3423     // register.
3424 
3425     Mods |= SISrcMods::OP_SEL_1;
3426     if (isExtractHiElt(Src, Src)) {
3427       Mods |= SISrcMods::OP_SEL_0;
3428 
3429       // TODO: Should we try to look for neg/abs here?
3430     }
3431 
3432     return true;
3433   }
3434 
3435   return false;
3436 }
3437 
3438 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
3439                                                   SDValue &SrcMods) const {
3440   unsigned Mods = 0;
3441   if (!SelectVOP3PMadMixModsImpl(In, Src, Mods))
3442     return false;
3443   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3444   return true;
3445 }
3446 
3447 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
3448                                                SDValue &SrcMods) const {
3449   unsigned Mods = 0;
3450   SelectVOP3PMadMixModsImpl(In, Src, Mods);
3451   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3452   return true;
3453 }
3454 
3455 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
3456   if (In.isUndef())
3457     return CurDAG->getUNDEF(MVT::i32);
3458 
3459   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
3460     SDLoc SL(In);
3461     return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
3462   }
3463 
3464   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
3465     SDLoc SL(In);
3466     return CurDAG->getConstant(
3467       C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
3468   }
3469 
3470   SDValue Src;
3471   if (isExtractHiElt(In, Src))
3472     return Src;
3473 
3474   return SDValue();
3475 }
3476 
3477 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
3478   assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
3479 
3480   const SIRegisterInfo *SIRI =
3481     static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
3482   const SIInstrInfo * SII =
3483     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
3484 
3485   unsigned Limit = 0;
3486   bool AllUsesAcceptSReg = true;
3487   for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
3488     Limit < 10 && U != E; ++U, ++Limit) {
3489     const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
3490 
3491     // If the register class is unknown, it could be an unknown
3492     // register class that needs to be an SGPR, e.g. an inline asm
3493     // constraint
3494     if (!RC || SIRI->isSGPRClass(RC))
3495       return false;
3496 
3497     if (RC != &AMDGPU::VS_32RegClass && RC != &AMDGPU::VS_64RegClass) {
3498       AllUsesAcceptSReg = false;
3499       SDNode * User = *U;
3500       if (User->isMachineOpcode()) {
3501         unsigned Opc = User->getMachineOpcode();
3502         const MCInstrDesc &Desc = SII->get(Opc);
3503         if (Desc.isCommutable()) {
3504           unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
3505           unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
3506           if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
3507             unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
3508             const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
3509             if (CommutedRC == &AMDGPU::VS_32RegClass ||
3510                 CommutedRC == &AMDGPU::VS_64RegClass)
3511               AllUsesAcceptSReg = true;
3512           }
3513         }
3514       }
3515       // If "AllUsesAcceptSReg == false" so far we haven't succeeded
3516       // commuting current user. This means have at least one use
3517       // that strictly require VGPR. Thus, we will not attempt to commute
3518       // other user instructions.
3519       if (!AllUsesAcceptSReg)
3520         break;
3521     }
3522   }
3523   return !AllUsesAcceptSReg && (Limit < 10);
3524 }
3525 
3526 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
3527   auto Ld = cast<LoadSDNode>(N);
3528 
3529   const MachineMemOperand *MMO = Ld->getMemOperand();
3530   if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(MMO))
3531     return false;
3532 
3533   return Ld->getAlign() >= Align(std::min(MMO->getSize(), uint64_t(4))) &&
3534          ((Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
3535            Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
3536           (Subtarget->getScalarizeGlobalBehavior() &&
3537            Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
3538            Ld->isSimple() &&
3539            static_cast<const SITargetLowering *>(getTargetLowering())
3540                ->isMemOpHasNoClobberedMemOperand(N)));
3541 }
3542 
3543 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
3544   const AMDGPUTargetLowering& Lowering =
3545     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
3546   bool IsModified = false;
3547   do {
3548     IsModified = false;
3549 
3550     // Go over all selected nodes and try to fold them a bit more
3551     SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
3552     while (Position != CurDAG->allnodes_end()) {
3553       SDNode *Node = &*Position++;
3554       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
3555       if (!MachineNode)
3556         continue;
3557 
3558       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
3559       if (ResNode != Node) {
3560         if (ResNode)
3561           ReplaceUses(Node, ResNode);
3562         IsModified = true;
3563       }
3564     }
3565     CurDAG->RemoveDeadNodes();
3566   } while (IsModified);
3567 }
3568 
3569 char AMDGPUDAGToDAGISel::ID = 0;
3570