xref: /freebsd/contrib/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp (revision 2f2ebe758bea201830bd021525424813f7fc6c6b)
1 //===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that NVPTX uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "NVPTXISelLowering.h"
15 #include "MCTargetDesc/NVPTXBaseInfo.h"
16 #include "NVPTX.h"
17 #include "NVPTXSubtarget.h"
18 #include "NVPTXTargetMachine.h"
19 #include "NVPTXTargetObjectFile.h"
20 #include "NVPTXUtilities.h"
21 #include "llvm/ADT/APInt.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/CodeGen/Analysis.h"
26 #include "llvm/CodeGen/ISDOpcodes.h"
27 #include "llvm/CodeGen/MachineFunction.h"
28 #include "llvm/CodeGen/MachineMemOperand.h"
29 #include "llvm/CodeGen/SelectionDAG.h"
30 #include "llvm/CodeGen/SelectionDAGNodes.h"
31 #include "llvm/CodeGen/TargetCallingConv.h"
32 #include "llvm/CodeGen/TargetLowering.h"
33 #include "llvm/CodeGen/ValueTypes.h"
34 #include "llvm/CodeGenTypes/MachineValueType.h"
35 #include "llvm/IR/Argument.h"
36 #include "llvm/IR/Attributes.h"
37 #include "llvm/IR/Constants.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/IR/DerivedTypes.h"
40 #include "llvm/IR/DiagnosticInfo.h"
41 #include "llvm/IR/FPEnv.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/GlobalValue.h"
44 #include "llvm/IR/Instruction.h"
45 #include "llvm/IR/Instructions.h"
46 #include "llvm/IR/IntrinsicsNVPTX.h"
47 #include "llvm/IR/Module.h"
48 #include "llvm/IR/Type.h"
49 #include "llvm/IR/Value.h"
50 #include "llvm/Support/Alignment.h"
51 #include "llvm/Support/Casting.h"
52 #include "llvm/Support/CodeGen.h"
53 #include "llvm/Support/CommandLine.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/raw_ostream.h"
56 #include "llvm/Target/TargetMachine.h"
57 #include "llvm/Target/TargetOptions.h"
58 #include <algorithm>
59 #include <cassert>
60 #include <cmath>
61 #include <cstdint>
62 #include <iterator>
63 #include <optional>
64 #include <sstream>
65 #include <string>
66 #include <utility>
67 #include <vector>
68 
69 #define DEBUG_TYPE "nvptx-lower"
70 
71 using namespace llvm;
72 
73 static std::atomic<unsigned> GlobalUniqueCallSite;
74 
75 static cl::opt<bool> sched4reg(
76     "nvptx-sched4reg",
77     cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));
78 
79 static cl::opt<unsigned> FMAContractLevelOpt(
80     "nvptx-fma-level", cl::Hidden,
81     cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
82              " 1: do it  2: do it aggressively"),
83     cl::init(2));
84 
85 static cl::opt<int> UsePrecDivF32(
86     "nvptx-prec-divf32", cl::Hidden,
87     cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
88              " IEEE Compliant F32 div.rnd if available."),
89     cl::init(2));
90 
91 static cl::opt<bool> UsePrecSqrtF32(
92     "nvptx-prec-sqrtf32", cl::Hidden,
93     cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
94     cl::init(true));
95 
96 static cl::opt<bool> ForceMinByValParamAlign(
97     "nvptx-force-min-byval-param-align", cl::Hidden,
98     cl::desc("NVPTX Specific: force 4-byte minimal alignment for byval"
99              " params of device functions."),
100     cl::init(false));
101 
102 int NVPTXTargetLowering::getDivF32Level() const {
103   if (UsePrecDivF32.getNumOccurrences() > 0) {
104     // If nvptx-prec-div32=N is used on the command-line, always honor it
105     return UsePrecDivF32;
106   } else {
107     // Otherwise, use div.approx if fast math is enabled
108     if (getTargetMachine().Options.UnsafeFPMath)
109       return 0;
110     else
111       return 2;
112   }
113 }
114 
115 bool NVPTXTargetLowering::usePrecSqrtF32() const {
116   if (UsePrecSqrtF32.getNumOccurrences() > 0) {
117     // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
118     return UsePrecSqrtF32;
119   } else {
120     // Otherwise, use sqrt.approx if fast math is enabled
121     return !getTargetMachine().Options.UnsafeFPMath;
122   }
123 }
124 
125 bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {
126   return MF.getDenormalMode(APFloat::IEEEsingle()).Output ==
127          DenormalMode::PreserveSign;
128 }
129 
130 static bool IsPTXVectorType(MVT VT) {
131   switch (VT.SimpleTy) {
132   default:
133     return false;
134   case MVT::v2i1:
135   case MVT::v4i1:
136   case MVT::v2i8:
137   case MVT::v4i8:
138   case MVT::v2i16:
139   case MVT::v4i16:
140   case MVT::v8i16: // <4 x i16x2>
141   case MVT::v2i32:
142   case MVT::v4i32:
143   case MVT::v2i64:
144   case MVT::v2f16:
145   case MVT::v4f16:
146   case MVT::v8f16: // <4 x f16x2>
147   case MVT::v2bf16:
148   case MVT::v4bf16:
149   case MVT::v8bf16: // <4 x bf16x2>
150   case MVT::v2f32:
151   case MVT::v4f32:
152   case MVT::v2f64:
153     return true;
154   }
155 }
156 
157 static bool Is16bitsType(MVT VT) {
158   return (VT.SimpleTy == MVT::f16 || VT.SimpleTy == MVT::bf16 ||
159           VT.SimpleTy == MVT::i16);
160 }
161 
162 /// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
163 /// EVTs that compose it.  Unlike ComputeValueVTs, this will break apart vectors
164 /// into their primitive components.
165 /// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
166 /// same number of types as the Ins/Outs arrays in LowerFormalArguments,
167 /// LowerCall, and LowerReturn.
168 static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
169                                Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
170                                SmallVectorImpl<uint64_t> *Offsets = nullptr,
171                                uint64_t StartingOffset = 0) {
172   SmallVector<EVT, 16> TempVTs;
173   SmallVector<uint64_t, 16> TempOffsets;
174 
175   // Special case for i128 - decompose to (i64, i64)
176   if (Ty->isIntegerTy(128)) {
177     ValueVTs.push_back(EVT(MVT::i64));
178     ValueVTs.push_back(EVT(MVT::i64));
179 
180     if (Offsets) {
181       Offsets->push_back(StartingOffset + 0);
182       Offsets->push_back(StartingOffset + 8);
183     }
184 
185     return;
186   }
187 
188   // Given a struct type, recursively traverse the elements with custom ComputePTXValueVTs.
189   if (StructType *STy = dyn_cast<StructType>(Ty)) {
190     auto const *SL = DL.getStructLayout(STy);
191     auto ElementNum = 0;
192     for(auto *EI : STy->elements()) {
193       ComputePTXValueVTs(TLI, DL, EI, ValueVTs, Offsets,
194                          StartingOffset + SL->getElementOffset(ElementNum));
195       ++ElementNum;
196     }
197     return;
198   }
199 
200   ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset);
201   for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {
202     EVT VT = TempVTs[i];
203     uint64_t Off = TempOffsets[i];
204     // Split vectors into individual elements, except for v2f16, which
205     // we will pass as a single scalar.
206     if (VT.isVector()) {
207       unsigned NumElts = VT.getVectorNumElements();
208       EVT EltVT = VT.getVectorElementType();
209       // Vectors with an even number of f16 elements will be passed to
210       // us as an array of v2f16/v2bf16 elements. We must match this so we
211       // stay in sync with Ins/Outs.
212       if ((Is16bitsType(EltVT.getSimpleVT())) && NumElts % 2 == 0) {
213         switch (EltVT.getSimpleVT().SimpleTy) {
214         case MVT::f16:
215           EltVT = MVT::v2f16;
216           break;
217         case MVT::bf16:
218           EltVT = MVT::v2bf16;
219           break;
220         case MVT::i16:
221           EltVT = MVT::v2i16;
222           break;
223         default:
224           llvm_unreachable("Unexpected type");
225         }
226         NumElts /= 2;
227       } else if (EltVT.getSimpleVT() == MVT::i8 &&
228                  (NumElts % 4 == 0 || NumElts == 3)) {
229         // v*i8 are formally lowered as v4i8
230         EltVT = MVT::v4i8;
231         NumElts = (NumElts + 3) / 4;
232       }
233       for (unsigned j = 0; j != NumElts; ++j) {
234         ValueVTs.push_back(EltVT);
235         if (Offsets)
236           Offsets->push_back(Off + j * EltVT.getStoreSize());
237       }
238     } else {
239       ValueVTs.push_back(VT);
240       if (Offsets)
241         Offsets->push_back(Off);
242     }
243   }
244 }
245 
246 /// PromoteScalarIntegerPTX
247 /// Used to make sure the arguments/returns are suitable for passing
248 /// and promote them to a larger size if they're not.
249 ///
250 /// The promoted type is placed in \p PromoteVT if the function returns true.
251 static bool PromoteScalarIntegerPTX(const EVT &VT, MVT *PromotedVT) {
252   if (VT.isScalarInteger()) {
253     switch (PowerOf2Ceil(VT.getFixedSizeInBits())) {
254     default:
255       llvm_unreachable(
256           "Promotion is not suitable for scalars of size larger than 64-bits");
257     case 1:
258       *PromotedVT = MVT::i1;
259       break;
260     case 2:
261     case 4:
262     case 8:
263       *PromotedVT = MVT::i8;
264       break;
265     case 16:
266       *PromotedVT = MVT::i16;
267       break;
268     case 32:
269       *PromotedVT = MVT::i32;
270       break;
271     case 64:
272       *PromotedVT = MVT::i64;
273       break;
274     }
275     return EVT(*PromotedVT) != VT;
276   }
277   return false;
278 }
279 
280 // Check whether we can merge loads/stores of some of the pieces of a
281 // flattened function parameter or return value into a single vector
282 // load/store.
283 //
284 // The flattened parameter is represented as a list of EVTs and
285 // offsets, and the whole structure is aligned to ParamAlignment. This
286 // function determines whether we can load/store pieces of the
287 // parameter starting at index Idx using a single vectorized op of
288 // size AccessSize. If so, it returns the number of param pieces
289 // covered by the vector op. Otherwise, it returns 1.
290 static unsigned CanMergeParamLoadStoresStartingAt(
291     unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
292     const SmallVectorImpl<uint64_t> &Offsets, Align ParamAlignment) {
293 
294   // Can't vectorize if param alignment is not sufficient.
295   if (ParamAlignment < AccessSize)
296     return 1;
297   // Can't vectorize if offset is not aligned.
298   if (Offsets[Idx] & (AccessSize - 1))
299     return 1;
300 
301   EVT EltVT = ValueVTs[Idx];
302   unsigned EltSize = EltVT.getStoreSize();
303 
304   // Element is too large to vectorize.
305   if (EltSize >= AccessSize)
306     return 1;
307 
308   unsigned NumElts = AccessSize / EltSize;
309   // Can't vectorize if AccessBytes if not a multiple of EltSize.
310   if (AccessSize != EltSize * NumElts)
311     return 1;
312 
313   // We don't have enough elements to vectorize.
314   if (Idx + NumElts > ValueVTs.size())
315     return 1;
316 
317   // PTX ISA can only deal with 2- and 4-element vector ops.
318   if (NumElts != 4 && NumElts != 2)
319     return 1;
320 
321   for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) {
322     // Types do not match.
323     if (ValueVTs[j] != EltVT)
324       return 1;
325 
326     // Elements are not contiguous.
327     if (Offsets[j] - Offsets[j - 1] != EltSize)
328       return 1;
329   }
330   // OK. We can vectorize ValueVTs[i..i+NumElts)
331   return NumElts;
332 }
333 
334 // Flags for tracking per-element vectorization state of loads/stores
335 // of a flattened function parameter or return value.
336 enum ParamVectorizationFlags {
337   PVF_INNER = 0x0, // Middle elements of a vector.
338   PVF_FIRST = 0x1, // First element of the vector.
339   PVF_LAST = 0x2,  // Last element of the vector.
340   // Scalar is effectively a 1-element vector.
341   PVF_SCALAR = PVF_FIRST | PVF_LAST
342 };
343 
344 // Computes whether and how we can vectorize the loads/stores of a
345 // flattened function parameter or return value.
346 //
347 // The flattened parameter is represented as the list of ValueVTs and
348 // Offsets, and is aligned to ParamAlignment bytes. We return a vector
349 // of the same size as ValueVTs indicating how each piece should be
350 // loaded/stored (i.e. as a scalar, or as part of a vector
351 // load/store).
352 static SmallVector<ParamVectorizationFlags, 16>
353 VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
354                      const SmallVectorImpl<uint64_t> &Offsets,
355                      Align ParamAlignment, bool IsVAArg = false) {
356   // Set vector size to match ValueVTs and mark all elements as
357   // scalars by default.
358   SmallVector<ParamVectorizationFlags, 16> VectorInfo;
359   VectorInfo.assign(ValueVTs.size(), PVF_SCALAR);
360 
361   if (IsVAArg)
362     return VectorInfo;
363 
364   // Check what we can vectorize using 128/64/32-bit accesses.
365   for (int I = 0, E = ValueVTs.size(); I != E; ++I) {
366     // Skip elements we've already processed.
367     assert(VectorInfo[I] == PVF_SCALAR && "Unexpected vector info state.");
368     for (unsigned AccessSize : {16, 8, 4, 2}) {
369       unsigned NumElts = CanMergeParamLoadStoresStartingAt(
370           I, AccessSize, ValueVTs, Offsets, ParamAlignment);
371       // Mark vectorized elements.
372       switch (NumElts) {
373       default:
374         llvm_unreachable("Unexpected return value");
375       case 1:
376         // Can't vectorize using this size, try next smaller size.
377         continue;
378       case 2:
379         assert(I + 1 < E && "Not enough elements.");
380         VectorInfo[I] = PVF_FIRST;
381         VectorInfo[I + 1] = PVF_LAST;
382         I += 1;
383         break;
384       case 4:
385         assert(I + 3 < E && "Not enough elements.");
386         VectorInfo[I] = PVF_FIRST;
387         VectorInfo[I + 1] = PVF_INNER;
388         VectorInfo[I + 2] = PVF_INNER;
389         VectorInfo[I + 3] = PVF_LAST;
390         I += 3;
391         break;
392       }
393       // Break out of the inner loop because we've already succeeded
394       // using largest possible AccessSize.
395       break;
396     }
397   }
398   return VectorInfo;
399 }
400 
401 // NVPTXTargetLowering Constructor.
402 NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
403                                          const NVPTXSubtarget &STI)
404     : TargetLowering(TM), nvTM(&TM), STI(STI) {
405   // always lower memset, memcpy, and memmove intrinsics to load/store
406   // instructions, rather
407   // then generating calls to memset, mempcy or memmove.
408   MaxStoresPerMemset = MaxStoresPerMemsetOptSize = (unsigned)0xFFFFFFFF;
409   MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = (unsigned) 0xFFFFFFFF;
410   MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = (unsigned) 0xFFFFFFFF;
411 
412   setBooleanContents(ZeroOrNegativeOneBooleanContent);
413   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
414 
415   // Jump is Expensive. Don't create extra control flow for 'and', 'or'
416   // condition branches.
417   setJumpIsExpensive(true);
418 
419   // Wide divides are _very_ slow. Try to reduce the width of the divide if
420   // possible.
421   addBypassSlowDiv(64, 32);
422 
423   // By default, use the Source scheduling
424   if (sched4reg)
425     setSchedulingPreference(Sched::RegPressure);
426   else
427     setSchedulingPreference(Sched::Source);
428 
429   auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
430                                     LegalizeAction NoF16Action) {
431     setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action);
432   };
433 
434   auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
435                                     LegalizeAction NoBF16Action) {
436     bool IsOpSupported = STI.hasBF16Math();
437     // Few instructions are available on sm_90 only
438     switch(Op) {
439       case ISD::FADD:
440       case ISD::FMUL:
441       case ISD::FSUB:
442       case ISD::SELECT:
443       case ISD::SELECT_CC:
444       case ISD::SETCC:
445       case ISD::FEXP2:
446       case ISD::FCEIL:
447       case ISD::FFLOOR:
448       case ISD::FNEARBYINT:
449       case ISD::FRINT:
450       case ISD::FROUNDEVEN:
451       case ISD::FTRUNC:
452         IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
453         break;
454     }
455     setOperationAction(
456         Op, VT, IsOpSupported ? Action : NoBF16Action);
457   };
458 
459   auto setI16x2OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
460                                      LegalizeAction NoI16x2Action) {
461     bool IsOpSupported = false;
462     // instructions are available on sm_90 only
463     switch (Op) {
464     case ISD::ADD:
465     case ISD::SMAX:
466     case ISD::SMIN:
467     case ISD::UMIN:
468     case ISD::UMAX:
469       IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 80;
470       break;
471     }
472     setOperationAction(Op, VT, IsOpSupported ? Action : NoI16x2Action);
473   };
474 
475   addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass);
476   addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass);
477   addRegisterClass(MVT::v2i16, &NVPTX::Int32RegsRegClass);
478   addRegisterClass(MVT::v4i8, &NVPTX::Int32RegsRegClass);
479   addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass);
480   addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass);
481   addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass);
482   addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass);
483   addRegisterClass(MVT::f16, &NVPTX::Int16RegsRegClass);
484   addRegisterClass(MVT::v2f16, &NVPTX::Int32RegsRegClass);
485   addRegisterClass(MVT::bf16, &NVPTX::Int16RegsRegClass);
486   addRegisterClass(MVT::v2bf16, &NVPTX::Int32RegsRegClass);
487 
488   // Conversion to/from FP16/FP16x2 is always legal.
489   setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom);
490   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom);
491   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Expand);
492   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f16, Expand);
493 
494   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
495   if (STI.getSmVersion() >= 30 && STI.getPTXVersion() > 31)
496     setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Legal);
497 
498   setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote);
499   setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand);
500 
501   // Conversion to/from BFP16/BFP16x2 is always legal.
502   setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Custom);
503   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2bf16, Custom);
504   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2bf16, Expand);
505   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2bf16, Expand);
506 
507   setBF16OperationAction(ISD::SETCC, MVT::v2bf16, Legal, Expand);
508   setBF16OperationAction(ISD::SETCC, MVT::bf16, Legal, Promote);
509   if (getOperationAction(ISD::SETCC, MVT::bf16) == Promote)
510     AddPromotedToType(ISD::SETCC, MVT::bf16, MVT::f32);
511 
512   // Conversion to/from i16/i16x2 is always legal.
513   setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom);
514   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom);
515   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Expand);
516   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i16, Expand);
517 
518   setOperationAction(ISD::BUILD_VECTOR, MVT::v4i8, Custom);
519   setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom);
520   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom);
521   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8, Custom);
522   // Only logical ops can be done on v4i8 directly, others must be done
523   // elementwise.
524   setOperationAction(
525       {ISD::ABS,         ISD::ADD,        ISD::ADDC,        ISD::ADDE,
526        ISD::BITREVERSE,  ISD::CTLZ,       ISD::CTPOP,       ISD::CTTZ,
527        ISD::FP_TO_SINT,  ISD::FP_TO_UINT, ISD::FSHL,        ISD::FSHR,
528        ISD::MUL,         ISD::MULHS,      ISD::MULHU,       ISD::PARITY,
529        ISD::ROTL,        ISD::ROTR,       ISD::SADDO,       ISD::SADDO_CARRY,
530        ISD::SADDSAT,     ISD::SDIV,       ISD::SDIVREM,     ISD::SELECT_CC,
531        ISD::SETCC,       ISD::SHL,        ISD::SINT_TO_FP,  ISD::SMAX,
532        ISD::SMIN,        ISD::SMULO,      ISD::SMUL_LOHI,   ISD::SRA,
533        ISD::SREM,        ISD::SRL,        ISD::SSHLSAT,     ISD::SSUBO,
534        ISD::SSUBO_CARRY, ISD::SSUBSAT,    ISD::SUB,         ISD::SUBC,
535        ISD::SUBE,        ISD::UADDO,      ISD::UADDO_CARRY, ISD::UADDSAT,
536        ISD::UDIV,        ISD::UDIVREM,    ISD::UINT_TO_FP,  ISD::UMAX,
537        ISD::UMIN,        ISD::UMULO,      ISD::UMUL_LOHI,   ISD::UREM,
538        ISD::USHLSAT,     ISD::USUBO,      ISD::USUBO_CARRY, ISD::VSELECT,
539        ISD::USUBSAT},
540       MVT::v4i8, Expand);
541 
542   // Operations not directly supported by NVPTX.
543   for (MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,
544                  MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16, MVT::v4i8,
545                  MVT::i32, MVT::i64}) {
546     setOperationAction(ISD::SELECT_CC, VT, Expand);
547     setOperationAction(ISD::BR_CC, VT, Expand);
548   }
549 
550   // Some SIGN_EXTEND_INREG can be done using cvt instruction.
551   // For others we will expand to a SHL/SRA pair.
552   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);
553   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
554   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
555   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
556   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
557   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Expand);
558 
559   setOperationAction(ISD::SHL_PARTS, MVT::i32  , Custom);
560   setOperationAction(ISD::SRA_PARTS, MVT::i32  , Custom);
561   setOperationAction(ISD::SRL_PARTS, MVT::i32  , Custom);
562   setOperationAction(ISD::SHL_PARTS, MVT::i64  , Custom);
563   setOperationAction(ISD::SRA_PARTS, MVT::i64  , Custom);
564   setOperationAction(ISD::SRL_PARTS, MVT::i64  , Custom);
565 
566   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
567   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
568 
569   // TODO: we may consider expanding ROTL/ROTR on older GPUs.  Currently on GPUs
570   // that don't have h/w rotation we lower them to multi-instruction assembly.
571   // See ROT*_sw in NVPTXIntrInfo.td
572   setOperationAction(ISD::ROTL, MVT::i64, Legal);
573   setOperationAction(ISD::ROTR, MVT::i64, Legal);
574   setOperationAction(ISD::ROTL, MVT::i32, Legal);
575   setOperationAction(ISD::ROTR, MVT::i32, Legal);
576 
577   setOperationAction(ISD::ROTL, MVT::i16, Expand);
578   setOperationAction(ISD::ROTL, MVT::v2i16, Expand);
579   setOperationAction(ISD::ROTR, MVT::i16, Expand);
580   setOperationAction(ISD::ROTR, MVT::v2i16, Expand);
581   setOperationAction(ISD::ROTL, MVT::i8, Expand);
582   setOperationAction(ISD::ROTR, MVT::i8, Expand);
583   setOperationAction(ISD::BSWAP, MVT::i16, Expand);
584 
585   // Indirect branch is not supported.
586   // This also disables Jump Table creation.
587   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
588   setOperationAction(ISD::BRIND, MVT::Other, Expand);
589 
590   setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
591   setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
592 
593   // We want to legalize constant related memmove and memcopy
594   // intrinsics.
595   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
596 
597   // Turn FP extload into load/fpextend
598   setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
599   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
600   setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
601   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
602   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
603   setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
604   setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
605   setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2bf16, Expand);
606   setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2bf16, Expand);
607   setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
608   setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
609   setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
610   setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4bf16, Expand);
611   setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4bf16, Expand);
612   setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
613   setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand);
614   setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand);
615   setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8bf16, Expand);
616   setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8bf16, Expand);
617   // Turn FP truncstore into trunc + store.
618   // FIXME: vector types should also be expanded
619   setTruncStoreAction(MVT::f32, MVT::f16, Expand);
620   setTruncStoreAction(MVT::f64, MVT::f16, Expand);
621   setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
622   setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
623   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
624 
625   // PTX does not support load / store predicate registers
626   setOperationAction(ISD::LOAD, MVT::i1, Custom);
627   setOperationAction(ISD::STORE, MVT::i1, Custom);
628 
629   for (MVT VT : MVT::integer_valuetypes()) {
630     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
631     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
632     setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
633     setTruncStoreAction(VT, MVT::i1, Expand);
634   }
635 
636   // expand extload of vector of integers.
637   setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::v2i16,
638                    MVT::v2i8, Expand);
639   setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
640 
641   // This is legal in NVPTX
642   setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
643   setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
644   setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
645   setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);
646 
647   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
648   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
649 
650   // TRAP can be lowered to PTX trap
651   setOperationAction(ISD::TRAP, MVT::Other, Legal);
652 
653   // Register custom handling for vector loads/stores
654   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
655     if (IsPTXVectorType(VT)) {
656       setOperationAction(ISD::LOAD, VT, Custom);
657       setOperationAction(ISD::STORE, VT, Custom);
658       setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);
659     }
660   }
661 
662   // Support varargs.
663   setOperationAction(ISD::VASTART, MVT::Other, Custom);
664   setOperationAction(ISD::VAARG, MVT::Other, Custom);
665   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
666   setOperationAction(ISD::VAEND, MVT::Other, Expand);
667 
668   // Custom handling for i8 intrinsics
669   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
670 
671   for (const auto& Ty : {MVT::i16, MVT::i32, MVT::i64}) {
672     setOperationAction(ISD::ABS,  Ty, Legal);
673     setOperationAction(ISD::SMIN, Ty, Legal);
674     setOperationAction(ISD::SMAX, Ty, Legal);
675     setOperationAction(ISD::UMIN, Ty, Legal);
676     setOperationAction(ISD::UMAX, Ty, Legal);
677 
678     setOperationAction(ISD::CTPOP, Ty, Legal);
679     setOperationAction(ISD::CTLZ, Ty, Legal);
680   }
681 
682   setI16x2OperationAction(ISD::ABS, MVT::v2i16, Legal, Custom);
683   setI16x2OperationAction(ISD::SMIN, MVT::v2i16, Legal, Custom);
684   setI16x2OperationAction(ISD::SMAX, MVT::v2i16, Legal, Custom);
685   setI16x2OperationAction(ISD::UMIN, MVT::v2i16, Legal, Custom);
686   setI16x2OperationAction(ISD::UMAX, MVT::v2i16, Legal, Custom);
687   setI16x2OperationAction(ISD::CTPOP, MVT::v2i16, Legal, Expand);
688   setI16x2OperationAction(ISD::CTLZ, MVT::v2i16, Legal, Expand);
689 
690   setI16x2OperationAction(ISD::ADD, MVT::v2i16, Legal, Custom);
691   setI16x2OperationAction(ISD::SUB, MVT::v2i16, Legal, Custom);
692   setI16x2OperationAction(ISD::MUL, MVT::v2i16, Legal, Custom);
693   setI16x2OperationAction(ISD::SHL, MVT::v2i16, Legal, Custom);
694   setI16x2OperationAction(ISD::SREM, MVT::v2i16, Legal, Custom);
695   setI16x2OperationAction(ISD::UREM, MVT::v2i16, Legal, Custom);
696 
697   // Other arithmetic and logic ops are unsupported.
698   setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SRA, ISD::SRL, ISD::MULHS,
699                       ISD::MULHU, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
700                       ISD::SINT_TO_FP, ISD::UINT_TO_FP},
701                      MVT::v2i16, Expand);
702 
703   setOperationAction(ISD::ADDC, MVT::i32, Legal);
704   setOperationAction(ISD::ADDE, MVT::i32, Legal);
705   setOperationAction(ISD::SUBC, MVT::i32, Legal);
706   setOperationAction(ISD::SUBE, MVT::i32, Legal);
707   if (STI.getPTXVersion() >= 43) {
708     setOperationAction(ISD::ADDC, MVT::i64, Legal);
709     setOperationAction(ISD::ADDE, MVT::i64, Legal);
710     setOperationAction(ISD::SUBC, MVT::i64, Legal);
711     setOperationAction(ISD::SUBE, MVT::i64, Legal);
712   }
713 
714   setOperationAction(ISD::CTTZ, MVT::i16, Expand);
715   setOperationAction(ISD::CTTZ, MVT::v2i16, Expand);
716   setOperationAction(ISD::CTTZ, MVT::i32, Expand);
717   setOperationAction(ISD::CTTZ, MVT::i64, Expand);
718 
719   // PTX does not directly support SELP of i1, so promote to i32 first
720   setOperationAction(ISD::SELECT, MVT::i1, Custom);
721 
722   // PTX cannot multiply two i64s in a single instruction.
723   setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
724   setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
725 
726   // We have some custom DAG combine patterns for these nodes
727   setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::EXTRACT_VECTOR_ELT, ISD::FADD,
728                        ISD::LOAD, ISD::MUL, ISD::SHL, ISD::SREM, ISD::UREM,
729                        ISD::VSELECT});
730 
731   // setcc for f16x2 and bf16x2 needs special handling to prevent
732   // legalizer's attempt to scalarize it due to v2i1 not being legal.
733   if (STI.allowFP16Math() || STI.hasBF16Math())
734     setTargetDAGCombine(ISD::SETCC);
735 
736   // Promote fp16 arithmetic if fp16 hardware isn't available or the
737   // user passed --nvptx-no-fp16-math. The flag is useful because,
738   // although sm_53+ GPUs have some sort of FP16 support in
739   // hardware, only sm_53 and sm_60 have full implementation. Others
740   // only have token amount of hardware and are likely to run faster
741   // by using fp32 units instead.
742   for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) {
743     setFP16OperationAction(Op, MVT::f16, Legal, Promote);
744     setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
745     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
746     // bf16 must be promoted to f32.
747     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
748     if (getOperationAction(Op, MVT::bf16) == Promote)
749       AddPromotedToType(Op, MVT::bf16, MVT::f32);
750   }
751 
752   // f16/f16x2 neg was introduced in PTX 60, SM_53.
753   const bool IsFP16FP16x2NegAvailable = STI.getSmVersion() >= 53 &&
754                                         STI.getPTXVersion() >= 60 &&
755                                         STI.allowFP16Math();
756   for (const auto &VT : {MVT::f16, MVT::v2f16})
757     setOperationAction(ISD::FNEG, VT,
758                        IsFP16FP16x2NegAvailable ? Legal : Expand);
759 
760   setBF16OperationAction(ISD::FNEG, MVT::bf16, Legal, Expand);
761   setBF16OperationAction(ISD::FNEG, MVT::v2bf16, Legal, Expand);
762   // (would be) Library functions.
763 
764   // These map to conversion instructions for scalar FP types.
765   for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
766                          ISD::FROUNDEVEN, ISD::FTRUNC}) {
767     setOperationAction(Op, MVT::f16, Legal);
768     setOperationAction(Op, MVT::f32, Legal);
769     setOperationAction(Op, MVT::f64, Legal);
770     setOperationAction(Op, MVT::v2f16, Expand);
771     setOperationAction(Op, MVT::v2bf16, Expand);
772     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
773     if (getOperationAction(Op, MVT::bf16) == Promote)
774       AddPromotedToType(Op, MVT::bf16, MVT::f32);
775   }
776 
777   if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71) {
778     setOperationAction(ISD::BF16_TO_FP, MVT::f32, Expand);
779   }
780   if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
781     for (MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {
782       setOperationAction(ISD::FP_EXTEND, VT, Custom);
783       setOperationAction(ISD::FP_ROUND, VT, Custom);
784     }
785   }
786 
787   // sm_80 only has conversions between f32 and bf16. Custom lower all other
788   // bf16 conversions.
789   if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
790     for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {
791       setOperationAction(
792           {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
793           VT, Custom);
794     }
795     setOperationAction(
796         {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
797         MVT::bf16, Custom);
798   }
799 
800   setOperationAction(ISD::FROUND, MVT::f16, Promote);
801   setOperationAction(ISD::FROUND, MVT::v2f16, Expand);
802   setOperationAction(ISD::FROUND, MVT::v2bf16, Expand);
803   setOperationAction(ISD::FROUND, MVT::f32, Custom);
804   setOperationAction(ISD::FROUND, MVT::f64, Custom);
805   setOperationAction(ISD::FROUND, MVT::bf16, Promote);
806   AddPromotedToType(ISD::FROUND, MVT::bf16, MVT::f32);
807 
808   // 'Expand' implements FCOPYSIGN without calling an external library.
809   setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
810   setOperationAction(ISD::FCOPYSIGN, MVT::v2f16, Expand);
811   setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand);
812   setOperationAction(ISD::FCOPYSIGN, MVT::v2bf16, Expand);
813   setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
814   setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
815 
816   // These map to corresponding instructions for f32/f64. f16 must be
817   // promoted to f32. v2f16 is expanded to f16, which is then promoted
818   // to f32.
819   for (const auto &Op :
820        {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS}) {
821     setOperationAction(Op, MVT::f16, Promote);
822     setOperationAction(Op, MVT::f32, Legal);
823     setOperationAction(Op, MVT::f64, Legal);
824     setOperationAction(Op, MVT::v2f16, Expand);
825     setOperationAction(Op, MVT::v2bf16, Expand);
826     setOperationAction(Op, MVT::bf16, Promote);
827     AddPromotedToType(Op, MVT::bf16, MVT::f32);
828   }
829   for (const auto &Op : {ISD::FABS}) {
830     setOperationAction(Op, MVT::f16, Promote);
831     setOperationAction(Op, MVT::f32, Legal);
832     setOperationAction(Op, MVT::f64, Legal);
833     setOperationAction(Op, MVT::v2f16, Expand);
834     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
835     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
836     if (getOperationAction(Op, MVT::bf16) == Promote)
837       AddPromotedToType(Op, MVT::bf16, MVT::f32);
838   }
839 
840   // max.f16, max.f16x2 and max.NaN are supported on sm_80+.
841   auto GetMinMaxAction = [&](LegalizeAction NotSm80Action) {
842     bool IsAtLeastSm80 = STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
843     return IsAtLeastSm80 ? Legal : NotSm80Action;
844   };
845   for (const auto &Op : {ISD::FMINNUM, ISD::FMAXNUM}) {
846     setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Promote), Promote);
847     setOperationAction(Op, MVT::f32, Legal);
848     setOperationAction(Op, MVT::f64, Legal);
849     setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
850     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
851     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
852     if (getOperationAction(Op, MVT::bf16) == Promote)
853       AddPromotedToType(Op, MVT::bf16, MVT::f32);
854   }
855   for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
856     setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Expand), Expand);
857     setFP16OperationAction(Op, MVT::bf16, Legal, Expand);
858     setOperationAction(Op, MVT::f32, GetMinMaxAction(Expand));
859     setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
860     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
861   }
862 
863   // Custom lowering for inline asm with 128-bit operands
864   setOperationAction(ISD::CopyToReg, MVT::i128, Custom);
865   setOperationAction(ISD::CopyFromReg, MVT::i128, Custom);
866 
867   // No FEXP2, FLOG2.  The PTX ex2 and log2 functions are always approximate.
868   // No FPOW or FREM in PTX.
869 
870   // Now deduce the information based on the above mentioned
871   // actions
872   computeRegisterProperties(STI.getRegisterInfo());
873 
874   setMinCmpXchgSizeInBits(32);
875   setMaxAtomicSizeInBitsSupported(64);
876   setMaxDivRemBitWidthSupported(64);
877 }
878 
879 const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
880 
881 #define MAKE_CASE(V)                                                           \
882   case V:                                                                      \
883     return #V;
884 
885   switch ((NVPTXISD::NodeType)Opcode) {
886   case NVPTXISD::FIRST_NUMBER:
887     break;
888 
889     MAKE_CASE(NVPTXISD::CALL)
890     MAKE_CASE(NVPTXISD::RET_GLUE)
891     MAKE_CASE(NVPTXISD::LOAD_PARAM)
892     MAKE_CASE(NVPTXISD::Wrapper)
893     MAKE_CASE(NVPTXISD::DeclareParam)
894     MAKE_CASE(NVPTXISD::DeclareScalarParam)
895     MAKE_CASE(NVPTXISD::DeclareRet)
896     MAKE_CASE(NVPTXISD::DeclareScalarRet)
897     MAKE_CASE(NVPTXISD::DeclareRetParam)
898     MAKE_CASE(NVPTXISD::PrintCall)
899     MAKE_CASE(NVPTXISD::PrintConvergentCall)
900     MAKE_CASE(NVPTXISD::PrintCallUni)
901     MAKE_CASE(NVPTXISD::PrintConvergentCallUni)
902     MAKE_CASE(NVPTXISD::LoadParam)
903     MAKE_CASE(NVPTXISD::LoadParamV2)
904     MAKE_CASE(NVPTXISD::LoadParamV4)
905     MAKE_CASE(NVPTXISD::StoreParam)
906     MAKE_CASE(NVPTXISD::StoreParamV2)
907     MAKE_CASE(NVPTXISD::StoreParamV4)
908     MAKE_CASE(NVPTXISD::StoreParamS32)
909     MAKE_CASE(NVPTXISD::StoreParamU32)
910     MAKE_CASE(NVPTXISD::CallArgBegin)
911     MAKE_CASE(NVPTXISD::CallArg)
912     MAKE_CASE(NVPTXISD::LastCallArg)
913     MAKE_CASE(NVPTXISD::CallArgEnd)
914     MAKE_CASE(NVPTXISD::CallVoid)
915     MAKE_CASE(NVPTXISD::CallVal)
916     MAKE_CASE(NVPTXISD::CallSymbol)
917     MAKE_CASE(NVPTXISD::Prototype)
918     MAKE_CASE(NVPTXISD::MoveParam)
919     MAKE_CASE(NVPTXISD::StoreRetval)
920     MAKE_CASE(NVPTXISD::StoreRetvalV2)
921     MAKE_CASE(NVPTXISD::StoreRetvalV4)
922     MAKE_CASE(NVPTXISD::PseudoUseParam)
923     MAKE_CASE(NVPTXISD::RETURN)
924     MAKE_CASE(NVPTXISD::CallSeqBegin)
925     MAKE_CASE(NVPTXISD::CallSeqEnd)
926     MAKE_CASE(NVPTXISD::CallPrototype)
927     MAKE_CASE(NVPTXISD::ProxyReg)
928     MAKE_CASE(NVPTXISD::LoadV2)
929     MAKE_CASE(NVPTXISD::LoadV4)
930     MAKE_CASE(NVPTXISD::LDGV2)
931     MAKE_CASE(NVPTXISD::LDGV4)
932     MAKE_CASE(NVPTXISD::LDUV2)
933     MAKE_CASE(NVPTXISD::LDUV4)
934     MAKE_CASE(NVPTXISD::StoreV2)
935     MAKE_CASE(NVPTXISD::StoreV4)
936     MAKE_CASE(NVPTXISD::FUN_SHFL_CLAMP)
937     MAKE_CASE(NVPTXISD::FUN_SHFR_CLAMP)
938     MAKE_CASE(NVPTXISD::IMAD)
939     MAKE_CASE(NVPTXISD::BFE)
940     MAKE_CASE(NVPTXISD::BFI)
941     MAKE_CASE(NVPTXISD::PRMT)
942     MAKE_CASE(NVPTXISD::DYNAMIC_STACKALLOC)
943     MAKE_CASE(NVPTXISD::SETP_F16X2)
944     MAKE_CASE(NVPTXISD::SETP_BF16X2)
945     MAKE_CASE(NVPTXISD::Dummy)
946     MAKE_CASE(NVPTXISD::MUL_WIDE_SIGNED)
947     MAKE_CASE(NVPTXISD::MUL_WIDE_UNSIGNED)
948     MAKE_CASE(NVPTXISD::Tex1DFloatS32)
949     MAKE_CASE(NVPTXISD::Tex1DFloatFloat)
950     MAKE_CASE(NVPTXISD::Tex1DFloatFloatLevel)
951     MAKE_CASE(NVPTXISD::Tex1DFloatFloatGrad)
952     MAKE_CASE(NVPTXISD::Tex1DS32S32)
953     MAKE_CASE(NVPTXISD::Tex1DS32Float)
954     MAKE_CASE(NVPTXISD::Tex1DS32FloatLevel)
955     MAKE_CASE(NVPTXISD::Tex1DS32FloatGrad)
956     MAKE_CASE(NVPTXISD::Tex1DU32S32)
957     MAKE_CASE(NVPTXISD::Tex1DU32Float)
958     MAKE_CASE(NVPTXISD::Tex1DU32FloatLevel)
959     MAKE_CASE(NVPTXISD::Tex1DU32FloatGrad)
960     MAKE_CASE(NVPTXISD::Tex1DArrayFloatS32)
961     MAKE_CASE(NVPTXISD::Tex1DArrayFloatFloat)
962     MAKE_CASE(NVPTXISD::Tex1DArrayFloatFloatLevel)
963     MAKE_CASE(NVPTXISD::Tex1DArrayFloatFloatGrad)
964     MAKE_CASE(NVPTXISD::Tex1DArrayS32S32)
965     MAKE_CASE(NVPTXISD::Tex1DArrayS32Float)
966     MAKE_CASE(NVPTXISD::Tex1DArrayS32FloatLevel)
967     MAKE_CASE(NVPTXISD::Tex1DArrayS32FloatGrad)
968     MAKE_CASE(NVPTXISD::Tex1DArrayU32S32)
969     MAKE_CASE(NVPTXISD::Tex1DArrayU32Float)
970     MAKE_CASE(NVPTXISD::Tex1DArrayU32FloatLevel)
971     MAKE_CASE(NVPTXISD::Tex1DArrayU32FloatGrad)
972     MAKE_CASE(NVPTXISD::Tex2DFloatS32)
973     MAKE_CASE(NVPTXISD::Tex2DFloatFloat)
974     MAKE_CASE(NVPTXISD::Tex2DFloatFloatLevel)
975     MAKE_CASE(NVPTXISD::Tex2DFloatFloatGrad)
976     MAKE_CASE(NVPTXISD::Tex2DS32S32)
977     MAKE_CASE(NVPTXISD::Tex2DS32Float)
978     MAKE_CASE(NVPTXISD::Tex2DS32FloatLevel)
979     MAKE_CASE(NVPTXISD::Tex2DS32FloatGrad)
980     MAKE_CASE(NVPTXISD::Tex2DU32S32)
981     MAKE_CASE(NVPTXISD::Tex2DU32Float)
982     MAKE_CASE(NVPTXISD::Tex2DU32FloatLevel)
983     MAKE_CASE(NVPTXISD::Tex2DU32FloatGrad)
984     MAKE_CASE(NVPTXISD::Tex2DArrayFloatS32)
985     MAKE_CASE(NVPTXISD::Tex2DArrayFloatFloat)
986     MAKE_CASE(NVPTXISD::Tex2DArrayFloatFloatLevel)
987     MAKE_CASE(NVPTXISD::Tex2DArrayFloatFloatGrad)
988     MAKE_CASE(NVPTXISD::Tex2DArrayS32S32)
989     MAKE_CASE(NVPTXISD::Tex2DArrayS32Float)
990     MAKE_CASE(NVPTXISD::Tex2DArrayS32FloatLevel)
991     MAKE_CASE(NVPTXISD::Tex2DArrayS32FloatGrad)
992     MAKE_CASE(NVPTXISD::Tex2DArrayU32S32)
993     MAKE_CASE(NVPTXISD::Tex2DArrayU32Float)
994     MAKE_CASE(NVPTXISD::Tex2DArrayU32FloatLevel)
995     MAKE_CASE(NVPTXISD::Tex2DArrayU32FloatGrad)
996     MAKE_CASE(NVPTXISD::Tex3DFloatS32)
997     MAKE_CASE(NVPTXISD::Tex3DFloatFloat)
998     MAKE_CASE(NVPTXISD::Tex3DFloatFloatLevel)
999     MAKE_CASE(NVPTXISD::Tex3DFloatFloatGrad)
1000     MAKE_CASE(NVPTXISD::Tex3DS32S32)
1001     MAKE_CASE(NVPTXISD::Tex3DS32Float)
1002     MAKE_CASE(NVPTXISD::Tex3DS32FloatLevel)
1003     MAKE_CASE(NVPTXISD::Tex3DS32FloatGrad)
1004     MAKE_CASE(NVPTXISD::Tex3DU32S32)
1005     MAKE_CASE(NVPTXISD::Tex3DU32Float)
1006     MAKE_CASE(NVPTXISD::Tex3DU32FloatLevel)
1007     MAKE_CASE(NVPTXISD::Tex3DU32FloatGrad)
1008     MAKE_CASE(NVPTXISD::TexCubeFloatFloat)
1009     MAKE_CASE(NVPTXISD::TexCubeFloatFloatLevel)
1010     MAKE_CASE(NVPTXISD::TexCubeS32Float)
1011     MAKE_CASE(NVPTXISD::TexCubeS32FloatLevel)
1012     MAKE_CASE(NVPTXISD::TexCubeU32Float)
1013     MAKE_CASE(NVPTXISD::TexCubeU32FloatLevel)
1014     MAKE_CASE(NVPTXISD::TexCubeArrayFloatFloat)
1015     MAKE_CASE(NVPTXISD::TexCubeArrayFloatFloatLevel)
1016     MAKE_CASE(NVPTXISD::TexCubeArrayS32Float)
1017     MAKE_CASE(NVPTXISD::TexCubeArrayS32FloatLevel)
1018     MAKE_CASE(NVPTXISD::TexCubeArrayU32Float)
1019     MAKE_CASE(NVPTXISD::TexCubeArrayU32FloatLevel)
1020     MAKE_CASE(NVPTXISD::Tld4R2DFloatFloat)
1021     MAKE_CASE(NVPTXISD::Tld4G2DFloatFloat)
1022     MAKE_CASE(NVPTXISD::Tld4B2DFloatFloat)
1023     MAKE_CASE(NVPTXISD::Tld4A2DFloatFloat)
1024     MAKE_CASE(NVPTXISD::Tld4R2DS64Float)
1025     MAKE_CASE(NVPTXISD::Tld4G2DS64Float)
1026     MAKE_CASE(NVPTXISD::Tld4B2DS64Float)
1027     MAKE_CASE(NVPTXISD::Tld4A2DS64Float)
1028     MAKE_CASE(NVPTXISD::Tld4R2DU64Float)
1029     MAKE_CASE(NVPTXISD::Tld4G2DU64Float)
1030     MAKE_CASE(NVPTXISD::Tld4B2DU64Float)
1031     MAKE_CASE(NVPTXISD::Tld4A2DU64Float)
1032 
1033     MAKE_CASE(NVPTXISD::TexUnified1DFloatS32)
1034     MAKE_CASE(NVPTXISD::TexUnified1DFloatFloat)
1035     MAKE_CASE(NVPTXISD::TexUnified1DFloatFloatLevel)
1036     MAKE_CASE(NVPTXISD::TexUnified1DFloatFloatGrad)
1037     MAKE_CASE(NVPTXISD::TexUnified1DS32S32)
1038     MAKE_CASE(NVPTXISD::TexUnified1DS32Float)
1039     MAKE_CASE(NVPTXISD::TexUnified1DS32FloatLevel)
1040     MAKE_CASE(NVPTXISD::TexUnified1DS32FloatGrad)
1041     MAKE_CASE(NVPTXISD::TexUnified1DU32S32)
1042     MAKE_CASE(NVPTXISD::TexUnified1DU32Float)
1043     MAKE_CASE(NVPTXISD::TexUnified1DU32FloatLevel)
1044     MAKE_CASE(NVPTXISD::TexUnified1DU32FloatGrad)
1045     MAKE_CASE(NVPTXISD::TexUnified1DArrayFloatS32)
1046     MAKE_CASE(NVPTXISD::TexUnified1DArrayFloatFloat)
1047     MAKE_CASE(NVPTXISD::TexUnified1DArrayFloatFloatLevel)
1048     MAKE_CASE(NVPTXISD::TexUnified1DArrayFloatFloatGrad)
1049     MAKE_CASE(NVPTXISD::TexUnified1DArrayS32S32)
1050     MAKE_CASE(NVPTXISD::TexUnified1DArrayS32Float)
1051     MAKE_CASE(NVPTXISD::TexUnified1DArrayS32FloatLevel)
1052     MAKE_CASE(NVPTXISD::TexUnified1DArrayS32FloatGrad)
1053     MAKE_CASE(NVPTXISD::TexUnified1DArrayU32S32)
1054     MAKE_CASE(NVPTXISD::TexUnified1DArrayU32Float)
1055     MAKE_CASE(NVPTXISD::TexUnified1DArrayU32FloatLevel)
1056     MAKE_CASE(NVPTXISD::TexUnified1DArrayU32FloatGrad)
1057     MAKE_CASE(NVPTXISD::TexUnified2DFloatS32)
1058     MAKE_CASE(NVPTXISD::TexUnified2DFloatFloat)
1059     MAKE_CASE(NVPTXISD::TexUnified2DFloatFloatLevel)
1060     MAKE_CASE(NVPTXISD::TexUnified2DFloatFloatGrad)
1061     MAKE_CASE(NVPTXISD::TexUnified2DS32S32)
1062     MAKE_CASE(NVPTXISD::TexUnified2DS32Float)
1063     MAKE_CASE(NVPTXISD::TexUnified2DS32FloatLevel)
1064     MAKE_CASE(NVPTXISD::TexUnified2DS32FloatGrad)
1065     MAKE_CASE(NVPTXISD::TexUnified2DU32S32)
1066     MAKE_CASE(NVPTXISD::TexUnified2DU32Float)
1067     MAKE_CASE(NVPTXISD::TexUnified2DU32FloatLevel)
1068     MAKE_CASE(NVPTXISD::TexUnified2DU32FloatGrad)
1069     MAKE_CASE(NVPTXISD::TexUnified2DArrayFloatS32)
1070     MAKE_CASE(NVPTXISD::TexUnified2DArrayFloatFloat)
1071     MAKE_CASE(NVPTXISD::TexUnified2DArrayFloatFloatLevel)
1072     MAKE_CASE(NVPTXISD::TexUnified2DArrayFloatFloatGrad)
1073     MAKE_CASE(NVPTXISD::TexUnified2DArrayS32S32)
1074     MAKE_CASE(NVPTXISD::TexUnified2DArrayS32Float)
1075     MAKE_CASE(NVPTXISD::TexUnified2DArrayS32FloatLevel)
1076     MAKE_CASE(NVPTXISD::TexUnified2DArrayS32FloatGrad)
1077     MAKE_CASE(NVPTXISD::TexUnified2DArrayU32S32)
1078     MAKE_CASE(NVPTXISD::TexUnified2DArrayU32Float)
1079     MAKE_CASE(NVPTXISD::TexUnified2DArrayU32FloatLevel)
1080     MAKE_CASE(NVPTXISD::TexUnified2DArrayU32FloatGrad)
1081     MAKE_CASE(NVPTXISD::TexUnified3DFloatS32)
1082     MAKE_CASE(NVPTXISD::TexUnified3DFloatFloat)
1083     MAKE_CASE(NVPTXISD::TexUnified3DFloatFloatLevel)
1084     MAKE_CASE(NVPTXISD::TexUnified3DFloatFloatGrad)
1085     MAKE_CASE(NVPTXISD::TexUnified3DS32S32)
1086     MAKE_CASE(NVPTXISD::TexUnified3DS32Float)
1087     MAKE_CASE(NVPTXISD::TexUnified3DS32FloatLevel)
1088     MAKE_CASE(NVPTXISD::TexUnified3DS32FloatGrad)
1089     MAKE_CASE(NVPTXISD::TexUnified3DU32S32)
1090     MAKE_CASE(NVPTXISD::TexUnified3DU32Float)
1091     MAKE_CASE(NVPTXISD::TexUnified3DU32FloatLevel)
1092     MAKE_CASE(NVPTXISD::TexUnified3DU32FloatGrad)
1093     MAKE_CASE(NVPTXISD::TexUnifiedCubeFloatFloat)
1094     MAKE_CASE(NVPTXISD::TexUnifiedCubeFloatFloatLevel)
1095     MAKE_CASE(NVPTXISD::TexUnifiedCubeS32Float)
1096     MAKE_CASE(NVPTXISD::TexUnifiedCubeS32FloatLevel)
1097     MAKE_CASE(NVPTXISD::TexUnifiedCubeU32Float)
1098     MAKE_CASE(NVPTXISD::TexUnifiedCubeU32FloatLevel)
1099     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayFloatFloat)
1100     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel)
1101     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayS32Float)
1102     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayS32FloatLevel)
1103     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayU32Float)
1104     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayU32FloatLevel)
1105     MAKE_CASE(NVPTXISD::TexUnifiedCubeFloatFloatGrad)
1106     MAKE_CASE(NVPTXISD::TexUnifiedCubeS32FloatGrad)
1107     MAKE_CASE(NVPTXISD::TexUnifiedCubeU32FloatGrad)
1108     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayFloatFloatGrad)
1109     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayS32FloatGrad)
1110     MAKE_CASE(NVPTXISD::TexUnifiedCubeArrayU32FloatGrad)
1111     MAKE_CASE(NVPTXISD::Tld4UnifiedR2DFloatFloat)
1112     MAKE_CASE(NVPTXISD::Tld4UnifiedG2DFloatFloat)
1113     MAKE_CASE(NVPTXISD::Tld4UnifiedB2DFloatFloat)
1114     MAKE_CASE(NVPTXISD::Tld4UnifiedA2DFloatFloat)
1115     MAKE_CASE(NVPTXISD::Tld4UnifiedR2DS64Float)
1116     MAKE_CASE(NVPTXISD::Tld4UnifiedG2DS64Float)
1117     MAKE_CASE(NVPTXISD::Tld4UnifiedB2DS64Float)
1118     MAKE_CASE(NVPTXISD::Tld4UnifiedA2DS64Float)
1119     MAKE_CASE(NVPTXISD::Tld4UnifiedR2DU64Float)
1120     MAKE_CASE(NVPTXISD::Tld4UnifiedG2DU64Float)
1121     MAKE_CASE(NVPTXISD::Tld4UnifiedB2DU64Float)
1122     MAKE_CASE(NVPTXISD::Tld4UnifiedA2DU64Float)
1123 
1124     MAKE_CASE(NVPTXISD::Suld1DI8Clamp)
1125     MAKE_CASE(NVPTXISD::Suld1DI16Clamp)
1126     MAKE_CASE(NVPTXISD::Suld1DI32Clamp)
1127     MAKE_CASE(NVPTXISD::Suld1DI64Clamp)
1128     MAKE_CASE(NVPTXISD::Suld1DV2I8Clamp)
1129     MAKE_CASE(NVPTXISD::Suld1DV2I16Clamp)
1130     MAKE_CASE(NVPTXISD::Suld1DV2I32Clamp)
1131     MAKE_CASE(NVPTXISD::Suld1DV2I64Clamp)
1132     MAKE_CASE(NVPTXISD::Suld1DV4I8Clamp)
1133     MAKE_CASE(NVPTXISD::Suld1DV4I16Clamp)
1134     MAKE_CASE(NVPTXISD::Suld1DV4I32Clamp)
1135 
1136     MAKE_CASE(NVPTXISD::Suld1DArrayI8Clamp)
1137     MAKE_CASE(NVPTXISD::Suld1DArrayI16Clamp)
1138     MAKE_CASE(NVPTXISD::Suld1DArrayI32Clamp)
1139     MAKE_CASE(NVPTXISD::Suld1DArrayI64Clamp)
1140     MAKE_CASE(NVPTXISD::Suld1DArrayV2I8Clamp)
1141     MAKE_CASE(NVPTXISD::Suld1DArrayV2I16Clamp)
1142     MAKE_CASE(NVPTXISD::Suld1DArrayV2I32Clamp)
1143     MAKE_CASE(NVPTXISD::Suld1DArrayV2I64Clamp)
1144     MAKE_CASE(NVPTXISD::Suld1DArrayV4I8Clamp)
1145     MAKE_CASE(NVPTXISD::Suld1DArrayV4I16Clamp)
1146     MAKE_CASE(NVPTXISD::Suld1DArrayV4I32Clamp)
1147 
1148     MAKE_CASE(NVPTXISD::Suld2DI8Clamp)
1149     MAKE_CASE(NVPTXISD::Suld2DI16Clamp)
1150     MAKE_CASE(NVPTXISD::Suld2DI32Clamp)
1151     MAKE_CASE(NVPTXISD::Suld2DI64Clamp)
1152     MAKE_CASE(NVPTXISD::Suld2DV2I8Clamp)
1153     MAKE_CASE(NVPTXISD::Suld2DV2I16Clamp)
1154     MAKE_CASE(NVPTXISD::Suld2DV2I32Clamp)
1155     MAKE_CASE(NVPTXISD::Suld2DV2I64Clamp)
1156     MAKE_CASE(NVPTXISD::Suld2DV4I8Clamp)
1157     MAKE_CASE(NVPTXISD::Suld2DV4I16Clamp)
1158     MAKE_CASE(NVPTXISD::Suld2DV4I32Clamp)
1159 
1160     MAKE_CASE(NVPTXISD::Suld2DArrayI8Clamp)
1161     MAKE_CASE(NVPTXISD::Suld2DArrayI16Clamp)
1162     MAKE_CASE(NVPTXISD::Suld2DArrayI32Clamp)
1163     MAKE_CASE(NVPTXISD::Suld2DArrayI64Clamp)
1164     MAKE_CASE(NVPTXISD::Suld2DArrayV2I8Clamp)
1165     MAKE_CASE(NVPTXISD::Suld2DArrayV2I16Clamp)
1166     MAKE_CASE(NVPTXISD::Suld2DArrayV2I32Clamp)
1167     MAKE_CASE(NVPTXISD::Suld2DArrayV2I64Clamp)
1168     MAKE_CASE(NVPTXISD::Suld2DArrayV4I8Clamp)
1169     MAKE_CASE(NVPTXISD::Suld2DArrayV4I16Clamp)
1170     MAKE_CASE(NVPTXISD::Suld2DArrayV4I32Clamp)
1171 
1172     MAKE_CASE(NVPTXISD::Suld3DI8Clamp)
1173     MAKE_CASE(NVPTXISD::Suld3DI16Clamp)
1174     MAKE_CASE(NVPTXISD::Suld3DI32Clamp)
1175     MAKE_CASE(NVPTXISD::Suld3DI64Clamp)
1176     MAKE_CASE(NVPTXISD::Suld3DV2I8Clamp)
1177     MAKE_CASE(NVPTXISD::Suld3DV2I16Clamp)
1178     MAKE_CASE(NVPTXISD::Suld3DV2I32Clamp)
1179     MAKE_CASE(NVPTXISD::Suld3DV2I64Clamp)
1180     MAKE_CASE(NVPTXISD::Suld3DV4I8Clamp)
1181     MAKE_CASE(NVPTXISD::Suld3DV4I16Clamp)
1182     MAKE_CASE(NVPTXISD::Suld3DV4I32Clamp)
1183 
1184     MAKE_CASE(NVPTXISD::Suld1DI8Trap)
1185     MAKE_CASE(NVPTXISD::Suld1DI16Trap)
1186     MAKE_CASE(NVPTXISD::Suld1DI32Trap)
1187     MAKE_CASE(NVPTXISD::Suld1DI64Trap)
1188     MAKE_CASE(NVPTXISD::Suld1DV2I8Trap)
1189     MAKE_CASE(NVPTXISD::Suld1DV2I16Trap)
1190     MAKE_CASE(NVPTXISD::Suld1DV2I32Trap)
1191     MAKE_CASE(NVPTXISD::Suld1DV2I64Trap)
1192     MAKE_CASE(NVPTXISD::Suld1DV4I8Trap)
1193     MAKE_CASE(NVPTXISD::Suld1DV4I16Trap)
1194     MAKE_CASE(NVPTXISD::Suld1DV4I32Trap)
1195 
1196     MAKE_CASE(NVPTXISD::Suld1DArrayI8Trap)
1197     MAKE_CASE(NVPTXISD::Suld1DArrayI16Trap)
1198     MAKE_CASE(NVPTXISD::Suld1DArrayI32Trap)
1199     MAKE_CASE(NVPTXISD::Suld1DArrayI64Trap)
1200     MAKE_CASE(NVPTXISD::Suld1DArrayV2I8Trap)
1201     MAKE_CASE(NVPTXISD::Suld1DArrayV2I16Trap)
1202     MAKE_CASE(NVPTXISD::Suld1DArrayV2I32Trap)
1203     MAKE_CASE(NVPTXISD::Suld1DArrayV2I64Trap)
1204     MAKE_CASE(NVPTXISD::Suld1DArrayV4I8Trap)
1205     MAKE_CASE(NVPTXISD::Suld1DArrayV4I16Trap)
1206     MAKE_CASE(NVPTXISD::Suld1DArrayV4I32Trap)
1207 
1208     MAKE_CASE(NVPTXISD::Suld2DI8Trap)
1209     MAKE_CASE(NVPTXISD::Suld2DI16Trap)
1210     MAKE_CASE(NVPTXISD::Suld2DI32Trap)
1211     MAKE_CASE(NVPTXISD::Suld2DI64Trap)
1212     MAKE_CASE(NVPTXISD::Suld2DV2I8Trap)
1213     MAKE_CASE(NVPTXISD::Suld2DV2I16Trap)
1214     MAKE_CASE(NVPTXISD::Suld2DV2I32Trap)
1215     MAKE_CASE(NVPTXISD::Suld2DV2I64Trap)
1216     MAKE_CASE(NVPTXISD::Suld2DV4I8Trap)
1217     MAKE_CASE(NVPTXISD::Suld2DV4I16Trap)
1218     MAKE_CASE(NVPTXISD::Suld2DV4I32Trap)
1219 
1220     MAKE_CASE(NVPTXISD::Suld2DArrayI8Trap)
1221     MAKE_CASE(NVPTXISD::Suld2DArrayI16Trap)
1222     MAKE_CASE(NVPTXISD::Suld2DArrayI32Trap)
1223     MAKE_CASE(NVPTXISD::Suld2DArrayI64Trap)
1224     MAKE_CASE(NVPTXISD::Suld2DArrayV2I8Trap)
1225     MAKE_CASE(NVPTXISD::Suld2DArrayV2I16Trap)
1226     MAKE_CASE(NVPTXISD::Suld2DArrayV2I32Trap)
1227     MAKE_CASE(NVPTXISD::Suld2DArrayV2I64Trap)
1228     MAKE_CASE(NVPTXISD::Suld2DArrayV4I8Trap)
1229     MAKE_CASE(NVPTXISD::Suld2DArrayV4I16Trap)
1230     MAKE_CASE(NVPTXISD::Suld2DArrayV4I32Trap)
1231 
1232     MAKE_CASE(NVPTXISD::Suld3DI8Trap)
1233     MAKE_CASE(NVPTXISD::Suld3DI16Trap)
1234     MAKE_CASE(NVPTXISD::Suld3DI32Trap)
1235     MAKE_CASE(NVPTXISD::Suld3DI64Trap)
1236     MAKE_CASE(NVPTXISD::Suld3DV2I8Trap)
1237     MAKE_CASE(NVPTXISD::Suld3DV2I16Trap)
1238     MAKE_CASE(NVPTXISD::Suld3DV2I32Trap)
1239     MAKE_CASE(NVPTXISD::Suld3DV2I64Trap)
1240     MAKE_CASE(NVPTXISD::Suld3DV4I8Trap)
1241     MAKE_CASE(NVPTXISD::Suld3DV4I16Trap)
1242     MAKE_CASE(NVPTXISD::Suld3DV4I32Trap)
1243 
1244     MAKE_CASE(NVPTXISD::Suld1DI8Zero)
1245     MAKE_CASE(NVPTXISD::Suld1DI16Zero)
1246     MAKE_CASE(NVPTXISD::Suld1DI32Zero)
1247     MAKE_CASE(NVPTXISD::Suld1DI64Zero)
1248     MAKE_CASE(NVPTXISD::Suld1DV2I8Zero)
1249     MAKE_CASE(NVPTXISD::Suld1DV2I16Zero)
1250     MAKE_CASE(NVPTXISD::Suld1DV2I32Zero)
1251     MAKE_CASE(NVPTXISD::Suld1DV2I64Zero)
1252     MAKE_CASE(NVPTXISD::Suld1DV4I8Zero)
1253     MAKE_CASE(NVPTXISD::Suld1DV4I16Zero)
1254     MAKE_CASE(NVPTXISD::Suld1DV4I32Zero)
1255 
1256     MAKE_CASE(NVPTXISD::Suld1DArrayI8Zero)
1257     MAKE_CASE(NVPTXISD::Suld1DArrayI16Zero)
1258     MAKE_CASE(NVPTXISD::Suld1DArrayI32Zero)
1259     MAKE_CASE(NVPTXISD::Suld1DArrayI64Zero)
1260     MAKE_CASE(NVPTXISD::Suld1DArrayV2I8Zero)
1261     MAKE_CASE(NVPTXISD::Suld1DArrayV2I16Zero)
1262     MAKE_CASE(NVPTXISD::Suld1DArrayV2I32Zero)
1263     MAKE_CASE(NVPTXISD::Suld1DArrayV2I64Zero)
1264     MAKE_CASE(NVPTXISD::Suld1DArrayV4I8Zero)
1265     MAKE_CASE(NVPTXISD::Suld1DArrayV4I16Zero)
1266     MAKE_CASE(NVPTXISD::Suld1DArrayV4I32Zero)
1267 
1268     MAKE_CASE(NVPTXISD::Suld2DI8Zero)
1269     MAKE_CASE(NVPTXISD::Suld2DI16Zero)
1270     MAKE_CASE(NVPTXISD::Suld2DI32Zero)
1271     MAKE_CASE(NVPTXISD::Suld2DI64Zero)
1272     MAKE_CASE(NVPTXISD::Suld2DV2I8Zero)
1273     MAKE_CASE(NVPTXISD::Suld2DV2I16Zero)
1274     MAKE_CASE(NVPTXISD::Suld2DV2I32Zero)
1275     MAKE_CASE(NVPTXISD::Suld2DV2I64Zero)
1276     MAKE_CASE(NVPTXISD::Suld2DV4I8Zero)
1277     MAKE_CASE(NVPTXISD::Suld2DV4I16Zero)
1278     MAKE_CASE(NVPTXISD::Suld2DV4I32Zero)
1279 
1280     MAKE_CASE(NVPTXISD::Suld2DArrayI8Zero)
1281     MAKE_CASE(NVPTXISD::Suld2DArrayI16Zero)
1282     MAKE_CASE(NVPTXISD::Suld2DArrayI32Zero)
1283     MAKE_CASE(NVPTXISD::Suld2DArrayI64Zero)
1284     MAKE_CASE(NVPTXISD::Suld2DArrayV2I8Zero)
1285     MAKE_CASE(NVPTXISD::Suld2DArrayV2I16Zero)
1286     MAKE_CASE(NVPTXISD::Suld2DArrayV2I32Zero)
1287     MAKE_CASE(NVPTXISD::Suld2DArrayV2I64Zero)
1288     MAKE_CASE(NVPTXISD::Suld2DArrayV4I8Zero)
1289     MAKE_CASE(NVPTXISD::Suld2DArrayV4I16Zero)
1290     MAKE_CASE(NVPTXISD::Suld2DArrayV4I32Zero)
1291 
1292     MAKE_CASE(NVPTXISD::Suld3DI8Zero)
1293     MAKE_CASE(NVPTXISD::Suld3DI16Zero)
1294     MAKE_CASE(NVPTXISD::Suld3DI32Zero)
1295     MAKE_CASE(NVPTXISD::Suld3DI64Zero)
1296     MAKE_CASE(NVPTXISD::Suld3DV2I8Zero)
1297     MAKE_CASE(NVPTXISD::Suld3DV2I16Zero)
1298     MAKE_CASE(NVPTXISD::Suld3DV2I32Zero)
1299     MAKE_CASE(NVPTXISD::Suld3DV2I64Zero)
1300     MAKE_CASE(NVPTXISD::Suld3DV4I8Zero)
1301     MAKE_CASE(NVPTXISD::Suld3DV4I16Zero)
1302     MAKE_CASE(NVPTXISD::Suld3DV4I32Zero)
1303   }
1304   return nullptr;
1305 
1306 #undef MAKE_CASE
1307 }
1308 
1309 TargetLoweringBase::LegalizeTypeAction
1310 NVPTXTargetLowering::getPreferredVectorAction(MVT VT) const {
1311   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
1312       VT.getScalarType() == MVT::i1)
1313     return TypeSplitVector;
1314   if (Isv2x16VT(VT))
1315     return TypeLegal;
1316   return TargetLoweringBase::getPreferredVectorAction(VT);
1317 }
1318 
1319 SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
1320                                              int Enabled, int &ExtraSteps,
1321                                              bool &UseOneConst,
1322                                              bool Reciprocal) const {
1323   if (!(Enabled == ReciprocalEstimate::Enabled ||
1324         (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
1325     return SDValue();
1326 
1327   if (ExtraSteps == ReciprocalEstimate::Unspecified)
1328     ExtraSteps = 0;
1329 
1330   SDLoc DL(Operand);
1331   EVT VT = Operand.getValueType();
1332   bool Ftz = useF32FTZ(DAG.getMachineFunction());
1333 
1334   auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {
1335     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
1336                        DAG.getConstant(IID, DL, MVT::i32), Operand);
1337   };
1338 
1339   // The sqrt and rsqrt refinement processes assume we always start out with an
1340   // approximation of the rsqrt.  Therefore, if we're going to do any refinement
1341   // (i.e. ExtraSteps > 0), we must return an rsqrt.  But if we're *not* doing
1342   // any refinement, we must return a regular sqrt.
1343   if (Reciprocal || ExtraSteps > 0) {
1344     if (VT == MVT::f32)
1345       return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1346                                    : Intrinsic::nvvm_rsqrt_approx_f);
1347     else if (VT == MVT::f64)
1348       return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1349     else
1350       return SDValue();
1351   } else {
1352     if (VT == MVT::f32)
1353       return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1354                                    : Intrinsic::nvvm_sqrt_approx_f);
1355     else {
1356       // There's no sqrt.approx.f64 instruction, so we emit
1357       // reciprocal(rsqrt(x)).  This is faster than
1358       // select(x == 0, 0, x * rsqrt(x)).  (In fact, it's faster than plain
1359       // x * rsqrt(x).)
1360       return DAG.getNode(
1361           ISD::INTRINSIC_WO_CHAIN, DL, VT,
1362           DAG.getConstant(Intrinsic::nvvm_rcp_approx_ftz_d, DL, MVT::i32),
1363           MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1364     }
1365   }
1366 }
1367 
1368 SDValue
1369 NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
1370   SDLoc dl(Op);
1371   const GlobalAddressSDNode *GAN = cast<GlobalAddressSDNode>(Op);
1372   auto PtrVT = getPointerTy(DAG.getDataLayout(), GAN->getAddressSpace());
1373   Op = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, PtrVT);
1374   return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op);
1375 }
1376 
1377 static bool IsTypePassedAsArray(const Type *Ty) {
1378   return Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128) ||
1379          Ty->isHalfTy() || Ty->isBFloatTy();
1380 }
1381 
1382 std::string NVPTXTargetLowering::getPrototype(
1383     const DataLayout &DL, Type *retTy, const ArgListTy &Args,
1384     const SmallVectorImpl<ISD::OutputArg> &Outs, MaybeAlign retAlignment,
1385     std::optional<std::pair<unsigned, const APInt &>> VAInfo,
1386     const CallBase &CB, unsigned UniqueCallSite) const {
1387   auto PtrVT = getPointerTy(DL);
1388 
1389   bool isABI = (STI.getSmVersion() >= 20);
1390   assert(isABI && "Non-ABI compilation is not supported");
1391   if (!isABI)
1392     return "";
1393 
1394   std::string Prototype;
1395   raw_string_ostream O(Prototype);
1396   O << "prototype_" << UniqueCallSite << " : .callprototype ";
1397 
1398   if (retTy->getTypeID() == Type::VoidTyID) {
1399     O << "()";
1400   } else {
1401     O << "(";
1402     if ((retTy->isFloatingPointTy() || retTy->isIntegerTy()) &&
1403         !IsTypePassedAsArray(retTy)) {
1404       unsigned size = 0;
1405       if (auto *ITy = dyn_cast<IntegerType>(retTy)) {
1406         size = ITy->getBitWidth();
1407       } else {
1408         assert(retTy->isFloatingPointTy() &&
1409                "Floating point type expected here");
1410         size = retTy->getPrimitiveSizeInBits();
1411       }
1412       // PTX ABI requires all scalar return values to be at least 32
1413       // bits in size.  fp16 normally uses .b16 as its storage type in
1414       // PTX, so its size must be adjusted here, too.
1415       size = promoteScalarArgumentSize(size);
1416 
1417       O << ".param .b" << size << " _";
1418     } else if (isa<PointerType>(retTy)) {
1419       O << ".param .b" << PtrVT.getSizeInBits() << " _";
1420     } else if (IsTypePassedAsArray(retTy)) {
1421       O << ".param .align " << (retAlignment ? retAlignment->value() : 0)
1422         << " .b8 _[" << DL.getTypeAllocSize(retTy) << "]";
1423     } else {
1424       llvm_unreachable("Unknown return type");
1425     }
1426     O << ") ";
1427   }
1428   O << "_ (";
1429 
1430   bool first = true;
1431 
1432   unsigned NumArgs = VAInfo ? VAInfo->first : Args.size();
1433   for (unsigned i = 0, OIdx = 0; i != NumArgs; ++i, ++OIdx) {
1434     Type *Ty = Args[i].Ty;
1435     if (!first) {
1436       O << ", ";
1437     }
1438     first = false;
1439 
1440     if (!Outs[OIdx].Flags.isByVal()) {
1441       if (IsTypePassedAsArray(Ty)) {
1442         Align ParamAlign =
1443             getArgumentAlignment(&CB, Ty, i + AttributeList::FirstArgIndex, DL);
1444         O << ".param .align " << ParamAlign.value() << " .b8 ";
1445         O << "_";
1446         O << "[" << DL.getTypeAllocSize(Ty) << "]";
1447         // update the index for Outs
1448         SmallVector<EVT, 16> vtparts;
1449         ComputeValueVTs(*this, DL, Ty, vtparts);
1450         if (unsigned len = vtparts.size())
1451           OIdx += len - 1;
1452         continue;
1453       }
1454       // i8 types in IR will be i16 types in SDAG
1455       assert((getValueType(DL, Ty) == Outs[OIdx].VT ||
1456               (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
1457              "type mismatch between callee prototype and arguments");
1458       // scalar type
1459       unsigned sz = 0;
1460       if (isa<IntegerType>(Ty)) {
1461         sz = cast<IntegerType>(Ty)->getBitWidth();
1462         sz = promoteScalarArgumentSize(sz);
1463       } else if (isa<PointerType>(Ty)) {
1464         sz = PtrVT.getSizeInBits();
1465       } else {
1466         sz = Ty->getPrimitiveSizeInBits();
1467       }
1468       O << ".param .b" << sz << " ";
1469       O << "_";
1470       continue;
1471     }
1472 
1473     // Indirect calls need strict ABI alignment so we disable optimizations by
1474     // not providing a function to optimize.
1475     Type *ETy = Args[i].IndirectType;
1476     Align InitialAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
1477     Align ParamByValAlign =
1478         getFunctionByValParamAlign(/*F=*/nullptr, ETy, InitialAlign, DL);
1479 
1480     O << ".param .align " << ParamByValAlign.value() << " .b8 ";
1481     O << "_";
1482     O << "[" << Outs[OIdx].Flags.getByValSize() << "]";
1483   }
1484 
1485   if (VAInfo)
1486     O << (first ? "" : ",") << " .param .align " << VAInfo->second
1487       << " .b8 _[]\n";
1488   O << ")";
1489   if (shouldEmitPTXNoReturn(&CB, *nvTM))
1490     O << " .noreturn";
1491   O << ";";
1492 
1493   return Prototype;
1494 }
1495 
1496 Align NVPTXTargetLowering::getFunctionArgumentAlignment(
1497     const Function *F, Type *Ty, unsigned Idx, const DataLayout &DL) const {
1498   return getAlign(*F, Idx).value_or(getFunctionParamOptimizedAlign(F, Ty, DL));
1499 }
1500 
1501 Align NVPTXTargetLowering::getArgumentAlignment(const CallBase *CB, Type *Ty,
1502                                                 unsigned Idx,
1503                                                 const DataLayout &DL) const {
1504   if (!CB) {
1505     // CallSite is zero, fallback to ABI type alignment
1506     return DL.getABITypeAlign(Ty);
1507   }
1508 
1509   const Function *DirectCallee = CB->getCalledFunction();
1510 
1511   if (!DirectCallee) {
1512     // We don't have a direct function symbol, but that may be because of
1513     // constant cast instructions in the call.
1514 
1515     // With bitcast'd call targets, the instruction will be the call
1516     if (const auto *CI = dyn_cast<CallInst>(CB)) {
1517       // Check if we have call alignment metadata
1518       if (MaybeAlign StackAlign = getAlign(*CI, Idx))
1519         return StackAlign.value();
1520     }
1521     DirectCallee = getMaybeBitcastedCallee(CB);
1522   }
1523 
1524   // Check for function alignment information if we found that the
1525   // ultimate target is a Function
1526   if (DirectCallee)
1527     return getFunctionArgumentAlignment(DirectCallee, Ty, Idx, DL);
1528 
1529   // Call is indirect, fall back to the ABI type alignment
1530   return DL.getABITypeAlign(Ty);
1531 }
1532 
1533 static bool adjustElementType(EVT &ElementType) {
1534   switch (ElementType.getSimpleVT().SimpleTy) {
1535   default:
1536     return false;
1537   case MVT::f16:
1538   case MVT::bf16:
1539     ElementType = MVT::i16;
1540     return true;
1541   case MVT::f32:
1542   case MVT::v2f16:
1543   case MVT::v2bf16:
1544     ElementType = MVT::i32;
1545     return true;
1546   case MVT::f64:
1547     ElementType = MVT::i64;
1548     return true;
1549   }
1550 }
1551 
1552 // Use byte-store when the param address of the argument value is unaligned.
1553 // This may happen when the return value is a field of a packed structure.
1554 //
1555 // This is called in LowerCall() when passing the param values.
1556 static SDValue LowerUnalignedStoreParam(SelectionDAG &DAG, SDValue Chain,
1557                                         uint64_t Offset, EVT ElementType,
1558                                         SDValue StVal, SDValue &InGlue,
1559                                         unsigned ArgID, const SDLoc &dl) {
1560   // Bit logic only works on integer types
1561   if (adjustElementType(ElementType))
1562     StVal = DAG.getNode(ISD::BITCAST, dl, ElementType, StVal);
1563 
1564   // Store each byte
1565   SDVTList StoreVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1566   for (unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
1567     // Shift the byte to the last byte position
1568     SDValue ShiftVal = DAG.getNode(ISD::SRL, dl, ElementType, StVal,
1569                                    DAG.getConstant(i * 8, dl, MVT::i32));
1570     SDValue StoreOperands[] = {Chain, DAG.getConstant(ArgID, dl, MVT::i32),
1571                                DAG.getConstant(Offset + i, dl, MVT::i32),
1572                                ShiftVal, InGlue};
1573     // Trunc store only the last byte by using
1574     //     st.param.b8
1575     // The register type can be larger than b8.
1576     Chain = DAG.getMemIntrinsicNode(
1577         NVPTXISD::StoreParam, dl, StoreVTs, StoreOperands, MVT::i8,
1578         MachinePointerInfo(), Align(1), MachineMemOperand::MOStore);
1579     InGlue = Chain.getValue(1);
1580   }
1581   return Chain;
1582 }
1583 
1584 // Use byte-load when the param adress of the returned value is unaligned.
1585 // This may happen when the returned value is a field of a packed structure.
1586 static SDValue
1587 LowerUnalignedLoadRetParam(SelectionDAG &DAG, SDValue &Chain, uint64_t Offset,
1588                            EVT ElementType, SDValue &InGlue,
1589                            SmallVectorImpl<SDValue> &TempProxyRegOps,
1590                            const SDLoc &dl) {
1591   // Bit logic only works on integer types
1592   EVT MergedType = ElementType;
1593   adjustElementType(MergedType);
1594 
1595   // Load each byte and construct the whole value. Initial value to 0
1596   SDValue RetVal = DAG.getConstant(0, dl, MergedType);
1597   // LoadParamMemI8 loads into i16 register only
1598   SDVTList LoadVTs = DAG.getVTList(MVT::i16, MVT::Other, MVT::Glue);
1599   for (unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
1600     SDValue LoadOperands[] = {Chain, DAG.getConstant(1, dl, MVT::i32),
1601                               DAG.getConstant(Offset + i, dl, MVT::i32),
1602                               InGlue};
1603     // This will be selected to LoadParamMemI8
1604     SDValue LdVal =
1605         DAG.getMemIntrinsicNode(NVPTXISD::LoadParam, dl, LoadVTs, LoadOperands,
1606                                 MVT::i8, MachinePointerInfo(), Align(1));
1607     SDValue TmpLdVal = LdVal.getValue(0);
1608     Chain = LdVal.getValue(1);
1609     InGlue = LdVal.getValue(2);
1610 
1611     TmpLdVal = DAG.getNode(NVPTXISD::ProxyReg, dl,
1612                            TmpLdVal.getSimpleValueType(), TmpLdVal);
1613     TempProxyRegOps.push_back(TmpLdVal);
1614 
1615     SDValue CMask = DAG.getConstant(255, dl, MergedType);
1616     SDValue CShift = DAG.getConstant(i * 8, dl, MVT::i32);
1617     // Need to extend the i16 register to the whole width.
1618     TmpLdVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MergedType, TmpLdVal);
1619     // Mask off the high bits. Leave only the lower 8bits.
1620     // Do this because we are using loadparam.b8.
1621     TmpLdVal = DAG.getNode(ISD::AND, dl, MergedType, TmpLdVal, CMask);
1622     // Shift and merge
1623     TmpLdVal = DAG.getNode(ISD::SHL, dl, MergedType, TmpLdVal, CShift);
1624     RetVal = DAG.getNode(ISD::OR, dl, MergedType, RetVal, TmpLdVal);
1625   }
1626   if (ElementType != MergedType)
1627     RetVal = DAG.getNode(ISD::BITCAST, dl, ElementType, RetVal);
1628 
1629   return RetVal;
1630 }
1631 
1632 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1633                                        SmallVectorImpl<SDValue> &InVals) const {
1634 
1635   if (CLI.IsVarArg && (STI.getPTXVersion() < 60 || STI.getSmVersion() < 30))
1636     report_fatal_error(
1637         "Support for variadic functions (unsized array parameter) introduced "
1638         "in PTX ISA version 6.0 and requires target sm_30.");
1639 
1640   SelectionDAG &DAG = CLI.DAG;
1641   SDLoc dl = CLI.DL;
1642   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1643   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1644   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1645   SDValue Chain = CLI.Chain;
1646   SDValue Callee = CLI.Callee;
1647   bool &isTailCall = CLI.IsTailCall;
1648   ArgListTy &Args = CLI.getArgs();
1649   Type *RetTy = CLI.RetTy;
1650   const CallBase *CB = CLI.CB;
1651   const DataLayout &DL = DAG.getDataLayout();
1652 
1653   bool isABI = (STI.getSmVersion() >= 20);
1654   assert(isABI && "Non-ABI compilation is not supported");
1655   if (!isABI)
1656     return Chain;
1657 
1658   // Variadic arguments.
1659   //
1660   // Normally, for each argument, we declare a param scalar or a param
1661   // byte array in the .param space, and store the argument value to that
1662   // param scalar or array starting at offset 0.
1663   //
1664   // In the case of the first variadic argument, we declare a vararg byte array
1665   // with size 0. The exact size of this array isn't known at this point, so
1666   // it'll be patched later. All the variadic arguments will be stored to this
1667   // array at a certain offset (which gets tracked by 'VAOffset'). The offset is
1668   // initially set to 0, so it can be used for non-variadic arguments (which use
1669   // 0 offset) to simplify the code.
1670   //
1671   // After all vararg is processed, 'VAOffset' holds the size of the
1672   // vararg byte array.
1673 
1674   SDValue VADeclareParam;                 // vararg byte array
1675   unsigned FirstVAArg = CLI.NumFixedArgs; // position of the first variadic
1676   unsigned VAOffset = 0;                  // current offset in the param array
1677 
1678   unsigned UniqueCallSite = GlobalUniqueCallSite.fetch_add(1);
1679   SDValue TempChain = Chain;
1680   Chain = DAG.getCALLSEQ_START(Chain, UniqueCallSite, 0, dl);
1681   SDValue InGlue = Chain.getValue(1);
1682 
1683   unsigned ParamCount = 0;
1684   // Args.size() and Outs.size() need not match.
1685   // Outs.size() will be larger
1686   //   * if there is an aggregate argument with multiple fields (each field
1687   //     showing up separately in Outs)
1688   //   * if there is a vector argument with more than typical vector-length
1689   //     elements (generally if more than 4) where each vector element is
1690   //     individually present in Outs.
1691   // So a different index should be used for indexing into Outs/OutVals.
1692   // See similar issue in LowerFormalArguments.
1693   unsigned OIdx = 0;
1694   // Declare the .params or .reg need to pass values
1695   // to the function
1696   for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
1697     EVT VT = Outs[OIdx].VT;
1698     Type *Ty = Args[i].Ty;
1699     bool IsVAArg = (i >= CLI.NumFixedArgs);
1700     bool IsByVal = Outs[OIdx].Flags.isByVal();
1701 
1702     SmallVector<EVT, 16> VTs;
1703     SmallVector<uint64_t, 16> Offsets;
1704 
1705     assert((!IsByVal || Args[i].IndirectType) &&
1706            "byval arg must have indirect type");
1707     Type *ETy = (IsByVal ? Args[i].IndirectType : Ty);
1708     ComputePTXValueVTs(*this, DL, ETy, VTs, &Offsets, IsByVal ? 0 : VAOffset);
1709 
1710     Align ArgAlign;
1711     if (IsByVal) {
1712       // The ByValAlign in the Outs[OIdx].Flags is always set at this point,
1713       // so we don't need to worry whether it's naturally aligned or not.
1714       // See TargetLowering::LowerCallTo().
1715       Align InitialAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
1716       ArgAlign = getFunctionByValParamAlign(CB->getCalledFunction(), ETy,
1717                                             InitialAlign, DL);
1718       if (IsVAArg)
1719         VAOffset = alignTo(VAOffset, ArgAlign);
1720     } else {
1721       ArgAlign = getArgumentAlignment(CB, Ty, ParamCount + 1, DL);
1722     }
1723 
1724     unsigned TypeSize =
1725         (IsByVal ? Outs[OIdx].Flags.getByValSize() : DL.getTypeAllocSize(Ty));
1726     SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1727 
1728     bool NeedAlign; // Does argument declaration specify alignment?
1729     bool PassAsArray = IsByVal || IsTypePassedAsArray(Ty);
1730     if (IsVAArg) {
1731       if (ParamCount == FirstVAArg) {
1732         SDValue DeclareParamOps[] = {
1733             Chain, DAG.getConstant(STI.getMaxRequiredAlignment(), dl, MVT::i32),
1734             DAG.getConstant(ParamCount, dl, MVT::i32),
1735             DAG.getConstant(1, dl, MVT::i32), InGlue};
1736         VADeclareParam = Chain = DAG.getNode(NVPTXISD::DeclareParam, dl,
1737                                              DeclareParamVTs, DeclareParamOps);
1738       }
1739       NeedAlign = PassAsArray;
1740     } else if (PassAsArray) {
1741       // declare .param .align <align> .b8 .param<n>[<size>];
1742       SDValue DeclareParamOps[] = {
1743           Chain, DAG.getConstant(ArgAlign.value(), dl, MVT::i32),
1744           DAG.getConstant(ParamCount, dl, MVT::i32),
1745           DAG.getConstant(TypeSize, dl, MVT::i32), InGlue};
1746       Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
1747                           DeclareParamOps);
1748       NeedAlign = true;
1749     } else {
1750       // declare .param .b<size> .param<n>;
1751       if (VT.isInteger() || VT.isFloatingPoint()) {
1752         // PTX ABI requires integral types to be at least 32 bits in
1753         // size. FP16 is loaded/stored using i16, so it's handled
1754         // here as well.
1755         TypeSize = promoteScalarArgumentSize(TypeSize * 8) / 8;
1756       }
1757       SDValue DeclareScalarParamOps[] = {
1758           Chain, DAG.getConstant(ParamCount, dl, MVT::i32),
1759           DAG.getConstant(TypeSize * 8, dl, MVT::i32),
1760           DAG.getConstant(0, dl, MVT::i32), InGlue};
1761       Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,
1762                           DeclareScalarParamOps);
1763       NeedAlign = false;
1764     }
1765     InGlue = Chain.getValue(1);
1766 
1767     // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
1768     // than 32-bits are sign extended or zero extended, depending on
1769     // whether they are signed or unsigned types. This case applies
1770     // only to scalar parameters and not to aggregate values.
1771     bool ExtendIntegerParam =
1772         Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty) < 32;
1773 
1774     auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, ArgAlign, IsVAArg);
1775     SmallVector<SDValue, 6> StoreOperands;
1776     for (unsigned j = 0, je = VTs.size(); j != je; ++j) {
1777       EVT EltVT = VTs[j];
1778       int CurOffset = Offsets[j];
1779       MaybeAlign PartAlign;
1780       if (NeedAlign)
1781         PartAlign = commonAlignment(ArgAlign, CurOffset);
1782 
1783       SDValue StVal = OutVals[OIdx];
1784 
1785       MVT PromotedVT;
1786       if (PromoteScalarIntegerPTX(EltVT, &PromotedVT)) {
1787         EltVT = EVT(PromotedVT);
1788       }
1789       if (PromoteScalarIntegerPTX(StVal.getValueType(), &PromotedVT)) {
1790         llvm::ISD::NodeType Ext =
1791             Outs[OIdx].Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1792         StVal = DAG.getNode(Ext, dl, PromotedVT, StVal);
1793       }
1794 
1795       if (IsByVal) {
1796         auto PtrVT = getPointerTy(DL);
1797         SDValue srcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StVal,
1798                                       DAG.getConstant(CurOffset, dl, PtrVT));
1799         StVal = DAG.getLoad(EltVT, dl, TempChain, srcAddr, MachinePointerInfo(),
1800                             PartAlign);
1801       } else if (ExtendIntegerParam) {
1802         assert(VTs.size() == 1 && "Scalar can't have multiple parts.");
1803         // zext/sext to i32
1804         StVal = DAG.getNode(Outs[OIdx].Flags.isSExt() ? ISD::SIGN_EXTEND
1805                                                       : ISD::ZERO_EXTEND,
1806                             dl, MVT::i32, StVal);
1807       }
1808 
1809       if (!ExtendIntegerParam && EltVT.getSizeInBits() < 16) {
1810         // Use 16-bit registers for small stores as it's the
1811         // smallest general purpose register size supported by NVPTX.
1812         StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal);
1813       }
1814 
1815       // If we have a PVF_SCALAR entry, it may not be sufficiently aligned for a
1816       // scalar store. In such cases, fall back to byte stores.
1817       if (VectorInfo[j] == PVF_SCALAR && !IsVAArg && PartAlign.has_value() &&
1818           PartAlign.value() <
1819               DL.getABITypeAlign(EltVT.getTypeForEVT(*DAG.getContext()))) {
1820         assert(StoreOperands.empty() && "Unfinished preceeding store.");
1821         Chain = LowerUnalignedStoreParam(
1822             DAG, Chain, IsByVal ? CurOffset + VAOffset : CurOffset, EltVT,
1823             StVal, InGlue, ParamCount, dl);
1824 
1825         // LowerUnalignedStoreParam took care of inserting the necessary nodes
1826         // into the SDAG, so just move on to the next element.
1827         if (!IsByVal)
1828           ++OIdx;
1829         continue;
1830       }
1831 
1832       // New store.
1833       if (VectorInfo[j] & PVF_FIRST) {
1834         assert(StoreOperands.empty() && "Unfinished preceding store.");
1835         StoreOperands.push_back(Chain);
1836         StoreOperands.push_back(
1837             DAG.getConstant(IsVAArg ? FirstVAArg : ParamCount, dl, MVT::i32));
1838 
1839         StoreOperands.push_back(DAG.getConstant(
1840             IsByVal ? CurOffset + VAOffset : (IsVAArg ? VAOffset : CurOffset),
1841             dl, MVT::i32));
1842       }
1843 
1844       // Record the value to store.
1845       StoreOperands.push_back(StVal);
1846 
1847       if (VectorInfo[j] & PVF_LAST) {
1848         unsigned NumElts = StoreOperands.size() - 3;
1849         NVPTXISD::NodeType Op;
1850         switch (NumElts) {
1851         case 1:
1852           Op = NVPTXISD::StoreParam;
1853           break;
1854         case 2:
1855           Op = NVPTXISD::StoreParamV2;
1856           break;
1857         case 4:
1858           Op = NVPTXISD::StoreParamV4;
1859           break;
1860         default:
1861           llvm_unreachable("Invalid vector info.");
1862         }
1863 
1864         StoreOperands.push_back(InGlue);
1865 
1866         // Adjust type of the store op if we've extended the scalar
1867         // return value.
1868         EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
1869 
1870         Chain = DAG.getMemIntrinsicNode(
1871             Op, dl, DAG.getVTList(MVT::Other, MVT::Glue), StoreOperands,
1872             TheStoreType, MachinePointerInfo(), PartAlign,
1873             MachineMemOperand::MOStore);
1874         InGlue = Chain.getValue(1);
1875 
1876         // Cleanup.
1877         StoreOperands.clear();
1878 
1879         // TODO: We may need to support vector types that can be passed
1880         // as scalars in variadic arguments.
1881         if (!IsByVal && IsVAArg) {
1882           assert(NumElts == 1 &&
1883                  "Vectorization is expected to be disabled for variadics.");
1884           VAOffset += DL.getTypeAllocSize(
1885               TheStoreType.getTypeForEVT(*DAG.getContext()));
1886         }
1887       }
1888       if (!IsByVal)
1889         ++OIdx;
1890     }
1891     assert(StoreOperands.empty() && "Unfinished parameter store.");
1892     if (!IsByVal && VTs.size() > 0)
1893       --OIdx;
1894     ++ParamCount;
1895     if (IsByVal && IsVAArg)
1896       VAOffset += TypeSize;
1897   }
1898 
1899   GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
1900   MaybeAlign retAlignment = std::nullopt;
1901 
1902   // Handle Result
1903   if (Ins.size() > 0) {
1904     SmallVector<EVT, 16> resvtparts;
1905     ComputeValueVTs(*this, DL, RetTy, resvtparts);
1906 
1907     // Declare
1908     //  .param .align N .b8 retval0[<size-in-bytes>], or
1909     //  .param .b<size-in-bits> retval0
1910     unsigned resultsz = DL.getTypeAllocSizeInBits(RetTy);
1911     if (!IsTypePassedAsArray(RetTy)) {
1912       resultsz = promoteScalarArgumentSize(resultsz);
1913       SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1914       SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
1915                                   DAG.getConstant(resultsz, dl, MVT::i32),
1916                                   DAG.getConstant(0, dl, MVT::i32), InGlue };
1917       Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,
1918                           DeclareRetOps);
1919       InGlue = Chain.getValue(1);
1920     } else {
1921       retAlignment = getArgumentAlignment(CB, RetTy, 0, DL);
1922       assert(retAlignment && "retAlignment is guaranteed to be set");
1923       SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1924       SDValue DeclareRetOps[] = {
1925           Chain, DAG.getConstant(retAlignment->value(), dl, MVT::i32),
1926           DAG.getConstant(resultsz / 8, dl, MVT::i32),
1927           DAG.getConstant(0, dl, MVT::i32), InGlue};
1928       Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
1929                           DeclareRetOps);
1930       InGlue = Chain.getValue(1);
1931     }
1932   }
1933 
1934   bool HasVAArgs = CLI.IsVarArg && (CLI.Args.size() > CLI.NumFixedArgs);
1935   // Set the size of the vararg param byte array if the callee is a variadic
1936   // function and the variadic part is not empty.
1937   if (HasVAArgs) {
1938     SDValue DeclareParamOps[] = {
1939         VADeclareParam.getOperand(0), VADeclareParam.getOperand(1),
1940         VADeclareParam.getOperand(2), DAG.getConstant(VAOffset, dl, MVT::i32),
1941         VADeclareParam.getOperand(4)};
1942     DAG.MorphNodeTo(VADeclareParam.getNode(), VADeclareParam.getOpcode(),
1943                     VADeclareParam->getVTList(), DeclareParamOps);
1944   }
1945 
1946   // Both indirect calls and libcalls have nullptr Func. In order to distinguish
1947   // between them we must rely on the call site value which is valid for
1948   // indirect calls but is always null for libcalls.
1949   bool isIndirectCall = !Func && CB;
1950 
1951   if (isa<ExternalSymbolSDNode>(Callee)) {
1952     Function* CalleeFunc = nullptr;
1953 
1954     // Try to find the callee in the current module.
1955     Callee = DAG.getSymbolFunctionGlobalAddress(Callee, &CalleeFunc);
1956     assert(CalleeFunc != nullptr && "Libcall callee must be set.");
1957 
1958     // Set the "libcall callee" attribute to indicate that the function
1959     // must always have a declaration.
1960     CalleeFunc->addFnAttr("nvptx-libcall-callee", "true");
1961   }
1962 
1963   if (isIndirectCall) {
1964     // This is indirect function call case : PTX requires a prototype of the
1965     // form
1966     // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
1967     // to be emitted, and the label has to used as the last arg of call
1968     // instruction.
1969     // The prototype is embedded in a string and put as the operand for a
1970     // CallPrototype SDNode which will print out to the value of the string.
1971     SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1972     std::string Proto = getPrototype(
1973         DL, RetTy, Args, Outs, retAlignment,
1974         HasVAArgs
1975             ? std::optional<std::pair<unsigned, const APInt &>>(std::make_pair(
1976                   CLI.NumFixedArgs, VADeclareParam->getConstantOperandAPInt(1)))
1977             : std::nullopt,
1978         *CB, UniqueCallSite);
1979     const char *ProtoStr = nvTM->getStrPool().save(Proto).data();
1980     SDValue ProtoOps[] = {
1981         Chain,
1982         DAG.getTargetExternalSymbol(ProtoStr, MVT::i32),
1983         InGlue,
1984     };
1985     Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, ProtoVTs, ProtoOps);
1986     InGlue = Chain.getValue(1);
1987   }
1988   // Op to just print "call"
1989   SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1990   SDValue PrintCallOps[] = {
1991     Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, dl, MVT::i32), InGlue
1992   };
1993   // We model convergent calls as separate opcodes.
1994   unsigned Opcode = isIndirectCall ? NVPTXISD::PrintCall : NVPTXISD::PrintCallUni;
1995   if (CLI.IsConvergent)
1996     Opcode = Opcode == NVPTXISD::PrintCallUni ? NVPTXISD::PrintConvergentCallUni
1997                                               : NVPTXISD::PrintConvergentCall;
1998   Chain = DAG.getNode(Opcode, dl, PrintCallVTs, PrintCallOps);
1999   InGlue = Chain.getValue(1);
2000 
2001   // Ops to print out the function name
2002   SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue);
2003   SDValue CallVoidOps[] = { Chain, Callee, InGlue };
2004   Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps);
2005   InGlue = Chain.getValue(1);
2006 
2007   // Ops to print out the param list
2008   SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue);
2009   SDValue CallArgBeginOps[] = { Chain, InGlue };
2010   Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs,
2011                       CallArgBeginOps);
2012   InGlue = Chain.getValue(1);
2013 
2014   for (unsigned i = 0, e = std::min(CLI.NumFixedArgs + 1, ParamCount); i != e;
2015        ++i) {
2016     unsigned opcode;
2017     if (i == (e - 1))
2018       opcode = NVPTXISD::LastCallArg;
2019     else
2020       opcode = NVPTXISD::CallArg;
2021     SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);
2022     SDValue CallArgOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),
2023                              DAG.getConstant(i, dl, MVT::i32), InGlue };
2024     Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps);
2025     InGlue = Chain.getValue(1);
2026   }
2027   SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);
2028   SDValue CallArgEndOps[] = { Chain,
2029                               DAG.getConstant(isIndirectCall ? 0 : 1, dl, MVT::i32),
2030                               InGlue };
2031   Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps);
2032   InGlue = Chain.getValue(1);
2033 
2034   if (isIndirectCall) {
2035     SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);
2036     SDValue PrototypeOps[] = {
2037         Chain, DAG.getConstant(UniqueCallSite, dl, MVT::i32), InGlue};
2038     Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps);
2039     InGlue = Chain.getValue(1);
2040   }
2041 
2042   SmallVector<SDValue, 16> ProxyRegOps;
2043   SmallVector<std::optional<MVT>, 16> ProxyRegTruncates;
2044   // An item of the vector is filled if the element does not need a ProxyReg
2045   // operation on it and should be added to InVals as is. ProxyRegOps and
2046   // ProxyRegTruncates contain empty/none items at the same index.
2047   SmallVector<SDValue, 16> RetElts;
2048   // A temporary ProxyReg operations inserted in `LowerUnalignedLoadRetParam()`
2049   // to use the values of `LoadParam`s and to be replaced later then
2050   // `CALLSEQ_END` is added.
2051   SmallVector<SDValue, 16> TempProxyRegOps;
2052 
2053   // Generate loads from param memory/moves from registers for result
2054   if (Ins.size() > 0) {
2055     SmallVector<EVT, 16> VTs;
2056     SmallVector<uint64_t, 16> Offsets;
2057     ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets, 0);
2058     assert(VTs.size() == Ins.size() && "Bad value decomposition");
2059 
2060     Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
2061     auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign);
2062 
2063     SmallVector<EVT, 6> LoadVTs;
2064     int VecIdx = -1; // Index of the first element of the vector.
2065 
2066     // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
2067     // 32-bits are sign extended or zero extended, depending on whether
2068     // they are signed or unsigned types.
2069     bool ExtendIntegerRetVal =
2070         RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
2071 
2072     for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
2073       bool needTruncate = false;
2074       EVT TheLoadType = VTs[i];
2075       EVT EltType = Ins[i].VT;
2076       Align EltAlign = commonAlignment(RetAlign, Offsets[i]);
2077       MVT PromotedVT;
2078 
2079       if (PromoteScalarIntegerPTX(TheLoadType, &PromotedVT)) {
2080         TheLoadType = EVT(PromotedVT);
2081         EltType = EVT(PromotedVT);
2082         needTruncate = true;
2083       }
2084 
2085       if (ExtendIntegerRetVal) {
2086         TheLoadType = MVT::i32;
2087         EltType = MVT::i32;
2088         needTruncate = true;
2089       } else if (TheLoadType.getSizeInBits() < 16) {
2090         if (VTs[i].isInteger())
2091           needTruncate = true;
2092         EltType = MVT::i16;
2093       }
2094 
2095       // If we have a PVF_SCALAR entry, it may not be sufficiently aligned for a
2096       // scalar load. In such cases, fall back to byte loads.
2097       if (VectorInfo[i] == PVF_SCALAR && RetTy->isAggregateType() &&
2098           EltAlign < DL.getABITypeAlign(
2099                          TheLoadType.getTypeForEVT(*DAG.getContext()))) {
2100         assert(VecIdx == -1 && LoadVTs.empty() && "Orphaned operand list.");
2101         SDValue Ret = LowerUnalignedLoadRetParam(
2102             DAG, Chain, Offsets[i], TheLoadType, InGlue, TempProxyRegOps, dl);
2103         ProxyRegOps.push_back(SDValue());
2104         ProxyRegTruncates.push_back(std::optional<MVT>());
2105         RetElts.resize(i);
2106         RetElts.push_back(Ret);
2107 
2108         continue;
2109       }
2110 
2111       // Record index of the very first element of the vector.
2112       if (VectorInfo[i] & PVF_FIRST) {
2113         assert(VecIdx == -1 && LoadVTs.empty() && "Orphaned operand list.");
2114         VecIdx = i;
2115       }
2116 
2117       LoadVTs.push_back(EltType);
2118 
2119       if (VectorInfo[i] & PVF_LAST) {
2120         unsigned NumElts = LoadVTs.size();
2121         LoadVTs.push_back(MVT::Other);
2122         LoadVTs.push_back(MVT::Glue);
2123         NVPTXISD::NodeType Op;
2124         switch (NumElts) {
2125         case 1:
2126           Op = NVPTXISD::LoadParam;
2127           break;
2128         case 2:
2129           Op = NVPTXISD::LoadParamV2;
2130           break;
2131         case 4:
2132           Op = NVPTXISD::LoadParamV4;
2133           break;
2134         default:
2135           llvm_unreachable("Invalid vector info.");
2136         }
2137 
2138         SDValue LoadOperands[] = {
2139             Chain, DAG.getConstant(1, dl, MVT::i32),
2140             DAG.getConstant(Offsets[VecIdx], dl, MVT::i32), InGlue};
2141         SDValue RetVal = DAG.getMemIntrinsicNode(
2142             Op, dl, DAG.getVTList(LoadVTs), LoadOperands, TheLoadType,
2143             MachinePointerInfo(), EltAlign,
2144             MachineMemOperand::MOLoad);
2145 
2146         for (unsigned j = 0; j < NumElts; ++j) {
2147           ProxyRegOps.push_back(RetVal.getValue(j));
2148 
2149           if (needTruncate)
2150             ProxyRegTruncates.push_back(std::optional<MVT>(Ins[VecIdx + j].VT));
2151           else
2152             ProxyRegTruncates.push_back(std::optional<MVT>());
2153         }
2154 
2155         Chain = RetVal.getValue(NumElts);
2156         InGlue = RetVal.getValue(NumElts + 1);
2157 
2158         // Cleanup
2159         VecIdx = -1;
2160         LoadVTs.clear();
2161       }
2162     }
2163   }
2164 
2165   Chain =
2166       DAG.getCALLSEQ_END(Chain, UniqueCallSite, UniqueCallSite + 1, InGlue, dl);
2167   InGlue = Chain.getValue(1);
2168 
2169   // Append ProxyReg instructions to the chain to make sure that `callseq_end`
2170   // will not get lost. Otherwise, during libcalls expansion, the nodes can become
2171   // dangling.
2172   for (unsigned i = 0; i < ProxyRegOps.size(); ++i) {
2173     if (i < RetElts.size() && RetElts[i]) {
2174       InVals.push_back(RetElts[i]);
2175       continue;
2176     }
2177 
2178     SDValue Ret = DAG.getNode(
2179       NVPTXISD::ProxyReg, dl,
2180       DAG.getVTList(ProxyRegOps[i].getSimpleValueType(), MVT::Other, MVT::Glue),
2181       { Chain, ProxyRegOps[i], InGlue }
2182     );
2183 
2184     Chain = Ret.getValue(1);
2185     InGlue = Ret.getValue(2);
2186 
2187     if (ProxyRegTruncates[i]) {
2188       Ret = DAG.getNode(ISD::TRUNCATE, dl, *ProxyRegTruncates[i], Ret);
2189     }
2190 
2191     InVals.push_back(Ret);
2192   }
2193 
2194   for (SDValue &T : TempProxyRegOps) {
2195     SDValue Repl = DAG.getNode(
2196         NVPTXISD::ProxyReg, dl,
2197         DAG.getVTList(T.getSimpleValueType(), MVT::Other, MVT::Glue),
2198         {Chain, T.getOperand(0), InGlue});
2199     DAG.ReplaceAllUsesWith(T, Repl);
2200     DAG.RemoveDeadNode(T.getNode());
2201 
2202     Chain = Repl.getValue(1);
2203     InGlue = Repl.getValue(2);
2204   }
2205 
2206   // set isTailCall to false for now, until we figure out how to express
2207   // tail call optimization in PTX
2208   isTailCall = false;
2209   return Chain;
2210 }
2211 
2212 SDValue NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
2213                                                      SelectionDAG &DAG) const {
2214 
2215   if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {
2216     const Function &Fn = DAG.getMachineFunction().getFunction();
2217 
2218     DiagnosticInfoUnsupported NoDynamicAlloca(
2219         Fn,
2220         "Support for dynamic alloca introduced in PTX ISA version 7.3 and "
2221         "requires target sm_52.",
2222         SDLoc(Op).getDebugLoc());
2223     DAG.getContext()->diagnose(NoDynamicAlloca);
2224     auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()),
2225                 Op.getOperand(0)};
2226     return DAG.getMergeValues(Ops, SDLoc());
2227   }
2228 
2229   SDValue Chain = Op.getOperand(0);
2230   SDValue Size = Op.getOperand(1);
2231   uint64_t Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
2232   SDLoc DL(Op.getNode());
2233 
2234   // The size for ptx alloca instruction is 64-bit for m64 and 32-bit for m32.
2235   if (nvTM->is64Bit())
2236     Size = DAG.getZExtOrTrunc(Size, DL, MVT::i64);
2237   else
2238     Size = DAG.getZExtOrTrunc(Size, DL, MVT::i32);
2239 
2240   SDValue AllocOps[] = {Chain, Size,
2241                         DAG.getTargetConstant(Align, DL, MVT::i32)};
2242   SDValue Alloca = DAG.getNode(NVPTXISD::DYNAMIC_STACKALLOC, DL,
2243                                nvTM->is64Bit() ? MVT::i64 : MVT::i32, AllocOps);
2244 
2245   SDValue MergeOps[] = {Alloca, Chain};
2246   return DAG.getMergeValues(MergeOps, DL);
2247 }
2248 
2249 // By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
2250 // (see LegalizeDAG.cpp). This is slow and uses local memory.
2251 // We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
2252 SDValue
2253 NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
2254   SDNode *Node = Op.getNode();
2255   SDLoc dl(Node);
2256   SmallVector<SDValue, 8> Ops;
2257   unsigned NumOperands = Node->getNumOperands();
2258   for (unsigned i = 0; i < NumOperands; ++i) {
2259     SDValue SubOp = Node->getOperand(i);
2260     EVT VVT = SubOp.getNode()->getValueType(0);
2261     EVT EltVT = VVT.getVectorElementType();
2262     unsigned NumSubElem = VVT.getVectorNumElements();
2263     for (unsigned j = 0; j < NumSubElem; ++j) {
2264       Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
2265                                 DAG.getIntPtrConstant(j, dl)));
2266     }
2267   }
2268   return DAG.getBuildVector(Node->getValueType(0), dl, Ops);
2269 }
2270 
2271 // We can init constant f16x2/v2i16/v4i8 with a single .b32 move.  Normally it
2272 // would get lowered as two constant loads and vector-packing move.
2273 // Instead we want just a constant move:
2274 //        mov.b32         %r2, 0x40003C00
2275 SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op,
2276                                                SelectionDAG &DAG) const {
2277   EVT VT = Op->getValueType(0);
2278   if (!(Isv2x16VT(VT) || VT == MVT::v4i8))
2279     return Op;
2280 
2281   SDLoc DL(Op);
2282 
2283   if (!llvm::all_of(Op->ops(), [](SDValue Operand) {
2284         return Operand->isUndef() || isa<ConstantSDNode>(Operand) ||
2285                isa<ConstantFPSDNode>(Operand);
2286       })) {
2287     // Lower non-const v4i8 vector as byte-wise constructed i32, which allows us
2288     // to optimize calculation of constant parts.
2289     if (VT == MVT::v4i8) {
2290       SDValue C8 = DAG.getConstant(8, DL, MVT::i32);
2291       SDValue E01 = DAG.getNode(
2292           NVPTXISD::BFI, DL, MVT::i32,
2293           DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32),
2294           DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32), C8, C8);
2295       SDValue E012 =
2296           DAG.getNode(NVPTXISD::BFI, DL, MVT::i32,
2297                       DAG.getAnyExtOrTrunc(Op->getOperand(2), DL, MVT::i32),
2298                       E01, DAG.getConstant(16, DL, MVT::i32), C8);
2299       SDValue E0123 =
2300           DAG.getNode(NVPTXISD::BFI, DL, MVT::i32,
2301                       DAG.getAnyExtOrTrunc(Op->getOperand(3), DL, MVT::i32),
2302                       E012, DAG.getConstant(24, DL, MVT::i32), C8);
2303       return DAG.getNode(ISD::BITCAST, DL, VT, E0123);
2304     }
2305     return Op;
2306   }
2307 
2308   // Get value or the Nth operand as an APInt(32). Undef values treated as 0.
2309   auto GetOperand = [](SDValue Op, int N) -> APInt {
2310     const SDValue &Operand = Op->getOperand(N);
2311     EVT VT = Op->getValueType(0);
2312     if (Operand->isUndef())
2313       return APInt(32, 0);
2314     APInt Value;
2315     if (VT == MVT::v2f16 || VT == MVT::v2bf16)
2316       Value = cast<ConstantFPSDNode>(Operand)->getValueAPF().bitcastToAPInt();
2317     else if (VT == MVT::v2i16 || VT == MVT::v4i8)
2318       Value = Operand->getAsAPIntVal();
2319     else
2320       llvm_unreachable("Unsupported type");
2321     // i8 values are carried around as i16, so we need to zero out upper bits,
2322     // so they do not get in the way of combining individual byte values
2323     if (VT == MVT::v4i8)
2324       Value = Value.trunc(8);
2325     return Value.zext(32);
2326   };
2327   APInt Value;
2328   if (Isv2x16VT(VT)) {
2329     Value = GetOperand(Op, 0) | GetOperand(Op, 1).shl(16);
2330   } else if (VT == MVT::v4i8) {
2331     Value = GetOperand(Op, 0) | GetOperand(Op, 1).shl(8) |
2332             GetOperand(Op, 2).shl(16) | GetOperand(Op, 3).shl(24);
2333   } else {
2334     llvm_unreachable("Unsupported type");
2335   }
2336   SDValue Const = DAG.getConstant(Value, SDLoc(Op), MVT::i32);
2337   return DAG.getNode(ISD::BITCAST, SDLoc(Op), Op->getValueType(0), Const);
2338 }
2339 
2340 SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
2341                                                      SelectionDAG &DAG) const {
2342   SDValue Index = Op->getOperand(1);
2343   SDValue Vector = Op->getOperand(0);
2344   SDLoc DL(Op);
2345   EVT VectorVT = Vector.getValueType();
2346 
2347   if (VectorVT == MVT::v4i8) {
2348     SDValue BFE =
2349         DAG.getNode(NVPTXISD::BFE, DL, MVT::i32,
2350                     {Vector,
2351                      DAG.getNode(ISD::MUL, DL, MVT::i32,
2352                                  DAG.getZExtOrTrunc(Index, DL, MVT::i32),
2353                                  DAG.getConstant(8, DL, MVT::i32)),
2354                      DAG.getConstant(8, DL, MVT::i32)});
2355     return DAG.getAnyExtOrTrunc(BFE, DL, Op->getValueType(0));
2356   }
2357 
2358   // Constant index will be matched by tablegen.
2359   if (isa<ConstantSDNode>(Index.getNode()))
2360     return Op;
2361 
2362   // Extract individual elements and select one of them.
2363   assert(Isv2x16VT(VectorVT) && "Unexpected vector type.");
2364   EVT EltVT = VectorVT.getVectorElementType();
2365 
2366   SDLoc dl(Op.getNode());
2367   SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,
2368                            DAG.getIntPtrConstant(0, dl));
2369   SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector,
2370                            DAG.getIntPtrConstant(1, dl));
2371   return DAG.getSelectCC(dl, Index, DAG.getIntPtrConstant(0, dl), E0, E1,
2372                          ISD::CondCode::SETEQ);
2373 }
2374 
2375 SDValue NVPTXTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
2376                                                     SelectionDAG &DAG) const {
2377   SDValue Vector = Op->getOperand(0);
2378   EVT VectorVT = Vector.getValueType();
2379 
2380   if (VectorVT != MVT::v4i8)
2381     return Op;
2382   SDLoc DL(Op);
2383   SDValue Value = Op->getOperand(1);
2384   if (Value->isUndef())
2385     return Vector;
2386 
2387   SDValue Index = Op->getOperand(2);
2388 
2389   SDValue BFI =
2390       DAG.getNode(NVPTXISD::BFI, DL, MVT::i32,
2391                   {DAG.getZExtOrTrunc(Value, DL, MVT::i32), Vector,
2392                    DAG.getNode(ISD::MUL, DL, MVT::i32,
2393                                DAG.getZExtOrTrunc(Index, DL, MVT::i32),
2394                                DAG.getConstant(8, DL, MVT::i32)),
2395                    DAG.getConstant(8, DL, MVT::i32)});
2396   return DAG.getNode(ISD::BITCAST, DL, Op->getValueType(0), BFI);
2397 }
2398 
2399 SDValue NVPTXTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2400                                                  SelectionDAG &DAG) const {
2401   SDValue V1 = Op.getOperand(0);
2402   EVT VectorVT = V1.getValueType();
2403   if (VectorVT != MVT::v4i8 || Op.getValueType() != MVT::v4i8)
2404     return Op;
2405 
2406   // Lower shuffle to PRMT instruction.
2407   const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
2408   SDValue V2 = Op.getOperand(1);
2409   uint32_t Selector = 0;
2410   for (auto I : llvm::enumerate(SVN->getMask())) {
2411     if (I.value() != -1) // -1 is a placeholder for undef.
2412       Selector |= (I.value() << (I.index() * 4));
2413   }
2414 
2415   SDLoc DL(Op);
2416   return DAG.getNode(NVPTXISD::PRMT, DL, MVT::v4i8, V1, V2,
2417                      DAG.getConstant(Selector, DL, MVT::i32),
2418                      DAG.getConstant(NVPTX::PTXPrmtMode::NONE, DL, MVT::i32));
2419 }
2420 /// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which
2421 /// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2422 ///    amount, or
2423 /// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2424 ///    amount.
2425 SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
2426                                                   SelectionDAG &DAG) const {
2427   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2428   assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
2429 
2430   EVT VT = Op.getValueType();
2431   unsigned VTBits = VT.getSizeInBits();
2432   SDLoc dl(Op);
2433   SDValue ShOpLo = Op.getOperand(0);
2434   SDValue ShOpHi = Op.getOperand(1);
2435   SDValue ShAmt  = Op.getOperand(2);
2436   unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
2437 
2438   if (VTBits == 32 && STI.getSmVersion() >= 35) {
2439     // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2440     // {dHi, dLo} = {aHi, aLo} >> Amt
2441     //   dHi = aHi >> Amt
2442     //   dLo = shf.r.clamp aLo, aHi, Amt
2443 
2444     SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
2445     SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi,
2446                              ShAmt);
2447 
2448     SDValue Ops[2] = { Lo, Hi };
2449     return DAG.getMergeValues(Ops, dl);
2450   }
2451   else {
2452     // {dHi, dLo} = {aHi, aLo} >> Amt
2453     // - if (Amt>=size) then
2454     //      dLo = aHi >> (Amt-size)
2455     //      dHi = aHi >> Amt (this is either all 0 or all 1)
2456     //   else
2457     //      dLo = (aLo >>logic Amt) | (aHi << (size-Amt))
2458     //      dHi = aHi >> Amt
2459 
2460     SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
2461                                    DAG.getConstant(VTBits, dl, MVT::i32),
2462                                    ShAmt);
2463     SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
2464     SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
2465                                      DAG.getConstant(VTBits, dl, MVT::i32));
2466     SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
2467     SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2468     SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
2469 
2470     SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
2471                                DAG.getConstant(VTBits, dl, MVT::i32),
2472                                ISD::SETGE);
2473     SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
2474     SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
2475 
2476     SDValue Ops[2] = { Lo, Hi };
2477     return DAG.getMergeValues(Ops, dl);
2478   }
2479 }
2480 
2481 /// LowerShiftLeftParts - Lower SHL_PARTS, which
2482 /// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
2483 ///    amount, or
2484 /// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
2485 ///    amount.
2486 SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
2487                                                  SelectionDAG &DAG) const {
2488   assert(Op.getNumOperands() == 3 && "Not a double-shift!");
2489   assert(Op.getOpcode() == ISD::SHL_PARTS);
2490 
2491   EVT VT = Op.getValueType();
2492   unsigned VTBits = VT.getSizeInBits();
2493   SDLoc dl(Op);
2494   SDValue ShOpLo = Op.getOperand(0);
2495   SDValue ShOpHi = Op.getOperand(1);
2496   SDValue ShAmt  = Op.getOperand(2);
2497 
2498   if (VTBits == 32 && STI.getSmVersion() >= 35) {
2499     // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
2500     // {dHi, dLo} = {aHi, aLo} << Amt
2501     //   dHi = shf.l.clamp aLo, aHi, Amt
2502     //   dLo = aLo << Amt
2503 
2504     SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi,
2505                              ShAmt);
2506     SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
2507 
2508     SDValue Ops[2] = { Lo, Hi };
2509     return DAG.getMergeValues(Ops, dl);
2510   }
2511   else {
2512     // {dHi, dLo} = {aHi, aLo} << Amt
2513     // - if (Amt>=size) then
2514     //      dLo = aLo << Amt (all 0)
2515     //      dLo = aLo << (Amt-size)
2516     //   else
2517     //      dLo = aLo << Amt
2518     //      dHi = (aHi << Amt) | (aLo >> (size-Amt))
2519 
2520     SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
2521                                    DAG.getConstant(VTBits, dl, MVT::i32),
2522                                    ShAmt);
2523     SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
2524     SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
2525                                      DAG.getConstant(VTBits, dl, MVT::i32));
2526     SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
2527     SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2528     SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
2529 
2530     SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
2531                                DAG.getConstant(VTBits, dl, MVT::i32),
2532                                ISD::SETGE);
2533     SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
2534     SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
2535 
2536     SDValue Ops[2] = { Lo, Hi };
2537     return DAG.getMergeValues(Ops, dl);
2538   }
2539 }
2540 
2541 SDValue NVPTXTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2542   EVT VT = Op.getValueType();
2543 
2544   if (VT == MVT::f32)
2545     return LowerFROUND32(Op, DAG);
2546 
2547   if (VT == MVT::f64)
2548     return LowerFROUND64(Op, DAG);
2549 
2550   llvm_unreachable("unhandled type");
2551 }
2552 
2553 // This is the the rounding method used in CUDA libdevice in C like code:
2554 // float roundf(float A)
2555 // {
2556 //   float RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f));
2557 //   RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2558 //   return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2559 // }
2560 SDValue NVPTXTargetLowering::LowerFROUND32(SDValue Op,
2561                                            SelectionDAG &DAG) const {
2562   SDLoc SL(Op);
2563   SDValue A = Op.getOperand(0);
2564   EVT VT = Op.getValueType();
2565 
2566   SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);
2567 
2568   // RoundedA = (float) (int) ( A > 0 ? (A + 0.5f) : (A - 0.5f))
2569   SDValue Bitcast  = DAG.getNode(ISD::BITCAST, SL, MVT::i32, A);
2570   const int SignBitMask = 0x80000000;
2571   SDValue Sign = DAG.getNode(ISD::AND, SL, MVT::i32, Bitcast,
2572                              DAG.getConstant(SignBitMask, SL, MVT::i32));
2573   const int PointFiveInBits = 0x3F000000;
2574   SDValue PointFiveWithSignRaw =
2575       DAG.getNode(ISD::OR, SL, MVT::i32, Sign,
2576                   DAG.getConstant(PointFiveInBits, SL, MVT::i32));
2577   SDValue PointFiveWithSign =
2578       DAG.getNode(ISD::BITCAST, SL, VT, PointFiveWithSignRaw);
2579   SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, A, PointFiveWithSign);
2580   SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);
2581 
2582   // RoundedA = abs(A) > 0x1.0p23 ? A : RoundedA;
2583   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2584   SDValue IsLarge =
2585       DAG.getSetCC(SL, SetCCVT, AbsA, DAG.getConstantFP(pow(2.0, 23.0), SL, VT),
2586                    ISD::SETOGT);
2587   RoundedA = DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA);
2588 
2589   // return abs(A) < 0.5 ? (float)(int)A : RoundedA;
2590   SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,
2591                                 DAG.getConstantFP(0.5, SL, VT), ISD::SETOLT);
2592   SDValue RoundedAForSmallA = DAG.getNode(ISD::FTRUNC, SL, VT, A);
2593   return DAG.getNode(ISD::SELECT, SL, VT, IsSmall, RoundedAForSmallA, RoundedA);
2594 }
2595 
2596 // The implementation of round(double) is similar to that of round(float) in
2597 // that they both separate the value range into three regions and use a method
2598 // specific to the region to round the values. However, round(double) first
2599 // calculates the round of the absolute value and then adds the sign back while
2600 // round(float) directly rounds the value with sign.
2601 SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op,
2602                                            SelectionDAG &DAG) const {
2603   SDLoc SL(Op);
2604   SDValue A = Op.getOperand(0);
2605   EVT VT = Op.getValueType();
2606 
2607   SDValue AbsA = DAG.getNode(ISD::FABS, SL, VT, A);
2608 
2609   // double RoundedA = (double) (int) (abs(A) + 0.5f);
2610   SDValue AdjustedA = DAG.getNode(ISD::FADD, SL, VT, AbsA,
2611                                   DAG.getConstantFP(0.5, SL, VT));
2612   SDValue RoundedA = DAG.getNode(ISD::FTRUNC, SL, VT, AdjustedA);
2613 
2614   // RoundedA = abs(A) < 0.5 ? (double)0 : RoundedA;
2615   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2616   SDValue IsSmall =DAG.getSetCC(SL, SetCCVT, AbsA,
2617                                 DAG.getConstantFP(0.5, SL, VT), ISD::SETOLT);
2618   RoundedA = DAG.getNode(ISD::SELECT, SL, VT, IsSmall,
2619                          DAG.getConstantFP(0, SL, VT),
2620                          RoundedA);
2621 
2622   // Add sign to rounded_A
2623   RoundedA = DAG.getNode(ISD::FCOPYSIGN, SL, VT, RoundedA, A);
2624   DAG.getNode(ISD::FTRUNC, SL, VT, A);
2625 
2626   // RoundedA = abs(A) > 0x1.0p52 ? A : RoundedA;
2627   SDValue IsLarge =
2628       DAG.getSetCC(SL, SetCCVT, AbsA, DAG.getConstantFP(pow(2.0, 52.0), SL, VT),
2629                    ISD::SETOGT);
2630   return DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA);
2631 }
2632 
2633 SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op,
2634                                             SelectionDAG &DAG) const {
2635   assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78);
2636 
2637   if (Op.getValueType() == MVT::bf16) {
2638     SDLoc Loc(Op);
2639     return DAG.getNode(
2640         ISD::FP_ROUND, Loc, MVT::bf16,
2641         DAG.getNode(Op.getOpcode(), Loc, MVT::f32, Op.getOperand(0)),
2642         DAG.getIntPtrConstant(0, Loc));
2643   }
2644 
2645   // Everything else is considered legal.
2646   return Op;
2647 }
2648 
2649 SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op,
2650                                             SelectionDAG &DAG) const {
2651   assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78);
2652 
2653   if (Op.getOperand(0).getValueType() == MVT::bf16) {
2654     SDLoc Loc(Op);
2655     return DAG.getNode(
2656         Op.getOpcode(), Loc, Op.getValueType(),
2657         DAG.getNode(ISD::FP_EXTEND, Loc, MVT::f32, Op.getOperand(0)));
2658   }
2659 
2660   // Everything else is considered legal.
2661   return Op;
2662 }
2663 
2664 SDValue NVPTXTargetLowering::LowerFP_ROUND(SDValue Op,
2665                                            SelectionDAG &DAG) const {
2666   EVT NarrowVT = Op.getValueType();
2667   SDValue Wide = Op.getOperand(0);
2668   EVT WideVT = Wide.getValueType();
2669   if (NarrowVT.getScalarType() == MVT::bf16) {
2670     const TargetLowering *TLI = STI.getTargetLowering();
2671     if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 70) {
2672       return TLI->expandFP_ROUND(Op.getNode(), DAG);
2673     }
2674     if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
2675       // This combination was the first to support f32 -> bf16.
2676       if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70) {
2677         if (WideVT.getScalarType() == MVT::f32) {
2678           return Op;
2679         }
2680         if (WideVT.getScalarType() == MVT::f64) {
2681           SDLoc Loc(Op);
2682           // Round-inexact-to-odd f64 to f32, then do the final rounding using
2683           // the hardware f32 -> bf16 instruction.
2684           SDValue rod = TLI->expandRoundInexactToOdd(
2685               WideVT.isVector() ? WideVT.changeVectorElementType(MVT::f32)
2686                                 : MVT::f32,
2687               Wide, Loc, DAG);
2688           return DAG.getFPExtendOrRound(rod, Loc, NarrowVT);
2689         }
2690       }
2691       return TLI->expandFP_ROUND(Op.getNode(), DAG);
2692     }
2693   }
2694 
2695   // Everything else is considered legal.
2696   return Op;
2697 }
2698 
2699 SDValue NVPTXTargetLowering::LowerFP_EXTEND(SDValue Op,
2700                                             SelectionDAG &DAG) const {
2701   SDValue Narrow = Op.getOperand(0);
2702   EVT NarrowVT = Narrow.getValueType();
2703   EVT WideVT = Op.getValueType();
2704   if (NarrowVT.getScalarType() == MVT::bf16) {
2705     if (WideVT.getScalarType() == MVT::f32 &&
2706         (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71)) {
2707       SDLoc Loc(Op);
2708       return DAG.getNode(ISD::BF16_TO_FP, Loc, WideVT, Narrow);
2709     }
2710     if (WideVT.getScalarType() == MVT::f64 &&
2711         (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78)) {
2712       EVT F32 = NarrowVT.isVector() ? NarrowVT.changeVectorElementType(MVT::f32)
2713                                     : MVT::f32;
2714       SDLoc Loc(Op);
2715       if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 71) {
2716         Op = DAG.getNode(ISD::FP_EXTEND, Loc, F32, Narrow);
2717       } else {
2718         Op = DAG.getNode(ISD::BF16_TO_FP, Loc, F32, Narrow);
2719       }
2720       return DAG.getNode(ISD::FP_EXTEND, Loc, WideVT, Op);
2721     }
2722   }
2723 
2724   // Everything else is considered legal.
2725   return Op;
2726 }
2727 
2728 static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) {
2729   SDLoc DL(Op);
2730   if (Op.getValueType() != MVT::v2i16)
2731     return Op;
2732   EVT EltVT = Op.getValueType().getVectorElementType();
2733   SmallVector<SDValue> VecElements;
2734   for (int I = 0, E = Op.getValueType().getVectorNumElements(); I < E; I++) {
2735     SmallVector<SDValue> ScalarArgs;
2736     llvm::transform(Op->ops(), std::back_inserter(ScalarArgs),
2737                     [&](const SDUse &O) {
2738                       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
2739                                          O.get(), DAG.getIntPtrConstant(I, DL));
2740                     });
2741     VecElements.push_back(DAG.getNode(Op.getOpcode(), DL, EltVT, ScalarArgs));
2742   }
2743   SDValue V =
2744       DAG.getNode(ISD::BUILD_VECTOR, DL, Op.getValueType(), VecElements);
2745   return V;
2746 }
2747 
2748 SDValue
2749 NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
2750   switch (Op.getOpcode()) {
2751   case ISD::RETURNADDR:
2752     return SDValue();
2753   case ISD::FRAMEADDR:
2754     return SDValue();
2755   case ISD::GlobalAddress:
2756     return LowerGlobalAddress(Op, DAG);
2757   case ISD::INTRINSIC_W_CHAIN:
2758     return Op;
2759   case ISD::BUILD_VECTOR:
2760     return LowerBUILD_VECTOR(Op, DAG);
2761   case ISD::EXTRACT_SUBVECTOR:
2762     return Op;
2763   case ISD::EXTRACT_VECTOR_ELT:
2764     return LowerEXTRACT_VECTOR_ELT(Op, DAG);
2765   case ISD::INSERT_VECTOR_ELT:
2766     return LowerINSERT_VECTOR_ELT(Op, DAG);
2767   case ISD::VECTOR_SHUFFLE:
2768     return LowerVECTOR_SHUFFLE(Op, DAG);
2769   case ISD::CONCAT_VECTORS:
2770     return LowerCONCAT_VECTORS(Op, DAG);
2771   case ISD::STORE:
2772     return LowerSTORE(Op, DAG);
2773   case ISD::LOAD:
2774     return LowerLOAD(Op, DAG);
2775   case ISD::SHL_PARTS:
2776     return LowerShiftLeftParts(Op, DAG);
2777   case ISD::SRA_PARTS:
2778   case ISD::SRL_PARTS:
2779     return LowerShiftRightParts(Op, DAG);
2780   case ISD::SELECT:
2781     return LowerSelect(Op, DAG);
2782   case ISD::FROUND:
2783     return LowerFROUND(Op, DAG);
2784   case ISD::SINT_TO_FP:
2785   case ISD::UINT_TO_FP:
2786     return LowerINT_TO_FP(Op, DAG);
2787   case ISD::FP_TO_SINT:
2788   case ISD::FP_TO_UINT:
2789     return LowerFP_TO_INT(Op, DAG);
2790   case ISD::FP_ROUND:
2791     return LowerFP_ROUND(Op, DAG);
2792   case ISD::FP_EXTEND:
2793     return LowerFP_EXTEND(Op, DAG);
2794   case ISD::VAARG:
2795     return LowerVAARG(Op, DAG);
2796   case ISD::VASTART:
2797     return LowerVASTART(Op, DAG);
2798   case ISD::ABS:
2799   case ISD::SMIN:
2800   case ISD::SMAX:
2801   case ISD::UMIN:
2802   case ISD::UMAX:
2803   case ISD::ADD:
2804   case ISD::SUB:
2805   case ISD::MUL:
2806   case ISD::SHL:
2807   case ISD::SREM:
2808   case ISD::UREM:
2809     return LowerVectorArith(Op, DAG);
2810   case ISD::DYNAMIC_STACKALLOC:
2811     return LowerDYNAMIC_STACKALLOC(Op, DAG);
2812   case ISD::CopyToReg:
2813     return LowerCopyToReg_128(Op, DAG);
2814   default:
2815     llvm_unreachable("Custom lowering not defined for operation");
2816   }
2817 }
2818 
2819 // This function is almost a copy of SelectionDAG::expandVAArg().
2820 // The only diff is that this one produces loads from local address space.
2821 SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
2822   const TargetLowering *TLI = STI.getTargetLowering();
2823   SDLoc DL(Op);
2824 
2825   SDNode *Node = Op.getNode();
2826   const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2827   EVT VT = Node->getValueType(0);
2828   auto *Ty = VT.getTypeForEVT(*DAG.getContext());
2829   SDValue Tmp1 = Node->getOperand(0);
2830   SDValue Tmp2 = Node->getOperand(1);
2831   const MaybeAlign MA(Node->getConstantOperandVal(3));
2832 
2833   SDValue VAListLoad = DAG.getLoad(TLI->getPointerTy(DAG.getDataLayout()), DL,
2834                                    Tmp1, Tmp2, MachinePointerInfo(V));
2835   SDValue VAList = VAListLoad;
2836 
2837   if (MA && *MA > TLI->getMinStackArgumentAlignment()) {
2838     VAList = DAG.getNode(
2839         ISD::ADD, DL, VAList.getValueType(), VAList,
2840         DAG.getConstant(MA->value() - 1, DL, VAList.getValueType()));
2841 
2842     VAList = DAG.getNode(
2843         ISD::AND, DL, VAList.getValueType(), VAList,
2844         DAG.getConstant(-(int64_t)MA->value(), DL, VAList.getValueType()));
2845   }
2846 
2847   // Increment the pointer, VAList, to the next vaarg
2848   Tmp1 = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
2849                      DAG.getConstant(DAG.getDataLayout().getTypeAllocSize(Ty),
2850                                      DL, VAList.getValueType()));
2851 
2852   // Store the incremented VAList to the legalized pointer
2853   Tmp1 = DAG.getStore(VAListLoad.getValue(1), DL, Tmp1, Tmp2,
2854                       MachinePointerInfo(V));
2855 
2856   const Value *SrcV =
2857       Constant::getNullValue(PointerType::get(Ty, ADDRESS_SPACE_LOCAL));
2858 
2859   // Load the actual argument out of the pointer VAList
2860   return DAG.getLoad(VT, DL, Tmp1, VAList, MachinePointerInfo(SrcV));
2861 }
2862 
2863 SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
2864   const TargetLowering *TLI = STI.getTargetLowering();
2865   SDLoc DL(Op);
2866   EVT PtrVT = TLI->getPointerTy(DAG.getDataLayout());
2867 
2868   // Store the address of unsized array <function>_vararg[] in the ap object.
2869   SDValue Arg = getParamSymbol(DAG, /* vararg */ -1, PtrVT);
2870   SDValue VAReg = DAG.getNode(NVPTXISD::Wrapper, DL, PtrVT, Arg);
2871 
2872   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
2873   return DAG.getStore(Op.getOperand(0), DL, VAReg, Op.getOperand(1),
2874                       MachinePointerInfo(SV));
2875 }
2876 
2877 SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const {
2878   SDValue Op0 = Op->getOperand(0);
2879   SDValue Op1 = Op->getOperand(1);
2880   SDValue Op2 = Op->getOperand(2);
2881   SDLoc DL(Op.getNode());
2882 
2883   assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1");
2884 
2885   Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
2886   Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
2887   SDValue Select = DAG.getNode(ISD::SELECT, DL, MVT::i32, Op0, Op1, Op2);
2888   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);
2889 
2890   return Trunc;
2891 }
2892 
2893 SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
2894   if (Op.getValueType() == MVT::i1)
2895     return LowerLOADi1(Op, DAG);
2896 
2897   // v2f16/v2bf16/v2i16/v4i8 are legal, so we can't rely on legalizer to handle
2898   // unaligned loads and have to handle it here.
2899   EVT VT = Op.getValueType();
2900   if (Isv2x16VT(VT) || VT == MVT::v4i8) {
2901     LoadSDNode *Load = cast<LoadSDNode>(Op);
2902     EVT MemVT = Load->getMemoryVT();
2903     if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2904                                         MemVT, *Load->getMemOperand())) {
2905       SDValue Ops[2];
2906       std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG);
2907       return DAG.getMergeValues(Ops, SDLoc(Op));
2908     }
2909   }
2910 
2911   return SDValue();
2912 }
2913 
2914 // v = ld i1* addr
2915 //   =>
2916 // v1 = ld i8* addr (-> i16)
2917 // v = trunc i16 to i1
2918 SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
2919   SDNode *Node = Op.getNode();
2920   LoadSDNode *LD = cast<LoadSDNode>(Node);
2921   SDLoc dl(Node);
2922   assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
2923   assert(Node->getValueType(0) == MVT::i1 &&
2924          "Custom lowering for i1 load only");
2925   SDValue newLD = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i16, LD->getChain(),
2926                                  LD->getBasePtr(), LD->getPointerInfo(),
2927                                  MVT::i8, LD->getAlign(),
2928                                  LD->getMemOperand()->getFlags());
2929   SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
2930   // The legalizer (the caller) is expecting two values from the legalized
2931   // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
2932   // in LegalizeDAG.cpp which also uses MergeValues.
2933   SDValue Ops[] = { result, LD->getChain() };
2934   return DAG.getMergeValues(Ops, dl);
2935 }
2936 
2937 SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
2938   StoreSDNode *Store = cast<StoreSDNode>(Op);
2939   EVT VT = Store->getMemoryVT();
2940 
2941   if (VT == MVT::i1)
2942     return LowerSTOREi1(Op, DAG);
2943 
2944   // v2f16 is legal, so we can't rely on legalizer to handle unaligned
2945   // stores and have to handle it here.
2946   if ((Isv2x16VT(VT) || VT == MVT::v4i8) &&
2947       !allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2948                                       VT, *Store->getMemOperand()))
2949     return expandUnalignedStore(Store, DAG);
2950 
2951   // v2f16, v2bf16 and v2i16 don't need special handling.
2952   if (Isv2x16VT(VT) || VT == MVT::v4i8)
2953     return SDValue();
2954 
2955   if (VT.isVector())
2956     return LowerSTOREVector(Op, DAG);
2957 
2958   return SDValue();
2959 }
2960 
2961 SDValue
2962 NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
2963   SDNode *N = Op.getNode();
2964   SDValue Val = N->getOperand(1);
2965   SDLoc DL(N);
2966   EVT ValVT = Val.getValueType();
2967 
2968   if (ValVT.isVector()) {
2969     // We only handle "native" vector sizes for now, e.g. <4 x double> is not
2970     // legal.  We can (and should) split that into 2 stores of <2 x double> here
2971     // but I'm leaving that as a TODO for now.
2972     if (!ValVT.isSimple())
2973       return SDValue();
2974     switch (ValVT.getSimpleVT().SimpleTy) {
2975     default:
2976       return SDValue();
2977     case MVT::v2i8:
2978     case MVT::v2i16:
2979     case MVT::v2i32:
2980     case MVT::v2i64:
2981     case MVT::v2f16:
2982     case MVT::v2bf16:
2983     case MVT::v2f32:
2984     case MVT::v2f64:
2985     case MVT::v4i8:
2986     case MVT::v4i16:
2987     case MVT::v4i32:
2988     case MVT::v4f16:
2989     case MVT::v4bf16:
2990     case MVT::v4f32:
2991     case MVT::v8f16: // <4 x f16x2>
2992     case MVT::v8bf16: // <4 x bf16x2>
2993     case MVT::v8i16:  // <4 x i16x2>
2994       // This is a "native" vector type
2995       break;
2996     }
2997 
2998     MemSDNode *MemSD = cast<MemSDNode>(N);
2999     const DataLayout &TD = DAG.getDataLayout();
3000 
3001     Align Alignment = MemSD->getAlign();
3002     Align PrefAlign =
3003         TD.getPrefTypeAlign(ValVT.getTypeForEVT(*DAG.getContext()));
3004     if (Alignment < PrefAlign) {
3005       // This store is not sufficiently aligned, so bail out and let this vector
3006       // store be scalarized.  Note that we may still be able to emit smaller
3007       // vector stores.  For example, if we are storing a <4 x float> with an
3008       // alignment of 8, this check will fail but the legalizer will try again
3009       // with 2 x <2 x float>, which will succeed with an alignment of 8.
3010       return SDValue();
3011     }
3012 
3013     unsigned Opcode = 0;
3014     EVT EltVT = ValVT.getVectorElementType();
3015     unsigned NumElts = ValVT.getVectorNumElements();
3016 
3017     // Since StoreV2 is a target node, we cannot rely on DAG type legalization.
3018     // Therefore, we must ensure the type is legal.  For i1 and i8, we set the
3019     // stored type to i16 and propagate the "real" type as the memory type.
3020     bool NeedExt = false;
3021     if (EltVT.getSizeInBits() < 16)
3022       NeedExt = true;
3023 
3024     bool StoreF16x2 = false;
3025     switch (NumElts) {
3026     default:
3027       return SDValue();
3028     case 2:
3029       Opcode = NVPTXISD::StoreV2;
3030       break;
3031     case 4:
3032       Opcode = NVPTXISD::StoreV4;
3033       break;
3034     case 8:
3035       // v8f16 is a special case. PTX doesn't have st.v8.f16
3036       // instruction. Instead, we split the vector into v2f16 chunks and
3037       // store them with st.v4.b32.
3038       assert(Is16bitsType(EltVT.getSimpleVT()) && "Wrong type for the vector.");
3039       Opcode = NVPTXISD::StoreV4;
3040       StoreF16x2 = true;
3041       break;
3042     }
3043 
3044     SmallVector<SDValue, 8> Ops;
3045 
3046     // First is the chain
3047     Ops.push_back(N->getOperand(0));
3048 
3049     if (StoreF16x2) {
3050       // Combine f16,f16 -> v2f16
3051       NumElts /= 2;
3052       for (unsigned i = 0; i < NumElts; ++i) {
3053         SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
3054                                  DAG.getIntPtrConstant(i * 2, DL));
3055         SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
3056                                  DAG.getIntPtrConstant(i * 2 + 1, DL));
3057         EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 2);
3058         SDValue V2 = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, E0, E1);
3059         Ops.push_back(V2);
3060       }
3061     } else {
3062       // Then the split values
3063       for (unsigned i = 0; i < NumElts; ++i) {
3064         SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
3065                                      DAG.getIntPtrConstant(i, DL));
3066         if (NeedExt)
3067           ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);
3068         Ops.push_back(ExtVal);
3069       }
3070     }
3071 
3072     // Then any remaining arguments
3073     Ops.append(N->op_begin() + 2, N->op_end());
3074 
3075     SDValue NewSt =
3076         DAG.getMemIntrinsicNode(Opcode, DL, DAG.getVTList(MVT::Other), Ops,
3077                                 MemSD->getMemoryVT(), MemSD->getMemOperand());
3078 
3079     // return DCI.CombineTo(N, NewSt, true);
3080     return NewSt;
3081   }
3082 
3083   return SDValue();
3084 }
3085 
3086 // st i1 v, addr
3087 //    =>
3088 // v1 = zxt v to i16
3089 // st.u8 i16, addr
3090 SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
3091   SDNode *Node = Op.getNode();
3092   SDLoc dl(Node);
3093   StoreSDNode *ST = cast<StoreSDNode>(Node);
3094   SDValue Tmp1 = ST->getChain();
3095   SDValue Tmp2 = ST->getBasePtr();
3096   SDValue Tmp3 = ST->getValue();
3097   assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
3098   Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);
3099   SDValue Result =
3100       DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), MVT::i8,
3101                         ST->getAlign(), ST->getMemOperand()->getFlags());
3102   return Result;
3103 }
3104 
3105 SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
3106                                                 SelectionDAG &DAG) const {
3107   // Change the CopyToReg to take in two 64-bit operands instead of a 128-bit
3108   // operand so that it can pass the legalization.
3109 
3110   assert(Op.getOperand(1).getValueType() == MVT::i128 &&
3111          "Custom lowering for 128-bit CopyToReg only");
3112 
3113   SDNode *Node = Op.getNode();
3114   SDLoc DL(Node);
3115 
3116   SDValue Cast = DAG.getBitcast(MVT::v2i64, Op->getOperand(2));
3117   SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Cast,
3118                            DAG.getIntPtrConstant(0, DL));
3119   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Cast,
3120                            DAG.getIntPtrConstant(1, DL));
3121 
3122   SmallVector<SDValue, 5> NewOps(Op->getNumOperands() + 1);
3123   SmallVector<EVT, 3> ResultsType(Node->values());
3124 
3125   NewOps[0] = Op->getOperand(0); // Chain
3126   NewOps[1] = Op->getOperand(1); // Dst Reg
3127   NewOps[2] = Lo;                // Lower 64-bit
3128   NewOps[3] = Hi;                // Higher 64-bit
3129   if (Op.getNumOperands() == 4)
3130     NewOps[4] = Op->getOperand(3); // Glue if exists
3131 
3132   return DAG.getNode(ISD::CopyToReg, DL, ResultsType, NewOps);
3133 }
3134 
3135 unsigned NVPTXTargetLowering::getNumRegisters(
3136     LLVMContext &Context, EVT VT,
3137     std::optional<MVT> RegisterVT = std::nullopt) const {
3138   if (VT == MVT::i128 && RegisterVT == MVT::i128)
3139     return 1;
3140   return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
3141 }
3142 
3143 bool NVPTXTargetLowering::splitValueIntoRegisterParts(
3144     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
3145     unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
3146   if (Val.getValueType() == MVT::i128 && NumParts == 1) {
3147     Parts[0] = Val;
3148     return true;
3149   }
3150   return false;
3151 }
3152 
3153 // This creates target external symbol for a function parameter.
3154 // Name of the symbol is composed from its index and the function name.
3155 // Negative index corresponds to special parameter (unsized array) used for
3156 // passing variable arguments.
3157 SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx,
3158                                             EVT v) const {
3159   StringRef SavedStr = nvTM->getStrPool().save(
3160       getParamName(&DAG.getMachineFunction().getFunction(), idx));
3161   return DAG.getTargetExternalSymbol(SavedStr.data(), v);
3162 }
3163 
3164 SDValue NVPTXTargetLowering::LowerFormalArguments(
3165     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3166     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
3167     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3168   MachineFunction &MF = DAG.getMachineFunction();
3169   const DataLayout &DL = DAG.getDataLayout();
3170   auto PtrVT = getPointerTy(DAG.getDataLayout());
3171 
3172   const Function *F = &MF.getFunction();
3173   const AttributeList &PAL = F->getAttributes();
3174   const TargetLowering *TLI = STI.getTargetLowering();
3175 
3176   SDValue Root = DAG.getRoot();
3177   std::vector<SDValue> OutChains;
3178 
3179   bool isABI = (STI.getSmVersion() >= 20);
3180   assert(isABI && "Non-ABI compilation is not supported");
3181   if (!isABI)
3182     return Chain;
3183 
3184   std::vector<Type *> argTypes;
3185   std::vector<const Argument *> theArgs;
3186   for (const Argument &I : F->args()) {
3187     theArgs.push_back(&I);
3188     argTypes.push_back(I.getType());
3189   }
3190   // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
3191   // Ins.size() will be larger
3192   //   * if there is an aggregate argument with multiple fields (each field
3193   //     showing up separately in Ins)
3194   //   * if there is a vector argument with more than typical vector-length
3195   //     elements (generally if more than 4) where each vector element is
3196   //     individually present in Ins.
3197   // So a different index should be used for indexing into Ins.
3198   // See similar issue in LowerCall.
3199   unsigned InsIdx = 0;
3200 
3201   for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++InsIdx) {
3202     Type *Ty = argTypes[i];
3203 
3204     if (theArgs[i]->use_empty()) {
3205       // argument is dead
3206       if (IsTypePassedAsArray(Ty) && !Ty->isVectorTy()) {
3207         SmallVector<EVT, 16> vtparts;
3208 
3209         ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts);
3210         if (vtparts.empty())
3211           report_fatal_error("Empty parameter types are not supported");
3212 
3213         for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
3214              ++parti) {
3215           InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
3216           ++InsIdx;
3217         }
3218         if (vtparts.size() > 0)
3219           --InsIdx;
3220         continue;
3221       }
3222       if (Ty->isVectorTy()) {
3223         EVT ObjectVT = getValueType(DL, Ty);
3224         unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);
3225         for (unsigned parti = 0; parti < NumRegs; ++parti) {
3226           InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
3227           ++InsIdx;
3228         }
3229         if (NumRegs > 0)
3230           --InsIdx;
3231         continue;
3232       }
3233       InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
3234       continue;
3235     }
3236 
3237     // In the following cases, assign a node order of "i+1"
3238     // to newly created nodes. The SDNodes for params have to
3239     // appear in the same order as their order of appearance
3240     // in the original function. "i+1" holds that order.
3241     if (!PAL.hasParamAttr(i, Attribute::ByVal)) {
3242       bool aggregateIsPacked = false;
3243       if (StructType *STy = dyn_cast<StructType>(Ty))
3244         aggregateIsPacked = STy->isPacked();
3245 
3246       SmallVector<EVT, 16> VTs;
3247       SmallVector<uint64_t, 16> Offsets;
3248       ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets, 0);
3249       if (VTs.empty())
3250         report_fatal_error("Empty parameter types are not supported");
3251 
3252       Align ArgAlign = getFunctionArgumentAlignment(
3253           F, Ty, i + AttributeList::FirstArgIndex, DL);
3254       auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, ArgAlign);
3255 
3256       SDValue Arg = getParamSymbol(DAG, i, PtrVT);
3257       int VecIdx = -1; // Index of the first element of the current vector.
3258       for (unsigned parti = 0, parte = VTs.size(); parti != parte; ++parti) {
3259         if (VectorInfo[parti] & PVF_FIRST) {
3260           assert(VecIdx == -1 && "Orphaned vector.");
3261           VecIdx = parti;
3262         }
3263 
3264         // That's the last element of this store op.
3265         if (VectorInfo[parti] & PVF_LAST) {
3266           unsigned NumElts = parti - VecIdx + 1;
3267           EVT EltVT = VTs[parti];
3268           // i1 is loaded/stored as i8.
3269           EVT LoadVT = EltVT;
3270           if (EltVT == MVT::i1)
3271             LoadVT = MVT::i8;
3272           else if (Isv2x16VT(EltVT) || EltVT == MVT::v4i8)
3273             // getLoad needs a vector type, but it can't handle
3274             // vectors which contain v2f16 or v2bf16 elements. So we must load
3275             // using i32 here and then bitcast back.
3276             LoadVT = MVT::i32;
3277 
3278           EVT VecVT = EVT::getVectorVT(F->getContext(), LoadVT, NumElts);
3279           SDValue VecAddr =
3280               DAG.getNode(ISD::ADD, dl, PtrVT, Arg,
3281                           DAG.getConstant(Offsets[VecIdx], dl, PtrVT));
3282           Value *srcValue = Constant::getNullValue(PointerType::get(
3283               EltVT.getTypeForEVT(F->getContext()), ADDRESS_SPACE_PARAM));
3284 
3285           const MaybeAlign PartAlign = [&]() -> MaybeAlign {
3286             if (aggregateIsPacked)
3287               return Align(1);
3288             if (NumElts != 1)
3289               return std::nullopt;
3290             Align PartAlign =
3291                 DL.getABITypeAlign(EltVT.getTypeForEVT(F->getContext()));
3292             return commonAlignment(PartAlign, Offsets[parti]);
3293           }();
3294           SDValue P = DAG.getLoad(VecVT, dl, Root, VecAddr,
3295                                   MachinePointerInfo(srcValue), PartAlign,
3296                                   MachineMemOperand::MODereferenceable |
3297                                       MachineMemOperand::MOInvariant);
3298           if (P.getNode())
3299             P.getNode()->setIROrder(i + 1);
3300           for (unsigned j = 0; j < NumElts; ++j) {
3301             SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LoadVT, P,
3302                                       DAG.getIntPtrConstant(j, dl));
3303             // We've loaded i1 as an i8 and now must truncate it back to i1
3304             if (EltVT == MVT::i1)
3305               Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Elt);
3306             // v2f16 was loaded as an i32. Now we must bitcast it back.
3307             else if (EltVT != LoadVT)
3308               Elt = DAG.getNode(ISD::BITCAST, dl, EltVT, Elt);
3309 
3310             // If a promoted integer type is used, truncate down to the original
3311             MVT PromotedVT;
3312             if (PromoteScalarIntegerPTX(EltVT, &PromotedVT)) {
3313               Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
3314             }
3315 
3316             // Extend the element if necessary (e.g. an i8 is loaded
3317             // into an i16 register)
3318             if (Ins[InsIdx].VT.isInteger() &&
3319                 Ins[InsIdx].VT.getFixedSizeInBits() >
3320                     LoadVT.getFixedSizeInBits()) {
3321               unsigned Extend = Ins[InsIdx].Flags.isSExt() ? ISD::SIGN_EXTEND
3322                                                            : ISD::ZERO_EXTEND;
3323               Elt = DAG.getNode(Extend, dl, Ins[InsIdx].VT, Elt);
3324             }
3325             InVals.push_back(Elt);
3326           }
3327 
3328           // Reset vector tracking state.
3329           VecIdx = -1;
3330         }
3331         ++InsIdx;
3332       }
3333       if (VTs.size() > 0)
3334         --InsIdx;
3335       continue;
3336     }
3337 
3338     // Param has ByVal attribute
3339     // Return MoveParam(param symbol).
3340     // Ideally, the param symbol can be returned directly,
3341     // but when SDNode builder decides to use it in a CopyToReg(),
3342     // machine instruction fails because TargetExternalSymbol
3343     // (not lowered) is target dependent, and CopyToReg assumes
3344     // the source is lowered.
3345     EVT ObjectVT = getValueType(DL, Ty);
3346     assert(ObjectVT == Ins[InsIdx].VT &&
3347            "Ins type did not match function type");
3348     SDValue Arg = getParamSymbol(DAG, i, PtrVT);
3349     SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
3350     if (p.getNode())
3351       p.getNode()->setIROrder(i + 1);
3352     InVals.push_back(p);
3353   }
3354 
3355   if (!OutChains.empty())
3356     DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains));
3357 
3358   return Chain;
3359 }
3360 
3361 // Use byte-store when the param adress of the return value is unaligned.
3362 // This may happen when the return value is a field of a packed structure.
3363 static SDValue LowerUnalignedStoreRet(SelectionDAG &DAG, SDValue Chain,
3364                                       uint64_t Offset, EVT ElementType,
3365                                       SDValue RetVal, const SDLoc &dl) {
3366   // Bit logic only works on integer types
3367   if (adjustElementType(ElementType))
3368     RetVal = DAG.getNode(ISD::BITCAST, dl, ElementType, RetVal);
3369 
3370   // Store each byte
3371   for (unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
3372     // Shift the byte to the last byte position
3373     SDValue ShiftVal = DAG.getNode(ISD::SRL, dl, ElementType, RetVal,
3374                                    DAG.getConstant(i * 8, dl, MVT::i32));
3375     SDValue StoreOperands[] = {Chain, DAG.getConstant(Offset + i, dl, MVT::i32),
3376                                ShiftVal};
3377     // Trunc store only the last byte by using
3378     //     st.param.b8
3379     // The register type can be larger than b8.
3380     Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
3381                                     DAG.getVTList(MVT::Other), StoreOperands,
3382                                     MVT::i8, MachinePointerInfo(), std::nullopt,
3383                                     MachineMemOperand::MOStore);
3384   }
3385   return Chain;
3386 }
3387 
3388 SDValue
3389 NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3390                                  bool isVarArg,
3391                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
3392                                  const SmallVectorImpl<SDValue> &OutVals,
3393                                  const SDLoc &dl, SelectionDAG &DAG) const {
3394   const MachineFunction &MF = DAG.getMachineFunction();
3395   const Function &F = MF.getFunction();
3396   Type *RetTy = MF.getFunction().getReturnType();
3397 
3398   bool isABI = (STI.getSmVersion() >= 20);
3399   assert(isABI && "Non-ABI compilation is not supported");
3400   if (!isABI)
3401     return Chain;
3402 
3403   const DataLayout &DL = DAG.getDataLayout();
3404   SmallVector<SDValue, 16> PromotedOutVals;
3405   SmallVector<EVT, 16> VTs;
3406   SmallVector<uint64_t, 16> Offsets;
3407   ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets);
3408   assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
3409 
3410   for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3411     SDValue PromotedOutVal = OutVals[i];
3412     MVT PromotedVT;
3413     if (PromoteScalarIntegerPTX(VTs[i], &PromotedVT)) {
3414       VTs[i] = EVT(PromotedVT);
3415     }
3416     if (PromoteScalarIntegerPTX(PromotedOutVal.getValueType(), &PromotedVT)) {
3417       llvm::ISD::NodeType Ext =
3418           Outs[i].Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3419       PromotedOutVal = DAG.getNode(Ext, dl, PromotedVT, PromotedOutVal);
3420     }
3421     PromotedOutVals.push_back(PromotedOutVal);
3422   }
3423 
3424   auto VectorInfo = VectorizePTXValueVTs(
3425       VTs, Offsets,
3426       RetTy->isSized() ? getFunctionParamOptimizedAlign(&F, RetTy, DL)
3427                        : Align(1));
3428 
3429   // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
3430   // 32-bits are sign extended or zero extended, depending on whether
3431   // they are signed or unsigned types.
3432   bool ExtendIntegerRetVal =
3433       RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
3434 
3435   SmallVector<SDValue, 6> StoreOperands;
3436   for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
3437     SDValue OutVal = OutVals[i];
3438     SDValue RetVal = PromotedOutVals[i];
3439 
3440     if (ExtendIntegerRetVal) {
3441       RetVal = DAG.getNode(Outs[i].Flags.isSExt() ? ISD::SIGN_EXTEND
3442                                                   : ISD::ZERO_EXTEND,
3443                            dl, MVT::i32, RetVal);
3444     } else if (OutVal.getValueSizeInBits() < 16) {
3445       // Use 16-bit registers for small load-stores as it's the
3446       // smallest general purpose register size supported by NVPTX.
3447       RetVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, RetVal);
3448     }
3449 
3450     // If we have a PVF_SCALAR entry, it may not even be sufficiently aligned
3451     // for a scalar store. In such cases, fall back to byte stores.
3452     if (VectorInfo[i] == PVF_SCALAR && RetTy->isAggregateType()) {
3453       EVT ElementType = ExtendIntegerRetVal ? MVT::i32 : VTs[i];
3454       Align ElementTypeAlign =
3455           DL.getABITypeAlign(ElementType.getTypeForEVT(RetTy->getContext()));
3456       Align ElementAlign =
3457           commonAlignment(DL.getABITypeAlign(RetTy), Offsets[i]);
3458       if (ElementAlign < ElementTypeAlign) {
3459         assert(StoreOperands.empty() && "Orphaned operand list.");
3460         Chain = LowerUnalignedStoreRet(DAG, Chain, Offsets[i], ElementType,
3461                                        RetVal, dl);
3462 
3463         // The call to LowerUnalignedStoreRet inserted the necessary SDAG nodes
3464         // into the graph, so just move on to the next element.
3465         continue;
3466       }
3467     }
3468 
3469     // New load/store. Record chain and offset operands.
3470     if (VectorInfo[i] & PVF_FIRST) {
3471       assert(StoreOperands.empty() && "Orphaned operand list.");
3472       StoreOperands.push_back(Chain);
3473       StoreOperands.push_back(DAG.getConstant(Offsets[i], dl, MVT::i32));
3474     }
3475 
3476     // Record the value to return.
3477     StoreOperands.push_back(RetVal);
3478 
3479     // That's the last element of this store op.
3480     if (VectorInfo[i] & PVF_LAST) {
3481       NVPTXISD::NodeType Op;
3482       unsigned NumElts = StoreOperands.size() - 2;
3483       switch (NumElts) {
3484       case 1:
3485         Op = NVPTXISD::StoreRetval;
3486         break;
3487       case 2:
3488         Op = NVPTXISD::StoreRetvalV2;
3489         break;
3490       case 4:
3491         Op = NVPTXISD::StoreRetvalV4;
3492         break;
3493       default:
3494         llvm_unreachable("Invalid vector info.");
3495       }
3496 
3497       // Adjust type of load/store op if we've extended the scalar
3498       // return value.
3499       EVT TheStoreType = ExtendIntegerRetVal ? MVT::i32 : VTs[i];
3500       Chain = DAG.getMemIntrinsicNode(
3501           Op, dl, DAG.getVTList(MVT::Other), StoreOperands, TheStoreType,
3502           MachinePointerInfo(), Align(1), MachineMemOperand::MOStore);
3503       // Cleanup vector state.
3504       StoreOperands.clear();
3505     }
3506   }
3507 
3508   return DAG.getNode(NVPTXISD::RET_GLUE, dl, MVT::Other, Chain);
3509 }
3510 
3511 void NVPTXTargetLowering::LowerAsmOperandForConstraint(
3512     SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
3513     SelectionDAG &DAG) const {
3514   if (Constraint.size() > 1)
3515     return;
3516   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
3517 }
3518 
3519 static unsigned getOpcForTextureInstr(unsigned Intrinsic) {
3520   switch (Intrinsic) {
3521   default:
3522     return 0;
3523 
3524   case Intrinsic::nvvm_tex_1d_v4f32_s32:
3525     return NVPTXISD::Tex1DFloatS32;
3526   case Intrinsic::nvvm_tex_1d_v4f32_f32:
3527     return NVPTXISD::Tex1DFloatFloat;
3528   case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
3529     return NVPTXISD::Tex1DFloatFloatLevel;
3530   case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
3531     return NVPTXISD::Tex1DFloatFloatGrad;
3532   case Intrinsic::nvvm_tex_1d_v4s32_s32:
3533     return NVPTXISD::Tex1DS32S32;
3534   case Intrinsic::nvvm_tex_1d_v4s32_f32:
3535     return NVPTXISD::Tex1DS32Float;
3536   case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
3537     return NVPTXISD::Tex1DS32FloatLevel;
3538   case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
3539     return NVPTXISD::Tex1DS32FloatGrad;
3540   case Intrinsic::nvvm_tex_1d_v4u32_s32:
3541     return NVPTXISD::Tex1DU32S32;
3542   case Intrinsic::nvvm_tex_1d_v4u32_f32:
3543     return NVPTXISD::Tex1DU32Float;
3544   case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
3545     return NVPTXISD::Tex1DU32FloatLevel;
3546   case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
3547     return NVPTXISD::Tex1DU32FloatGrad;
3548 
3549   case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
3550     return NVPTXISD::Tex1DArrayFloatS32;
3551   case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
3552     return NVPTXISD::Tex1DArrayFloatFloat;
3553   case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
3554     return NVPTXISD::Tex1DArrayFloatFloatLevel;
3555   case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
3556     return NVPTXISD::Tex1DArrayFloatFloatGrad;
3557   case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
3558     return NVPTXISD::Tex1DArrayS32S32;
3559   case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
3560     return NVPTXISD::Tex1DArrayS32Float;
3561   case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
3562     return NVPTXISD::Tex1DArrayS32FloatLevel;
3563   case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
3564     return NVPTXISD::Tex1DArrayS32FloatGrad;
3565   case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
3566     return NVPTXISD::Tex1DArrayU32S32;
3567   case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
3568     return NVPTXISD::Tex1DArrayU32Float;
3569   case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
3570     return NVPTXISD::Tex1DArrayU32FloatLevel;
3571   case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
3572     return NVPTXISD::Tex1DArrayU32FloatGrad;
3573 
3574   case Intrinsic::nvvm_tex_2d_v4f32_s32:
3575     return NVPTXISD::Tex2DFloatS32;
3576   case Intrinsic::nvvm_tex_2d_v4f32_f32:
3577     return NVPTXISD::Tex2DFloatFloat;
3578   case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
3579     return NVPTXISD::Tex2DFloatFloatLevel;
3580   case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
3581     return NVPTXISD::Tex2DFloatFloatGrad;
3582   case Intrinsic::nvvm_tex_2d_v4s32_s32:
3583     return NVPTXISD::Tex2DS32S32;
3584   case Intrinsic::nvvm_tex_2d_v4s32_f32:
3585     return NVPTXISD::Tex2DS32Float;
3586   case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
3587     return NVPTXISD::Tex2DS32FloatLevel;
3588   case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
3589     return NVPTXISD::Tex2DS32FloatGrad;
3590   case Intrinsic::nvvm_tex_2d_v4u32_s32:
3591     return NVPTXISD::Tex2DU32S32;
3592   case Intrinsic::nvvm_tex_2d_v4u32_f32:
3593     return NVPTXISD::Tex2DU32Float;
3594   case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
3595     return NVPTXISD::Tex2DU32FloatLevel;
3596   case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
3597     return NVPTXISD::Tex2DU32FloatGrad;
3598 
3599   case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
3600     return NVPTXISD::Tex2DArrayFloatS32;
3601   case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
3602     return NVPTXISD::Tex2DArrayFloatFloat;
3603   case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
3604     return NVPTXISD::Tex2DArrayFloatFloatLevel;
3605   case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
3606     return NVPTXISD::Tex2DArrayFloatFloatGrad;
3607   case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
3608     return NVPTXISD::Tex2DArrayS32S32;
3609   case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
3610     return NVPTXISD::Tex2DArrayS32Float;
3611   case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
3612     return NVPTXISD::Tex2DArrayS32FloatLevel;
3613   case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
3614     return NVPTXISD::Tex2DArrayS32FloatGrad;
3615   case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
3616     return NVPTXISD::Tex2DArrayU32S32;
3617   case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
3618     return NVPTXISD::Tex2DArrayU32Float;
3619   case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
3620     return NVPTXISD::Tex2DArrayU32FloatLevel;
3621   case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
3622     return NVPTXISD::Tex2DArrayU32FloatGrad;
3623 
3624   case Intrinsic::nvvm_tex_3d_v4f32_s32:
3625     return NVPTXISD::Tex3DFloatS32;
3626   case Intrinsic::nvvm_tex_3d_v4f32_f32:
3627     return NVPTXISD::Tex3DFloatFloat;
3628   case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
3629     return NVPTXISD::Tex3DFloatFloatLevel;
3630   case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
3631     return NVPTXISD::Tex3DFloatFloatGrad;
3632   case Intrinsic::nvvm_tex_3d_v4s32_s32:
3633     return NVPTXISD::Tex3DS32S32;
3634   case Intrinsic::nvvm_tex_3d_v4s32_f32:
3635     return NVPTXISD::Tex3DS32Float;
3636   case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
3637     return NVPTXISD::Tex3DS32FloatLevel;
3638   case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
3639     return NVPTXISD::Tex3DS32FloatGrad;
3640   case Intrinsic::nvvm_tex_3d_v4u32_s32:
3641     return NVPTXISD::Tex3DU32S32;
3642   case Intrinsic::nvvm_tex_3d_v4u32_f32:
3643     return NVPTXISD::Tex3DU32Float;
3644   case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
3645     return NVPTXISD::Tex3DU32FloatLevel;
3646   case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
3647     return NVPTXISD::Tex3DU32FloatGrad;
3648 
3649   case Intrinsic::nvvm_tex_cube_v4f32_f32:
3650     return NVPTXISD::TexCubeFloatFloat;
3651   case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
3652     return NVPTXISD::TexCubeFloatFloatLevel;
3653   case Intrinsic::nvvm_tex_cube_v4s32_f32:
3654     return NVPTXISD::TexCubeS32Float;
3655   case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
3656     return NVPTXISD::TexCubeS32FloatLevel;
3657   case Intrinsic::nvvm_tex_cube_v4u32_f32:
3658     return NVPTXISD::TexCubeU32Float;
3659   case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
3660     return NVPTXISD::TexCubeU32FloatLevel;
3661 
3662   case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
3663     return NVPTXISD::TexCubeArrayFloatFloat;
3664   case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
3665     return NVPTXISD::TexCubeArrayFloatFloatLevel;
3666   case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
3667     return NVPTXISD::TexCubeArrayS32Float;
3668   case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
3669     return NVPTXISD::TexCubeArrayS32FloatLevel;
3670   case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
3671     return NVPTXISD::TexCubeArrayU32Float;
3672   case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
3673     return NVPTXISD::TexCubeArrayU32FloatLevel;
3674 
3675   case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
3676     return NVPTXISD::Tld4R2DFloatFloat;
3677   case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
3678     return NVPTXISD::Tld4G2DFloatFloat;
3679   case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
3680     return NVPTXISD::Tld4B2DFloatFloat;
3681   case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
3682     return NVPTXISD::Tld4A2DFloatFloat;
3683   case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
3684     return NVPTXISD::Tld4R2DS64Float;
3685   case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
3686     return NVPTXISD::Tld4G2DS64Float;
3687   case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
3688     return NVPTXISD::Tld4B2DS64Float;
3689   case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
3690     return NVPTXISD::Tld4A2DS64Float;
3691   case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
3692     return NVPTXISD::Tld4R2DU64Float;
3693   case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
3694     return NVPTXISD::Tld4G2DU64Float;
3695   case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
3696     return NVPTXISD::Tld4B2DU64Float;
3697   case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
3698     return NVPTXISD::Tld4A2DU64Float;
3699 
3700   case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
3701     return NVPTXISD::TexUnified1DFloatS32;
3702   case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
3703     return NVPTXISD::TexUnified1DFloatFloat;
3704   case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
3705     return NVPTXISD::TexUnified1DFloatFloatLevel;
3706   case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
3707     return NVPTXISD::TexUnified1DFloatFloatGrad;
3708   case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
3709     return NVPTXISD::TexUnified1DS32S32;
3710   case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
3711     return NVPTXISD::TexUnified1DS32Float;
3712   case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
3713     return NVPTXISD::TexUnified1DS32FloatLevel;
3714   case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
3715     return NVPTXISD::TexUnified1DS32FloatGrad;
3716   case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
3717     return NVPTXISD::TexUnified1DU32S32;
3718   case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
3719     return NVPTXISD::TexUnified1DU32Float;
3720   case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
3721     return NVPTXISD::TexUnified1DU32FloatLevel;
3722   case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
3723     return NVPTXISD::TexUnified1DU32FloatGrad;
3724 
3725   case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
3726     return NVPTXISD::TexUnified1DArrayFloatS32;
3727   case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
3728     return NVPTXISD::TexUnified1DArrayFloatFloat;
3729   case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
3730     return NVPTXISD::TexUnified1DArrayFloatFloatLevel;
3731   case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
3732     return NVPTXISD::TexUnified1DArrayFloatFloatGrad;
3733   case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
3734     return NVPTXISD::TexUnified1DArrayS32S32;
3735   case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
3736     return NVPTXISD::TexUnified1DArrayS32Float;
3737   case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
3738     return NVPTXISD::TexUnified1DArrayS32FloatLevel;
3739   case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
3740     return NVPTXISD::TexUnified1DArrayS32FloatGrad;
3741   case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
3742     return NVPTXISD::TexUnified1DArrayU32S32;
3743   case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
3744     return NVPTXISD::TexUnified1DArrayU32Float;
3745   case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
3746     return NVPTXISD::TexUnified1DArrayU32FloatLevel;
3747   case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
3748     return NVPTXISD::TexUnified1DArrayU32FloatGrad;
3749 
3750   case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
3751     return NVPTXISD::TexUnified2DFloatS32;
3752   case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
3753     return NVPTXISD::TexUnified2DFloatFloat;
3754   case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
3755     return NVPTXISD::TexUnified2DFloatFloatLevel;
3756   case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
3757     return NVPTXISD::TexUnified2DFloatFloatGrad;
3758   case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
3759     return NVPTXISD::TexUnified2DS32S32;
3760   case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
3761     return NVPTXISD::TexUnified2DS32Float;
3762   case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
3763     return NVPTXISD::TexUnified2DS32FloatLevel;
3764   case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
3765     return NVPTXISD::TexUnified2DS32FloatGrad;
3766   case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
3767     return NVPTXISD::TexUnified2DU32S32;
3768   case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
3769     return NVPTXISD::TexUnified2DU32Float;
3770   case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
3771     return NVPTXISD::TexUnified2DU32FloatLevel;
3772   case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
3773     return NVPTXISD::TexUnified2DU32FloatGrad;
3774 
3775   case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
3776     return NVPTXISD::TexUnified2DArrayFloatS32;
3777   case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
3778     return NVPTXISD::TexUnified2DArrayFloatFloat;
3779   case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
3780     return NVPTXISD::TexUnified2DArrayFloatFloatLevel;
3781   case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
3782     return NVPTXISD::TexUnified2DArrayFloatFloatGrad;
3783   case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
3784     return NVPTXISD::TexUnified2DArrayS32S32;
3785   case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
3786     return NVPTXISD::TexUnified2DArrayS32Float;
3787   case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
3788     return NVPTXISD::TexUnified2DArrayS32FloatLevel;
3789   case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
3790     return NVPTXISD::TexUnified2DArrayS32FloatGrad;
3791   case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
3792     return NVPTXISD::TexUnified2DArrayU32S32;
3793   case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
3794     return NVPTXISD::TexUnified2DArrayU32Float;
3795   case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
3796     return NVPTXISD::TexUnified2DArrayU32FloatLevel;
3797   case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
3798     return NVPTXISD::TexUnified2DArrayU32FloatGrad;
3799 
3800   case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
3801     return NVPTXISD::TexUnified3DFloatS32;
3802   case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
3803     return NVPTXISD::TexUnified3DFloatFloat;
3804   case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
3805     return NVPTXISD::TexUnified3DFloatFloatLevel;
3806   case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
3807     return NVPTXISD::TexUnified3DFloatFloatGrad;
3808   case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
3809     return NVPTXISD::TexUnified3DS32S32;
3810   case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
3811     return NVPTXISD::TexUnified3DS32Float;
3812   case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
3813     return NVPTXISD::TexUnified3DS32FloatLevel;
3814   case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
3815     return NVPTXISD::TexUnified3DS32FloatGrad;
3816   case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
3817     return NVPTXISD::TexUnified3DU32S32;
3818   case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
3819     return NVPTXISD::TexUnified3DU32Float;
3820   case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
3821     return NVPTXISD::TexUnified3DU32FloatLevel;
3822   case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
3823     return NVPTXISD::TexUnified3DU32FloatGrad;
3824 
3825   case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
3826     return NVPTXISD::TexUnifiedCubeFloatFloat;
3827   case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
3828     return NVPTXISD::TexUnifiedCubeFloatFloatLevel;
3829   case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
3830     return NVPTXISD::TexUnifiedCubeS32Float;
3831   case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
3832     return NVPTXISD::TexUnifiedCubeS32FloatLevel;
3833   case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
3834     return NVPTXISD::TexUnifiedCubeU32Float;
3835   case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
3836     return NVPTXISD::TexUnifiedCubeU32FloatLevel;
3837 
3838   case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
3839     return NVPTXISD::TexUnifiedCubeArrayFloatFloat;
3840   case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
3841     return NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel;
3842   case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
3843     return NVPTXISD::TexUnifiedCubeArrayS32Float;
3844   case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
3845     return NVPTXISD::TexUnifiedCubeArrayS32FloatLevel;
3846   case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
3847     return NVPTXISD::TexUnifiedCubeArrayU32Float;
3848   case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
3849     return NVPTXISD::TexUnifiedCubeArrayU32FloatLevel;
3850 
3851   case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
3852     return NVPTXISD::TexUnifiedCubeFloatFloatGrad;
3853   case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
3854     return NVPTXISD::TexUnifiedCubeS32FloatGrad;
3855   case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
3856     return NVPTXISD::TexUnifiedCubeU32FloatGrad;
3857   case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
3858     return NVPTXISD::TexUnifiedCubeArrayFloatFloatGrad;
3859   case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
3860     return NVPTXISD::TexUnifiedCubeArrayS32FloatGrad;
3861   case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
3862     return NVPTXISD::TexUnifiedCubeArrayU32FloatGrad;
3863 
3864   case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
3865     return NVPTXISD::Tld4UnifiedR2DFloatFloat;
3866   case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
3867     return NVPTXISD::Tld4UnifiedG2DFloatFloat;
3868   case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
3869     return NVPTXISD::Tld4UnifiedB2DFloatFloat;
3870   case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
3871     return NVPTXISD::Tld4UnifiedA2DFloatFloat;
3872   case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
3873     return NVPTXISD::Tld4UnifiedR2DS64Float;
3874   case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
3875     return NVPTXISD::Tld4UnifiedG2DS64Float;
3876   case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
3877     return NVPTXISD::Tld4UnifiedB2DS64Float;
3878   case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
3879     return NVPTXISD::Tld4UnifiedA2DS64Float;
3880   case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
3881     return NVPTXISD::Tld4UnifiedR2DU64Float;
3882   case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
3883     return NVPTXISD::Tld4UnifiedG2DU64Float;
3884   case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
3885     return NVPTXISD::Tld4UnifiedB2DU64Float;
3886   case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
3887     return NVPTXISD::Tld4UnifiedA2DU64Float;
3888   }
3889 }
3890 
3891 static unsigned getOpcForSurfaceInstr(unsigned Intrinsic) {
3892   switch (Intrinsic) {
3893   default:
3894     return 0;
3895   case Intrinsic::nvvm_suld_1d_i8_clamp:
3896     return NVPTXISD::Suld1DI8Clamp;
3897   case Intrinsic::nvvm_suld_1d_i16_clamp:
3898     return NVPTXISD::Suld1DI16Clamp;
3899   case Intrinsic::nvvm_suld_1d_i32_clamp:
3900     return NVPTXISD::Suld1DI32Clamp;
3901   case Intrinsic::nvvm_suld_1d_i64_clamp:
3902     return NVPTXISD::Suld1DI64Clamp;
3903   case Intrinsic::nvvm_suld_1d_v2i8_clamp:
3904     return NVPTXISD::Suld1DV2I8Clamp;
3905   case Intrinsic::nvvm_suld_1d_v2i16_clamp:
3906     return NVPTXISD::Suld1DV2I16Clamp;
3907   case Intrinsic::nvvm_suld_1d_v2i32_clamp:
3908     return NVPTXISD::Suld1DV2I32Clamp;
3909   case Intrinsic::nvvm_suld_1d_v2i64_clamp:
3910     return NVPTXISD::Suld1DV2I64Clamp;
3911   case Intrinsic::nvvm_suld_1d_v4i8_clamp:
3912     return NVPTXISD::Suld1DV4I8Clamp;
3913   case Intrinsic::nvvm_suld_1d_v4i16_clamp:
3914     return NVPTXISD::Suld1DV4I16Clamp;
3915   case Intrinsic::nvvm_suld_1d_v4i32_clamp:
3916     return NVPTXISD::Suld1DV4I32Clamp;
3917   case Intrinsic::nvvm_suld_1d_array_i8_clamp:
3918     return NVPTXISD::Suld1DArrayI8Clamp;
3919   case Intrinsic::nvvm_suld_1d_array_i16_clamp:
3920     return NVPTXISD::Suld1DArrayI16Clamp;
3921   case Intrinsic::nvvm_suld_1d_array_i32_clamp:
3922     return NVPTXISD::Suld1DArrayI32Clamp;
3923   case Intrinsic::nvvm_suld_1d_array_i64_clamp:
3924     return NVPTXISD::Suld1DArrayI64Clamp;
3925   case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
3926     return NVPTXISD::Suld1DArrayV2I8Clamp;
3927   case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
3928     return NVPTXISD::Suld1DArrayV2I16Clamp;
3929   case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
3930     return NVPTXISD::Suld1DArrayV2I32Clamp;
3931   case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
3932     return NVPTXISD::Suld1DArrayV2I64Clamp;
3933   case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
3934     return NVPTXISD::Suld1DArrayV4I8Clamp;
3935   case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
3936     return NVPTXISD::Suld1DArrayV4I16Clamp;
3937   case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
3938     return NVPTXISD::Suld1DArrayV4I32Clamp;
3939   case Intrinsic::nvvm_suld_2d_i8_clamp:
3940     return NVPTXISD::Suld2DI8Clamp;
3941   case Intrinsic::nvvm_suld_2d_i16_clamp:
3942     return NVPTXISD::Suld2DI16Clamp;
3943   case Intrinsic::nvvm_suld_2d_i32_clamp:
3944     return NVPTXISD::Suld2DI32Clamp;
3945   case Intrinsic::nvvm_suld_2d_i64_clamp:
3946     return NVPTXISD::Suld2DI64Clamp;
3947   case Intrinsic::nvvm_suld_2d_v2i8_clamp:
3948     return NVPTXISD::Suld2DV2I8Clamp;
3949   case Intrinsic::nvvm_suld_2d_v2i16_clamp:
3950     return NVPTXISD::Suld2DV2I16Clamp;
3951   case Intrinsic::nvvm_suld_2d_v2i32_clamp:
3952     return NVPTXISD::Suld2DV2I32Clamp;
3953   case Intrinsic::nvvm_suld_2d_v2i64_clamp:
3954     return NVPTXISD::Suld2DV2I64Clamp;
3955   case Intrinsic::nvvm_suld_2d_v4i8_clamp:
3956     return NVPTXISD::Suld2DV4I8Clamp;
3957   case Intrinsic::nvvm_suld_2d_v4i16_clamp:
3958     return NVPTXISD::Suld2DV4I16Clamp;
3959   case Intrinsic::nvvm_suld_2d_v4i32_clamp:
3960     return NVPTXISD::Suld2DV4I32Clamp;
3961   case Intrinsic::nvvm_suld_2d_array_i8_clamp:
3962     return NVPTXISD::Suld2DArrayI8Clamp;
3963   case Intrinsic::nvvm_suld_2d_array_i16_clamp:
3964     return NVPTXISD::Suld2DArrayI16Clamp;
3965   case Intrinsic::nvvm_suld_2d_array_i32_clamp:
3966     return NVPTXISD::Suld2DArrayI32Clamp;
3967   case Intrinsic::nvvm_suld_2d_array_i64_clamp:
3968     return NVPTXISD::Suld2DArrayI64Clamp;
3969   case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
3970     return NVPTXISD::Suld2DArrayV2I8Clamp;
3971   case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
3972     return NVPTXISD::Suld2DArrayV2I16Clamp;
3973   case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
3974     return NVPTXISD::Suld2DArrayV2I32Clamp;
3975   case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
3976     return NVPTXISD::Suld2DArrayV2I64Clamp;
3977   case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
3978     return NVPTXISD::Suld2DArrayV4I8Clamp;
3979   case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
3980     return NVPTXISD::Suld2DArrayV4I16Clamp;
3981   case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
3982     return NVPTXISD::Suld2DArrayV4I32Clamp;
3983   case Intrinsic::nvvm_suld_3d_i8_clamp:
3984     return NVPTXISD::Suld3DI8Clamp;
3985   case Intrinsic::nvvm_suld_3d_i16_clamp:
3986     return NVPTXISD::Suld3DI16Clamp;
3987   case Intrinsic::nvvm_suld_3d_i32_clamp:
3988     return NVPTXISD::Suld3DI32Clamp;
3989   case Intrinsic::nvvm_suld_3d_i64_clamp:
3990     return NVPTXISD::Suld3DI64Clamp;
3991   case Intrinsic::nvvm_suld_3d_v2i8_clamp:
3992     return NVPTXISD::Suld3DV2I8Clamp;
3993   case Intrinsic::nvvm_suld_3d_v2i16_clamp:
3994     return NVPTXISD::Suld3DV2I16Clamp;
3995   case Intrinsic::nvvm_suld_3d_v2i32_clamp:
3996     return NVPTXISD::Suld3DV2I32Clamp;
3997   case Intrinsic::nvvm_suld_3d_v2i64_clamp:
3998     return NVPTXISD::Suld3DV2I64Clamp;
3999   case Intrinsic::nvvm_suld_3d_v4i8_clamp:
4000     return NVPTXISD::Suld3DV4I8Clamp;
4001   case Intrinsic::nvvm_suld_3d_v4i16_clamp:
4002     return NVPTXISD::Suld3DV4I16Clamp;
4003   case Intrinsic::nvvm_suld_3d_v4i32_clamp:
4004     return NVPTXISD::Suld3DV4I32Clamp;
4005   case Intrinsic::nvvm_suld_1d_i8_trap:
4006     return NVPTXISD::Suld1DI8Trap;
4007   case Intrinsic::nvvm_suld_1d_i16_trap:
4008     return NVPTXISD::Suld1DI16Trap;
4009   case Intrinsic::nvvm_suld_1d_i32_trap:
4010     return NVPTXISD::Suld1DI32Trap;
4011   case Intrinsic::nvvm_suld_1d_i64_trap:
4012     return NVPTXISD::Suld1DI64Trap;
4013   case Intrinsic::nvvm_suld_1d_v2i8_trap:
4014     return NVPTXISD::Suld1DV2I8Trap;
4015   case Intrinsic::nvvm_suld_1d_v2i16_trap:
4016     return NVPTXISD::Suld1DV2I16Trap;
4017   case Intrinsic::nvvm_suld_1d_v2i32_trap:
4018     return NVPTXISD::Suld1DV2I32Trap;
4019   case Intrinsic::nvvm_suld_1d_v2i64_trap:
4020     return NVPTXISD::Suld1DV2I64Trap;
4021   case Intrinsic::nvvm_suld_1d_v4i8_trap:
4022     return NVPTXISD::Suld1DV4I8Trap;
4023   case Intrinsic::nvvm_suld_1d_v4i16_trap:
4024     return NVPTXISD::Suld1DV4I16Trap;
4025   case Intrinsic::nvvm_suld_1d_v4i32_trap:
4026     return NVPTXISD::Suld1DV4I32Trap;
4027   case Intrinsic::nvvm_suld_1d_array_i8_trap:
4028     return NVPTXISD::Suld1DArrayI8Trap;
4029   case Intrinsic::nvvm_suld_1d_array_i16_trap:
4030     return NVPTXISD::Suld1DArrayI16Trap;
4031   case Intrinsic::nvvm_suld_1d_array_i32_trap:
4032     return NVPTXISD::Suld1DArrayI32Trap;
4033   case Intrinsic::nvvm_suld_1d_array_i64_trap:
4034     return NVPTXISD::Suld1DArrayI64Trap;
4035   case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
4036     return NVPTXISD::Suld1DArrayV2I8Trap;
4037   case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
4038     return NVPTXISD::Suld1DArrayV2I16Trap;
4039   case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
4040     return NVPTXISD::Suld1DArrayV2I32Trap;
4041   case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
4042     return NVPTXISD::Suld1DArrayV2I64Trap;
4043   case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
4044     return NVPTXISD::Suld1DArrayV4I8Trap;
4045   case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
4046     return NVPTXISD::Suld1DArrayV4I16Trap;
4047   case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
4048     return NVPTXISD::Suld1DArrayV4I32Trap;
4049   case Intrinsic::nvvm_suld_2d_i8_trap:
4050     return NVPTXISD::Suld2DI8Trap;
4051   case Intrinsic::nvvm_suld_2d_i16_trap:
4052     return NVPTXISD::Suld2DI16Trap;
4053   case Intrinsic::nvvm_suld_2d_i32_trap:
4054     return NVPTXISD::Suld2DI32Trap;
4055   case Intrinsic::nvvm_suld_2d_i64_trap:
4056     return NVPTXISD::Suld2DI64Trap;
4057   case Intrinsic::nvvm_suld_2d_v2i8_trap:
4058     return NVPTXISD::Suld2DV2I8Trap;
4059   case Intrinsic::nvvm_suld_2d_v2i16_trap:
4060     return NVPTXISD::Suld2DV2I16Trap;
4061   case Intrinsic::nvvm_suld_2d_v2i32_trap:
4062     return NVPTXISD::Suld2DV2I32Trap;
4063   case Intrinsic::nvvm_suld_2d_v2i64_trap:
4064     return NVPTXISD::Suld2DV2I64Trap;
4065   case Intrinsic::nvvm_suld_2d_v4i8_trap:
4066     return NVPTXISD::Suld2DV4I8Trap;
4067   case Intrinsic::nvvm_suld_2d_v4i16_trap:
4068     return NVPTXISD::Suld2DV4I16Trap;
4069   case Intrinsic::nvvm_suld_2d_v4i32_trap:
4070     return NVPTXISD::Suld2DV4I32Trap;
4071   case Intrinsic::nvvm_suld_2d_array_i8_trap:
4072     return NVPTXISD::Suld2DArrayI8Trap;
4073   case Intrinsic::nvvm_suld_2d_array_i16_trap:
4074     return NVPTXISD::Suld2DArrayI16Trap;
4075   case Intrinsic::nvvm_suld_2d_array_i32_trap:
4076     return NVPTXISD::Suld2DArrayI32Trap;
4077   case Intrinsic::nvvm_suld_2d_array_i64_trap:
4078     return NVPTXISD::Suld2DArrayI64Trap;
4079   case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
4080     return NVPTXISD::Suld2DArrayV2I8Trap;
4081   case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
4082     return NVPTXISD::Suld2DArrayV2I16Trap;
4083   case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
4084     return NVPTXISD::Suld2DArrayV2I32Trap;
4085   case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
4086     return NVPTXISD::Suld2DArrayV2I64Trap;
4087   case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
4088     return NVPTXISD::Suld2DArrayV4I8Trap;
4089   case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
4090     return NVPTXISD::Suld2DArrayV4I16Trap;
4091   case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
4092     return NVPTXISD::Suld2DArrayV4I32Trap;
4093   case Intrinsic::nvvm_suld_3d_i8_trap:
4094     return NVPTXISD::Suld3DI8Trap;
4095   case Intrinsic::nvvm_suld_3d_i16_trap:
4096     return NVPTXISD::Suld3DI16Trap;
4097   case Intrinsic::nvvm_suld_3d_i32_trap:
4098     return NVPTXISD::Suld3DI32Trap;
4099   case Intrinsic::nvvm_suld_3d_i64_trap:
4100     return NVPTXISD::Suld3DI64Trap;
4101   case Intrinsic::nvvm_suld_3d_v2i8_trap:
4102     return NVPTXISD::Suld3DV2I8Trap;
4103   case Intrinsic::nvvm_suld_3d_v2i16_trap:
4104     return NVPTXISD::Suld3DV2I16Trap;
4105   case Intrinsic::nvvm_suld_3d_v2i32_trap:
4106     return NVPTXISD::Suld3DV2I32Trap;
4107   case Intrinsic::nvvm_suld_3d_v2i64_trap:
4108     return NVPTXISD::Suld3DV2I64Trap;
4109   case Intrinsic::nvvm_suld_3d_v4i8_trap:
4110     return NVPTXISD::Suld3DV4I8Trap;
4111   case Intrinsic::nvvm_suld_3d_v4i16_trap:
4112     return NVPTXISD::Suld3DV4I16Trap;
4113   case Intrinsic::nvvm_suld_3d_v4i32_trap:
4114     return NVPTXISD::Suld3DV4I32Trap;
4115   case Intrinsic::nvvm_suld_1d_i8_zero:
4116     return NVPTXISD::Suld1DI8Zero;
4117   case Intrinsic::nvvm_suld_1d_i16_zero:
4118     return NVPTXISD::Suld1DI16Zero;
4119   case Intrinsic::nvvm_suld_1d_i32_zero:
4120     return NVPTXISD::Suld1DI32Zero;
4121   case Intrinsic::nvvm_suld_1d_i64_zero:
4122     return NVPTXISD::Suld1DI64Zero;
4123   case Intrinsic::nvvm_suld_1d_v2i8_zero:
4124     return NVPTXISD::Suld1DV2I8Zero;
4125   case Intrinsic::nvvm_suld_1d_v2i16_zero:
4126     return NVPTXISD::Suld1DV2I16Zero;
4127   case Intrinsic::nvvm_suld_1d_v2i32_zero:
4128     return NVPTXISD::Suld1DV2I32Zero;
4129   case Intrinsic::nvvm_suld_1d_v2i64_zero:
4130     return NVPTXISD::Suld1DV2I64Zero;
4131   case Intrinsic::nvvm_suld_1d_v4i8_zero:
4132     return NVPTXISD::Suld1DV4I8Zero;
4133   case Intrinsic::nvvm_suld_1d_v4i16_zero:
4134     return NVPTXISD::Suld1DV4I16Zero;
4135   case Intrinsic::nvvm_suld_1d_v4i32_zero:
4136     return NVPTXISD::Suld1DV4I32Zero;
4137   case Intrinsic::nvvm_suld_1d_array_i8_zero:
4138     return NVPTXISD::Suld1DArrayI8Zero;
4139   case Intrinsic::nvvm_suld_1d_array_i16_zero:
4140     return NVPTXISD::Suld1DArrayI16Zero;
4141   case Intrinsic::nvvm_suld_1d_array_i32_zero:
4142     return NVPTXISD::Suld1DArrayI32Zero;
4143   case Intrinsic::nvvm_suld_1d_array_i64_zero:
4144     return NVPTXISD::Suld1DArrayI64Zero;
4145   case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
4146     return NVPTXISD::Suld1DArrayV2I8Zero;
4147   case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
4148     return NVPTXISD::Suld1DArrayV2I16Zero;
4149   case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
4150     return NVPTXISD::Suld1DArrayV2I32Zero;
4151   case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
4152     return NVPTXISD::Suld1DArrayV2I64Zero;
4153   case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
4154     return NVPTXISD::Suld1DArrayV4I8Zero;
4155   case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
4156     return NVPTXISD::Suld1DArrayV4I16Zero;
4157   case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
4158     return NVPTXISD::Suld1DArrayV4I32Zero;
4159   case Intrinsic::nvvm_suld_2d_i8_zero:
4160     return NVPTXISD::Suld2DI8Zero;
4161   case Intrinsic::nvvm_suld_2d_i16_zero:
4162     return NVPTXISD::Suld2DI16Zero;
4163   case Intrinsic::nvvm_suld_2d_i32_zero:
4164     return NVPTXISD::Suld2DI32Zero;
4165   case Intrinsic::nvvm_suld_2d_i64_zero:
4166     return NVPTXISD::Suld2DI64Zero;
4167   case Intrinsic::nvvm_suld_2d_v2i8_zero:
4168     return NVPTXISD::Suld2DV2I8Zero;
4169   case Intrinsic::nvvm_suld_2d_v2i16_zero:
4170     return NVPTXISD::Suld2DV2I16Zero;
4171   case Intrinsic::nvvm_suld_2d_v2i32_zero:
4172     return NVPTXISD::Suld2DV2I32Zero;
4173   case Intrinsic::nvvm_suld_2d_v2i64_zero:
4174     return NVPTXISD::Suld2DV2I64Zero;
4175   case Intrinsic::nvvm_suld_2d_v4i8_zero:
4176     return NVPTXISD::Suld2DV4I8Zero;
4177   case Intrinsic::nvvm_suld_2d_v4i16_zero:
4178     return NVPTXISD::Suld2DV4I16Zero;
4179   case Intrinsic::nvvm_suld_2d_v4i32_zero:
4180     return NVPTXISD::Suld2DV4I32Zero;
4181   case Intrinsic::nvvm_suld_2d_array_i8_zero:
4182     return NVPTXISD::Suld2DArrayI8Zero;
4183   case Intrinsic::nvvm_suld_2d_array_i16_zero:
4184     return NVPTXISD::Suld2DArrayI16Zero;
4185   case Intrinsic::nvvm_suld_2d_array_i32_zero:
4186     return NVPTXISD::Suld2DArrayI32Zero;
4187   case Intrinsic::nvvm_suld_2d_array_i64_zero:
4188     return NVPTXISD::Suld2DArrayI64Zero;
4189   case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
4190     return NVPTXISD::Suld2DArrayV2I8Zero;
4191   case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
4192     return NVPTXISD::Suld2DArrayV2I16Zero;
4193   case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
4194     return NVPTXISD::Suld2DArrayV2I32Zero;
4195   case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
4196     return NVPTXISD::Suld2DArrayV2I64Zero;
4197   case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
4198     return NVPTXISD::Suld2DArrayV4I8Zero;
4199   case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
4200     return NVPTXISD::Suld2DArrayV4I16Zero;
4201   case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
4202     return NVPTXISD::Suld2DArrayV4I32Zero;
4203   case Intrinsic::nvvm_suld_3d_i8_zero:
4204     return NVPTXISD::Suld3DI8Zero;
4205   case Intrinsic::nvvm_suld_3d_i16_zero:
4206     return NVPTXISD::Suld3DI16Zero;
4207   case Intrinsic::nvvm_suld_3d_i32_zero:
4208     return NVPTXISD::Suld3DI32Zero;
4209   case Intrinsic::nvvm_suld_3d_i64_zero:
4210     return NVPTXISD::Suld3DI64Zero;
4211   case Intrinsic::nvvm_suld_3d_v2i8_zero:
4212     return NVPTXISD::Suld3DV2I8Zero;
4213   case Intrinsic::nvvm_suld_3d_v2i16_zero:
4214     return NVPTXISD::Suld3DV2I16Zero;
4215   case Intrinsic::nvvm_suld_3d_v2i32_zero:
4216     return NVPTXISD::Suld3DV2I32Zero;
4217   case Intrinsic::nvvm_suld_3d_v2i64_zero:
4218     return NVPTXISD::Suld3DV2I64Zero;
4219   case Intrinsic::nvvm_suld_3d_v4i8_zero:
4220     return NVPTXISD::Suld3DV4I8Zero;
4221   case Intrinsic::nvvm_suld_3d_v4i16_zero:
4222     return NVPTXISD::Suld3DV4I16Zero;
4223   case Intrinsic::nvvm_suld_3d_v4i32_zero:
4224     return NVPTXISD::Suld3DV4I32Zero;
4225   }
4226 }
4227 
4228 // llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
4229 // TgtMemIntrinsic
4230 // because we need the information that is only available in the "Value" type
4231 // of destination
4232 // pointer. In particular, the address space information.
4233 bool NVPTXTargetLowering::getTgtMemIntrinsic(
4234     IntrinsicInfo &Info, const CallInst &I,
4235     MachineFunction &MF, unsigned Intrinsic) const {
4236   switch (Intrinsic) {
4237   default:
4238     return false;
4239   case Intrinsic::nvvm_match_all_sync_i32p:
4240   case Intrinsic::nvvm_match_all_sync_i64p:
4241     Info.opc = ISD::INTRINSIC_W_CHAIN;
4242     // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute
4243     // in order to model data exchange with other threads, but perform no real
4244     // memory accesses.
4245     Info.memVT = MVT::i1;
4246 
4247     // Our result depends on both our and other thread's arguments.
4248     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4249     return true;
4250   case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
4251   case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
4252   case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
4253   case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
4254   case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
4255   case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
4256   case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
4257   case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
4258   case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
4259   case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
4260   case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
4261   case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
4262   case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
4263   case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
4264   case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
4265   case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
4266   case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
4267   case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
4268   case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
4269   case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
4270   case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
4271   case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
4272   case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
4273   case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
4274     Info.opc = ISD::INTRINSIC_W_CHAIN;
4275     Info.memVT = MVT::v8f16;
4276     Info.ptrVal = I.getArgOperand(0);
4277     Info.offset = 0;
4278     Info.flags = MachineMemOperand::MOLoad;
4279     Info.align = Align(16);
4280     return true;
4281   }
4282   case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
4283   case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
4284   case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
4285   case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
4286   case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
4287   case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
4288   case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
4289   case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
4290   case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
4291   case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
4292   case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
4293   case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
4294   case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
4295   case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
4296   case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
4297   case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
4298   case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
4299   case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
4300   case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
4301   case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
4302   case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
4303   case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
4304   case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
4305   case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
4306     Info.opc = ISD::INTRINSIC_W_CHAIN;
4307     Info.memVT = MVT::v2i32;
4308     Info.ptrVal = I.getArgOperand(0);
4309     Info.offset = 0;
4310     Info.flags = MachineMemOperand::MOLoad;
4311     Info.align = Align(8);
4312     return true;
4313   }
4314 
4315   case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
4316   case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
4317   case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
4318   case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
4319   case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
4320   case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
4321   case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
4322   case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
4323   case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
4324   case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
4325   case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
4326   case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
4327   case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
4328   case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
4329   case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
4330   case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
4331 
4332   case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
4333   case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
4334   case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
4335   case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
4336   case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
4337   case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
4338   case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
4339   case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
4340   case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
4341   case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
4342   case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
4343   case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
4344   case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
4345   case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
4346   case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
4347   case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
4348   case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
4349   case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16: {
4350     Info.opc = ISD::INTRINSIC_W_CHAIN;
4351     Info.memVT = MVT::v4i32;
4352     Info.ptrVal = I.getArgOperand(0);
4353     Info.offset = 0;
4354     Info.flags = MachineMemOperand::MOLoad;
4355     Info.align = Align(16);
4356     return true;
4357   }
4358 
4359   case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
4360   case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
4361   case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
4362   case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
4363   case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
4364   case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
4365   case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
4366   case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
4367 
4368   case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
4369   case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
4370   case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
4371   case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
4372   case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
4373   case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
4374   case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
4375   case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
4376   case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
4377   case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
4378   case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
4379   case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
4380   case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
4381   case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
4382   case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
4383   case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
4384   case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
4385   case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
4386   case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
4387   case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
4388   case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
4389   case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16: {
4390     Info.opc = ISD::INTRINSIC_W_CHAIN;
4391     Info.memVT = MVT::i32;
4392     Info.ptrVal = I.getArgOperand(0);
4393     Info.offset = 0;
4394     Info.flags = MachineMemOperand::MOLoad;
4395     Info.align = Align(4);
4396     return true;
4397   }
4398 
4399   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
4400   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
4401   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
4402   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
4403   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
4404   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
4405   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
4406   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
4407   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
4408   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
4409   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
4410   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
4411     Info.opc = ISD::INTRINSIC_W_CHAIN;
4412     Info.memVT = MVT::v4f16;
4413     Info.ptrVal = I.getArgOperand(0);
4414     Info.offset = 0;
4415     Info.flags = MachineMemOperand::MOLoad;
4416     Info.align = Align(16);
4417     return true;
4418   }
4419 
4420   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
4421   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
4422   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
4423   case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
4424   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
4425   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
4426   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
4427   case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
4428   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
4429   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
4430   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
4431   case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
4432   case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
4433   case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
4434   case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
4435   case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
4436     Info.opc = ISD::INTRINSIC_W_CHAIN;
4437     Info.memVT = MVT::v8f32;
4438     Info.ptrVal = I.getArgOperand(0);
4439     Info.offset = 0;
4440     Info.flags = MachineMemOperand::MOLoad;
4441     Info.align = Align(16);
4442     return true;
4443   }
4444 
4445   case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
4446   case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
4447   case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
4448   case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
4449 
4450   case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
4451   case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
4452   case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
4453   case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
4454 
4455   case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
4456   case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
4457   case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
4458   case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
4459   case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
4460   case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
4461   case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
4462   case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
4463   case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
4464   case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
4465   case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
4466   case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
4467     Info.opc = ISD::INTRINSIC_W_CHAIN;
4468     Info.memVT = MVT::v8i32;
4469     Info.ptrVal = I.getArgOperand(0);
4470     Info.offset = 0;
4471     Info.flags = MachineMemOperand::MOLoad;
4472     Info.align = Align(16);
4473     return true;
4474   }
4475 
4476   case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
4477   case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
4478   case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
4479   case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
4480   case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
4481   case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
4482   case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
4483   case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
4484   case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
4485   case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16: {
4486     Info.opc = ISD::INTRINSIC_W_CHAIN;
4487     Info.memVT = MVT::v2i32;
4488     Info.ptrVal = I.getArgOperand(0);
4489     Info.offset = 0;
4490     Info.flags = MachineMemOperand::MOLoad;
4491     Info.align = Align(8);
4492     return true;
4493   }
4494 
4495   case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
4496   case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
4497   case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
4498   case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
4499 
4500   case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
4501   case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
4502   case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
4503   case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
4504     Info.opc = ISD::INTRINSIC_W_CHAIN;
4505     Info.memVT = MVT::f64;
4506     Info.ptrVal = I.getArgOperand(0);
4507     Info.offset = 0;
4508     Info.flags = MachineMemOperand::MOLoad;
4509     Info.align = Align(8);
4510     return true;
4511   }
4512 
4513   case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
4514   case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
4515   case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
4516   case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
4517     Info.opc = ISD::INTRINSIC_W_CHAIN;
4518     Info.memVT = MVT::v2f64;
4519     Info.ptrVal = I.getArgOperand(0);
4520     Info.offset = 0;
4521     Info.flags = MachineMemOperand::MOLoad;
4522     Info.align = Align(16);
4523     return true;
4524   }
4525 
4526   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
4527   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
4528   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
4529   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
4530   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
4531   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
4532   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
4533   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
4534   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
4535   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
4536   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
4537   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
4538     Info.opc = ISD::INTRINSIC_VOID;
4539     Info.memVT = MVT::v4f16;
4540     Info.ptrVal = I.getArgOperand(0);
4541     Info.offset = 0;
4542     Info.flags = MachineMemOperand::MOStore;
4543     Info.align = Align(16);
4544     return true;
4545   }
4546 
4547   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
4548   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
4549   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
4550   case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
4551   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
4552   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
4553   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
4554   case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
4555   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
4556   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
4557   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
4558   case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
4559   case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
4560   case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
4561   case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
4562   case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
4563     Info.opc = ISD::INTRINSIC_VOID;
4564     Info.memVT = MVT::v8f32;
4565     Info.ptrVal = I.getArgOperand(0);
4566     Info.offset = 0;
4567     Info.flags = MachineMemOperand::MOStore;
4568     Info.align = Align(16);
4569     return true;
4570   }
4571 
4572   case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
4573   case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
4574   case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
4575   case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
4576   case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
4577   case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
4578   case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
4579   case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
4580   case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
4581   case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
4582   case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
4583   case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
4584     Info.opc = ISD::INTRINSIC_VOID;
4585     Info.memVT = MVT::v8i32;
4586     Info.ptrVal = I.getArgOperand(0);
4587     Info.offset = 0;
4588     Info.flags = MachineMemOperand::MOStore;
4589     Info.align = Align(16);
4590     return true;
4591   }
4592 
4593   case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
4594   case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
4595   case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
4596   case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
4597   case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
4598   case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
4599   case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
4600   case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride: {
4601     Info.opc = ISD::INTRINSIC_VOID;
4602     Info.memVT = MVT::v2i32;
4603     Info.ptrVal = I.getArgOperand(0);
4604     Info.offset = 0;
4605     Info.flags = MachineMemOperand::MOStore;
4606     Info.align = Align(8);
4607     return true;
4608   }
4609 
4610   case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
4611   case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
4612   case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
4613   case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
4614     Info.opc = ISD::INTRINSIC_VOID;
4615     Info.memVT = MVT::v2f64;
4616     Info.ptrVal = I.getArgOperand(0);
4617     Info.offset = 0;
4618     Info.flags = MachineMemOperand::MOStore;
4619     Info.align = Align(16);
4620     return true;
4621   }
4622 
4623   case Intrinsic::nvvm_atomic_load_inc_32:
4624   case Intrinsic::nvvm_atomic_load_dec_32:
4625 
4626   case Intrinsic::nvvm_atomic_add_gen_f_cta:
4627   case Intrinsic::nvvm_atomic_add_gen_f_sys:
4628   case Intrinsic::nvvm_atomic_add_gen_i_cta:
4629   case Intrinsic::nvvm_atomic_add_gen_i_sys:
4630   case Intrinsic::nvvm_atomic_and_gen_i_cta:
4631   case Intrinsic::nvvm_atomic_and_gen_i_sys:
4632   case Intrinsic::nvvm_atomic_cas_gen_i_cta:
4633   case Intrinsic::nvvm_atomic_cas_gen_i_sys:
4634   case Intrinsic::nvvm_atomic_dec_gen_i_cta:
4635   case Intrinsic::nvvm_atomic_dec_gen_i_sys:
4636   case Intrinsic::nvvm_atomic_inc_gen_i_cta:
4637   case Intrinsic::nvvm_atomic_inc_gen_i_sys:
4638   case Intrinsic::nvvm_atomic_max_gen_i_cta:
4639   case Intrinsic::nvvm_atomic_max_gen_i_sys:
4640   case Intrinsic::nvvm_atomic_min_gen_i_cta:
4641   case Intrinsic::nvvm_atomic_min_gen_i_sys:
4642   case Intrinsic::nvvm_atomic_or_gen_i_cta:
4643   case Intrinsic::nvvm_atomic_or_gen_i_sys:
4644   case Intrinsic::nvvm_atomic_exch_gen_i_cta:
4645   case Intrinsic::nvvm_atomic_exch_gen_i_sys:
4646   case Intrinsic::nvvm_atomic_xor_gen_i_cta:
4647   case Intrinsic::nvvm_atomic_xor_gen_i_sys: {
4648     auto &DL = I.getDataLayout();
4649     Info.opc = ISD::INTRINSIC_W_CHAIN;
4650     Info.memVT = getValueType(DL, I.getType());
4651     Info.ptrVal = I.getArgOperand(0);
4652     Info.offset = 0;
4653     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4654     Info.align.reset();
4655     return true;
4656   }
4657 
4658   case Intrinsic::nvvm_ldu_global_i:
4659   case Intrinsic::nvvm_ldu_global_f:
4660   case Intrinsic::nvvm_ldu_global_p: {
4661     auto &DL = I.getDataLayout();
4662     Info.opc = ISD::INTRINSIC_W_CHAIN;
4663     if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
4664       Info.memVT = getValueType(DL, I.getType());
4665     else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)
4666       Info.memVT = getPointerTy(DL);
4667     else
4668       Info.memVT = getValueType(DL, I.getType());
4669     Info.ptrVal = I.getArgOperand(0);
4670     Info.offset = 0;
4671     Info.flags = MachineMemOperand::MOLoad;
4672     Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4673 
4674     return true;
4675   }
4676   case Intrinsic::nvvm_ldg_global_i:
4677   case Intrinsic::nvvm_ldg_global_f:
4678   case Intrinsic::nvvm_ldg_global_p: {
4679     auto &DL = I.getDataLayout();
4680 
4681     Info.opc = ISD::INTRINSIC_W_CHAIN;
4682     if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
4683       Info.memVT = getValueType(DL, I.getType());
4684     else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
4685       Info.memVT = getPointerTy(DL);
4686     else
4687       Info.memVT = getValueType(DL, I.getType());
4688     Info.ptrVal = I.getArgOperand(0);
4689     Info.offset = 0;
4690     Info.flags = MachineMemOperand::MOLoad;
4691     Info.align = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4692 
4693     return true;
4694   }
4695 
4696   case Intrinsic::nvvm_tex_1d_v4f32_s32:
4697   case Intrinsic::nvvm_tex_1d_v4f32_f32:
4698   case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
4699   case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
4700   case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
4701   case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
4702   case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
4703   case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
4704   case Intrinsic::nvvm_tex_2d_v4f32_s32:
4705   case Intrinsic::nvvm_tex_2d_v4f32_f32:
4706   case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
4707   case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
4708   case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
4709   case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
4710   case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
4711   case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
4712   case Intrinsic::nvvm_tex_3d_v4f32_s32:
4713   case Intrinsic::nvvm_tex_3d_v4f32_f32:
4714   case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
4715   case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
4716   case Intrinsic::nvvm_tex_cube_v4f32_f32:
4717   case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
4718   case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
4719   case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
4720   case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
4721   case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
4722   case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
4723   case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
4724   case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
4725   case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
4726   case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
4727   case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
4728   case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
4729   case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
4730   case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
4731   case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
4732   case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
4733   case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
4734   case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
4735   case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
4736   case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
4737   case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
4738   case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
4739   case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
4740   case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
4741   case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
4742   case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
4743   case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
4744   case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
4745   case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
4746   case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
4747   case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
4748   case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
4749   case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
4750   case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
4751   case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
4752   case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
4753   case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
4754     Info.opc = getOpcForTextureInstr(Intrinsic);
4755     Info.memVT = MVT::v4f32;
4756     Info.ptrVal = nullptr;
4757     Info.offset = 0;
4758     Info.flags = MachineMemOperand::MOLoad;
4759     Info.align = Align(16);
4760     return true;
4761 
4762   case Intrinsic::nvvm_tex_1d_v4s32_s32:
4763   case Intrinsic::nvvm_tex_1d_v4s32_f32:
4764   case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
4765   case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
4766   case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
4767   case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
4768   case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
4769   case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
4770   case Intrinsic::nvvm_tex_2d_v4s32_s32:
4771   case Intrinsic::nvvm_tex_2d_v4s32_f32:
4772   case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
4773   case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
4774   case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
4775   case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
4776   case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
4777   case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
4778   case Intrinsic::nvvm_tex_3d_v4s32_s32:
4779   case Intrinsic::nvvm_tex_3d_v4s32_f32:
4780   case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
4781   case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
4782   case Intrinsic::nvvm_tex_cube_v4s32_f32:
4783   case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
4784   case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
4785   case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
4786   case Intrinsic::nvvm_tex_cube_v4u32_f32:
4787   case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
4788   case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
4789   case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
4790   case Intrinsic::nvvm_tex_1d_v4u32_s32:
4791   case Intrinsic::nvvm_tex_1d_v4u32_f32:
4792   case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
4793   case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
4794   case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
4795   case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
4796   case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
4797   case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
4798   case Intrinsic::nvvm_tex_2d_v4u32_s32:
4799   case Intrinsic::nvvm_tex_2d_v4u32_f32:
4800   case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
4801   case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
4802   case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
4803   case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
4804   case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
4805   case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
4806   case Intrinsic::nvvm_tex_3d_v4u32_s32:
4807   case Intrinsic::nvvm_tex_3d_v4u32_f32:
4808   case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
4809   case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
4810   case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
4811   case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
4812   case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
4813   case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
4814   case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
4815   case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
4816   case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
4817   case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
4818   case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
4819   case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
4820   case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
4821   case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
4822   case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
4823   case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
4824   case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
4825   case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
4826   case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
4827   case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
4828   case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
4829   case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
4830   case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
4831   case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
4832   case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
4833   case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
4834   case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
4835   case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
4836   case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
4837   case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
4838   case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
4839   case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
4840   case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
4841   case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
4842   case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
4843   case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
4844   case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
4845   case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
4846   case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
4847   case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
4848   case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
4849   case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
4850   case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
4851   case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
4852   case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
4853   case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
4854   case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
4855   case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
4856   case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
4857   case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
4858   case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
4859   case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
4860   case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
4861   case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
4862   case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
4863   case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
4864   case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
4865   case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
4866   case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
4867   case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
4868   case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
4869   case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
4870   case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
4871   case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
4872   case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
4873   case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
4874   case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
4875   case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
4876   case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
4877   case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
4878     Info.opc = getOpcForTextureInstr(Intrinsic);
4879     Info.memVT = MVT::v4i32;
4880     Info.ptrVal = nullptr;
4881     Info.offset = 0;
4882     Info.flags = MachineMemOperand::MOLoad;
4883     Info.align = Align(16);
4884     return true;
4885 
4886   case Intrinsic::nvvm_suld_1d_i8_clamp:
4887   case Intrinsic::nvvm_suld_1d_v2i8_clamp:
4888   case Intrinsic::nvvm_suld_1d_v4i8_clamp:
4889   case Intrinsic::nvvm_suld_1d_array_i8_clamp:
4890   case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
4891   case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
4892   case Intrinsic::nvvm_suld_2d_i8_clamp:
4893   case Intrinsic::nvvm_suld_2d_v2i8_clamp:
4894   case Intrinsic::nvvm_suld_2d_v4i8_clamp:
4895   case Intrinsic::nvvm_suld_2d_array_i8_clamp:
4896   case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
4897   case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
4898   case Intrinsic::nvvm_suld_3d_i8_clamp:
4899   case Intrinsic::nvvm_suld_3d_v2i8_clamp:
4900   case Intrinsic::nvvm_suld_3d_v4i8_clamp:
4901   case Intrinsic::nvvm_suld_1d_i8_trap:
4902   case Intrinsic::nvvm_suld_1d_v2i8_trap:
4903   case Intrinsic::nvvm_suld_1d_v4i8_trap:
4904   case Intrinsic::nvvm_suld_1d_array_i8_trap:
4905   case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
4906   case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
4907   case Intrinsic::nvvm_suld_2d_i8_trap:
4908   case Intrinsic::nvvm_suld_2d_v2i8_trap:
4909   case Intrinsic::nvvm_suld_2d_v4i8_trap:
4910   case Intrinsic::nvvm_suld_2d_array_i8_trap:
4911   case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
4912   case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
4913   case Intrinsic::nvvm_suld_3d_i8_trap:
4914   case Intrinsic::nvvm_suld_3d_v2i8_trap:
4915   case Intrinsic::nvvm_suld_3d_v4i8_trap:
4916   case Intrinsic::nvvm_suld_1d_i8_zero:
4917   case Intrinsic::nvvm_suld_1d_v2i8_zero:
4918   case Intrinsic::nvvm_suld_1d_v4i8_zero:
4919   case Intrinsic::nvvm_suld_1d_array_i8_zero:
4920   case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
4921   case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
4922   case Intrinsic::nvvm_suld_2d_i8_zero:
4923   case Intrinsic::nvvm_suld_2d_v2i8_zero:
4924   case Intrinsic::nvvm_suld_2d_v4i8_zero:
4925   case Intrinsic::nvvm_suld_2d_array_i8_zero:
4926   case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
4927   case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
4928   case Intrinsic::nvvm_suld_3d_i8_zero:
4929   case Intrinsic::nvvm_suld_3d_v2i8_zero:
4930   case Intrinsic::nvvm_suld_3d_v4i8_zero:
4931     Info.opc = getOpcForSurfaceInstr(Intrinsic);
4932     Info.memVT = MVT::i8;
4933     Info.ptrVal = nullptr;
4934     Info.offset = 0;
4935     Info.flags = MachineMemOperand::MOLoad;
4936     Info.align = Align(16);
4937     return true;
4938 
4939   case Intrinsic::nvvm_suld_1d_i16_clamp:
4940   case Intrinsic::nvvm_suld_1d_v2i16_clamp:
4941   case Intrinsic::nvvm_suld_1d_v4i16_clamp:
4942   case Intrinsic::nvvm_suld_1d_array_i16_clamp:
4943   case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
4944   case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
4945   case Intrinsic::nvvm_suld_2d_i16_clamp:
4946   case Intrinsic::nvvm_suld_2d_v2i16_clamp:
4947   case Intrinsic::nvvm_suld_2d_v4i16_clamp:
4948   case Intrinsic::nvvm_suld_2d_array_i16_clamp:
4949   case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
4950   case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
4951   case Intrinsic::nvvm_suld_3d_i16_clamp:
4952   case Intrinsic::nvvm_suld_3d_v2i16_clamp:
4953   case Intrinsic::nvvm_suld_3d_v4i16_clamp:
4954   case Intrinsic::nvvm_suld_1d_i16_trap:
4955   case Intrinsic::nvvm_suld_1d_v2i16_trap:
4956   case Intrinsic::nvvm_suld_1d_v4i16_trap:
4957   case Intrinsic::nvvm_suld_1d_array_i16_trap:
4958   case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
4959   case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
4960   case Intrinsic::nvvm_suld_2d_i16_trap:
4961   case Intrinsic::nvvm_suld_2d_v2i16_trap:
4962   case Intrinsic::nvvm_suld_2d_v4i16_trap:
4963   case Intrinsic::nvvm_suld_2d_array_i16_trap:
4964   case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
4965   case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
4966   case Intrinsic::nvvm_suld_3d_i16_trap:
4967   case Intrinsic::nvvm_suld_3d_v2i16_trap:
4968   case Intrinsic::nvvm_suld_3d_v4i16_trap:
4969   case Intrinsic::nvvm_suld_1d_i16_zero:
4970   case Intrinsic::nvvm_suld_1d_v2i16_zero:
4971   case Intrinsic::nvvm_suld_1d_v4i16_zero:
4972   case Intrinsic::nvvm_suld_1d_array_i16_zero:
4973   case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
4974   case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
4975   case Intrinsic::nvvm_suld_2d_i16_zero:
4976   case Intrinsic::nvvm_suld_2d_v2i16_zero:
4977   case Intrinsic::nvvm_suld_2d_v4i16_zero:
4978   case Intrinsic::nvvm_suld_2d_array_i16_zero:
4979   case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
4980   case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
4981   case Intrinsic::nvvm_suld_3d_i16_zero:
4982   case Intrinsic::nvvm_suld_3d_v2i16_zero:
4983   case Intrinsic::nvvm_suld_3d_v4i16_zero:
4984     Info.opc = getOpcForSurfaceInstr(Intrinsic);
4985     Info.memVT = MVT::i16;
4986     Info.ptrVal = nullptr;
4987     Info.offset = 0;
4988     Info.flags = MachineMemOperand::MOLoad;
4989     Info.align = Align(16);
4990     return true;
4991 
4992   case Intrinsic::nvvm_suld_1d_i32_clamp:
4993   case Intrinsic::nvvm_suld_1d_v2i32_clamp:
4994   case Intrinsic::nvvm_suld_1d_v4i32_clamp:
4995   case Intrinsic::nvvm_suld_1d_array_i32_clamp:
4996   case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
4997   case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
4998   case Intrinsic::nvvm_suld_2d_i32_clamp:
4999   case Intrinsic::nvvm_suld_2d_v2i32_clamp:
5000   case Intrinsic::nvvm_suld_2d_v4i32_clamp:
5001   case Intrinsic::nvvm_suld_2d_array_i32_clamp:
5002   case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
5003   case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
5004   case Intrinsic::nvvm_suld_3d_i32_clamp:
5005   case Intrinsic::nvvm_suld_3d_v2i32_clamp:
5006   case Intrinsic::nvvm_suld_3d_v4i32_clamp:
5007   case Intrinsic::nvvm_suld_1d_i32_trap:
5008   case Intrinsic::nvvm_suld_1d_v2i32_trap:
5009   case Intrinsic::nvvm_suld_1d_v4i32_trap:
5010   case Intrinsic::nvvm_suld_1d_array_i32_trap:
5011   case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
5012   case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
5013   case Intrinsic::nvvm_suld_2d_i32_trap:
5014   case Intrinsic::nvvm_suld_2d_v2i32_trap:
5015   case Intrinsic::nvvm_suld_2d_v4i32_trap:
5016   case Intrinsic::nvvm_suld_2d_array_i32_trap:
5017   case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
5018   case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
5019   case Intrinsic::nvvm_suld_3d_i32_trap:
5020   case Intrinsic::nvvm_suld_3d_v2i32_trap:
5021   case Intrinsic::nvvm_suld_3d_v4i32_trap:
5022   case Intrinsic::nvvm_suld_1d_i32_zero:
5023   case Intrinsic::nvvm_suld_1d_v2i32_zero:
5024   case Intrinsic::nvvm_suld_1d_v4i32_zero:
5025   case Intrinsic::nvvm_suld_1d_array_i32_zero:
5026   case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
5027   case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
5028   case Intrinsic::nvvm_suld_2d_i32_zero:
5029   case Intrinsic::nvvm_suld_2d_v2i32_zero:
5030   case Intrinsic::nvvm_suld_2d_v4i32_zero:
5031   case Intrinsic::nvvm_suld_2d_array_i32_zero:
5032   case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
5033   case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
5034   case Intrinsic::nvvm_suld_3d_i32_zero:
5035   case Intrinsic::nvvm_suld_3d_v2i32_zero:
5036   case Intrinsic::nvvm_suld_3d_v4i32_zero:
5037     Info.opc = getOpcForSurfaceInstr(Intrinsic);
5038     Info.memVT = MVT::i32;
5039     Info.ptrVal = nullptr;
5040     Info.offset = 0;
5041     Info.flags = MachineMemOperand::MOLoad;
5042     Info.align = Align(16);
5043     return true;
5044 
5045   case Intrinsic::nvvm_suld_1d_i64_clamp:
5046   case Intrinsic::nvvm_suld_1d_v2i64_clamp:
5047   case Intrinsic::nvvm_suld_1d_array_i64_clamp:
5048   case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
5049   case Intrinsic::nvvm_suld_2d_i64_clamp:
5050   case Intrinsic::nvvm_suld_2d_v2i64_clamp:
5051   case Intrinsic::nvvm_suld_2d_array_i64_clamp:
5052   case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
5053   case Intrinsic::nvvm_suld_3d_i64_clamp:
5054   case Intrinsic::nvvm_suld_3d_v2i64_clamp:
5055   case Intrinsic::nvvm_suld_1d_i64_trap:
5056   case Intrinsic::nvvm_suld_1d_v2i64_trap:
5057   case Intrinsic::nvvm_suld_1d_array_i64_trap:
5058   case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
5059   case Intrinsic::nvvm_suld_2d_i64_trap:
5060   case Intrinsic::nvvm_suld_2d_v2i64_trap:
5061   case Intrinsic::nvvm_suld_2d_array_i64_trap:
5062   case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
5063   case Intrinsic::nvvm_suld_3d_i64_trap:
5064   case Intrinsic::nvvm_suld_3d_v2i64_trap:
5065   case Intrinsic::nvvm_suld_1d_i64_zero:
5066   case Intrinsic::nvvm_suld_1d_v2i64_zero:
5067   case Intrinsic::nvvm_suld_1d_array_i64_zero:
5068   case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
5069   case Intrinsic::nvvm_suld_2d_i64_zero:
5070   case Intrinsic::nvvm_suld_2d_v2i64_zero:
5071   case Intrinsic::nvvm_suld_2d_array_i64_zero:
5072   case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
5073   case Intrinsic::nvvm_suld_3d_i64_zero:
5074   case Intrinsic::nvvm_suld_3d_v2i64_zero:
5075     Info.opc = getOpcForSurfaceInstr(Intrinsic);
5076     Info.memVT = MVT::i64;
5077     Info.ptrVal = nullptr;
5078     Info.offset = 0;
5079     Info.flags = MachineMemOperand::MOLoad;
5080     Info.align = Align(16);
5081     return true;
5082   }
5083   return false;
5084 }
5085 
5086 /// getFunctionParamOptimizedAlign - since function arguments are passed via
5087 /// .param space, we may want to increase their alignment in a way that
5088 /// ensures that we can effectively vectorize their loads & stores. We can
5089 /// increase alignment only if the function has internal or has private
5090 /// linkage as for other linkage types callers may already rely on default
5091 /// alignment. To allow using 128-bit vectorized loads/stores, this function
5092 /// ensures that alignment is 16 or greater.
5093 Align NVPTXTargetLowering::getFunctionParamOptimizedAlign(
5094     const Function *F, Type *ArgTy, const DataLayout &DL) const {
5095   // Capping the alignment to 128 bytes as that is the maximum alignment
5096   // supported by PTX.
5097   const Align ABITypeAlign = std::min(Align(128), DL.getABITypeAlign(ArgTy));
5098 
5099   // If a function has linkage different from internal or private, we
5100   // must use default ABI alignment as external users rely on it. Same
5101   // for a function that may be called from a function pointer.
5102   if (!F || !F->hasLocalLinkage() ||
5103       F->hasAddressTaken(/*Users=*/nullptr,
5104                          /*IgnoreCallbackUses=*/false,
5105                          /*IgnoreAssumeLikeCalls=*/true,
5106                          /*IgnoreLLVMUsed=*/true))
5107     return ABITypeAlign;
5108 
5109   assert(!isKernelFunction(*F) && "Expect kernels to have non-local linkage");
5110   return std::max(Align(16), ABITypeAlign);
5111 }
5112 
5113 /// Helper for computing alignment of a device function byval parameter.
5114 Align NVPTXTargetLowering::getFunctionByValParamAlign(
5115     const Function *F, Type *ArgTy, Align InitialAlign,
5116     const DataLayout &DL) const {
5117   Align ArgAlign = InitialAlign;
5118   // Try to increase alignment to enhance vectorization options.
5119   if (F)
5120     ArgAlign = std::max(ArgAlign, getFunctionParamOptimizedAlign(F, ArgTy, DL));
5121 
5122   // Old ptx versions have a bug. When PTX code takes address of
5123   // byval parameter with alignment < 4, ptxas generates code to
5124   // spill argument into memory. Alas on sm_50+ ptxas generates
5125   // SASS code that fails with misaligned access. To work around
5126   // the problem, make sure that we align byval parameters by at
5127   // least 4. This bug seems to be fixed at least starting from
5128   // ptxas > 9.0.
5129   // TODO: remove this after verifying the bug is not reproduced
5130   // on non-deprecated ptxas versions.
5131   if (ForceMinByValParamAlign)
5132     ArgAlign = std::max(ArgAlign, Align(4));
5133 
5134   return ArgAlign;
5135 }
5136 
5137 // Helper for getting a function parameter name. Name is composed from
5138 // its index and the function name. Negative index corresponds to special
5139 // parameter (unsized array) used for passing variable arguments.
5140 std::string NVPTXTargetLowering::getParamName(const Function *F,
5141                                               int Idx) const {
5142   std::string ParamName;
5143   raw_string_ostream ParamStr(ParamName);
5144 
5145   ParamStr << getTargetMachine().getSymbol(F)->getName();
5146   if (Idx < 0)
5147     ParamStr << "_vararg";
5148   else
5149     ParamStr << "_param_" << Idx;
5150 
5151   return ParamName;
5152 }
5153 
5154 /// isLegalAddressingMode - Return true if the addressing mode represented
5155 /// by AM is legal for this target, for a load/store of the specified type.
5156 /// Used to guide target specific optimizations, like loop strength reduction
5157 /// (LoopStrengthReduce.cpp) and memory optimization for address mode
5158 /// (CodeGenPrepare.cpp)
5159 bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,
5160                                                 const AddrMode &AM, Type *Ty,
5161                                                 unsigned AS, Instruction *I) const {
5162   // AddrMode - This represents an addressing mode of:
5163   //    BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
5164   //
5165   // The legal address modes are
5166   // - [avar]
5167   // - [areg]
5168   // - [areg+immoff]
5169   // - [immAddr]
5170 
5171   // immoff must fit in a signed 32-bit int
5172   if (!APInt(64, AM.BaseOffs).isSignedIntN(32))
5173     return false;
5174 
5175   if (AM.BaseGV)
5176     return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;
5177 
5178   switch (AM.Scale) {
5179   case 0: // "r", "r+i" or "i" is allowed
5180     break;
5181   case 1:
5182     if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
5183       return false;
5184     // Otherwise we have r+i.
5185     break;
5186   default:
5187     // No scale > 1 is allowed
5188     return false;
5189   }
5190   return true;
5191 }
5192 
5193 //===----------------------------------------------------------------------===//
5194 //                         NVPTX Inline Assembly Support
5195 //===----------------------------------------------------------------------===//
5196 
5197 /// getConstraintType - Given a constraint letter, return the type of
5198 /// constraint it is for this target.
5199 NVPTXTargetLowering::ConstraintType
5200 NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {
5201   if (Constraint.size() == 1) {
5202     switch (Constraint[0]) {
5203     default:
5204       break;
5205     case 'b':
5206     case 'r':
5207     case 'h':
5208     case 'c':
5209     case 'l':
5210     case 'f':
5211     case 'd':
5212     case 'q':
5213     case '0':
5214     case 'N':
5215       return C_RegisterClass;
5216     }
5217   }
5218   return TargetLowering::getConstraintType(Constraint);
5219 }
5220 
5221 std::pair<unsigned, const TargetRegisterClass *>
5222 NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
5223                                                   StringRef Constraint,
5224                                                   MVT VT) const {
5225   if (Constraint.size() == 1) {
5226     switch (Constraint[0]) {
5227     case 'b':
5228       return std::make_pair(0U, &NVPTX::Int1RegsRegClass);
5229     case 'c':
5230       return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
5231     case 'h':
5232       return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
5233     case 'r':
5234       return std::make_pair(0U, &NVPTX::Int32RegsRegClass);
5235     case 'l':
5236     case 'N':
5237       return std::make_pair(0U, &NVPTX::Int64RegsRegClass);
5238     case 'q': {
5239       if (STI.getSmVersion() < 70)
5240         report_fatal_error("Inline asm with 128 bit operands is only "
5241                            "supported for sm_70 and higher!");
5242       return std::make_pair(0U, &NVPTX::Int128RegsRegClass);
5243     }
5244     case 'f':
5245       return std::make_pair(0U, &NVPTX::Float32RegsRegClass);
5246     case 'd':
5247       return std::make_pair(0U, &NVPTX::Float64RegsRegClass);
5248     }
5249   }
5250   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
5251 }
5252 
5253 //===----------------------------------------------------------------------===//
5254 //                         NVPTX DAG Combining
5255 //===----------------------------------------------------------------------===//
5256 
5257 bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,
5258                                    CodeGenOptLevel OptLevel) const {
5259   // Always honor command-line argument
5260   if (FMAContractLevelOpt.getNumOccurrences() > 0)
5261     return FMAContractLevelOpt > 0;
5262 
5263   // Do not contract if we're not optimizing the code.
5264   if (OptLevel == CodeGenOptLevel::None)
5265     return false;
5266 
5267   // Honor TargetOptions flags that explicitly say fusion is okay.
5268   if (MF.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast)
5269     return true;
5270 
5271   return allowUnsafeFPMath(MF);
5272 }
5273 
5274 bool NVPTXTargetLowering::allowUnsafeFPMath(MachineFunction &MF) const {
5275   // Honor TargetOptions flags that explicitly say unsafe math is okay.
5276   if (MF.getTarget().Options.UnsafeFPMath)
5277     return true;
5278 
5279   // Allow unsafe math if unsafe-fp-math attribute explicitly says so.
5280   const Function &F = MF.getFunction();
5281   return F.getFnAttribute("unsafe-fp-math").getValueAsBool();
5282 }
5283 
5284 static bool isConstZero(const SDValue &Operand) {
5285   const auto *Const = dyn_cast<ConstantSDNode>(Operand);
5286   return Const && Const->getZExtValue() == 0;
5287 }
5288 
5289 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
5290 /// operands N0 and N1.  This is a helper for PerformADDCombine that is
5291 /// called with the default operands, and if that fails, with commuted
5292 /// operands.
5293 static SDValue
5294 PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
5295                               TargetLowering::DAGCombinerInfo &DCI) {
5296   EVT VT = N0.getValueType();
5297 
5298   // Since integer multiply-add costs the same as integer multiply
5299   // but is more costly than integer add, do the fusion only when
5300   // the mul is only used in the add.
5301   // TODO: this may not be true for later architectures, consider relaxing this
5302   if (!N0.getNode()->hasOneUse())
5303     return SDValue();
5304 
5305   // fold (add (mul a, b), c) -> (mad a, b, c)
5306   //
5307   if (N0.getOpcode() == ISD::MUL)
5308     return DCI.DAG.getNode(NVPTXISD::IMAD, SDLoc(N), VT, N0.getOperand(0),
5309                            N0.getOperand(1), N1);
5310 
5311   // fold (add (select cond, 0, (mul a, b)), c)
5312   //   -> (select cond, c, (mad a, b, c))
5313   //
5314   if (N0.getOpcode() == ISD::SELECT) {
5315     unsigned ZeroOpNum;
5316     if (isConstZero(N0->getOperand(1)))
5317       ZeroOpNum = 1;
5318     else if (isConstZero(N0->getOperand(2)))
5319       ZeroOpNum = 2;
5320     else
5321       return SDValue();
5322 
5323     SDValue M = N0->getOperand((ZeroOpNum == 1) ? 2 : 1);
5324     if (M->getOpcode() != ISD::MUL || !M.getNode()->hasOneUse())
5325       return SDValue();
5326 
5327     SDValue MAD = DCI.DAG.getNode(NVPTXISD::IMAD, SDLoc(N), VT,
5328                                   M->getOperand(0), M->getOperand(1), N1);
5329     return DCI.DAG.getSelect(SDLoc(N), VT, N0->getOperand(0),
5330                              ((ZeroOpNum == 1) ? N1 : MAD),
5331                              ((ZeroOpNum == 1) ? MAD : N1));
5332   }
5333 
5334   return SDValue();
5335 }
5336 
5337 static SDValue
5338 PerformFADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
5339                                TargetLowering::DAGCombinerInfo &DCI,
5340                                CodeGenOptLevel OptLevel) {
5341   EVT VT = N0.getValueType();
5342   if (N0.getOpcode() == ISD::FMUL) {
5343     const auto *TLI = static_cast<const NVPTXTargetLowering *>(
5344         &DCI.DAG.getTargetLoweringInfo());
5345     if (!TLI->allowFMA(DCI.DAG.getMachineFunction(), OptLevel))
5346       return SDValue();
5347 
5348     // For floating point:
5349     // Do the fusion only when the mul has less than 5 uses and all
5350     // are add.
5351     // The heuristic is that if a use is not an add, then that use
5352     // cannot be fused into fma, therefore mul is still needed anyway.
5353     // If there are more than 4 uses, even if they are all add, fusing
5354     // them will increase register pressue.
5355     //
5356     int numUses = 0;
5357     int nonAddCount = 0;
5358     for (const SDNode *User : N0.getNode()->uses()) {
5359       numUses++;
5360       if (User->getOpcode() != ISD::FADD)
5361         ++nonAddCount;
5362       if (numUses >= 5)
5363         return SDValue();
5364     }
5365     if (nonAddCount) {
5366       int orderNo = N->getIROrder();
5367       int orderNo2 = N0.getNode()->getIROrder();
5368       // simple heuristics here for considering potential register
5369       // pressure, the logics here is that the differnce are used
5370       // to measure the distance between def and use, the longer distance
5371       // more likely cause register pressure.
5372       if (orderNo - orderNo2 < 500)
5373         return SDValue();
5374 
5375       // Now, check if at least one of the FMUL's operands is live beyond the
5376       // node N, which guarantees that the FMA will not increase register
5377       // pressure at node N.
5378       bool opIsLive = false;
5379       const SDNode *left = N0.getOperand(0).getNode();
5380       const SDNode *right = N0.getOperand(1).getNode();
5381 
5382       if (isa<ConstantSDNode>(left) || isa<ConstantSDNode>(right))
5383         opIsLive = true;
5384 
5385       if (!opIsLive)
5386         for (const SDNode *User : left->uses()) {
5387           int orderNo3 = User->getIROrder();
5388           if (orderNo3 > orderNo) {
5389             opIsLive = true;
5390             break;
5391           }
5392         }
5393 
5394       if (!opIsLive)
5395         for (const SDNode *User : right->uses()) {
5396           int orderNo3 = User->getIROrder();
5397           if (orderNo3 > orderNo) {
5398             opIsLive = true;
5399             break;
5400           }
5401         }
5402 
5403       if (!opIsLive)
5404         return SDValue();
5405     }
5406 
5407     return DCI.DAG.getNode(ISD::FMA, SDLoc(N), VT, N0.getOperand(0),
5408                            N0.getOperand(1), N1);
5409   }
5410 
5411   return SDValue();
5412 }
5413 
5414 static SDValue PerformStoreCombineHelper(SDNode *N, std::size_t Front,
5415                                          std::size_t Back) {
5416   if (all_of(N->ops().drop_front(Front).drop_back(Back),
5417              [](const SDUse &U) { return U.get()->isUndef(); }))
5418     // Operand 0 is the previous value in the chain. Cannot return EntryToken
5419     // as the previous value will become unused and eliminated later.
5420     return N->getOperand(0);
5421 
5422   return SDValue();
5423 }
5424 
5425 static SDValue PerformStoreParamCombine(SDNode *N) {
5426   // Operands from the 3rd to the 2nd last one are the values to be stored.
5427   //   {Chain, ArgID, Offset, Val, Glue}
5428   return PerformStoreCombineHelper(N, 3, 1);
5429 }
5430 
5431 static SDValue PerformStoreRetvalCombine(SDNode *N) {
5432   // Operands from the 2nd to the last one are the values to be stored
5433   return PerformStoreCombineHelper(N, 2, 0);
5434 }
5435 
5436 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
5437 ///
5438 static SDValue PerformADDCombine(SDNode *N,
5439                                  TargetLowering::DAGCombinerInfo &DCI,
5440                                  CodeGenOptLevel OptLevel) {
5441   if (OptLevel == CodeGenOptLevel::None)
5442     return SDValue();
5443 
5444   SDValue N0 = N->getOperand(0);
5445   SDValue N1 = N->getOperand(1);
5446 
5447   // Skip non-integer, non-scalar case
5448   EVT VT = N0.getValueType();
5449   if (VT.isVector() || VT != MVT::i32)
5450     return SDValue();
5451 
5452   // First try with the default operand order.
5453   if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI))
5454     return Result;
5455 
5456   // If that didn't work, try again with the operands commuted.
5457   return PerformADDCombineWithOperands(N, N1, N0, DCI);
5458 }
5459 
5460 /// PerformFADDCombine - Target-specific dag combine xforms for ISD::FADD.
5461 ///
5462 static SDValue PerformFADDCombine(SDNode *N,
5463                                  TargetLowering::DAGCombinerInfo &DCI,
5464                                  CodeGenOptLevel OptLevel) {
5465   SDValue N0 = N->getOperand(0);
5466   SDValue N1 = N->getOperand(1);
5467 
5468   EVT VT = N0.getValueType();
5469   if (VT.isVector() || !(VT == MVT::f32 || VT == MVT::f64))
5470     return SDValue();
5471 
5472   // First try with the default operand order.
5473   if (SDValue Result = PerformFADDCombineWithOperands(N, N0, N1, DCI, OptLevel))
5474     return Result;
5475 
5476   // If that didn't work, try again with the operands commuted.
5477   return PerformFADDCombineWithOperands(N, N1, N0, DCI, OptLevel);
5478 }
5479 
5480 static SDValue PerformANDCombine(SDNode *N,
5481                                  TargetLowering::DAGCombinerInfo &DCI) {
5482   // The type legalizer turns a vector load of i8 values into a zextload to i16
5483   // registers, optionally ANY_EXTENDs it (if target type is integer),
5484   // and ANDs off the high 8 bits. Since we turn this load into a
5485   // target-specific DAG node, the DAG combiner fails to eliminate these AND
5486   // nodes. Do that here.
5487   SDValue Val = N->getOperand(0);
5488   SDValue Mask = N->getOperand(1);
5489 
5490   if (isa<ConstantSDNode>(Val)) {
5491     std::swap(Val, Mask);
5492   }
5493 
5494   SDValue AExt;
5495 
5496   // Convert BFE-> truncate i16 -> and 255
5497   // To just BFE-> truncate i16, as the value already has all the bits in the
5498   // right places.
5499   if (Val.getOpcode() == ISD::TRUNCATE) {
5500     SDValue BFE = Val.getOperand(0);
5501     if (BFE.getOpcode() != NVPTXISD::BFE)
5502       return SDValue();
5503 
5504     ConstantSDNode *BFEBits = dyn_cast<ConstantSDNode>(BFE.getOperand(0));
5505     if (!BFEBits)
5506       return SDValue();
5507     uint64_t BFEBitsVal = BFEBits->getZExtValue();
5508 
5509     ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Mask);
5510     if (!MaskCnst) {
5511       // Not an AND with a constant
5512       return SDValue();
5513     }
5514     uint64_t MaskVal = MaskCnst->getZExtValue();
5515 
5516     if (MaskVal != (uint64_t(1) << BFEBitsVal) - 1)
5517       return SDValue();
5518     // If we get here, the AND is unnecessary.  Just replace it with the trunc
5519     DCI.CombineTo(N, Val, false);
5520   }
5521   // Generally, we will see zextload -> IMOV16rr -> ANY_EXTEND -> and
5522   if (Val.getOpcode() == ISD::ANY_EXTEND) {
5523     AExt = Val;
5524     Val = Val->getOperand(0);
5525   }
5526 
5527   if (Val->isMachineOpcode() && Val->getMachineOpcode() == NVPTX::IMOV16rr) {
5528     Val = Val->getOperand(0);
5529   }
5530 
5531   if (Val->getOpcode() == NVPTXISD::LoadV2 ||
5532       Val->getOpcode() == NVPTXISD::LoadV4) {
5533     ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Mask);
5534     if (!MaskCnst) {
5535       // Not an AND with a constant
5536       return SDValue();
5537     }
5538 
5539     uint64_t MaskVal = MaskCnst->getZExtValue();
5540     if (MaskVal != 0xff) {
5541       // Not an AND that chops off top 8 bits
5542       return SDValue();
5543     }
5544 
5545     MemSDNode *Mem = dyn_cast<MemSDNode>(Val);
5546     if (!Mem) {
5547       // Not a MemSDNode?!?
5548       return SDValue();
5549     }
5550 
5551     EVT MemVT = Mem->getMemoryVT();
5552     if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8) {
5553       // We only handle the i8 case
5554       return SDValue();
5555     }
5556 
5557     unsigned ExtType = Val->getConstantOperandVal(Val->getNumOperands() - 1);
5558     if (ExtType == ISD::SEXTLOAD) {
5559       // If for some reason the load is a sextload, the and is needed to zero
5560       // out the high 8 bits
5561       return SDValue();
5562     }
5563 
5564     bool AddTo = false;
5565     if (AExt.getNode() != nullptr) {
5566       // Re-insert the ext as a zext.
5567       Val = DCI.DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5568                             AExt.getValueType(), Val);
5569       AddTo = true;
5570     }
5571 
5572     // If we get here, the AND is unnecessary.  Just replace it with the load
5573     DCI.CombineTo(N, Val, AddTo);
5574   }
5575 
5576   return SDValue();
5577 }
5578 
5579 static SDValue PerformREMCombine(SDNode *N,
5580                                  TargetLowering::DAGCombinerInfo &DCI,
5581                                  CodeGenOptLevel OptLevel) {
5582   assert(N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM);
5583 
5584   // Don't do anything at less than -O2.
5585   if (OptLevel < CodeGenOptLevel::Default)
5586     return SDValue();
5587 
5588   SelectionDAG &DAG = DCI.DAG;
5589   SDLoc DL(N);
5590   EVT VT = N->getValueType(0);
5591   bool IsSigned = N->getOpcode() == ISD::SREM;
5592   unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV;
5593 
5594   const SDValue &Num = N->getOperand(0);
5595   const SDValue &Den = N->getOperand(1);
5596 
5597   for (const SDNode *U : Num->uses()) {
5598     if (U->getOpcode() == DivOpc && U->getOperand(0) == Num &&
5599         U->getOperand(1) == Den) {
5600       // Num % Den -> Num - (Num / Den) * Den
5601       return DAG.getNode(ISD::SUB, DL, VT, Num,
5602                          DAG.getNode(ISD::MUL, DL, VT,
5603                                      DAG.getNode(DivOpc, DL, VT, Num, Den),
5604                                      Den));
5605     }
5606   }
5607   return SDValue();
5608 }
5609 
5610 enum OperandSignedness {
5611   Signed = 0,
5612   Unsigned,
5613   Unknown
5614 };
5615 
5616 /// IsMulWideOperandDemotable - Checks if the provided DAG node is an operand
5617 /// that can be demoted to \p OptSize bits without loss of information. The
5618 /// signedness of the operand, if determinable, is placed in \p S.
5619 static bool IsMulWideOperandDemotable(SDValue Op,
5620                                       unsigned OptSize,
5621                                       OperandSignedness &S) {
5622   S = Unknown;
5623 
5624   if (Op.getOpcode() == ISD::SIGN_EXTEND ||
5625       Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
5626     EVT OrigVT = Op.getOperand(0).getValueType();
5627     if (OrigVT.getFixedSizeInBits() <= OptSize) {
5628       S = Signed;
5629       return true;
5630     }
5631   } else if (Op.getOpcode() == ISD::ZERO_EXTEND) {
5632     EVT OrigVT = Op.getOperand(0).getValueType();
5633     if (OrigVT.getFixedSizeInBits() <= OptSize) {
5634       S = Unsigned;
5635       return true;
5636     }
5637   }
5638 
5639   return false;
5640 }
5641 
5642 /// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can
5643 /// be demoted to \p OptSize bits without loss of information. If the operands
5644 /// contain a constant, it should appear as the RHS operand. The signedness of
5645 /// the operands is placed in \p IsSigned.
5646 static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
5647                                         unsigned OptSize,
5648                                         bool &IsSigned) {
5649   OperandSignedness LHSSign;
5650 
5651   // The LHS operand must be a demotable op
5652   if (!IsMulWideOperandDemotable(LHS, OptSize, LHSSign))
5653     return false;
5654 
5655   // We should have been able to determine the signedness from the LHS
5656   if (LHSSign == Unknown)
5657     return false;
5658 
5659   IsSigned = (LHSSign == Signed);
5660 
5661   // The RHS can be a demotable op or a constant
5662   if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(RHS)) {
5663     const APInt &Val = CI->getAPIntValue();
5664     if (LHSSign == Unsigned) {
5665       return Val.isIntN(OptSize);
5666     } else {
5667       return Val.isSignedIntN(OptSize);
5668     }
5669   } else {
5670     OperandSignedness RHSSign;
5671     if (!IsMulWideOperandDemotable(RHS, OptSize, RHSSign))
5672       return false;
5673 
5674     return LHSSign == RHSSign;
5675   }
5676 }
5677 
5678 /// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply
5679 /// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform
5680 /// works on both multiply DAG nodes and SHL DAG nodes with a constant shift
5681 /// amount.
5682 static SDValue TryMULWIDECombine(SDNode *N,
5683                                  TargetLowering::DAGCombinerInfo &DCI) {
5684   EVT MulType = N->getValueType(0);
5685   if (MulType != MVT::i32 && MulType != MVT::i64) {
5686     return SDValue();
5687   }
5688 
5689   SDLoc DL(N);
5690   unsigned OptSize = MulType.getSizeInBits() >> 1;
5691   SDValue LHS = N->getOperand(0);
5692   SDValue RHS = N->getOperand(1);
5693 
5694   // Canonicalize the multiply so the constant (if any) is on the right
5695   if (N->getOpcode() == ISD::MUL) {
5696     if (isa<ConstantSDNode>(LHS)) {
5697       std::swap(LHS, RHS);
5698     }
5699   }
5700 
5701   // If we have a SHL, determine the actual multiply amount
5702   if (N->getOpcode() == ISD::SHL) {
5703     ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(RHS);
5704     if (!ShlRHS) {
5705       return SDValue();
5706     }
5707 
5708     APInt ShiftAmt = ShlRHS->getAPIntValue();
5709     unsigned BitWidth = MulType.getSizeInBits();
5710     if (ShiftAmt.sge(0) && ShiftAmt.slt(BitWidth)) {
5711       APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;
5712       RHS = DCI.DAG.getConstant(MulVal, DL, MulType);
5713     } else {
5714       return SDValue();
5715     }
5716   }
5717 
5718   bool Signed;
5719   // Verify that our operands are demotable
5720   if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, Signed)) {
5721     return SDValue();
5722   }
5723 
5724   EVT DemotedVT;
5725   if (MulType == MVT::i32) {
5726     DemotedVT = MVT::i16;
5727   } else {
5728     DemotedVT = MVT::i32;
5729   }
5730 
5731   // Truncate the operands to the correct size. Note that these are just for
5732   // type consistency and will (likely) be eliminated in later phases.
5733   SDValue TruncLHS =
5734     DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, LHS);
5735   SDValue TruncRHS =
5736     DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, RHS);
5737 
5738   unsigned Opc;
5739   if (Signed) {
5740     Opc = NVPTXISD::MUL_WIDE_SIGNED;
5741   } else {
5742     Opc = NVPTXISD::MUL_WIDE_UNSIGNED;
5743   }
5744 
5745   return DCI.DAG.getNode(Opc, DL, MulType, TruncLHS, TruncRHS);
5746 }
5747 
5748 static bool isConstOne(const SDValue &Operand) {
5749   const auto *Const = dyn_cast<ConstantSDNode>(Operand);
5750   return Const && Const->getZExtValue() == 1;
5751 }
5752 
5753 static SDValue matchMADConstOnePattern(SDValue Add) {
5754   if (Add->getOpcode() != ISD::ADD)
5755     return SDValue();
5756 
5757   if (isConstOne(Add->getOperand(0)))
5758     return Add->getOperand(1);
5759 
5760   if (isConstOne(Add->getOperand(1)))
5761     return Add->getOperand(0);
5762 
5763   return SDValue();
5764 }
5765 
5766 static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL,
5767                                   TargetLowering::DAGCombinerInfo &DCI) {
5768 
5769   if (SDValue Y = matchMADConstOnePattern(Add))
5770     return DCI.DAG.getNode(NVPTXISD::IMAD, DL, VT, X, Y, X);
5771 
5772   return SDValue();
5773 }
5774 
5775 static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT,
5776                                         SDLoc DL,
5777                                         TargetLowering::DAGCombinerInfo &DCI) {
5778   if (Select->getOpcode() != ISD::SELECT)
5779     return SDValue();
5780 
5781   SDValue Cond = Select->getOperand(0);
5782 
5783   unsigned ConstOpNo;
5784   if (isConstOne(Select->getOperand(1)))
5785     ConstOpNo = 1;
5786   else if (isConstOne(Select->getOperand(2)))
5787     ConstOpNo = 2;
5788   else
5789     return SDValue();
5790 
5791   SDValue Y = Select->getOperand((ConstOpNo == 1) ? 2 : 1);
5792 
5793   // Do not combine if the resulting sequence is not obviously profitable.
5794   if (!matchMADConstOnePattern(Y))
5795     return SDValue();
5796 
5797   SDValue NewMul = DCI.DAG.getNode(ISD::MUL, DL, VT, X, Y);
5798 
5799   return DCI.DAG.getNode(ISD::SELECT, DL, VT, Cond,
5800                          (ConstOpNo == 1) ? X : NewMul,
5801                          (ConstOpNo == 1) ? NewMul : X);
5802 }
5803 
5804 static SDValue
5805 PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
5806                               TargetLowering::DAGCombinerInfo &DCI) {
5807 
5808   EVT VT = N0.getValueType();
5809   if (VT.isVector())
5810     return SDValue();
5811 
5812   if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5813     return SDValue();
5814 
5815   SDLoc DL(N);
5816 
5817   // (mul x, (add y, 1)) -> (mad x, y, x)
5818   if (SDValue Res = combineMADConstOne(N0, N1, VT, DL, DCI))
5819     return Res;
5820   if (SDValue Res = combineMADConstOne(N1, N0, VT, DL, DCI))
5821     return Res;
5822 
5823   // (mul x, (select y, 1)) -> (select (mul x, y), x)
5824   if (SDValue Res = combineMulSelectConstOne(N0, N1, VT, DL, DCI))
5825     return Res;
5826   if (SDValue Res = combineMulSelectConstOne(N1, N0, VT, DL, DCI))
5827     return Res;
5828 
5829   return SDValue();
5830 }
5831 
5832 /// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.
5833 static SDValue PerformMULCombine(SDNode *N,
5834                                  TargetLowering::DAGCombinerInfo &DCI,
5835                                  CodeGenOptLevel OptLevel) {
5836   if (OptLevel == CodeGenOptLevel::None)
5837     return SDValue();
5838 
5839   if (SDValue Ret = TryMULWIDECombine(N, DCI))
5840     return Ret;
5841 
5842   SDValue N0 = N->getOperand(0);
5843   SDValue N1 = N->getOperand(1);
5844   return PerformMULCombineWithOperands(N, N0, N1, DCI);
5845 }
5846 
5847 /// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
5848 static SDValue PerformSHLCombine(SDNode *N,
5849                                  TargetLowering::DAGCombinerInfo &DCI,
5850                                  CodeGenOptLevel OptLevel) {
5851   if (OptLevel > CodeGenOptLevel::None) {
5852     // Try mul.wide combining at OptLevel > 0
5853     if (SDValue Ret = TryMULWIDECombine(N, DCI))
5854       return Ret;
5855   }
5856 
5857   return SDValue();
5858 }
5859 
5860 static SDValue PerformSETCCCombine(SDNode *N,
5861                                    TargetLowering::DAGCombinerInfo &DCI,
5862                                    unsigned int SmVersion) {
5863   EVT CCType = N->getValueType(0);
5864   SDValue A = N->getOperand(0);
5865   SDValue B = N->getOperand(1);
5866 
5867   EVT AType = A.getValueType();
5868   if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))
5869     return SDValue();
5870 
5871   if (A.getValueType() == MVT::v2bf16 && SmVersion < 90)
5872     return SDValue();
5873 
5874   SDLoc DL(N);
5875   // setp.f16x2 returns two scalar predicates, which we need to
5876   // convert back to v2i1. The returned result will be scalarized by
5877   // the legalizer, but the comparison will remain a single vector
5878   // instruction.
5879   SDValue CCNode = DCI.DAG.getNode(
5880       A.getValueType() == MVT::v2f16 ? NVPTXISD::SETP_F16X2
5881                                      : NVPTXISD::SETP_BF16X2,
5882       DL, DCI.DAG.getVTList(MVT::i1, MVT::i1), {A, B, N->getOperand(2)});
5883   return DCI.DAG.getNode(ISD::BUILD_VECTOR, DL, CCType, CCNode.getValue(0),
5884                          CCNode.getValue(1));
5885 }
5886 
5887 static SDValue PerformEXTRACTCombine(SDNode *N,
5888                                      TargetLowering::DAGCombinerInfo &DCI) {
5889   SDValue Vector = N->getOperand(0);
5890   SDLoc DL(N);
5891   EVT VectorVT = Vector.getValueType();
5892   if (Vector->getOpcode() == ISD::LOAD && VectorVT.isSimple() &&
5893       IsPTXVectorType(VectorVT.getSimpleVT()))
5894     return SDValue(); // Native vector loads already combine nicely w/
5895                       // extract_vector_elt.
5896   // Don't mess with singletons or v2*16, v4i8 and v8i8 types, we already
5897   // handle them OK.
5898   if (VectorVT.getVectorNumElements() == 1 || Isv2x16VT(VectorVT) ||
5899       VectorVT == MVT::v4i8 || VectorVT == MVT::v8i8)
5900     return SDValue();
5901 
5902   // Don't mess with undef values as sra may be simplified to 0, not undef.
5903   if (Vector->isUndef() || ISD::allOperandsUndef(Vector.getNode()))
5904     return SDValue();
5905 
5906   uint64_t VectorBits = VectorVT.getSizeInBits();
5907   // We only handle the types we can extract in-register.
5908   if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))
5909     return SDValue();
5910 
5911   ConstantSDNode *Index = dyn_cast<ConstantSDNode>(N->getOperand(1));
5912   // Index == 0 is handled by generic DAG combiner.
5913   if (!Index || Index->getZExtValue() == 0)
5914     return SDValue();
5915 
5916   MVT IVT = MVT::getIntegerVT(VectorBits);
5917   EVT EltVT = VectorVT.getVectorElementType();
5918   EVT EltIVT = EltVT.changeTypeToInteger();
5919   uint64_t EltBits = EltVT.getScalarSizeInBits();
5920 
5921   SDValue Result = DCI.DAG.getNode(
5922       ISD::TRUNCATE, DL, EltIVT,
5923       DCI.DAG.getNode(
5924           ISD::SRA, DL, IVT, DCI.DAG.getNode(ISD::BITCAST, DL, IVT, Vector),
5925           DCI.DAG.getConstant(Index->getZExtValue() * EltBits, DL, IVT)));
5926 
5927   // If element has non-integer type, bitcast it back to the expected type.
5928   if (EltVT != EltIVT)
5929     Result = DCI.DAG.getNode(ISD::BITCAST, DL, EltVT, Result);
5930   // Past legalizer, we may need to extent i8 -> i16 to match the register type.
5931   if (EltVT != N->getValueType(0))
5932     Result = DCI.DAG.getNode(ISD::ANY_EXTEND, DL, N->getValueType(0), Result);
5933 
5934   return Result;
5935 }
5936 
5937 static SDValue PerformVSELECTCombine(SDNode *N,
5938                                      TargetLowering::DAGCombinerInfo &DCI) {
5939   SDValue VA = N->getOperand(1);
5940   EVT VectorVT = VA.getValueType();
5941   if (VectorVT != MVT::v4i8)
5942     return SDValue();
5943 
5944   // We need to split vselect into individual per-element operations Because we
5945   // use BFE/BFI instruction for byte extraction/insertion, we do end up with
5946   // 32-bit values, so we may as well do comparison as i32 to avoid conversions
5947   // to/from i16 normally used for i8 values.
5948   SmallVector<SDValue, 4> E;
5949   SDLoc DL(N);
5950   SDValue VCond = N->getOperand(0);
5951   SDValue VB = N->getOperand(2);
5952   for (int I = 0; I < 4; ++I) {
5953     SDValue C = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i1, VCond,
5954                                 DCI.DAG.getConstant(I, DL, MVT::i32));
5955     SDValue EA = DCI.DAG.getAnyExtOrTrunc(
5956         DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, VA,
5957                         DCI.DAG.getConstant(I, DL, MVT::i32)),
5958         DL, MVT::i32);
5959     SDValue EB = DCI.DAG.getAnyExtOrTrunc(
5960         DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, VB,
5961                         DCI.DAG.getConstant(I, DL, MVT::i32)),
5962         DL, MVT::i32);
5963     E.push_back(DCI.DAG.getAnyExtOrTrunc(
5964         DCI.DAG.getNode(ISD::SELECT, DL, MVT::i32, C, EA, EB), DL, MVT::i8));
5965   }
5966   return DCI.DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i8, E);
5967 }
5968 
5969 static SDValue PerformLOADCombine(SDNode *N,
5970                                   TargetLowering::DAGCombinerInfo &DCI) {
5971   SelectionDAG &DAG = DCI.DAG;
5972   LoadSDNode *LD = cast<LoadSDNode>(N);
5973 
5974   // Lower a v16i8 load into a LoadV4 operation with i32 results instead of
5975   // letting ReplaceLoadVector split it into smaller loads during legalization.
5976   // This is done at dag-combine1 time, so that vector operations with i8
5977   // elements can be optimised away instead of being needlessly split during
5978   // legalization, which involves storing to the stack and loading it back.
5979   EVT VT = N->getValueType(0);
5980   if (VT != MVT::v16i8)
5981     return SDValue();
5982 
5983   SDLoc DL(N);
5984 
5985   // Create a v4i32 vector load operation, effectively <4 x v4i8>.
5986   unsigned Opc = NVPTXISD::LoadV4;
5987   EVT NewVT = MVT::v4i32;
5988   EVT EltVT = NewVT.getVectorElementType();
5989   unsigned NumElts = NewVT.getVectorNumElements();
5990   EVT RetVTs[] = {EltVT, EltVT, EltVT, EltVT, MVT::Other};
5991   SDVTList RetVTList = DAG.getVTList(RetVTs);
5992   SmallVector<SDValue, 8> Ops(N->ops());
5993   Ops.push_back(DAG.getIntPtrConstant(LD->getExtensionType(), DL));
5994   SDValue NewLoad = DAG.getMemIntrinsicNode(Opc, DL, RetVTList, Ops, NewVT,
5995                                             LD->getMemOperand());
5996   SDValue NewChain = NewLoad.getValue(NumElts);
5997 
5998   // Create a vector of the same type returned by the original load.
5999   SmallVector<SDValue, 4> Elts;
6000   for (unsigned i = 0; i < NumElts; i++)
6001     Elts.push_back(NewLoad.getValue(i));
6002   return DCI.DAG.getMergeValues(
6003       {DCI.DAG.getBitcast(VT, DCI.DAG.getBuildVector(NewVT, DL, Elts)),
6004        NewChain},
6005       DL);
6006 }
6007 
6008 SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
6009                                                DAGCombinerInfo &DCI) const {
6010   CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();
6011   switch (N->getOpcode()) {
6012     default: break;
6013     case ISD::ADD:
6014       return PerformADDCombine(N, DCI, OptLevel);
6015     case ISD::FADD:
6016       return PerformFADDCombine(N, DCI, OptLevel);
6017     case ISD::MUL:
6018       return PerformMULCombine(N, DCI, OptLevel);
6019     case ISD::SHL:
6020       return PerformSHLCombine(N, DCI, OptLevel);
6021     case ISD::AND:
6022       return PerformANDCombine(N, DCI);
6023     case ISD::UREM:
6024     case ISD::SREM:
6025       return PerformREMCombine(N, DCI, OptLevel);
6026     case ISD::SETCC:
6027       return PerformSETCCCombine(N, DCI, STI.getSmVersion());
6028     case ISD::LOAD:
6029       return PerformLOADCombine(N, DCI);
6030     case NVPTXISD::StoreRetval:
6031     case NVPTXISD::StoreRetvalV2:
6032     case NVPTXISD::StoreRetvalV4:
6033       return PerformStoreRetvalCombine(N);
6034     case NVPTXISD::StoreParam:
6035     case NVPTXISD::StoreParamV2:
6036     case NVPTXISD::StoreParamV4:
6037       return PerformStoreParamCombine(N);
6038     case ISD::EXTRACT_VECTOR_ELT:
6039       return PerformEXTRACTCombine(N, DCI);
6040     case ISD::VSELECT:
6041       return PerformVSELECTCombine(N, DCI);
6042   }
6043   return SDValue();
6044 }
6045 
6046 /// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.
6047 static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
6048                               SmallVectorImpl<SDValue> &Results) {
6049   EVT ResVT = N->getValueType(0);
6050   SDLoc DL(N);
6051 
6052   assert(ResVT.isVector() && "Vector load must have vector type");
6053 
6054   // We only handle "native" vector sizes for now, e.g. <4 x double> is not
6055   // legal.  We can (and should) split that into 2 loads of <2 x double> here
6056   // but I'm leaving that as a TODO for now.
6057   assert(ResVT.isSimple() && "Can only handle simple types");
6058   switch (ResVT.getSimpleVT().SimpleTy) {
6059   default:
6060     return;
6061   case MVT::v2i8:
6062   case MVT::v2i16:
6063   case MVT::v2i32:
6064   case MVT::v2i64:
6065   case MVT::v2f16:
6066   case MVT::v2f32:
6067   case MVT::v2f64:
6068   case MVT::v4i8:
6069   case MVT::v4i16:
6070   case MVT::v4i32:
6071   case MVT::v4f16:
6072   case MVT::v4f32:
6073   case MVT::v8f16:  // <4 x f16x2>
6074   case MVT::v8bf16: // <4 x bf16x2>
6075   case MVT::v8i16:  // <4 x i16x2>
6076     // This is a "native" vector type
6077     break;
6078   }
6079 
6080   LoadSDNode *LD = cast<LoadSDNode>(N);
6081 
6082   Align Alignment = LD->getAlign();
6083   auto &TD = DAG.getDataLayout();
6084   Align PrefAlign =
6085       TD.getPrefTypeAlign(LD->getMemoryVT().getTypeForEVT(*DAG.getContext()));
6086   if (Alignment < PrefAlign) {
6087     // This load is not sufficiently aligned, so bail out and let this vector
6088     // load be scalarized.  Note that we may still be able to emit smaller
6089     // vector loads.  For example, if we are loading a <4 x float> with an
6090     // alignment of 8, this check will fail but the legalizer will try again
6091     // with 2 x <2 x float>, which will succeed with an alignment of 8.
6092     return;
6093   }
6094 
6095   EVT EltVT = ResVT.getVectorElementType();
6096   unsigned NumElts = ResVT.getVectorNumElements();
6097 
6098   // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
6099   // Therefore, we must ensure the type is legal.  For i1 and i8, we set the
6100   // loaded type to i16 and propagate the "real" type as the memory type.
6101   bool NeedTrunc = false;
6102   if (EltVT.getSizeInBits() < 16) {
6103     EltVT = MVT::i16;
6104     NeedTrunc = true;
6105   }
6106 
6107   unsigned Opcode = 0;
6108   SDVTList LdResVTs;
6109   bool Load16x2 = false;
6110 
6111   switch (NumElts) {
6112   default:
6113     return;
6114   case 2:
6115     Opcode = NVPTXISD::LoadV2;
6116     LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
6117     break;
6118   case 4: {
6119     Opcode = NVPTXISD::LoadV4;
6120     EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
6121     LdResVTs = DAG.getVTList(ListVTs);
6122     break;
6123   }
6124   case 8: {
6125     // v8f16 is a special case. PTX doesn't have ld.v8.f16
6126     // instruction. Instead, we split the vector into v2f16 chunks and
6127     // load them with ld.v4.b32.
6128     assert(Is16bitsType(EltVT.getSimpleVT()) && "Unsupported v8 vector type.");
6129     Load16x2 = true;
6130     Opcode = NVPTXISD::LoadV4;
6131     EVT VVT;
6132     switch (EltVT.getSimpleVT().SimpleTy) {
6133     case MVT::f16:
6134       VVT = MVT::v2f16;
6135       break;
6136     case MVT::bf16:
6137       VVT = MVT::v2bf16;
6138       break;
6139     case MVT::i16:
6140       VVT = MVT::v2i16;
6141       break;
6142     default:
6143       llvm_unreachable("Unsupported v8 vector type.");
6144     }
6145     EVT ListVTs[] = {VVT, VVT, VVT, VVT, MVT::Other};
6146     LdResVTs = DAG.getVTList(ListVTs);
6147     break;
6148   }
6149   }
6150 
6151   // Copy regular operands
6152   SmallVector<SDValue, 8> OtherOps(N->op_begin(), N->op_end());
6153 
6154   // The select routine does not have access to the LoadSDNode instance, so
6155   // pass along the extension information
6156   OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType(), DL));
6157 
6158   SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
6159                                           LD->getMemoryVT(),
6160                                           LD->getMemOperand());
6161 
6162   SmallVector<SDValue, 8> ScalarRes;
6163   if (Load16x2) {
6164     // Split v2f16 subvectors back into individual elements.
6165     NumElts /= 2;
6166     for (unsigned i = 0; i < NumElts; ++i) {
6167       SDValue SubVector = NewLD.getValue(i);
6168       SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SubVector,
6169                                DAG.getIntPtrConstant(0, DL));
6170       SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SubVector,
6171                                DAG.getIntPtrConstant(1, DL));
6172       ScalarRes.push_back(E0);
6173       ScalarRes.push_back(E1);
6174     }
6175   } else {
6176     for (unsigned i = 0; i < NumElts; ++i) {
6177       SDValue Res = NewLD.getValue(i);
6178       if (NeedTrunc)
6179         Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
6180       ScalarRes.push_back(Res);
6181     }
6182   }
6183 
6184   SDValue LoadChain = NewLD.getValue(NumElts);
6185 
6186   SDValue BuildVec = DAG.getBuildVector(ResVT, DL, ScalarRes);
6187 
6188   Results.push_back(BuildVec);
6189   Results.push_back(LoadChain);
6190 }
6191 
6192 static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
6193                                      SmallVectorImpl<SDValue> &Results) {
6194   SDValue Chain = N->getOperand(0);
6195   SDValue Intrin = N->getOperand(1);
6196   SDLoc DL(N);
6197 
6198   // Get the intrinsic ID
6199   unsigned IntrinNo = Intrin.getNode()->getAsZExtVal();
6200   switch (IntrinNo) {
6201   default:
6202     return;
6203   case Intrinsic::nvvm_ldg_global_i:
6204   case Intrinsic::nvvm_ldg_global_f:
6205   case Intrinsic::nvvm_ldg_global_p:
6206   case Intrinsic::nvvm_ldu_global_i:
6207   case Intrinsic::nvvm_ldu_global_f:
6208   case Intrinsic::nvvm_ldu_global_p: {
6209     EVT ResVT = N->getValueType(0);
6210 
6211     if (ResVT.isVector()) {
6212       // Vector LDG/LDU
6213 
6214       unsigned NumElts = ResVT.getVectorNumElements();
6215       EVT EltVT = ResVT.getVectorElementType();
6216 
6217       // Since LDU/LDG are target nodes, we cannot rely on DAG type
6218       // legalization.
6219       // Therefore, we must ensure the type is legal.  For i1 and i8, we set the
6220       // loaded type to i16 and propagate the "real" type as the memory type.
6221       bool NeedTrunc = false;
6222       if (EltVT.getSizeInBits() < 16) {
6223         EltVT = MVT::i16;
6224         NeedTrunc = true;
6225       }
6226 
6227       unsigned Opcode = 0;
6228       SDVTList LdResVTs;
6229 
6230       switch (NumElts) {
6231       default:
6232         return;
6233       case 2:
6234         switch (IntrinNo) {
6235         default:
6236           return;
6237         case Intrinsic::nvvm_ldg_global_i:
6238         case Intrinsic::nvvm_ldg_global_f:
6239         case Intrinsic::nvvm_ldg_global_p:
6240           Opcode = NVPTXISD::LDGV2;
6241           break;
6242         case Intrinsic::nvvm_ldu_global_i:
6243         case Intrinsic::nvvm_ldu_global_f:
6244         case Intrinsic::nvvm_ldu_global_p:
6245           Opcode = NVPTXISD::LDUV2;
6246           break;
6247         }
6248         LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
6249         break;
6250       case 4: {
6251         switch (IntrinNo) {
6252         default:
6253           return;
6254         case Intrinsic::nvvm_ldg_global_i:
6255         case Intrinsic::nvvm_ldg_global_f:
6256         case Intrinsic::nvvm_ldg_global_p:
6257           Opcode = NVPTXISD::LDGV4;
6258           break;
6259         case Intrinsic::nvvm_ldu_global_i:
6260         case Intrinsic::nvvm_ldu_global_f:
6261         case Intrinsic::nvvm_ldu_global_p:
6262           Opcode = NVPTXISD::LDUV4;
6263           break;
6264         }
6265         EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
6266         LdResVTs = DAG.getVTList(ListVTs);
6267         break;
6268       }
6269       }
6270 
6271       SmallVector<SDValue, 8> OtherOps;
6272 
6273       // Copy regular operands
6274 
6275       OtherOps.push_back(Chain); // Chain
6276                                  // Skip operand 1 (intrinsic ID)
6277       // Others
6278       OtherOps.append(N->op_begin() + 2, N->op_end());
6279 
6280       MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
6281 
6282       SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
6283                                               MemSD->getMemoryVT(),
6284                                               MemSD->getMemOperand());
6285 
6286       SmallVector<SDValue, 4> ScalarRes;
6287 
6288       for (unsigned i = 0; i < NumElts; ++i) {
6289         SDValue Res = NewLD.getValue(i);
6290         if (NeedTrunc)
6291           Res =
6292               DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
6293         ScalarRes.push_back(Res);
6294       }
6295 
6296       SDValue LoadChain = NewLD.getValue(NumElts);
6297 
6298       SDValue BuildVec =
6299           DAG.getBuildVector(ResVT, DL, ScalarRes);
6300 
6301       Results.push_back(BuildVec);
6302       Results.push_back(LoadChain);
6303     } else {
6304       // i8 LDG/LDU
6305       assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
6306              "Custom handling of non-i8 ldu/ldg?");
6307 
6308       // Just copy all operands as-is
6309       SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
6310 
6311       // Force output to i16
6312       SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other);
6313 
6314       MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
6315 
6316       // We make sure the memory type is i8, which will be used during isel
6317       // to select the proper instruction.
6318       SDValue NewLD =
6319           DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, Ops,
6320                                   MVT::i8, MemSD->getMemOperand());
6321 
6322       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
6323                                     NewLD.getValue(0)));
6324       Results.push_back(NewLD.getValue(1));
6325     }
6326   }
6327   }
6328 }
6329 
6330 static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,
6331                                    SmallVectorImpl<SDValue> &Results) {
6332   // Change the CopyFromReg to output 2 64-bit results instead of a 128-bit
6333   // result so that it can pass the legalization
6334   SDLoc DL(N);
6335   SDValue Chain = N->getOperand(0);
6336   SDValue Reg = N->getOperand(1);
6337   SDValue Glue = N->getOperand(2);
6338 
6339   assert(Reg.getValueType() == MVT::i128 &&
6340          "Custom lowering for CopyFromReg with 128-bit reg only");
6341   SmallVector<EVT, 4> ResultsType = {MVT::i64, MVT::i64, N->getValueType(1),
6342                                      N->getValueType(2)};
6343   SmallVector<SDValue, 3> NewOps = {Chain, Reg, Glue};
6344 
6345   SDValue NewValue = DAG.getNode(ISD::CopyFromReg, DL, ResultsType, NewOps);
6346   SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128,
6347                              {NewValue.getValue(0), NewValue.getValue(1)});
6348 
6349   Results.push_back(Pair);
6350   Results.push_back(NewValue.getValue(2));
6351   Results.push_back(NewValue.getValue(3));
6352 }
6353 
6354 void NVPTXTargetLowering::ReplaceNodeResults(
6355     SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
6356   switch (N->getOpcode()) {
6357   default:
6358     report_fatal_error("Unhandled custom legalization");
6359   case ISD::LOAD:
6360     ReplaceLoadVector(N, DAG, Results);
6361     return;
6362   case ISD::INTRINSIC_W_CHAIN:
6363     ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
6364     return;
6365   case ISD::CopyFromReg:
6366     ReplaceCopyFromReg_128(N, DAG, Results);
6367     return;
6368   }
6369 }
6370 
6371 NVPTXTargetLowering::AtomicExpansionKind
6372 NVPTXTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
6373   Type *Ty = AI->getValOperand()->getType();
6374 
6375   if (AI->isFloatingPointOperation()) {
6376     if (AI->getOperation() == AtomicRMWInst::BinOp::FAdd) {
6377       if (Ty->isHalfTy() && STI.getSmVersion() >= 70 &&
6378           STI.getPTXVersion() >= 63)
6379         return AtomicExpansionKind::None;
6380       if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 &&
6381           STI.getPTXVersion() >= 78)
6382         return AtomicExpansionKind::None;
6383       if (Ty->isFloatTy())
6384         return AtomicExpansionKind::None;
6385       if (Ty->isDoubleTy() && STI.hasAtomAddF64())
6386         return AtomicExpansionKind::None;
6387     }
6388     return AtomicExpansionKind::CmpXChg;
6389   }
6390 
6391   assert(Ty->isIntegerTy() && "Ty should be integer at this point");
6392   auto ITy = cast<llvm::IntegerType>(Ty);
6393 
6394   switch (AI->getOperation()) {
6395   default:
6396     return AtomicExpansionKind::CmpXChg;
6397   case AtomicRMWInst::BinOp::And:
6398   case AtomicRMWInst::BinOp::Or:
6399   case AtomicRMWInst::BinOp::Xor:
6400   case AtomicRMWInst::BinOp::Xchg:
6401     switch (ITy->getBitWidth()) {
6402     case 8:
6403     case 16:
6404       return AtomicExpansionKind::CmpXChg;
6405     case 32:
6406       return AtomicExpansionKind::None;
6407     case 64:
6408       if (STI.hasAtomBitwise64())
6409         return AtomicExpansionKind::None;
6410       return AtomicExpansionKind::CmpXChg;
6411     default:
6412       llvm_unreachable("unsupported width encountered");
6413     }
6414   case AtomicRMWInst::BinOp::Add:
6415   case AtomicRMWInst::BinOp::Sub:
6416   case AtomicRMWInst::BinOp::Max:
6417   case AtomicRMWInst::BinOp::Min:
6418   case AtomicRMWInst::BinOp::UMax:
6419   case AtomicRMWInst::BinOp::UMin:
6420     switch (ITy->getBitWidth()) {
6421     case 8:
6422     case 16:
6423       return AtomicExpansionKind::CmpXChg;
6424     case 32:
6425       return AtomicExpansionKind::None;
6426     case 64:
6427       if (STI.hasAtomMinMax64())
6428         return AtomicExpansionKind::None;
6429       return AtomicExpansionKind::CmpXChg;
6430     default:
6431       llvm_unreachable("unsupported width encountered");
6432     }
6433   }
6434 
6435   return AtomicExpansionKind::CmpXChg;
6436 }
6437 
6438 // Pin NVPTXTargetObjectFile's vtables to this file.
6439 NVPTXTargetObjectFile::~NVPTXTargetObjectFile() = default;
6440 
6441 MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal(
6442     const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
6443   return getDataSection();
6444 }
6445