xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp (revision c57c26179033f64c2011a2d2a904ee3fa62e826a)
1 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the SelectionDAG class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/CodeGen/SelectionDAG.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/APSInt.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/BitVector.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/ADT/FoldingSet.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/Analysis/AliasAnalysis.h"
27 #include "llvm/Analysis/MemoryLocation.h"
28 #include "llvm/Analysis/ValueTracking.h"
29 #include "llvm/Analysis/VectorUtils.h"
30 #include "llvm/BinaryFormat/Dwarf.h"
31 #include "llvm/CodeGen/Analysis.h"
32 #include "llvm/CodeGen/FunctionLoweringInfo.h"
33 #include "llvm/CodeGen/ISDOpcodes.h"
34 #include "llvm/CodeGen/MachineBasicBlock.h"
35 #include "llvm/CodeGen/MachineConstantPool.h"
36 #include "llvm/CodeGen/MachineFrameInfo.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineMemOperand.h"
39 #include "llvm/CodeGen/MachineValueType.h"
40 #include "llvm/CodeGen/RuntimeLibcalls.h"
41 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
42 #include "llvm/CodeGen/SelectionDAGNodes.h"
43 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
44 #include "llvm/CodeGen/TargetFrameLowering.h"
45 #include "llvm/CodeGen/TargetLowering.h"
46 #include "llvm/CodeGen/TargetRegisterInfo.h"
47 #include "llvm/CodeGen/TargetSubtargetInfo.h"
48 #include "llvm/CodeGen/ValueTypes.h"
49 #include "llvm/IR/Constant.h"
50 #include "llvm/IR/ConstantRange.h"
51 #include "llvm/IR/Constants.h"
52 #include "llvm/IR/DataLayout.h"
53 #include "llvm/IR/DebugInfoMetadata.h"
54 #include "llvm/IR/DebugLoc.h"
55 #include "llvm/IR/DerivedTypes.h"
56 #include "llvm/IR/Function.h"
57 #include "llvm/IR/GlobalValue.h"
58 #include "llvm/IR/Metadata.h"
59 #include "llvm/IR/Type.h"
60 #include "llvm/Support/Casting.h"
61 #include "llvm/Support/CodeGen.h"
62 #include "llvm/Support/Compiler.h"
63 #include "llvm/Support/Debug.h"
64 #include "llvm/Support/ErrorHandling.h"
65 #include "llvm/Support/KnownBits.h"
66 #include "llvm/Support/MathExtras.h"
67 #include "llvm/Support/Mutex.h"
68 #include "llvm/Support/raw_ostream.h"
69 #include "llvm/Target/TargetMachine.h"
70 #include "llvm/Target/TargetOptions.h"
71 #include "llvm/TargetParser/Triple.h"
72 #include "llvm/Transforms/Utils/SizeOpts.h"
73 #include <algorithm>
74 #include <cassert>
75 #include <cstdint>
76 #include <cstdlib>
77 #include <limits>
78 #include <set>
79 #include <string>
80 #include <utility>
81 #include <vector>
82 
83 using namespace llvm;
84 
85 /// makeVTList - Return an instance of the SDVTList struct initialized with the
86 /// specified members.
87 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
88   SDVTList Res = {VTs, NumVTs};
89   return Res;
90 }
91 
92 // Default null implementations of the callbacks.
93 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
94 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
95 void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
96 
97 void SelectionDAG::DAGNodeDeletedListener::anchor() {}
98 void SelectionDAG::DAGNodeInsertedListener::anchor() {}
99 
100 #define DEBUG_TYPE "selectiondag"
101 
102 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
103        cl::Hidden, cl::init(true),
104        cl::desc("Gang up loads and stores generated by inlining of memcpy"));
105 
106 static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
107        cl::desc("Number limit for gluing ld/st of memcpy."),
108        cl::Hidden, cl::init(0));
109 
110 static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
111   LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
112 }
113 
114 //===----------------------------------------------------------------------===//
115 //                              ConstantFPSDNode Class
116 //===----------------------------------------------------------------------===//
117 
118 /// isExactlyValue - We don't rely on operator== working on double values, as
119 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
120 /// As such, this method can be used to do an exact bit-for-bit comparison of
121 /// two floating point values.
122 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
123   return getValueAPF().bitwiseIsEqual(V);
124 }
125 
126 bool ConstantFPSDNode::isValueValidForType(EVT VT,
127                                            const APFloat& Val) {
128   assert(VT.isFloatingPoint() && "Can only convert between FP types");
129 
130   // convert modifies in place, so make a copy.
131   APFloat Val2 = APFloat(Val);
132   bool losesInfo;
133   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
134                       APFloat::rmNearestTiesToEven,
135                       &losesInfo);
136   return !losesInfo;
137 }
138 
139 //===----------------------------------------------------------------------===//
140 //                              ISD Namespace
141 //===----------------------------------------------------------------------===//
142 
143 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
144   if (N->getOpcode() == ISD::SPLAT_VECTOR) {
145     unsigned EltSize =
146         N->getValueType(0).getVectorElementType().getSizeInBits();
147     if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
148       SplatVal = Op0->getAPIntValue().trunc(EltSize);
149       return true;
150     }
151     if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
152       SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize);
153       return true;
154     }
155   }
156 
157   auto *BV = dyn_cast<BuildVectorSDNode>(N);
158   if (!BV)
159     return false;
160 
161   APInt SplatUndef;
162   unsigned SplatBitSize;
163   bool HasUndefs;
164   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
165   // Endianness does not matter here. We are checking for a splat given the
166   // element size of the vector, and if we find such a splat for little endian
167   // layout, then that should be valid also for big endian (as the full vector
168   // size is known to be a multiple of the element size).
169   const bool IsBigEndian = false;
170   return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
171                              EltSize, IsBigEndian) &&
172          EltSize == SplatBitSize;
173 }
174 
175 // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
176 // specializations of the more general isConstantSplatVector()?
177 
178 bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
179   // Look through a bit convert.
180   while (N->getOpcode() == ISD::BITCAST)
181     N = N->getOperand(0).getNode();
182 
183   if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
184     APInt SplatVal;
185     return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
186   }
187 
188   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
189 
190   unsigned i = 0, e = N->getNumOperands();
191 
192   // Skip over all of the undef values.
193   while (i != e && N->getOperand(i).isUndef())
194     ++i;
195 
196   // Do not accept an all-undef vector.
197   if (i == e) return false;
198 
199   // Do not accept build_vectors that aren't all constants or which have non-~0
200   // elements. We have to be a bit careful here, as the type of the constant
201   // may not be the same as the type of the vector elements due to type
202   // legalization (the elements are promoted to a legal type for the target and
203   // a vector of a type may be legal when the base element type is not).
204   // We only want to check enough bits to cover the vector elements, because
205   // we care if the resultant vector is all ones, not whether the individual
206   // constants are.
207   SDValue NotZero = N->getOperand(i);
208   unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
209   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
210     if (CN->getAPIntValue().countr_one() < EltSize)
211       return false;
212   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
213     if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
214       return false;
215   } else
216     return false;
217 
218   // Okay, we have at least one ~0 value, check to see if the rest match or are
219   // undefs. Even with the above element type twiddling, this should be OK, as
220   // the same type legalization should have applied to all the elements.
221   for (++i; i != e; ++i)
222     if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
223       return false;
224   return true;
225 }
226 
227 bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
228   // Look through a bit convert.
229   while (N->getOpcode() == ISD::BITCAST)
230     N = N->getOperand(0).getNode();
231 
232   if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
233     APInt SplatVal;
234     return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
235   }
236 
237   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
238 
239   bool IsAllUndef = true;
240   for (const SDValue &Op : N->op_values()) {
241     if (Op.isUndef())
242       continue;
243     IsAllUndef = false;
244     // Do not accept build_vectors that aren't all constants or which have non-0
245     // elements. We have to be a bit careful here, as the type of the constant
246     // may not be the same as the type of the vector elements due to type
247     // legalization (the elements are promoted to a legal type for the target
248     // and a vector of a type may be legal when the base element type is not).
249     // We only want to check enough bits to cover the vector elements, because
250     // we care if the resultant vector is all zeros, not whether the individual
251     // constants are.
252     unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
253     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
254       if (CN->getAPIntValue().countr_zero() < EltSize)
255         return false;
256     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
257       if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
258         return false;
259     } else
260       return false;
261   }
262 
263   // Do not accept an all-undef vector.
264   if (IsAllUndef)
265     return false;
266   return true;
267 }
268 
269 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
270   return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
271 }
272 
273 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
274   return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
275 }
276 
277 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
278   if (N->getOpcode() != ISD::BUILD_VECTOR)
279     return false;
280 
281   for (const SDValue &Op : N->op_values()) {
282     if (Op.isUndef())
283       continue;
284     if (!isa<ConstantSDNode>(Op))
285       return false;
286   }
287   return true;
288 }
289 
290 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
291   if (N->getOpcode() != ISD::BUILD_VECTOR)
292     return false;
293 
294   for (const SDValue &Op : N->op_values()) {
295     if (Op.isUndef())
296       continue;
297     if (!isa<ConstantFPSDNode>(Op))
298       return false;
299   }
300   return true;
301 }
302 
303 bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
304                              bool Signed) {
305   assert(N->getValueType(0).isVector() && "Expected a vector!");
306 
307   unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
308   if (EltSize <= NewEltSize)
309     return false;
310 
311   if (N->getOpcode() == ISD::ZERO_EXTEND) {
312     return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
313             NewEltSize) &&
314            !Signed;
315   }
316   if (N->getOpcode() == ISD::SIGN_EXTEND) {
317     return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
318             NewEltSize) &&
319            Signed;
320   }
321   if (N->getOpcode() != ISD::BUILD_VECTOR)
322     return false;
323 
324   for (const SDValue &Op : N->op_values()) {
325     if (Op.isUndef())
326       continue;
327     if (!isa<ConstantSDNode>(Op))
328       return false;
329 
330     APInt C = Op->getAsAPIntVal().trunc(EltSize);
331     if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)
332       return false;
333     if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)
334       return false;
335   }
336 
337   return true;
338 }
339 
340 bool ISD::allOperandsUndef(const SDNode *N) {
341   // Return false if the node has no operands.
342   // This is "logically inconsistent" with the definition of "all" but
343   // is probably the desired behavior.
344   if (N->getNumOperands() == 0)
345     return false;
346   return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
347 }
348 
349 bool ISD::isFreezeUndef(const SDNode *N) {
350   return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();
351 }
352 
353 template <typename ConstNodeType>
354 bool ISD::matchUnaryPredicateImpl(SDValue Op,
355                                   std::function<bool(ConstNodeType *)> Match,
356                                   bool AllowUndefs) {
357   // FIXME: Add support for scalar UNDEF cases?
358   if (auto *C = dyn_cast<ConstNodeType>(Op))
359     return Match(C);
360 
361   // FIXME: Add support for vector UNDEF cases?
362   if (ISD::BUILD_VECTOR != Op.getOpcode() &&
363       ISD::SPLAT_VECTOR != Op.getOpcode())
364     return false;
365 
366   EVT SVT = Op.getValueType().getScalarType();
367   for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
368     if (AllowUndefs && Op.getOperand(i).isUndef()) {
369       if (!Match(nullptr))
370         return false;
371       continue;
372     }
373 
374     auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
375     if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
376       return false;
377   }
378   return true;
379 }
380 // Build used template types.
381 template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
382     SDValue, std::function<bool(ConstantSDNode *)>, bool);
383 template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
384     SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
385 
386 bool ISD::matchBinaryPredicate(
387     SDValue LHS, SDValue RHS,
388     std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
389     bool AllowUndefs, bool AllowTypeMismatch) {
390   if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
391     return false;
392 
393   // TODO: Add support for scalar UNDEF cases?
394   if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
395     if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
396       return Match(LHSCst, RHSCst);
397 
398   // TODO: Add support for vector UNDEF cases?
399   if (LHS.getOpcode() != RHS.getOpcode() ||
400       (LHS.getOpcode() != ISD::BUILD_VECTOR &&
401        LHS.getOpcode() != ISD::SPLAT_VECTOR))
402     return false;
403 
404   EVT SVT = LHS.getValueType().getScalarType();
405   for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
406     SDValue LHSOp = LHS.getOperand(i);
407     SDValue RHSOp = RHS.getOperand(i);
408     bool LHSUndef = AllowUndefs && LHSOp.isUndef();
409     bool RHSUndef = AllowUndefs && RHSOp.isUndef();
410     auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
411     auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
412     if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
413       return false;
414     if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
415                                LHSOp.getValueType() != RHSOp.getValueType()))
416       return false;
417     if (!Match(LHSCst, RHSCst))
418       return false;
419   }
420   return true;
421 }
422 
423 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
424   switch (VecReduceOpcode) {
425   default:
426     llvm_unreachable("Expected VECREDUCE opcode");
427   case ISD::VECREDUCE_FADD:
428   case ISD::VECREDUCE_SEQ_FADD:
429   case ISD::VP_REDUCE_FADD:
430   case ISD::VP_REDUCE_SEQ_FADD:
431     return ISD::FADD;
432   case ISD::VECREDUCE_FMUL:
433   case ISD::VECREDUCE_SEQ_FMUL:
434   case ISD::VP_REDUCE_FMUL:
435   case ISD::VP_REDUCE_SEQ_FMUL:
436     return ISD::FMUL;
437   case ISD::VECREDUCE_ADD:
438   case ISD::VP_REDUCE_ADD:
439     return ISD::ADD;
440   case ISD::VECREDUCE_MUL:
441   case ISD::VP_REDUCE_MUL:
442     return ISD::MUL;
443   case ISD::VECREDUCE_AND:
444   case ISD::VP_REDUCE_AND:
445     return ISD::AND;
446   case ISD::VECREDUCE_OR:
447   case ISD::VP_REDUCE_OR:
448     return ISD::OR;
449   case ISD::VECREDUCE_XOR:
450   case ISD::VP_REDUCE_XOR:
451     return ISD::XOR;
452   case ISD::VECREDUCE_SMAX:
453   case ISD::VP_REDUCE_SMAX:
454     return ISD::SMAX;
455   case ISD::VECREDUCE_SMIN:
456   case ISD::VP_REDUCE_SMIN:
457     return ISD::SMIN;
458   case ISD::VECREDUCE_UMAX:
459   case ISD::VP_REDUCE_UMAX:
460     return ISD::UMAX;
461   case ISD::VECREDUCE_UMIN:
462   case ISD::VP_REDUCE_UMIN:
463     return ISD::UMIN;
464   case ISD::VECREDUCE_FMAX:
465   case ISD::VP_REDUCE_FMAX:
466     return ISD::FMAXNUM;
467   case ISD::VECREDUCE_FMIN:
468   case ISD::VP_REDUCE_FMIN:
469     return ISD::FMINNUM;
470   case ISD::VECREDUCE_FMAXIMUM:
471     return ISD::FMAXIMUM;
472   case ISD::VECREDUCE_FMINIMUM:
473     return ISD::FMINIMUM;
474   }
475 }
476 
477 bool ISD::isVPOpcode(unsigned Opcode) {
478   switch (Opcode) {
479   default:
480     return false;
481 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...)                                    \
482   case ISD::VPSD:                                                              \
483     return true;
484 #include "llvm/IR/VPIntrinsics.def"
485   }
486 }
487 
488 bool ISD::isVPBinaryOp(unsigned Opcode) {
489   switch (Opcode) {
490   default:
491     break;
492 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
493 #define VP_PROPERTY_BINARYOP return true;
494 #define END_REGISTER_VP_SDNODE(VPSD) break;
495 #include "llvm/IR/VPIntrinsics.def"
496   }
497   return false;
498 }
499 
500 bool ISD::isVPReduction(unsigned Opcode) {
501   switch (Opcode) {
502   default:
503     break;
504 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
505 #define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
506 #define END_REGISTER_VP_SDNODE(VPSD) break;
507 #include "llvm/IR/VPIntrinsics.def"
508   }
509   return false;
510 }
511 
512 /// The operand position of the vector mask.
513 std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
514   switch (Opcode) {
515   default:
516     return std::nullopt;
517 #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...)         \
518   case ISD::VPSD:                                                              \
519     return MASKPOS;
520 #include "llvm/IR/VPIntrinsics.def"
521   }
522 }
523 
524 /// The operand position of the explicit vector length parameter.
525 std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
526   switch (Opcode) {
527   default:
528     return std::nullopt;
529 #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS)      \
530   case ISD::VPSD:                                                              \
531     return EVLPOS;
532 #include "llvm/IR/VPIntrinsics.def"
533   }
534 }
535 
536 std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
537                                                 bool hasFPExcept) {
538   // FIXME: Return strict opcodes in case of fp exceptions.
539   switch (VPOpcode) {
540   default:
541     return std::nullopt;
542 #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
543 #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
544 #define END_REGISTER_VP_SDNODE(VPOPC) break;
545 #include "llvm/IR/VPIntrinsics.def"
546   }
547   return std::nullopt;
548 }
549 
550 unsigned ISD::getVPForBaseOpcode(unsigned Opcode) {
551   switch (Opcode) {
552   default:
553     llvm_unreachable("can not translate this Opcode to VP.");
554 #define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
555 #define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
556 #define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
557 #include "llvm/IR/VPIntrinsics.def"
558   }
559 }
560 
561 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
562   switch (ExtType) {
563   case ISD::EXTLOAD:
564     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
565   case ISD::SEXTLOAD:
566     return ISD::SIGN_EXTEND;
567   case ISD::ZEXTLOAD:
568     return ISD::ZERO_EXTEND;
569   default:
570     break;
571   }
572 
573   llvm_unreachable("Invalid LoadExtType");
574 }
575 
576 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
577   // To perform this operation, we just need to swap the L and G bits of the
578   // operation.
579   unsigned OldL = (Operation >> 2) & 1;
580   unsigned OldG = (Operation >> 1) & 1;
581   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
582                        (OldL << 1) |       // New G bit
583                        (OldG << 2));       // New L bit.
584 }
585 
586 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
587   unsigned Operation = Op;
588   if (isIntegerLike)
589     Operation ^= 7;   // Flip L, G, E bits, but not U.
590   else
591     Operation ^= 15;  // Flip all of the condition bits.
592 
593   if (Operation > ISD::SETTRUE2)
594     Operation &= ~8;  // Don't let N and U bits get set.
595 
596   return ISD::CondCode(Operation);
597 }
598 
599 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
600   return getSetCCInverseImpl(Op, Type.isInteger());
601 }
602 
603 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
604                                                bool isIntegerLike) {
605   return getSetCCInverseImpl(Op, isIntegerLike);
606 }
607 
608 /// For an integer comparison, return 1 if the comparison is a signed operation
609 /// and 2 if the result is an unsigned comparison. Return zero if the operation
610 /// does not depend on the sign of the input (setne and seteq).
611 static int isSignedOp(ISD::CondCode Opcode) {
612   switch (Opcode) {
613   default: llvm_unreachable("Illegal integer setcc operation!");
614   case ISD::SETEQ:
615   case ISD::SETNE: return 0;
616   case ISD::SETLT:
617   case ISD::SETLE:
618   case ISD::SETGT:
619   case ISD::SETGE: return 1;
620   case ISD::SETULT:
621   case ISD::SETULE:
622   case ISD::SETUGT:
623   case ISD::SETUGE: return 2;
624   }
625 }
626 
627 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
628                                        EVT Type) {
629   bool IsInteger = Type.isInteger();
630   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
631     // Cannot fold a signed integer setcc with an unsigned integer setcc.
632     return ISD::SETCC_INVALID;
633 
634   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
635 
636   // If the N and U bits get set, then the resultant comparison DOES suddenly
637   // care about orderedness, and it is true when ordered.
638   if (Op > ISD::SETTRUE2)
639     Op &= ~16;     // Clear the U bit if the N bit is set.
640 
641   // Canonicalize illegal integer setcc's.
642   if (IsInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
643     Op = ISD::SETNE;
644 
645   return ISD::CondCode(Op);
646 }
647 
648 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
649                                         EVT Type) {
650   bool IsInteger = Type.isInteger();
651   if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
652     // Cannot fold a signed setcc with an unsigned setcc.
653     return ISD::SETCC_INVALID;
654 
655   // Combine all of the condition bits.
656   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
657 
658   // Canonicalize illegal integer setcc's.
659   if (IsInteger) {
660     switch (Result) {
661     default: break;
662     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
663     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
664     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
665     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
666     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
667     }
668   }
669 
670   return Result;
671 }
672 
673 //===----------------------------------------------------------------------===//
674 //                           SDNode Profile Support
675 //===----------------------------------------------------------------------===//
676 
677 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
678 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
679   ID.AddInteger(OpC);
680 }
681 
682 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
683 /// solely with their pointer.
684 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
685   ID.AddPointer(VTList.VTs);
686 }
687 
688 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
689 static void AddNodeIDOperands(FoldingSetNodeID &ID,
690                               ArrayRef<SDValue> Ops) {
691   for (const auto &Op : Ops) {
692     ID.AddPointer(Op.getNode());
693     ID.AddInteger(Op.getResNo());
694   }
695 }
696 
697 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
698 static void AddNodeIDOperands(FoldingSetNodeID &ID,
699                               ArrayRef<SDUse> Ops) {
700   for (const auto &Op : Ops) {
701     ID.AddPointer(Op.getNode());
702     ID.AddInteger(Op.getResNo());
703   }
704 }
705 
706 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
707                           SDVTList VTList, ArrayRef<SDValue> OpList) {
708   AddNodeIDOpcode(ID, OpC);
709   AddNodeIDValueTypes(ID, VTList);
710   AddNodeIDOperands(ID, OpList);
711 }
712 
713 /// If this is an SDNode with special info, add this info to the NodeID data.
714 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
715   switch (N->getOpcode()) {
716   case ISD::TargetExternalSymbol:
717   case ISD::ExternalSymbol:
718   case ISD::MCSymbol:
719     llvm_unreachable("Should only be used on nodes with operands");
720   default: break;  // Normal nodes don't need extra info.
721   case ISD::TargetConstant:
722   case ISD::Constant: {
723     const ConstantSDNode *C = cast<ConstantSDNode>(N);
724     ID.AddPointer(C->getConstantIntValue());
725     ID.AddBoolean(C->isOpaque());
726     break;
727   }
728   case ISD::TargetConstantFP:
729   case ISD::ConstantFP:
730     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
731     break;
732   case ISD::TargetGlobalAddress:
733   case ISD::GlobalAddress:
734   case ISD::TargetGlobalTLSAddress:
735   case ISD::GlobalTLSAddress: {
736     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
737     ID.AddPointer(GA->getGlobal());
738     ID.AddInteger(GA->getOffset());
739     ID.AddInteger(GA->getTargetFlags());
740     break;
741   }
742   case ISD::BasicBlock:
743     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
744     break;
745   case ISD::Register:
746     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
747     break;
748   case ISD::RegisterMask:
749     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
750     break;
751   case ISD::SRCVALUE:
752     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
753     break;
754   case ISD::FrameIndex:
755   case ISD::TargetFrameIndex:
756     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
757     break;
758   case ISD::LIFETIME_START:
759   case ISD::LIFETIME_END:
760     if (cast<LifetimeSDNode>(N)->hasOffset()) {
761       ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
762       ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
763     }
764     break;
765   case ISD::PSEUDO_PROBE:
766     ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
767     ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
768     ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
769     break;
770   case ISD::JumpTable:
771   case ISD::TargetJumpTable:
772     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
773     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
774     break;
775   case ISD::ConstantPool:
776   case ISD::TargetConstantPool: {
777     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
778     ID.AddInteger(CP->getAlign().value());
779     ID.AddInteger(CP->getOffset());
780     if (CP->isMachineConstantPoolEntry())
781       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
782     else
783       ID.AddPointer(CP->getConstVal());
784     ID.AddInteger(CP->getTargetFlags());
785     break;
786   }
787   case ISD::TargetIndex: {
788     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
789     ID.AddInteger(TI->getIndex());
790     ID.AddInteger(TI->getOffset());
791     ID.AddInteger(TI->getTargetFlags());
792     break;
793   }
794   case ISD::LOAD: {
795     const LoadSDNode *LD = cast<LoadSDNode>(N);
796     ID.AddInteger(LD->getMemoryVT().getRawBits());
797     ID.AddInteger(LD->getRawSubclassData());
798     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
799     ID.AddInteger(LD->getMemOperand()->getFlags());
800     break;
801   }
802   case ISD::STORE: {
803     const StoreSDNode *ST = cast<StoreSDNode>(N);
804     ID.AddInteger(ST->getMemoryVT().getRawBits());
805     ID.AddInteger(ST->getRawSubclassData());
806     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
807     ID.AddInteger(ST->getMemOperand()->getFlags());
808     break;
809   }
810   case ISD::VP_LOAD: {
811     const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
812     ID.AddInteger(ELD->getMemoryVT().getRawBits());
813     ID.AddInteger(ELD->getRawSubclassData());
814     ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
815     ID.AddInteger(ELD->getMemOperand()->getFlags());
816     break;
817   }
818   case ISD::VP_STORE: {
819     const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
820     ID.AddInteger(EST->getMemoryVT().getRawBits());
821     ID.AddInteger(EST->getRawSubclassData());
822     ID.AddInteger(EST->getPointerInfo().getAddrSpace());
823     ID.AddInteger(EST->getMemOperand()->getFlags());
824     break;
825   }
826   case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
827     const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(N);
828     ID.AddInteger(SLD->getMemoryVT().getRawBits());
829     ID.AddInteger(SLD->getRawSubclassData());
830     ID.AddInteger(SLD->getPointerInfo().getAddrSpace());
831     break;
832   }
833   case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
834     const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
835     ID.AddInteger(SST->getMemoryVT().getRawBits());
836     ID.AddInteger(SST->getRawSubclassData());
837     ID.AddInteger(SST->getPointerInfo().getAddrSpace());
838     break;
839   }
840   case ISD::VP_GATHER: {
841     const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);
842     ID.AddInteger(EG->getMemoryVT().getRawBits());
843     ID.AddInteger(EG->getRawSubclassData());
844     ID.AddInteger(EG->getPointerInfo().getAddrSpace());
845     ID.AddInteger(EG->getMemOperand()->getFlags());
846     break;
847   }
848   case ISD::VP_SCATTER: {
849     const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);
850     ID.AddInteger(ES->getMemoryVT().getRawBits());
851     ID.AddInteger(ES->getRawSubclassData());
852     ID.AddInteger(ES->getPointerInfo().getAddrSpace());
853     ID.AddInteger(ES->getMemOperand()->getFlags());
854     break;
855   }
856   case ISD::MLOAD: {
857     const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
858     ID.AddInteger(MLD->getMemoryVT().getRawBits());
859     ID.AddInteger(MLD->getRawSubclassData());
860     ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
861     ID.AddInteger(MLD->getMemOperand()->getFlags());
862     break;
863   }
864   case ISD::MSTORE: {
865     const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
866     ID.AddInteger(MST->getMemoryVT().getRawBits());
867     ID.AddInteger(MST->getRawSubclassData());
868     ID.AddInteger(MST->getPointerInfo().getAddrSpace());
869     ID.AddInteger(MST->getMemOperand()->getFlags());
870     break;
871   }
872   case ISD::MGATHER: {
873     const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
874     ID.AddInteger(MG->getMemoryVT().getRawBits());
875     ID.AddInteger(MG->getRawSubclassData());
876     ID.AddInteger(MG->getPointerInfo().getAddrSpace());
877     ID.AddInteger(MG->getMemOperand()->getFlags());
878     break;
879   }
880   case ISD::MSCATTER: {
881     const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
882     ID.AddInteger(MS->getMemoryVT().getRawBits());
883     ID.AddInteger(MS->getRawSubclassData());
884     ID.AddInteger(MS->getPointerInfo().getAddrSpace());
885     ID.AddInteger(MS->getMemOperand()->getFlags());
886     break;
887   }
888   case ISD::ATOMIC_CMP_SWAP:
889   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
890   case ISD::ATOMIC_SWAP:
891   case ISD::ATOMIC_LOAD_ADD:
892   case ISD::ATOMIC_LOAD_SUB:
893   case ISD::ATOMIC_LOAD_AND:
894   case ISD::ATOMIC_LOAD_CLR:
895   case ISD::ATOMIC_LOAD_OR:
896   case ISD::ATOMIC_LOAD_XOR:
897   case ISD::ATOMIC_LOAD_NAND:
898   case ISD::ATOMIC_LOAD_MIN:
899   case ISD::ATOMIC_LOAD_MAX:
900   case ISD::ATOMIC_LOAD_UMIN:
901   case ISD::ATOMIC_LOAD_UMAX:
902   case ISD::ATOMIC_LOAD:
903   case ISD::ATOMIC_STORE: {
904     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
905     ID.AddInteger(AT->getMemoryVT().getRawBits());
906     ID.AddInteger(AT->getRawSubclassData());
907     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
908     ID.AddInteger(AT->getMemOperand()->getFlags());
909     break;
910   }
911   case ISD::VECTOR_SHUFFLE: {
912     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
913     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
914          i != e; ++i)
915       ID.AddInteger(SVN->getMaskElt(i));
916     break;
917   }
918   case ISD::TargetBlockAddress:
919   case ISD::BlockAddress: {
920     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
921     ID.AddPointer(BA->getBlockAddress());
922     ID.AddInteger(BA->getOffset());
923     ID.AddInteger(BA->getTargetFlags());
924     break;
925   }
926   case ISD::AssertAlign:
927     ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());
928     break;
929   case ISD::PREFETCH:
930   case ISD::INTRINSIC_VOID:
931   case ISD::INTRINSIC_W_CHAIN:
932     // Handled by MemIntrinsicSDNode check after the switch.
933     break;
934   } // end switch (N->getOpcode())
935 
936   // MemIntrinsic nodes could also have subclass data, address spaces, and flags
937   // to check.
938   if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {
939     ID.AddInteger(MN->getRawSubclassData());
940     ID.AddInteger(MN->getPointerInfo().getAddrSpace());
941     ID.AddInteger(MN->getMemOperand()->getFlags());
942     ID.AddInteger(MN->getMemoryVT().getRawBits());
943   }
944 }
945 
946 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
947 /// data.
948 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
949   AddNodeIDOpcode(ID, N->getOpcode());
950   // Add the return value info.
951   AddNodeIDValueTypes(ID, N->getVTList());
952   // Add the operand info.
953   AddNodeIDOperands(ID, N->ops());
954 
955   // Handle SDNode leafs with special info.
956   AddNodeIDCustom(ID, N);
957 }
958 
959 //===----------------------------------------------------------------------===//
960 //                              SelectionDAG Class
961 //===----------------------------------------------------------------------===//
962 
963 /// doNotCSE - Return true if CSE should not be performed for this node.
964 static bool doNotCSE(SDNode *N) {
965   if (N->getValueType(0) == MVT::Glue)
966     return true; // Never CSE anything that produces a glue result.
967 
968   switch (N->getOpcode()) {
969   default: break;
970   case ISD::HANDLENODE:
971   case ISD::EH_LABEL:
972     return true;   // Never CSE these nodes.
973   }
974 
975   // Check that remaining values produced are not flags.
976   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
977     if (N->getValueType(i) == MVT::Glue)
978       return true; // Never CSE anything that produces a glue result.
979 
980   return false;
981 }
982 
983 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
984 /// SelectionDAG.
985 void SelectionDAG::RemoveDeadNodes() {
986   // Create a dummy node (which is not added to allnodes), that adds a reference
987   // to the root node, preventing it from being deleted.
988   HandleSDNode Dummy(getRoot());
989 
990   SmallVector<SDNode*, 128> DeadNodes;
991 
992   // Add all obviously-dead nodes to the DeadNodes worklist.
993   for (SDNode &Node : allnodes())
994     if (Node.use_empty())
995       DeadNodes.push_back(&Node);
996 
997   RemoveDeadNodes(DeadNodes);
998 
999   // If the root changed (e.g. it was a dead load, update the root).
1000   setRoot(Dummy.getValue());
1001 }
1002 
1003 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
1004 /// given list, and any nodes that become unreachable as a result.
1005 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1006 
1007   // Process the worklist, deleting the nodes and adding their uses to the
1008   // worklist.
1009   while (!DeadNodes.empty()) {
1010     SDNode *N = DeadNodes.pop_back_val();
1011     // Skip to next node if we've already managed to delete the node. This could
1012     // happen if replacing a node causes a node previously added to the node to
1013     // be deleted.
1014     if (N->getOpcode() == ISD::DELETED_NODE)
1015       continue;
1016 
1017     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1018       DUL->NodeDeleted(N, nullptr);
1019 
1020     // Take the node out of the appropriate CSE map.
1021     RemoveNodeFromCSEMaps(N);
1022 
1023     // Next, brutally remove the operand list.  This is safe to do, as there are
1024     // no cycles in the graph.
1025     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1026       SDUse &Use = *I++;
1027       SDNode *Operand = Use.getNode();
1028       Use.set(SDValue());
1029 
1030       // Now that we removed this operand, see if there are no uses of it left.
1031       if (Operand->use_empty())
1032         DeadNodes.push_back(Operand);
1033     }
1034 
1035     DeallocateNode(N);
1036   }
1037 }
1038 
1039 void SelectionDAG::RemoveDeadNode(SDNode *N){
1040   SmallVector<SDNode*, 16> DeadNodes(1, N);
1041 
1042   // Create a dummy node that adds a reference to the root node, preventing
1043   // it from being deleted.  (This matters if the root is an operand of the
1044   // dead node.)
1045   HandleSDNode Dummy(getRoot());
1046 
1047   RemoveDeadNodes(DeadNodes);
1048 }
1049 
1050 void SelectionDAG::DeleteNode(SDNode *N) {
1051   // First take this out of the appropriate CSE map.
1052   RemoveNodeFromCSEMaps(N);
1053 
1054   // Finally, remove uses due to operands of this node, remove from the
1055   // AllNodes list, and delete the node.
1056   DeleteNodeNotInCSEMaps(N);
1057 }
1058 
1059 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1060   assert(N->getIterator() != AllNodes.begin() &&
1061          "Cannot delete the entry node!");
1062   assert(N->use_empty() && "Cannot delete a node that is not dead!");
1063 
1064   // Drop all of the operands and decrement used node's use counts.
1065   N->DropOperands();
1066 
1067   DeallocateNode(N);
1068 }
1069 
1070 void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1071   assert(!(V->isVariadic() && isParameter));
1072   if (isParameter)
1073     ByvalParmDbgValues.push_back(V);
1074   else
1075     DbgValues.push_back(V);
1076   for (const SDNode *Node : V->getSDNodes())
1077     if (Node)
1078       DbgValMap[Node].push_back(V);
1079 }
1080 
1081 void SDDbgInfo::erase(const SDNode *Node) {
1082   DbgValMapType::iterator I = DbgValMap.find(Node);
1083   if (I == DbgValMap.end())
1084     return;
1085   for (auto &Val: I->second)
1086     Val->setIsInvalidated();
1087   DbgValMap.erase(I);
1088 }
1089 
1090 void SelectionDAG::DeallocateNode(SDNode *N) {
1091   // If we have operands, deallocate them.
1092   removeOperands(N);
1093 
1094   NodeAllocator.Deallocate(AllNodes.remove(N));
1095 
1096   // Set the opcode to DELETED_NODE to help catch bugs when node
1097   // memory is reallocated.
1098   // FIXME: There are places in SDag that have grown a dependency on the opcode
1099   // value in the released node.
1100   __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1101   N->NodeType = ISD::DELETED_NODE;
1102 
1103   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1104   // them and forget about that node.
1105   DbgInfo->erase(N);
1106 
1107   // Invalidate extra info.
1108   SDEI.erase(N);
1109 }
1110 
1111 #ifndef NDEBUG
1112 /// VerifySDNode - Check the given SDNode.  Aborts if it is invalid.
1113 static void VerifySDNode(SDNode *N) {
1114   switch (N->getOpcode()) {
1115   default:
1116     break;
1117   case ISD::BUILD_PAIR: {
1118     EVT VT = N->getValueType(0);
1119     assert(N->getNumValues() == 1 && "Too many results!");
1120     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1121            "Wrong return type!");
1122     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1123     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1124            "Mismatched operand types!");
1125     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1126            "Wrong operand type!");
1127     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1128            "Wrong return type size");
1129     break;
1130   }
1131   case ISD::BUILD_VECTOR: {
1132     assert(N->getNumValues() == 1 && "Too many results!");
1133     assert(N->getValueType(0).isVector() && "Wrong return type!");
1134     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1135            "Wrong number of operands!");
1136     EVT EltVT = N->getValueType(0).getVectorElementType();
1137     for (const SDUse &Op : N->ops()) {
1138       assert((Op.getValueType() == EltVT ||
1139               (EltVT.isInteger() && Op.getValueType().isInteger() &&
1140                EltVT.bitsLE(Op.getValueType()))) &&
1141              "Wrong operand type!");
1142       assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1143              "Operands must all have the same type");
1144     }
1145     break;
1146   }
1147   }
1148 }
1149 #endif // NDEBUG
1150 
1151 /// Insert a newly allocated node into the DAG.
1152 ///
1153 /// Handles insertion into the all nodes list and CSE map, as well as
1154 /// verification and other common operations when a new node is allocated.
1155 void SelectionDAG::InsertNode(SDNode *N) {
1156   AllNodes.push_back(N);
1157 #ifndef NDEBUG
1158   N->PersistentId = NextPersistentId++;
1159   VerifySDNode(N);
1160 #endif
1161   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1162     DUL->NodeInserted(N);
1163 }
1164 
1165 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1166 /// correspond to it.  This is useful when we're about to delete or repurpose
1167 /// the node.  We don't want future request for structurally identical nodes
1168 /// to return N anymore.
1169 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1170   bool Erased = false;
1171   switch (N->getOpcode()) {
1172   case ISD::HANDLENODE: return false;  // noop.
1173   case ISD::CONDCODE:
1174     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1175            "Cond code doesn't exist!");
1176     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1177     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1178     break;
1179   case ISD::ExternalSymbol:
1180     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1181     break;
1182   case ISD::TargetExternalSymbol: {
1183     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1184     Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1185         ESN->getSymbol(), ESN->getTargetFlags()));
1186     break;
1187   }
1188   case ISD::MCSymbol: {
1189     auto *MCSN = cast<MCSymbolSDNode>(N);
1190     Erased = MCSymbols.erase(MCSN->getMCSymbol());
1191     break;
1192   }
1193   case ISD::VALUETYPE: {
1194     EVT VT = cast<VTSDNode>(N)->getVT();
1195     if (VT.isExtended()) {
1196       Erased = ExtendedValueTypeNodes.erase(VT);
1197     } else {
1198       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1199       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1200     }
1201     break;
1202   }
1203   default:
1204     // Remove it from the CSE Map.
1205     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1206     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1207     Erased = CSEMap.RemoveNode(N);
1208     break;
1209   }
1210 #ifndef NDEBUG
1211   // Verify that the node was actually in one of the CSE maps, unless it has a
1212   // glue result (which cannot be CSE'd) or is one of the special cases that are
1213   // not subject to CSE.
1214   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1215       !N->isMachineOpcode() && !doNotCSE(N)) {
1216     N->dump(this);
1217     dbgs() << "\n";
1218     llvm_unreachable("Node is not in map!");
1219   }
1220 #endif
1221   return Erased;
1222 }
1223 
1224 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1225 /// maps and modified in place. Add it back to the CSE maps, unless an identical
1226 /// node already exists, in which case transfer all its users to the existing
1227 /// node. This transfer can potentially trigger recursive merging.
1228 void
1229 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1230   // For node types that aren't CSE'd, just act as if no identical node
1231   // already exists.
1232   if (!doNotCSE(N)) {
1233     SDNode *Existing = CSEMap.GetOrInsertNode(N);
1234     if (Existing != N) {
1235       // If there was already an existing matching node, use ReplaceAllUsesWith
1236       // to replace the dead one with the existing one.  This can cause
1237       // recursive merging of other unrelated nodes down the line.
1238       ReplaceAllUsesWith(N, Existing);
1239 
1240       // N is now dead. Inform the listeners and delete it.
1241       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1242         DUL->NodeDeleted(N, Existing);
1243       DeleteNodeNotInCSEMaps(N);
1244       return;
1245     }
1246   }
1247 
1248   // If the node doesn't already exist, we updated it.  Inform listeners.
1249   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1250     DUL->NodeUpdated(N);
1251 }
1252 
1253 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1254 /// were replaced with those specified.  If this node is never memoized,
1255 /// return null, otherwise return a pointer to the slot it would take.  If a
1256 /// node already exists with these operands, the slot will be non-null.
1257 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1258                                            void *&InsertPos) {
1259   if (doNotCSE(N))
1260     return nullptr;
1261 
1262   SDValue Ops[] = { Op };
1263   FoldingSetNodeID ID;
1264   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1265   AddNodeIDCustom(ID, N);
1266   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1267   if (Node)
1268     Node->intersectFlagsWith(N->getFlags());
1269   return Node;
1270 }
1271 
1272 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1273 /// were replaced with those specified.  If this node is never memoized,
1274 /// return null, otherwise return a pointer to the slot it would take.  If a
1275 /// node already exists with these operands, the slot will be non-null.
1276 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1277                                            SDValue Op1, SDValue Op2,
1278                                            void *&InsertPos) {
1279   if (doNotCSE(N))
1280     return nullptr;
1281 
1282   SDValue Ops[] = { Op1, Op2 };
1283   FoldingSetNodeID ID;
1284   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1285   AddNodeIDCustom(ID, N);
1286   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1287   if (Node)
1288     Node->intersectFlagsWith(N->getFlags());
1289   return Node;
1290 }
1291 
1292 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1293 /// were replaced with those specified.  If this node is never memoized,
1294 /// return null, otherwise return a pointer to the slot it would take.  If a
1295 /// node already exists with these operands, the slot will be non-null.
1296 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1297                                            void *&InsertPos) {
1298   if (doNotCSE(N))
1299     return nullptr;
1300 
1301   FoldingSetNodeID ID;
1302   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1303   AddNodeIDCustom(ID, N);
1304   SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1305   if (Node)
1306     Node->intersectFlagsWith(N->getFlags());
1307   return Node;
1308 }
1309 
1310 Align SelectionDAG::getEVTAlign(EVT VT) const {
1311   Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)
1312                              : VT.getTypeForEVT(*getContext());
1313 
1314   return getDataLayout().getABITypeAlign(Ty);
1315 }
1316 
1317 // EntryNode could meaningfully have debug info if we can find it...
1318 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1319     : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1320                                       getVTList(MVT::Other, MVT::Glue)),
1321       Root(getEntryNode()) {
1322   InsertNode(&EntryNode);
1323   DbgInfo = new SDDbgInfo();
1324 }
1325 
1326 void SelectionDAG::init(MachineFunction &NewMF,
1327                         OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1328                         const TargetLibraryInfo *LibraryInfo,
1329                         UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1330                         BlockFrequencyInfo *BFIin,
1331                         FunctionVarLocs const *VarLocs) {
1332   MF = &NewMF;
1333   SDAGISelPass = PassPtr;
1334   ORE = &NewORE;
1335   TLI = getSubtarget().getTargetLowering();
1336   TSI = getSubtarget().getSelectionDAGInfo();
1337   LibInfo = LibraryInfo;
1338   Context = &MF->getFunction().getContext();
1339   UA = NewUA;
1340   PSI = PSIin;
1341   BFI = BFIin;
1342   FnVarLocs = VarLocs;
1343 }
1344 
1345 SelectionDAG::~SelectionDAG() {
1346   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1347   allnodes_clear();
1348   OperandRecycler.clear(OperandAllocator);
1349   delete DbgInfo;
1350 }
1351 
1352 bool SelectionDAG::shouldOptForSize() const {
1353   return MF->getFunction().hasOptSize() ||
1354       llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI);
1355 }
1356 
1357 void SelectionDAG::allnodes_clear() {
1358   assert(&*AllNodes.begin() == &EntryNode);
1359   AllNodes.remove(AllNodes.begin());
1360   while (!AllNodes.empty())
1361     DeallocateNode(&AllNodes.front());
1362 #ifndef NDEBUG
1363   NextPersistentId = 0;
1364 #endif
1365 }
1366 
1367 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1368                                           void *&InsertPos) {
1369   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1370   if (N) {
1371     switch (N->getOpcode()) {
1372     default: break;
1373     case ISD::Constant:
1374     case ISD::ConstantFP:
1375       llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1376                        "debug location.  Use another overload.");
1377     }
1378   }
1379   return N;
1380 }
1381 
1382 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1383                                           const SDLoc &DL, void *&InsertPos) {
1384   SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1385   if (N) {
1386     switch (N->getOpcode()) {
1387     case ISD::Constant:
1388     case ISD::ConstantFP:
1389       // Erase debug location from the node if the node is used at several
1390       // different places. Do not propagate one location to all uses as it
1391       // will cause a worse single stepping debugging experience.
1392       if (N->getDebugLoc() != DL.getDebugLoc())
1393         N->setDebugLoc(DebugLoc());
1394       break;
1395     default:
1396       // When the node's point of use is located earlier in the instruction
1397       // sequence than its prior point of use, update its debug info to the
1398       // earlier location.
1399       if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1400         N->setDebugLoc(DL.getDebugLoc());
1401       break;
1402     }
1403   }
1404   return N;
1405 }
1406 
1407 void SelectionDAG::clear() {
1408   allnodes_clear();
1409   OperandRecycler.clear(OperandAllocator);
1410   OperandAllocator.Reset();
1411   CSEMap.clear();
1412 
1413   ExtendedValueTypeNodes.clear();
1414   ExternalSymbols.clear();
1415   TargetExternalSymbols.clear();
1416   MCSymbols.clear();
1417   SDEI.clear();
1418   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1419             static_cast<CondCodeSDNode*>(nullptr));
1420   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1421             static_cast<SDNode*>(nullptr));
1422 
1423   EntryNode.UseList = nullptr;
1424   InsertNode(&EntryNode);
1425   Root = getEntryNode();
1426   DbgInfo->clear();
1427 }
1428 
1429 SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1430   return VT.bitsGT(Op.getValueType())
1431              ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1432              : getNode(ISD::FP_ROUND, DL, VT, Op,
1433                        getIntPtrConstant(0, DL, /*isTarget=*/true));
1434 }
1435 
1436 std::pair<SDValue, SDValue>
1437 SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1438                                        const SDLoc &DL, EVT VT) {
1439   assert(!VT.bitsEq(Op.getValueType()) &&
1440          "Strict no-op FP extend/round not allowed.");
1441   SDValue Res =
1442       VT.bitsGT(Op.getValueType())
1443           ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1444           : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1445                     {Chain, Op, getIntPtrConstant(0, DL)});
1446 
1447   return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1448 }
1449 
1450 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1451   return VT.bitsGT(Op.getValueType()) ?
1452     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1453     getNode(ISD::TRUNCATE, DL, VT, Op);
1454 }
1455 
1456 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1457   return VT.bitsGT(Op.getValueType()) ?
1458     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1459     getNode(ISD::TRUNCATE, DL, VT, Op);
1460 }
1461 
1462 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1463   return VT.bitsGT(Op.getValueType()) ?
1464     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1465     getNode(ISD::TRUNCATE, DL, VT, Op);
1466 }
1467 
1468 SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1469                                                  EVT VT) {
1470   assert(!VT.isVector());
1471   auto Type = Op.getValueType();
1472   SDValue DestOp;
1473   if (Type == VT)
1474     return Op;
1475   auto Size = Op.getValueSizeInBits();
1476   DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1477   if (DestOp.getValueType() == VT)
1478     return DestOp;
1479 
1480   return getAnyExtOrTrunc(DestOp, DL, VT);
1481 }
1482 
1483 SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1484                                                EVT VT) {
1485   assert(!VT.isVector());
1486   auto Type = Op.getValueType();
1487   SDValue DestOp;
1488   if (Type == VT)
1489     return Op;
1490   auto Size = Op.getValueSizeInBits();
1491   DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1492   if (DestOp.getValueType() == VT)
1493     return DestOp;
1494 
1495   return getSExtOrTrunc(DestOp, DL, VT);
1496 }
1497 
1498 SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1499                                                EVT VT) {
1500   assert(!VT.isVector());
1501   auto Type = Op.getValueType();
1502   SDValue DestOp;
1503   if (Type == VT)
1504     return Op;
1505   auto Size = Op.getValueSizeInBits();
1506   DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1507   if (DestOp.getValueType() == VT)
1508     return DestOp;
1509 
1510   return getZExtOrTrunc(DestOp, DL, VT);
1511 }
1512 
1513 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1514                                         EVT OpVT) {
1515   if (VT.bitsLE(Op.getValueType()))
1516     return getNode(ISD::TRUNCATE, SL, VT, Op);
1517 
1518   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1519   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1520 }
1521 
1522 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1523   EVT OpVT = Op.getValueType();
1524   assert(VT.isInteger() && OpVT.isInteger() &&
1525          "Cannot getZeroExtendInReg FP types");
1526   assert(VT.isVector() == OpVT.isVector() &&
1527          "getZeroExtendInReg type should be vector iff the operand "
1528          "type is vector!");
1529   assert((!VT.isVector() ||
1530           VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1531          "Vector element counts must match in getZeroExtendInReg");
1532   assert(VT.bitsLE(OpVT) && "Not extending!");
1533   if (OpVT == VT)
1534     return Op;
1535   APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(),
1536                                    VT.getScalarSizeInBits());
1537   return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1538 }
1539 
1540 SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1541   // Only unsigned pointer semantics are supported right now. In the future this
1542   // might delegate to TLI to check pointer signedness.
1543   return getZExtOrTrunc(Op, DL, VT);
1544 }
1545 
1546 SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1547   // Only unsigned pointer semantics are supported right now. In the future this
1548   // might delegate to TLI to check pointer signedness.
1549   return getZeroExtendInReg(Op, DL, VT);
1550 }
1551 
1552 SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1553   return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);
1554 }
1555 
1556 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1557 SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1558   return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1559 }
1560 
1561 SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1562   SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1563   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1564 }
1565 
1566 SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1567                                       SDValue Mask, SDValue EVL, EVT VT) {
1568   SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1569   return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL);
1570 }
1571 
1572 SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1573                                          SDValue Mask, SDValue EVL) {
1574   return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1575 }
1576 
1577 SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1578                                        SDValue Mask, SDValue EVL) {
1579   if (VT.bitsGT(Op.getValueType()))
1580     return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL);
1581   if (VT.bitsLT(Op.getValueType()))
1582     return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL);
1583   return Op;
1584 }
1585 
1586 SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1587                                       EVT OpVT) {
1588   if (!V)
1589     return getConstant(0, DL, VT);
1590 
1591   switch (TLI->getBooleanContents(OpVT)) {
1592   case TargetLowering::ZeroOrOneBooleanContent:
1593   case TargetLowering::UndefinedBooleanContent:
1594     return getConstant(1, DL, VT);
1595   case TargetLowering::ZeroOrNegativeOneBooleanContent:
1596     return getAllOnesConstant(DL, VT);
1597   }
1598   llvm_unreachable("Unexpected boolean content enum!");
1599 }
1600 
1601 SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1602                                   bool isT, bool isO) {
1603   EVT EltVT = VT.getScalarType();
1604   assert((EltVT.getSizeInBits() >= 64 ||
1605           (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1606          "getConstant with a uint64_t value that doesn't fit in the type!");
1607   return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1608 }
1609 
1610 SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1611                                   bool isT, bool isO) {
1612   return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1613 }
1614 
1615 SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1616                                   EVT VT, bool isT, bool isO) {
1617   assert(VT.isInteger() && "Cannot create FP integer constant!");
1618 
1619   EVT EltVT = VT.getScalarType();
1620   const ConstantInt *Elt = &Val;
1621 
1622   // In some cases the vector type is legal but the element type is illegal and
1623   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
1624   // inserted value (the type does not need to match the vector element type).
1625   // Any extra bits introduced will be truncated away.
1626   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1627                            TargetLowering::TypePromoteInteger) {
1628     EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1629     APInt NewVal;
1630     if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))
1631       NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());
1632     else
1633       NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1634     Elt = ConstantInt::get(*getContext(), NewVal);
1635   }
1636   // In other cases the element type is illegal and needs to be expanded, for
1637   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1638   // the value into n parts and use a vector type with n-times the elements.
1639   // Then bitcast to the type requested.
1640   // Legalizing constants too early makes the DAGCombiner's job harder so we
1641   // only legalize if the DAG tells us we must produce legal types.
1642   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1643            TLI->getTypeAction(*getContext(), EltVT) ==
1644                TargetLowering::TypeExpandInteger) {
1645     const APInt &NewVal = Elt->getValue();
1646     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1647     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1648 
1649     // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1650     if (VT.isScalableVector() ||
1651         TLI->isOperationLegal(ISD::SPLAT_VECTOR, VT)) {
1652       assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1653              "Can only handle an even split!");
1654       unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1655 
1656       SmallVector<SDValue, 2> ScalarParts;
1657       for (unsigned i = 0; i != Parts; ++i)
1658         ScalarParts.push_back(getConstant(
1659             NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1660             ViaEltVT, isT, isO));
1661 
1662       return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1663     }
1664 
1665     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1666     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1667 
1668     // Check the temporary vector is the correct size. If this fails then
1669     // getTypeToTransformTo() probably returned a type whose size (in bits)
1670     // isn't a power-of-2 factor of the requested type size.
1671     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1672 
1673     SmallVector<SDValue, 2> EltParts;
1674     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1675       EltParts.push_back(getConstant(
1676           NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1677           ViaEltVT, isT, isO));
1678 
1679     // EltParts is currently in little endian order. If we actually want
1680     // big-endian order then reverse it now.
1681     if (getDataLayout().isBigEndian())
1682       std::reverse(EltParts.begin(), EltParts.end());
1683 
1684     // The elements must be reversed when the element order is different
1685     // to the endianness of the elements (because the BITCAST is itself a
1686     // vector shuffle in this situation). However, we do not need any code to
1687     // perform this reversal because getConstant() is producing a vector
1688     // splat.
1689     // This situation occurs in MIPS MSA.
1690 
1691     SmallVector<SDValue, 8> Ops;
1692     for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1693       llvm::append_range(Ops, EltParts);
1694 
1695     SDValue V =
1696         getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1697     return V;
1698   }
1699 
1700   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1701          "APInt size does not match type size!");
1702   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1703   FoldingSetNodeID ID;
1704   AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1705   ID.AddPointer(Elt);
1706   ID.AddBoolean(isO);
1707   void *IP = nullptr;
1708   SDNode *N = nullptr;
1709   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1710     if (!VT.isVector())
1711       return SDValue(N, 0);
1712 
1713   if (!N) {
1714     N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1715     CSEMap.InsertNode(N, IP);
1716     InsertNode(N);
1717     NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1718   }
1719 
1720   SDValue Result(N, 0);
1721   if (VT.isVector())
1722     Result = getSplat(VT, DL, Result);
1723   return Result;
1724 }
1725 
1726 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1727                                         bool isTarget) {
1728   return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1729 }
1730 
1731 SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1732                                              const SDLoc &DL, bool LegalTypes) {
1733   assert(VT.isInteger() && "Shift amount is not an integer type!");
1734   EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1735   return getConstant(Val, DL, ShiftVT);
1736 }
1737 
1738 SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1739                                            bool isTarget) {
1740   return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1741 }
1742 
1743 SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1744                                     bool isTarget) {
1745   return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1746 }
1747 
1748 SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1749                                     EVT VT, bool isTarget) {
1750   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1751 
1752   EVT EltVT = VT.getScalarType();
1753 
1754   // Do the map lookup using the actual bit pattern for the floating point
1755   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1756   // we don't have issues with SNANs.
1757   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1758   FoldingSetNodeID ID;
1759   AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1760   ID.AddPointer(&V);
1761   void *IP = nullptr;
1762   SDNode *N = nullptr;
1763   if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1764     if (!VT.isVector())
1765       return SDValue(N, 0);
1766 
1767   if (!N) {
1768     N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1769     CSEMap.InsertNode(N, IP);
1770     InsertNode(N);
1771   }
1772 
1773   SDValue Result(N, 0);
1774   if (VT.isVector())
1775     Result = getSplat(VT, DL, Result);
1776   NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1777   return Result;
1778 }
1779 
1780 SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1781                                     bool isTarget) {
1782   EVT EltVT = VT.getScalarType();
1783   if (EltVT == MVT::f32)
1784     return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1785   if (EltVT == MVT::f64)
1786     return getConstantFP(APFloat(Val), DL, VT, isTarget);
1787   if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1788       EltVT == MVT::f16 || EltVT == MVT::bf16) {
1789     bool Ignored;
1790     APFloat APF = APFloat(Val);
1791     APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1792                 &Ignored);
1793     return getConstantFP(APF, DL, VT, isTarget);
1794   }
1795   llvm_unreachable("Unsupported type in getConstantFP");
1796 }
1797 
1798 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1799                                        EVT VT, int64_t Offset, bool isTargetGA,
1800                                        unsigned TargetFlags) {
1801   assert((TargetFlags == 0 || isTargetGA) &&
1802          "Cannot set target flags on target-independent globals");
1803 
1804   // Truncate (with sign-extension) the offset value to the pointer size.
1805   unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1806   if (BitWidth < 64)
1807     Offset = SignExtend64(Offset, BitWidth);
1808 
1809   unsigned Opc;
1810   if (GV->isThreadLocal())
1811     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1812   else
1813     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1814 
1815   FoldingSetNodeID ID;
1816   AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1817   ID.AddPointer(GV);
1818   ID.AddInteger(Offset);
1819   ID.AddInteger(TargetFlags);
1820   void *IP = nullptr;
1821   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1822     return SDValue(E, 0);
1823 
1824   auto *N = newSDNode<GlobalAddressSDNode>(
1825       Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1826   CSEMap.InsertNode(N, IP);
1827     InsertNode(N);
1828   return SDValue(N, 0);
1829 }
1830 
1831 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1832   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1833   FoldingSetNodeID ID;
1834   AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1835   ID.AddInteger(FI);
1836   void *IP = nullptr;
1837   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1838     return SDValue(E, 0);
1839 
1840   auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1841   CSEMap.InsertNode(N, IP);
1842   InsertNode(N);
1843   return SDValue(N, 0);
1844 }
1845 
1846 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1847                                    unsigned TargetFlags) {
1848   assert((TargetFlags == 0 || isTarget) &&
1849          "Cannot set target flags on target-independent jump tables");
1850   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1851   FoldingSetNodeID ID;
1852   AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1853   ID.AddInteger(JTI);
1854   ID.AddInteger(TargetFlags);
1855   void *IP = nullptr;
1856   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1857     return SDValue(E, 0);
1858 
1859   auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1860   CSEMap.InsertNode(N, IP);
1861   InsertNode(N);
1862   return SDValue(N, 0);
1863 }
1864 
1865 SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1866                                             const SDLoc &DL) {
1867   EVT PTy = getTargetLoweringInfo().getPointerTy(getDataLayout());
1868   return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1869                  getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));
1870 }
1871 
1872 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1873                                       MaybeAlign Alignment, int Offset,
1874                                       bool isTarget, unsigned TargetFlags) {
1875   assert((TargetFlags == 0 || isTarget) &&
1876          "Cannot set target flags on target-independent globals");
1877   if (!Alignment)
1878     Alignment = shouldOptForSize()
1879                     ? getDataLayout().getABITypeAlign(C->getType())
1880                     : getDataLayout().getPrefTypeAlign(C->getType());
1881   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1882   FoldingSetNodeID ID;
1883   AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1884   ID.AddInteger(Alignment->value());
1885   ID.AddInteger(Offset);
1886   ID.AddPointer(C);
1887   ID.AddInteger(TargetFlags);
1888   void *IP = nullptr;
1889   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1890     return SDValue(E, 0);
1891 
1892   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1893                                           TargetFlags);
1894   CSEMap.InsertNode(N, IP);
1895   InsertNode(N);
1896   SDValue V = SDValue(N, 0);
1897   NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1898   return V;
1899 }
1900 
1901 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1902                                       MaybeAlign Alignment, int Offset,
1903                                       bool isTarget, unsigned TargetFlags) {
1904   assert((TargetFlags == 0 || isTarget) &&
1905          "Cannot set target flags on target-independent globals");
1906   if (!Alignment)
1907     Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1908   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1909   FoldingSetNodeID ID;
1910   AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1911   ID.AddInteger(Alignment->value());
1912   ID.AddInteger(Offset);
1913   C->addSelectionDAGCSEId(ID);
1914   ID.AddInteger(TargetFlags);
1915   void *IP = nullptr;
1916   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1917     return SDValue(E, 0);
1918 
1919   auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1920                                           TargetFlags);
1921   CSEMap.InsertNode(N, IP);
1922   InsertNode(N);
1923   return SDValue(N, 0);
1924 }
1925 
1926 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1927   FoldingSetNodeID ID;
1928   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt);
1929   ID.AddPointer(MBB);
1930   void *IP = nullptr;
1931   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1932     return SDValue(E, 0);
1933 
1934   auto *N = newSDNode<BasicBlockSDNode>(MBB);
1935   CSEMap.InsertNode(N, IP);
1936   InsertNode(N);
1937   return SDValue(N, 0);
1938 }
1939 
1940 SDValue SelectionDAG::getValueType(EVT VT) {
1941   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1942       ValueTypeNodes.size())
1943     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1944 
1945   SDNode *&N = VT.isExtended() ?
1946     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1947 
1948   if (N) return SDValue(N, 0);
1949   N = newSDNode<VTSDNode>(VT);
1950   InsertNode(N);
1951   return SDValue(N, 0);
1952 }
1953 
1954 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1955   SDNode *&N = ExternalSymbols[Sym];
1956   if (N) return SDValue(N, 0);
1957   N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1958   InsertNode(N);
1959   return SDValue(N, 0);
1960 }
1961 
1962 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1963   SDNode *&N = MCSymbols[Sym];
1964   if (N)
1965     return SDValue(N, 0);
1966   N = newSDNode<MCSymbolSDNode>(Sym, VT);
1967   InsertNode(N);
1968   return SDValue(N, 0);
1969 }
1970 
1971 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1972                                               unsigned TargetFlags) {
1973   SDNode *&N =
1974       TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1975   if (N) return SDValue(N, 0);
1976   N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1977   InsertNode(N);
1978   return SDValue(N, 0);
1979 }
1980 
1981 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1982   if ((unsigned)Cond >= CondCodeNodes.size())
1983     CondCodeNodes.resize(Cond+1);
1984 
1985   if (!CondCodeNodes[Cond]) {
1986     auto *N = newSDNode<CondCodeSDNode>(Cond);
1987     CondCodeNodes[Cond] = N;
1988     InsertNode(N);
1989   }
1990 
1991   return SDValue(CondCodeNodes[Cond], 0);
1992 }
1993 
1994 SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
1995                                 bool ConstantFold) {
1996   assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
1997          "APInt size does not match type size!");
1998 
1999   if (MulImm == 0)
2000     return getConstant(0, DL, VT);
2001 
2002   if (ConstantFold) {
2003     const MachineFunction &MF = getMachineFunction();
2004     const Function &F = MF.getFunction();
2005     ConstantRange CR = getVScaleRange(&F, 64);
2006     if (const APInt *C = CR.getSingleElement())
2007       return getConstant(MulImm * C->getZExtValue(), DL, VT);
2008   }
2009 
2010   return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
2011 }
2012 
2013 SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC,
2014                                       bool ConstantFold) {
2015   if (EC.isScalable())
2016     return getVScale(DL, VT,
2017                      APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2018 
2019   return getConstant(EC.getKnownMinValue(), DL, VT);
2020 }
2021 
2022 SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2023   APInt One(ResVT.getScalarSizeInBits(), 1);
2024   return getStepVector(DL, ResVT, One);
2025 }
2026 
2027 SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
2028   assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2029   if (ResVT.isScalableVector())
2030     return getNode(
2031         ISD::STEP_VECTOR, DL, ResVT,
2032         getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
2033 
2034   SmallVector<SDValue, 16> OpsStepConstants;
2035   for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2036     OpsStepConstants.push_back(
2037         getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
2038   return getBuildVector(ResVT, DL, OpsStepConstants);
2039 }
2040 
2041 /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2042 /// point at N1 to point at N2 and indices that point at N2 to point at N1.
2043 static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2044   std::swap(N1, N2);
2045   ShuffleVectorSDNode::commuteMask(M);
2046 }
2047 
2048 SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2049                                        SDValue N2, ArrayRef<int> Mask) {
2050   assert(VT.getVectorNumElements() == Mask.size() &&
2051          "Must have the same number of vector elements as mask elements!");
2052   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2053          "Invalid VECTOR_SHUFFLE");
2054 
2055   // Canonicalize shuffle undef, undef -> undef
2056   if (N1.isUndef() && N2.isUndef())
2057     return getUNDEF(VT);
2058 
2059   // Validate that all indices in Mask are within the range of the elements
2060   // input to the shuffle.
2061   int NElts = Mask.size();
2062   assert(llvm::all_of(Mask,
2063                       [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2064          "Index out of range");
2065 
2066   // Copy the mask so we can do any needed cleanup.
2067   SmallVector<int, 8> MaskVec(Mask);
2068 
2069   // Canonicalize shuffle v, v -> v, undef
2070   if (N1 == N2) {
2071     N2 = getUNDEF(VT);
2072     for (int i = 0; i != NElts; ++i)
2073       if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2074   }
2075 
2076   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
2077   if (N1.isUndef())
2078     commuteShuffle(N1, N2, MaskVec);
2079 
2080   if (TLI->hasVectorBlend()) {
2081     // If shuffling a splat, try to blend the splat instead. We do this here so
2082     // that even when this arises during lowering we don't have to re-handle it.
2083     auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2084       BitVector UndefElements;
2085       SDValue Splat = BV->getSplatValue(&UndefElements);
2086       if (!Splat)
2087         return;
2088 
2089       for (int i = 0; i < NElts; ++i) {
2090         if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2091           continue;
2092 
2093         // If this input comes from undef, mark it as such.
2094         if (UndefElements[MaskVec[i] - Offset]) {
2095           MaskVec[i] = -1;
2096           continue;
2097         }
2098 
2099         // If we can blend a non-undef lane, use that instead.
2100         if (!UndefElements[i])
2101           MaskVec[i] = i + Offset;
2102       }
2103     };
2104     if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2105       BlendSplat(N1BV, 0);
2106     if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2107       BlendSplat(N2BV, NElts);
2108   }
2109 
2110   // Canonicalize all index into lhs, -> shuffle lhs, undef
2111   // Canonicalize all index into rhs, -> shuffle rhs, undef
2112   bool AllLHS = true, AllRHS = true;
2113   bool N2Undef = N2.isUndef();
2114   for (int i = 0; i != NElts; ++i) {
2115     if (MaskVec[i] >= NElts) {
2116       if (N2Undef)
2117         MaskVec[i] = -1;
2118       else
2119         AllLHS = false;
2120     } else if (MaskVec[i] >= 0) {
2121       AllRHS = false;
2122     }
2123   }
2124   if (AllLHS && AllRHS)
2125     return getUNDEF(VT);
2126   if (AllLHS && !N2Undef)
2127     N2 = getUNDEF(VT);
2128   if (AllRHS) {
2129     N1 = getUNDEF(VT);
2130     commuteShuffle(N1, N2, MaskVec);
2131   }
2132   // Reset our undef status after accounting for the mask.
2133   N2Undef = N2.isUndef();
2134   // Re-check whether both sides ended up undef.
2135   if (N1.isUndef() && N2Undef)
2136     return getUNDEF(VT);
2137 
2138   // If Identity shuffle return that node.
2139   bool Identity = true, AllSame = true;
2140   for (int i = 0; i != NElts; ++i) {
2141     if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2142     if (MaskVec[i] != MaskVec[0]) AllSame = false;
2143   }
2144   if (Identity && NElts)
2145     return N1;
2146 
2147   // Shuffling a constant splat doesn't change the result.
2148   if (N2Undef) {
2149     SDValue V = N1;
2150 
2151     // Look through any bitcasts. We check that these don't change the number
2152     // (and size) of elements and just changes their types.
2153     while (V.getOpcode() == ISD::BITCAST)
2154       V = V->getOperand(0);
2155 
2156     // A splat should always show up as a build vector node.
2157     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2158       BitVector UndefElements;
2159       SDValue Splat = BV->getSplatValue(&UndefElements);
2160       // If this is a splat of an undef, shuffling it is also undef.
2161       if (Splat && Splat.isUndef())
2162         return getUNDEF(VT);
2163 
2164       bool SameNumElts =
2165           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2166 
2167       // We only have a splat which can skip shuffles if there is a splatted
2168       // value and no undef lanes rearranged by the shuffle.
2169       if (Splat && UndefElements.none()) {
2170         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2171         // number of elements match or the value splatted is a zero constant.
2172         if (SameNumElts || isNullConstant(Splat))
2173           return N1;
2174       }
2175 
2176       // If the shuffle itself creates a splat, build the vector directly.
2177       if (AllSame && SameNumElts) {
2178         EVT BuildVT = BV->getValueType(0);
2179         const SDValue &Splatted = BV->getOperand(MaskVec[0]);
2180         SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
2181 
2182         // We may have jumped through bitcasts, so the type of the
2183         // BUILD_VECTOR may not match the type of the shuffle.
2184         if (BuildVT != VT)
2185           NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
2186         return NewBV;
2187       }
2188     }
2189   }
2190 
2191   FoldingSetNodeID ID;
2192   SDValue Ops[2] = { N1, N2 };
2193   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
2194   for (int i = 0; i != NElts; ++i)
2195     ID.AddInteger(MaskVec[i]);
2196 
2197   void* IP = nullptr;
2198   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2199     return SDValue(E, 0);
2200 
2201   // Allocate the mask array for the node out of the BumpPtrAllocator, since
2202   // SDNode doesn't have access to it.  This memory will be "leaked" when
2203   // the node is deallocated, but recovered when the NodeAllocator is released.
2204   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
2205   llvm::copy(MaskVec, MaskAlloc);
2206 
2207   auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
2208                                            dl.getDebugLoc(), MaskAlloc);
2209   createOperands(N, Ops);
2210 
2211   CSEMap.InsertNode(N, IP);
2212   InsertNode(N);
2213   SDValue V = SDValue(N, 0);
2214   NewSDValueDbgMsg(V, "Creating new node: ", this);
2215   return V;
2216 }
2217 
2218 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2219   EVT VT = SV.getValueType(0);
2220   SmallVector<int, 8> MaskVec(SV.getMask());
2221   ShuffleVectorSDNode::commuteMask(MaskVec);
2222 
2223   SDValue Op0 = SV.getOperand(0);
2224   SDValue Op1 = SV.getOperand(1);
2225   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
2226 }
2227 
2228 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
2229   FoldingSetNodeID ID;
2230   AddNodeIDNode(ID, ISD::Register, getVTList(VT), std::nullopt);
2231   ID.AddInteger(RegNo);
2232   void *IP = nullptr;
2233   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2234     return SDValue(E, 0);
2235 
2236   auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
2237   N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2238   CSEMap.InsertNode(N, IP);
2239   InsertNode(N);
2240   return SDValue(N, 0);
2241 }
2242 
2243 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2244   FoldingSetNodeID ID;
2245   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt);
2246   ID.AddPointer(RegMask);
2247   void *IP = nullptr;
2248   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2249     return SDValue(E, 0);
2250 
2251   auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2252   CSEMap.InsertNode(N, IP);
2253   InsertNode(N);
2254   return SDValue(N, 0);
2255 }
2256 
2257 SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2258                                  MCSymbol *Label) {
2259   return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2260 }
2261 
2262 SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2263                                    SDValue Root, MCSymbol *Label) {
2264   FoldingSetNodeID ID;
2265   SDValue Ops[] = { Root };
2266   AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2267   ID.AddPointer(Label);
2268   void *IP = nullptr;
2269   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2270     return SDValue(E, 0);
2271 
2272   auto *N =
2273       newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2274   createOperands(N, Ops);
2275 
2276   CSEMap.InsertNode(N, IP);
2277   InsertNode(N);
2278   return SDValue(N, 0);
2279 }
2280 
2281 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2282                                       int64_t Offset, bool isTarget,
2283                                       unsigned TargetFlags) {
2284   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2285 
2286   FoldingSetNodeID ID;
2287   AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
2288   ID.AddPointer(BA);
2289   ID.AddInteger(Offset);
2290   ID.AddInteger(TargetFlags);
2291   void *IP = nullptr;
2292   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2293     return SDValue(E, 0);
2294 
2295   auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2296   CSEMap.InsertNode(N, IP);
2297   InsertNode(N);
2298   return SDValue(N, 0);
2299 }
2300 
2301 SDValue SelectionDAG::getSrcValue(const Value *V) {
2302   FoldingSetNodeID ID;
2303   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt);
2304   ID.AddPointer(V);
2305 
2306   void *IP = nullptr;
2307   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2308     return SDValue(E, 0);
2309 
2310   auto *N = newSDNode<SrcValueSDNode>(V);
2311   CSEMap.InsertNode(N, IP);
2312   InsertNode(N);
2313   return SDValue(N, 0);
2314 }
2315 
2316 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2317   FoldingSetNodeID ID;
2318   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt);
2319   ID.AddPointer(MD);
2320 
2321   void *IP = nullptr;
2322   if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2323     return SDValue(E, 0);
2324 
2325   auto *N = newSDNode<MDNodeSDNode>(MD);
2326   CSEMap.InsertNode(N, IP);
2327   InsertNode(N);
2328   return SDValue(N, 0);
2329 }
2330 
2331 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2332   if (VT == V.getValueType())
2333     return V;
2334 
2335   return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2336 }
2337 
2338 SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2339                                        unsigned SrcAS, unsigned DestAS) {
2340   SDValue Ops[] = {Ptr};
2341   FoldingSetNodeID ID;
2342   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
2343   ID.AddInteger(SrcAS);
2344   ID.AddInteger(DestAS);
2345 
2346   void *IP = nullptr;
2347   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2348     return SDValue(E, 0);
2349 
2350   auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2351                                            VT, SrcAS, DestAS);
2352   createOperands(N, Ops);
2353 
2354   CSEMap.InsertNode(N, IP);
2355   InsertNode(N);
2356   return SDValue(N, 0);
2357 }
2358 
2359 SDValue SelectionDAG::getFreeze(SDValue V) {
2360   return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2361 }
2362 
2363 /// getShiftAmountOperand - Return the specified value casted to
2364 /// the target's desired shift amount type.
2365 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2366   EVT OpTy = Op.getValueType();
2367   EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2368   if (OpTy == ShTy || OpTy.isVector()) return Op;
2369 
2370   return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2371 }
2372 
2373 SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2374   SDLoc dl(Node);
2375   const TargetLowering &TLI = getTargetLoweringInfo();
2376   const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2377   EVT VT = Node->getValueType(0);
2378   SDValue Tmp1 = Node->getOperand(0);
2379   SDValue Tmp2 = Node->getOperand(1);
2380   const MaybeAlign MA(Node->getConstantOperandVal(3));
2381 
2382   SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2383                                Tmp2, MachinePointerInfo(V));
2384   SDValue VAList = VAListLoad;
2385 
2386   if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2387     VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2388                      getConstant(MA->value() - 1, dl, VAList.getValueType()));
2389 
2390     VAList =
2391         getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2392                 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2393   }
2394 
2395   // Increment the pointer, VAList, to the next vaarg
2396   Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2397                  getConstant(getDataLayout().getTypeAllocSize(
2398                                                VT.getTypeForEVT(*getContext())),
2399                              dl, VAList.getValueType()));
2400   // Store the incremented VAList to the legalized pointer
2401   Tmp1 =
2402       getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2403   // Load the actual argument out of the pointer VAList
2404   return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2405 }
2406 
2407 SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2408   SDLoc dl(Node);
2409   const TargetLowering &TLI = getTargetLoweringInfo();
2410   // This defaults to loading a pointer from the input and storing it to the
2411   // output, returning the chain.
2412   const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2413   const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2414   SDValue Tmp1 =
2415       getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2416               Node->getOperand(2), MachinePointerInfo(VS));
2417   return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2418                   MachinePointerInfo(VD));
2419 }
2420 
2421 Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2422   const DataLayout &DL = getDataLayout();
2423   Type *Ty = VT.getTypeForEVT(*getContext());
2424   Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2425 
2426   if (TLI->isTypeLegal(VT) || !VT.isVector())
2427     return RedAlign;
2428 
2429   const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2430   const Align StackAlign = TFI->getStackAlign();
2431 
2432   // See if we can choose a smaller ABI alignment in cases where it's an
2433   // illegal vector type that will get broken down.
2434   if (RedAlign > StackAlign) {
2435     EVT IntermediateVT;
2436     MVT RegisterVT;
2437     unsigned NumIntermediates;
2438     TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2439                                 NumIntermediates, RegisterVT);
2440     Ty = IntermediateVT.getTypeForEVT(*getContext());
2441     Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2442     if (RedAlign2 < RedAlign)
2443       RedAlign = RedAlign2;
2444   }
2445 
2446   return RedAlign;
2447 }
2448 
2449 SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2450   MachineFrameInfo &MFI = MF->getFrameInfo();
2451   const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2452   int StackID = 0;
2453   if (Bytes.isScalable())
2454     StackID = TFI->getStackIDForScalableVectors();
2455   // The stack id gives an indication of whether the object is scalable or
2456   // not, so it's safe to pass in the minimum size here.
2457   int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,
2458                                        false, nullptr, StackID);
2459   return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2460 }
2461 
2462 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2463   Type *Ty = VT.getTypeForEVT(*getContext());
2464   Align StackAlign =
2465       std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2466   return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2467 }
2468 
2469 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2470   TypeSize VT1Size = VT1.getStoreSize();
2471   TypeSize VT2Size = VT2.getStoreSize();
2472   assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2473          "Don't know how to choose the maximum size when creating a stack "
2474          "temporary");
2475   TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2476                        ? VT1Size
2477                        : VT2Size;
2478 
2479   Type *Ty1 = VT1.getTypeForEVT(*getContext());
2480   Type *Ty2 = VT2.getTypeForEVT(*getContext());
2481   const DataLayout &DL = getDataLayout();
2482   Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2483   return CreateStackTemporary(Bytes, Align);
2484 }
2485 
2486 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2487                                 ISD::CondCode Cond, const SDLoc &dl) {
2488   EVT OpVT = N1.getValueType();
2489 
2490   auto GetUndefBooleanConstant = [&]() {
2491     if (VT.getScalarType() == MVT::i1 ||
2492         TLI->getBooleanContents(OpVT) ==
2493             TargetLowering::UndefinedBooleanContent)
2494       return getUNDEF(VT);
2495     // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2496     // so we cannot use getUNDEF(). Return zero instead.
2497     return getConstant(0, dl, VT);
2498   };
2499 
2500   // These setcc operations always fold.
2501   switch (Cond) {
2502   default: break;
2503   case ISD::SETFALSE:
2504   case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2505   case ISD::SETTRUE:
2506   case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2507 
2508   case ISD::SETOEQ:
2509   case ISD::SETOGT:
2510   case ISD::SETOGE:
2511   case ISD::SETOLT:
2512   case ISD::SETOLE:
2513   case ISD::SETONE:
2514   case ISD::SETO:
2515   case ISD::SETUO:
2516   case ISD::SETUEQ:
2517   case ISD::SETUNE:
2518     assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2519     break;
2520   }
2521 
2522   if (OpVT.isInteger()) {
2523     // For EQ and NE, we can always pick a value for the undef to make the
2524     // predicate pass or fail, so we can return undef.
2525     // Matches behavior in llvm::ConstantFoldCompareInstruction.
2526     // icmp eq/ne X, undef -> undef.
2527     if ((N1.isUndef() || N2.isUndef()) &&
2528         (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2529       return GetUndefBooleanConstant();
2530 
2531     // If both operands are undef, we can return undef for int comparison.
2532     // icmp undef, undef -> undef.
2533     if (N1.isUndef() && N2.isUndef())
2534       return GetUndefBooleanConstant();
2535 
2536     // icmp X, X -> true/false
2537     // icmp X, undef -> true/false because undef could be X.
2538     if (N1.isUndef() || N2.isUndef() || N1 == N2)
2539       return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2540   }
2541 
2542   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2543     const APInt &C2 = N2C->getAPIntValue();
2544     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2545       const APInt &C1 = N1C->getAPIntValue();
2546 
2547       return getBoolConstant(ICmpInst::compare(C1, C2, getICmpCondCode(Cond)),
2548                              dl, VT, OpVT);
2549     }
2550   }
2551 
2552   auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2553   auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2554 
2555   if (N1CFP && N2CFP) {
2556     APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2557     switch (Cond) {
2558     default: break;
2559     case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
2560                         return GetUndefBooleanConstant();
2561                       [[fallthrough]];
2562     case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2563                                              OpVT);
2564     case ISD::SETNE:  if (R==APFloat::cmpUnordered)
2565                         return GetUndefBooleanConstant();
2566                       [[fallthrough]];
2567     case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2568                                              R==APFloat::cmpLessThan, dl, VT,
2569                                              OpVT);
2570     case ISD::SETLT:  if (R==APFloat::cmpUnordered)
2571                         return GetUndefBooleanConstant();
2572                       [[fallthrough]];
2573     case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2574                                              OpVT);
2575     case ISD::SETGT:  if (R==APFloat::cmpUnordered)
2576                         return GetUndefBooleanConstant();
2577                       [[fallthrough]];
2578     case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2579                                              VT, OpVT);
2580     case ISD::SETLE:  if (R==APFloat::cmpUnordered)
2581                         return GetUndefBooleanConstant();
2582                       [[fallthrough]];
2583     case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2584                                              R==APFloat::cmpEqual, dl, VT,
2585                                              OpVT);
2586     case ISD::SETGE:  if (R==APFloat::cmpUnordered)
2587                         return GetUndefBooleanConstant();
2588                       [[fallthrough]];
2589     case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2590                                          R==APFloat::cmpEqual, dl, VT, OpVT);
2591     case ISD::SETO:   return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2592                                              OpVT);
2593     case ISD::SETUO:  return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2594                                              OpVT);
2595     case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2596                                              R==APFloat::cmpEqual, dl, VT,
2597                                              OpVT);
2598     case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2599                                              OpVT);
2600     case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2601                                              R==APFloat::cmpLessThan, dl, VT,
2602                                              OpVT);
2603     case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2604                                              R==APFloat::cmpUnordered, dl, VT,
2605                                              OpVT);
2606     case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2607                                              VT, OpVT);
2608     case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2609                                              OpVT);
2610     }
2611   } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2612     // Ensure that the constant occurs on the RHS.
2613     ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2614     if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2615       return SDValue();
2616     return getSetCC(dl, VT, N2, N1, SwappedCond);
2617   } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2618              (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2619     // If an operand is known to be a nan (or undef that could be a nan), we can
2620     // fold it.
2621     // Choosing NaN for the undef will always make unordered comparison succeed
2622     // and ordered comparison fails.
2623     // Matches behavior in llvm::ConstantFoldCompareInstruction.
2624     switch (ISD::getUnorderedFlavor(Cond)) {
2625     default:
2626       llvm_unreachable("Unknown flavor!");
2627     case 0: // Known false.
2628       return getBoolConstant(false, dl, VT, OpVT);
2629     case 1: // Known true.
2630       return getBoolConstant(true, dl, VT, OpVT);
2631     case 2: // Undefined.
2632       return GetUndefBooleanConstant();
2633     }
2634   }
2635 
2636   // Could not fold it.
2637   return SDValue();
2638 }
2639 
2640 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
2641 /// use this predicate to simplify operations downstream.
2642 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2643   unsigned BitWidth = Op.getScalarValueSizeInBits();
2644   return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2645 }
2646 
2647 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
2648 /// this predicate to simplify operations downstream.  Mask is known to be zero
2649 /// for bits that V cannot have.
2650 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2651                                      unsigned Depth) const {
2652   return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2653 }
2654 
2655 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2656 /// DemandedElts.  We use this predicate to simplify operations downstream.
2657 /// Mask is known to be zero for bits that V cannot have.
2658 bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2659                                      const APInt &DemandedElts,
2660                                      unsigned Depth) const {
2661   return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2662 }
2663 
2664 /// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2665 /// DemandedElts.  We use this predicate to simplify operations downstream.
2666 bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2667                                       unsigned Depth /* = 0 */) const {
2668   return computeKnownBits(V, DemandedElts, Depth).isZero();
2669 }
2670 
2671 /// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2672 bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2673                                         unsigned Depth) const {
2674   return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2675 }
2676 
2677 APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2678                                                    const APInt &DemandedElts,
2679                                                    unsigned Depth) const {
2680   EVT VT = Op.getValueType();
2681   assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2682 
2683   unsigned NumElts = VT.getVectorNumElements();
2684   assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2685 
2686   APInt KnownZeroElements = APInt::getZero(NumElts);
2687   for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2688     if (!DemandedElts[EltIdx])
2689       continue; // Don't query elements that are not demanded.
2690     APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
2691     if (MaskedVectorIsZero(Op, Mask, Depth))
2692       KnownZeroElements.setBit(EltIdx);
2693   }
2694   return KnownZeroElements;
2695 }
2696 
2697 /// isSplatValue - Return true if the vector V has the same value
2698 /// across all DemandedElts. For scalable vectors, we don't know the
2699 /// number of lanes at compile time.  Instead, we use a 1 bit APInt
2700 /// to represent a conservative value for all lanes; that is, that
2701 /// one bit value is implicitly splatted across all lanes.
2702 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2703                                 APInt &UndefElts, unsigned Depth) const {
2704   unsigned Opcode = V.getOpcode();
2705   EVT VT = V.getValueType();
2706   assert(VT.isVector() && "Vector type expected");
2707   assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2708          "scalable demanded bits are ignored");
2709 
2710   if (!DemandedElts)
2711     return false; // No demanded elts, better to assume we don't know anything.
2712 
2713   if (Depth >= MaxRecursionDepth)
2714     return false; // Limit search depth.
2715 
2716   // Deal with some common cases here that work for both fixed and scalable
2717   // vector types.
2718   switch (Opcode) {
2719   case ISD::SPLAT_VECTOR:
2720     UndefElts = V.getOperand(0).isUndef()
2721                     ? APInt::getAllOnes(DemandedElts.getBitWidth())
2722                     : APInt(DemandedElts.getBitWidth(), 0);
2723     return true;
2724   case ISD::ADD:
2725   case ISD::SUB:
2726   case ISD::AND:
2727   case ISD::XOR:
2728   case ISD::OR: {
2729     APInt UndefLHS, UndefRHS;
2730     SDValue LHS = V.getOperand(0);
2731     SDValue RHS = V.getOperand(1);
2732     if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2733         isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2734       UndefElts = UndefLHS | UndefRHS;
2735       return true;
2736     }
2737     return false;
2738   }
2739   case ISD::ABS:
2740   case ISD::TRUNCATE:
2741   case ISD::SIGN_EXTEND:
2742   case ISD::ZERO_EXTEND:
2743     return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2744   default:
2745     if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2746         Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2747       return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,
2748                                             Depth);
2749     break;
2750 }
2751 
2752   // We don't support other cases than those above for scalable vectors at
2753   // the moment.
2754   if (VT.isScalableVector())
2755     return false;
2756 
2757   unsigned NumElts = VT.getVectorNumElements();
2758   assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2759   UndefElts = APInt::getZero(NumElts);
2760 
2761   switch (Opcode) {
2762   case ISD::BUILD_VECTOR: {
2763     SDValue Scl;
2764     for (unsigned i = 0; i != NumElts; ++i) {
2765       SDValue Op = V.getOperand(i);
2766       if (Op.isUndef()) {
2767         UndefElts.setBit(i);
2768         continue;
2769       }
2770       if (!DemandedElts[i])
2771         continue;
2772       if (Scl && Scl != Op)
2773         return false;
2774       Scl = Op;
2775     }
2776     return true;
2777   }
2778   case ISD::VECTOR_SHUFFLE: {
2779     // Check if this is a shuffle node doing a splat or a shuffle of a splat.
2780     APInt DemandedLHS = APInt::getZero(NumElts);
2781     APInt DemandedRHS = APInt::getZero(NumElts);
2782     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2783     for (int i = 0; i != (int)NumElts; ++i) {
2784       int M = Mask[i];
2785       if (M < 0) {
2786         UndefElts.setBit(i);
2787         continue;
2788       }
2789       if (!DemandedElts[i])
2790         continue;
2791       if (M < (int)NumElts)
2792         DemandedLHS.setBit(M);
2793       else
2794         DemandedRHS.setBit(M - NumElts);
2795     }
2796 
2797     // If we aren't demanding either op, assume there's no splat.
2798     // If we are demanding both ops, assume there's no splat.
2799     if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
2800         (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
2801       return false;
2802 
2803     // See if the demanded elts of the source op is a splat or we only demand
2804     // one element, which should always be a splat.
2805     // TODO: Handle source ops splats with undefs.
2806     auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
2807       APInt SrcUndefs;
2808       return (SrcElts.popcount() == 1) ||
2809              (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&
2810               (SrcElts & SrcUndefs).isZero());
2811     };
2812     if (!DemandedLHS.isZero())
2813       return CheckSplatSrc(V.getOperand(0), DemandedLHS);
2814     return CheckSplatSrc(V.getOperand(1), DemandedRHS);
2815   }
2816   case ISD::EXTRACT_SUBVECTOR: {
2817     // Offset the demanded elts by the subvector index.
2818     SDValue Src = V.getOperand(0);
2819     // We don't support scalable vectors at the moment.
2820     if (Src.getValueType().isScalableVector())
2821       return false;
2822     uint64_t Idx = V.getConstantOperandVal(1);
2823     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2824     APInt UndefSrcElts;
2825     APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
2826     if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2827       UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2828       return true;
2829     }
2830     break;
2831   }
2832   case ISD::ANY_EXTEND_VECTOR_INREG:
2833   case ISD::SIGN_EXTEND_VECTOR_INREG:
2834   case ISD::ZERO_EXTEND_VECTOR_INREG: {
2835     // Widen the demanded elts by the src element count.
2836     SDValue Src = V.getOperand(0);
2837     // We don't support scalable vectors at the moment.
2838     if (Src.getValueType().isScalableVector())
2839       return false;
2840     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2841     APInt UndefSrcElts;
2842     APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
2843     if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2844       UndefElts = UndefSrcElts.trunc(NumElts);
2845       return true;
2846     }
2847     break;
2848   }
2849   case ISD::BITCAST: {
2850     SDValue Src = V.getOperand(0);
2851     EVT SrcVT = Src.getValueType();
2852     unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
2853     unsigned BitWidth = VT.getScalarSizeInBits();
2854 
2855     // Ignore bitcasts from unsupported types.
2856     // TODO: Add fp support?
2857     if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
2858       break;
2859 
2860     // Bitcast 'small element' vector to 'large element' vector.
2861     if ((BitWidth % SrcBitWidth) == 0) {
2862       // See if each sub element is a splat.
2863       unsigned Scale = BitWidth / SrcBitWidth;
2864       unsigned NumSrcElts = SrcVT.getVectorNumElements();
2865       APInt ScaledDemandedElts =
2866           APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2867       for (unsigned I = 0; I != Scale; ++I) {
2868         APInt SubUndefElts;
2869         APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);
2870         APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);
2871         SubDemandedElts &= ScaledDemandedElts;
2872         if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))
2873           return false;
2874         // TODO: Add support for merging sub undef elements.
2875         if (!SubUndefElts.isZero())
2876           return false;
2877       }
2878       return true;
2879     }
2880     break;
2881   }
2882   }
2883 
2884   return false;
2885 }
2886 
2887 /// Helper wrapper to main isSplatValue function.
2888 bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
2889   EVT VT = V.getValueType();
2890   assert(VT.isVector() && "Vector type expected");
2891 
2892   APInt UndefElts;
2893   // Since the number of lanes in a scalable vector is unknown at compile time,
2894   // we track one bit which is implicitly broadcast to all lanes.  This means
2895   // that all lanes in a scalable vector are considered demanded.
2896   APInt DemandedElts
2897     = APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2898   return isSplatValue(V, DemandedElts, UndefElts) &&
2899          (AllowUndefs || !UndefElts);
2900 }
2901 
2902 SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2903   V = peekThroughExtractSubvectors(V);
2904 
2905   EVT VT = V.getValueType();
2906   unsigned Opcode = V.getOpcode();
2907   switch (Opcode) {
2908   default: {
2909     APInt UndefElts;
2910     // Since the number of lanes in a scalable vector is unknown at compile time,
2911     // we track one bit which is implicitly broadcast to all lanes.  This means
2912     // that all lanes in a scalable vector are considered demanded.
2913     APInt DemandedElts
2914       = APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements());
2915 
2916     if (isSplatValue(V, DemandedElts, UndefElts)) {
2917       if (VT.isScalableVector()) {
2918         // DemandedElts and UndefElts are ignored for scalable vectors, since
2919         // the only supported cases are SPLAT_VECTOR nodes.
2920         SplatIdx = 0;
2921       } else {
2922         // Handle case where all demanded elements are UNDEF.
2923         if (DemandedElts.isSubsetOf(UndefElts)) {
2924           SplatIdx = 0;
2925           return getUNDEF(VT);
2926         }
2927         SplatIdx = (UndefElts & DemandedElts).countr_one();
2928       }
2929       return V;
2930     }
2931     break;
2932   }
2933   case ISD::SPLAT_VECTOR:
2934     SplatIdx = 0;
2935     return V;
2936   case ISD::VECTOR_SHUFFLE: {
2937     assert(!VT.isScalableVector());
2938     // Check if this is a shuffle node doing a splat.
2939     // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2940     // getTargetVShiftNode currently struggles without the splat source.
2941     auto *SVN = cast<ShuffleVectorSDNode>(V);
2942     if (!SVN->isSplat())
2943       break;
2944     int Idx = SVN->getSplatIndex();
2945     int NumElts = V.getValueType().getVectorNumElements();
2946     SplatIdx = Idx % NumElts;
2947     return V.getOperand(Idx / NumElts);
2948   }
2949   }
2950 
2951   return SDValue();
2952 }
2953 
2954 SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
2955   int SplatIdx;
2956   if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2957     EVT SVT = SrcVector.getValueType().getScalarType();
2958     EVT LegalSVT = SVT;
2959     if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2960       if (!SVT.isInteger())
2961         return SDValue();
2962       LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2963       if (LegalSVT.bitsLT(SVT))
2964         return SDValue();
2965     }
2966     return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2967                    getVectorIdxConstant(SplatIdx, SDLoc(V)));
2968   }
2969   return SDValue();
2970 }
2971 
2972 const APInt *
2973 SelectionDAG::getValidShiftAmountConstant(SDValue V,
2974                                           const APInt &DemandedElts) const {
2975   assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2976           V.getOpcode() == ISD::SRA) &&
2977          "Unknown shift node");
2978   unsigned BitWidth = V.getScalarValueSizeInBits();
2979   if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2980     // Shifting more than the bitwidth is not valid.
2981     const APInt &ShAmt = SA->getAPIntValue();
2982     if (ShAmt.ult(BitWidth))
2983       return &ShAmt;
2984   }
2985   return nullptr;
2986 }
2987 
2988 const APInt *SelectionDAG::getValidMinimumShiftAmountConstant(
2989     SDValue V, const APInt &DemandedElts) const {
2990   assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2991           V.getOpcode() == ISD::SRA) &&
2992          "Unknown shift node");
2993   if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
2994     return ValidAmt;
2995   unsigned BitWidth = V.getScalarValueSizeInBits();
2996   auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2997   if (!BV)
2998     return nullptr;
2999   const APInt *MinShAmt = nullptr;
3000   for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3001     if (!DemandedElts[i])
3002       continue;
3003     auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3004     if (!SA)
3005       return nullptr;
3006     // Shifting more than the bitwidth is not valid.
3007     const APInt &ShAmt = SA->getAPIntValue();
3008     if (ShAmt.uge(BitWidth))
3009       return nullptr;
3010     if (MinShAmt && MinShAmt->ule(ShAmt))
3011       continue;
3012     MinShAmt = &ShAmt;
3013   }
3014   return MinShAmt;
3015 }
3016 
3017 const APInt *SelectionDAG::getValidMaximumShiftAmountConstant(
3018     SDValue V, const APInt &DemandedElts) const {
3019   assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3020           V.getOpcode() == ISD::SRA) &&
3021          "Unknown shift node");
3022   if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3023     return ValidAmt;
3024   unsigned BitWidth = V.getScalarValueSizeInBits();
3025   auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3026   if (!BV)
3027     return nullptr;
3028   const APInt *MaxShAmt = nullptr;
3029   for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3030     if (!DemandedElts[i])
3031       continue;
3032     auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3033     if (!SA)
3034       return nullptr;
3035     // Shifting more than the bitwidth is not valid.
3036     const APInt &ShAmt = SA->getAPIntValue();
3037     if (ShAmt.uge(BitWidth))
3038       return nullptr;
3039     if (MaxShAmt && MaxShAmt->uge(ShAmt))
3040       continue;
3041     MaxShAmt = &ShAmt;
3042   }
3043   return MaxShAmt;
3044 }
3045 
3046 /// Determine which bits of Op are known to be either zero or one and return
3047 /// them in Known. For vectors, the known bits are those that are shared by
3048 /// every vector element.
3049 KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3050   EVT VT = Op.getValueType();
3051 
3052   // Since the number of lanes in a scalable vector is unknown at compile time,
3053   // we track one bit which is implicitly broadcast to all lanes.  This means
3054   // that all lanes in a scalable vector are considered demanded.
3055   APInt DemandedElts = VT.isFixedLengthVector()
3056                            ? APInt::getAllOnes(VT.getVectorNumElements())
3057                            : APInt(1, 1);
3058   return computeKnownBits(Op, DemandedElts, Depth);
3059 }
3060 
3061 /// Determine which bits of Op are known to be either zero or one and return
3062 /// them in Known. The DemandedElts argument allows us to only collect the known
3063 /// bits that are shared by the requested vector elements.
3064 KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3065                                          unsigned Depth) const {
3066   unsigned BitWidth = Op.getScalarValueSizeInBits();
3067 
3068   KnownBits Known(BitWidth);   // Don't know anything.
3069 
3070   if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3071     // We know all of the bits for a constant!
3072     return KnownBits::makeConstant(C->getAPIntValue());
3073   }
3074   if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
3075     // We know all of the bits for a constant fp!
3076     return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
3077   }
3078 
3079   if (Depth >= MaxRecursionDepth)
3080     return Known;  // Limit search depth.
3081 
3082   KnownBits Known2;
3083   unsigned NumElts = DemandedElts.getBitWidth();
3084   assert((!Op.getValueType().isFixedLengthVector() ||
3085           NumElts == Op.getValueType().getVectorNumElements()) &&
3086          "Unexpected vector size");
3087 
3088   if (!DemandedElts)
3089     return Known;  // No demanded elts, better to assume we don't know anything.
3090 
3091   unsigned Opcode = Op.getOpcode();
3092   switch (Opcode) {
3093   case ISD::MERGE_VALUES:
3094     return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,
3095                             Depth + 1);
3096   case ISD::SPLAT_VECTOR: {
3097     SDValue SrcOp = Op.getOperand(0);
3098     assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3099            "Expected SPLAT_VECTOR implicit truncation");
3100     // Implicitly truncate the bits to match the official semantics of
3101     // SPLAT_VECTOR.
3102     Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth);
3103     break;
3104   }
3105   case ISD::SPLAT_VECTOR_PARTS: {
3106     unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();
3107     assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3108            "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3109     for (auto [I, SrcOp] : enumerate(Op->ops())) {
3110       Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);
3111     }
3112     break;
3113   }
3114   case ISD::BUILD_VECTOR:
3115     assert(!Op.getValueType().isScalableVector());
3116     // Collect the known bits that are shared by every demanded vector element.
3117     Known.Zero.setAllBits(); Known.One.setAllBits();
3118     for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3119       if (!DemandedElts[i])
3120         continue;
3121 
3122       SDValue SrcOp = Op.getOperand(i);
3123       Known2 = computeKnownBits(SrcOp, Depth + 1);
3124 
3125       // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3126       if (SrcOp.getValueSizeInBits() != BitWidth) {
3127         assert(SrcOp.getValueSizeInBits() > BitWidth &&
3128                "Expected BUILD_VECTOR implicit truncation");
3129         Known2 = Known2.trunc(BitWidth);
3130       }
3131 
3132       // Known bits are the values that are shared by every demanded element.
3133       Known = Known.intersectWith(Known2);
3134 
3135       // If we don't know any bits, early out.
3136       if (Known.isUnknown())
3137         break;
3138     }
3139     break;
3140   case ISD::VECTOR_SHUFFLE: {
3141     assert(!Op.getValueType().isScalableVector());
3142     // Collect the known bits that are shared by every vector element referenced
3143     // by the shuffle.
3144     APInt DemandedLHS, DemandedRHS;
3145     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3146     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3147     if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
3148                                 DemandedLHS, DemandedRHS))
3149       break;
3150 
3151     // Known bits are the values that are shared by every demanded element.
3152     Known.Zero.setAllBits(); Known.One.setAllBits();
3153     if (!!DemandedLHS) {
3154       SDValue LHS = Op.getOperand(0);
3155       Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
3156       Known = Known.intersectWith(Known2);
3157     }
3158     // If we don't know any bits, early out.
3159     if (Known.isUnknown())
3160       break;
3161     if (!!DemandedRHS) {
3162       SDValue RHS = Op.getOperand(1);
3163       Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
3164       Known = Known.intersectWith(Known2);
3165     }
3166     break;
3167   }
3168   case ISD::VSCALE: {
3169     const Function &F = getMachineFunction().getFunction();
3170     const APInt &Multiplier = Op.getConstantOperandAPInt(0);
3171     Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits();
3172     break;
3173   }
3174   case ISD::CONCAT_VECTORS: {
3175     if (Op.getValueType().isScalableVector())
3176       break;
3177     // Split DemandedElts and test each of the demanded subvectors.
3178     Known.Zero.setAllBits(); Known.One.setAllBits();
3179     EVT SubVectorVT = Op.getOperand(0).getValueType();
3180     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3181     unsigned NumSubVectors = Op.getNumOperands();
3182     for (unsigned i = 0; i != NumSubVectors; ++i) {
3183       APInt DemandedSub =
3184           DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
3185       if (!!DemandedSub) {
3186         SDValue Sub = Op.getOperand(i);
3187         Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
3188         Known = Known.intersectWith(Known2);
3189       }
3190       // If we don't know any bits, early out.
3191       if (Known.isUnknown())
3192         break;
3193     }
3194     break;
3195   }
3196   case ISD::INSERT_SUBVECTOR: {
3197     if (Op.getValueType().isScalableVector())
3198       break;
3199     // Demand any elements from the subvector and the remainder from the src its
3200     // inserted into.
3201     SDValue Src = Op.getOperand(0);
3202     SDValue Sub = Op.getOperand(1);
3203     uint64_t Idx = Op.getConstantOperandVal(2);
3204     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3205     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3206     APInt DemandedSrcElts = DemandedElts;
3207     DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
3208 
3209     Known.One.setAllBits();
3210     Known.Zero.setAllBits();
3211     if (!!DemandedSubElts) {
3212       Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
3213       if (Known.isUnknown())
3214         break; // early-out.
3215     }
3216     if (!!DemandedSrcElts) {
3217       Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3218       Known = Known.intersectWith(Known2);
3219     }
3220     break;
3221   }
3222   case ISD::EXTRACT_SUBVECTOR: {
3223     // Offset the demanded elts by the subvector index.
3224     SDValue Src = Op.getOperand(0);
3225     // Bail until we can represent demanded elements for scalable vectors.
3226     if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3227       break;
3228     uint64_t Idx = Op.getConstantOperandVal(1);
3229     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3230     APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3231     Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3232     break;
3233   }
3234   case ISD::SCALAR_TO_VECTOR: {
3235     if (Op.getValueType().isScalableVector())
3236       break;
3237     // We know about scalar_to_vector as much as we know about it source,
3238     // which becomes the first element of otherwise unknown vector.
3239     if (DemandedElts != 1)
3240       break;
3241 
3242     SDValue N0 = Op.getOperand(0);
3243     Known = computeKnownBits(N0, Depth + 1);
3244     if (N0.getValueSizeInBits() != BitWidth)
3245       Known = Known.trunc(BitWidth);
3246 
3247     break;
3248   }
3249   case ISD::BITCAST: {
3250     if (Op.getValueType().isScalableVector())
3251       break;
3252 
3253     SDValue N0 = Op.getOperand(0);
3254     EVT SubVT = N0.getValueType();
3255     unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3256 
3257     // Ignore bitcasts from unsupported types.
3258     if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3259       break;
3260 
3261     // Fast handling of 'identity' bitcasts.
3262     if (BitWidth == SubBitWidth) {
3263       Known = computeKnownBits(N0, DemandedElts, Depth + 1);
3264       break;
3265     }
3266 
3267     bool IsLE = getDataLayout().isLittleEndian();
3268 
3269     // Bitcast 'small element' vector to 'large element' scalar/vector.
3270     if ((BitWidth % SubBitWidth) == 0) {
3271       assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3272 
3273       // Collect known bits for the (larger) output by collecting the known
3274       // bits from each set of sub elements and shift these into place.
3275       // We need to separately call computeKnownBits for each set of
3276       // sub elements as the knownbits for each is likely to be different.
3277       unsigned SubScale = BitWidth / SubBitWidth;
3278       APInt SubDemandedElts(NumElts * SubScale, 0);
3279       for (unsigned i = 0; i != NumElts; ++i)
3280         if (DemandedElts[i])
3281           SubDemandedElts.setBit(i * SubScale);
3282 
3283       for (unsigned i = 0; i != SubScale; ++i) {
3284         Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
3285                          Depth + 1);
3286         unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3287         Known.insertBits(Known2, SubBitWidth * Shifts);
3288       }
3289     }
3290 
3291     // Bitcast 'large element' scalar/vector to 'small element' vector.
3292     if ((SubBitWidth % BitWidth) == 0) {
3293       assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3294 
3295       // Collect known bits for the (smaller) output by collecting the known
3296       // bits from the overlapping larger input elements and extracting the
3297       // sub sections we actually care about.
3298       unsigned SubScale = SubBitWidth / BitWidth;
3299       APInt SubDemandedElts =
3300           APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
3301       Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
3302 
3303       Known.Zero.setAllBits(); Known.One.setAllBits();
3304       for (unsigned i = 0; i != NumElts; ++i)
3305         if (DemandedElts[i]) {
3306           unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3307           unsigned Offset = (Shifts % SubScale) * BitWidth;
3308           Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));
3309           // If we don't know any bits, early out.
3310           if (Known.isUnknown())
3311             break;
3312         }
3313     }
3314     break;
3315   }
3316   case ISD::AND:
3317     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3318     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3319 
3320     Known &= Known2;
3321     break;
3322   case ISD::OR:
3323     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3324     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3325 
3326     Known |= Known2;
3327     break;
3328   case ISD::XOR:
3329     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3330     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3331 
3332     Known ^= Known2;
3333     break;
3334   case ISD::MUL: {
3335     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3336     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3337     bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3338     // TODO: SelfMultiply can be poison, but not undef.
3339     if (SelfMultiply)
3340       SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3341           Op.getOperand(0), DemandedElts, false, Depth + 1);
3342     Known = KnownBits::mul(Known, Known2, SelfMultiply);
3343 
3344     // If the multiplication is known not to overflow, the product of a number
3345     // with itself is non-negative. Only do this if we didn't already computed
3346     // the opposite value for the sign bit.
3347     if (Op->getFlags().hasNoSignedWrap() &&
3348         Op.getOperand(0) == Op.getOperand(1) &&
3349         !Known.isNegative())
3350       Known.makeNonNegative();
3351     break;
3352   }
3353   case ISD::MULHU: {
3354     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3355     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3356     Known = KnownBits::mulhu(Known, Known2);
3357     break;
3358   }
3359   case ISD::MULHS: {
3360     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3361     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3362     Known = KnownBits::mulhs(Known, Known2);
3363     break;
3364   }
3365   case ISD::UMUL_LOHI: {
3366     assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3367     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3368     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3369     bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3370     if (Op.getResNo() == 0)
3371       Known = KnownBits::mul(Known, Known2, SelfMultiply);
3372     else
3373       Known = KnownBits::mulhu(Known, Known2);
3374     break;
3375   }
3376   case ISD::SMUL_LOHI: {
3377     assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3378     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3379     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3380     bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3381     if (Op.getResNo() == 0)
3382       Known = KnownBits::mul(Known, Known2, SelfMultiply);
3383     else
3384       Known = KnownBits::mulhs(Known, Known2);
3385     break;
3386   }
3387   case ISD::AVGCEILU: {
3388     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3389     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3390     Known = Known.zext(BitWidth + 1);
3391     Known2 = Known2.zext(BitWidth + 1);
3392     KnownBits One = KnownBits::makeConstant(APInt(1, 1));
3393     Known = KnownBits::computeForAddCarry(Known, Known2, One);
3394     Known = Known.extractBits(BitWidth, 1);
3395     break;
3396   }
3397   case ISD::SELECT:
3398   case ISD::VSELECT:
3399     Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3400     // If we don't know any bits, early out.
3401     if (Known.isUnknown())
3402       break;
3403     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3404 
3405     // Only known if known in both the LHS and RHS.
3406     Known = Known.intersectWith(Known2);
3407     break;
3408   case ISD::SELECT_CC:
3409     Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3410     // If we don't know any bits, early out.
3411     if (Known.isUnknown())
3412       break;
3413     Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3414 
3415     // Only known if known in both the LHS and RHS.
3416     Known = Known.intersectWith(Known2);
3417     break;
3418   case ISD::SMULO:
3419   case ISD::UMULO:
3420     if (Op.getResNo() != 1)
3421       break;
3422     // The boolean result conforms to getBooleanContents.
3423     // If we know the result of a setcc has the top bits zero, use this info.
3424     // We know that we have an integer-based boolean since these operations
3425     // are only available for integer.
3426     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3427             TargetLowering::ZeroOrOneBooleanContent &&
3428         BitWidth > 1)
3429       Known.Zero.setBitsFrom(1);
3430     break;
3431   case ISD::SETCC:
3432   case ISD::SETCCCARRY:
3433   case ISD::STRICT_FSETCC:
3434   case ISD::STRICT_FSETCCS: {
3435     unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3436     // If we know the result of a setcc has the top bits zero, use this info.
3437     if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3438             TargetLowering::ZeroOrOneBooleanContent &&
3439         BitWidth > 1)
3440       Known.Zero.setBitsFrom(1);
3441     break;
3442   }
3443   case ISD::SHL:
3444     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3445     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3446     Known = KnownBits::shl(Known, Known2);
3447 
3448     // Minimum shift low bits are known zero.
3449     if (const APInt *ShMinAmt =
3450             getValidMinimumShiftAmountConstant(Op, DemandedElts))
3451       Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3452     break;
3453   case ISD::SRL:
3454     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3455     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3456     Known = KnownBits::lshr(Known, Known2);
3457 
3458     // Minimum shift high bits are known zero.
3459     if (const APInt *ShMinAmt =
3460             getValidMinimumShiftAmountConstant(Op, DemandedElts))
3461       Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3462     break;
3463   case ISD::SRA:
3464     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3465     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3466     Known = KnownBits::ashr(Known, Known2);
3467     break;
3468   case ISD::FSHL:
3469   case ISD::FSHR:
3470     if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3471       unsigned Amt = C->getAPIntValue().urem(BitWidth);
3472 
3473       // For fshl, 0-shift returns the 1st arg.
3474       // For fshr, 0-shift returns the 2nd arg.
3475       if (Amt == 0) {
3476         Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3477                                  DemandedElts, Depth + 1);
3478         break;
3479       }
3480 
3481       // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3482       // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3483       Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3484       Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3485       if (Opcode == ISD::FSHL) {
3486         Known.One <<= Amt;
3487         Known.Zero <<= Amt;
3488         Known2.One.lshrInPlace(BitWidth - Amt);
3489         Known2.Zero.lshrInPlace(BitWidth - Amt);
3490       } else {
3491         Known.One <<= BitWidth - Amt;
3492         Known.Zero <<= BitWidth - Amt;
3493         Known2.One.lshrInPlace(Amt);
3494         Known2.Zero.lshrInPlace(Amt);
3495       }
3496       Known = Known.unionWith(Known2);
3497     }
3498     break;
3499   case ISD::SHL_PARTS:
3500   case ISD::SRA_PARTS:
3501   case ISD::SRL_PARTS: {
3502     assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3503 
3504     // Collect lo/hi source values and concatenate.
3505     unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();
3506     unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();
3507     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3508     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3509     Known = Known2.concat(Known);
3510 
3511     // Collect shift amount.
3512     Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3513 
3514     if (Opcode == ISD::SHL_PARTS)
3515       Known = KnownBits::shl(Known, Known2);
3516     else if (Opcode == ISD::SRA_PARTS)
3517       Known = KnownBits::ashr(Known, Known2);
3518     else // if (Opcode == ISD::SRL_PARTS)
3519       Known = KnownBits::lshr(Known, Known2);
3520 
3521     // TODO: Minimum shift low/high bits are known zero.
3522 
3523     if (Op.getResNo() == 0)
3524       Known = Known.extractBits(LoBits, 0);
3525     else
3526       Known = Known.extractBits(HiBits, LoBits);
3527     break;
3528   }
3529   case ISD::SIGN_EXTEND_INREG: {
3530     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3531     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3532     Known = Known.sextInReg(EVT.getScalarSizeInBits());
3533     break;
3534   }
3535   case ISD::CTTZ:
3536   case ISD::CTTZ_ZERO_UNDEF: {
3537     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3538     // If we have a known 1, its position is our upper bound.
3539     unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3540     unsigned LowBits = llvm::bit_width(PossibleTZ);
3541     Known.Zero.setBitsFrom(LowBits);
3542     break;
3543   }
3544   case ISD::CTLZ:
3545   case ISD::CTLZ_ZERO_UNDEF: {
3546     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3547     // If we have a known 1, its position is our upper bound.
3548     unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3549     unsigned LowBits = llvm::bit_width(PossibleLZ);
3550     Known.Zero.setBitsFrom(LowBits);
3551     break;
3552   }
3553   case ISD::CTPOP: {
3554     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3555     // If we know some of the bits are zero, they can't be one.
3556     unsigned PossibleOnes = Known2.countMaxPopulation();
3557     Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));
3558     break;
3559   }
3560   case ISD::PARITY: {
3561     // Parity returns 0 everywhere but the LSB.
3562     Known.Zero.setBitsFrom(1);
3563     break;
3564   }
3565   case ISD::LOAD: {
3566     LoadSDNode *LD = cast<LoadSDNode>(Op);
3567     const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3568     if (ISD::isNON_EXTLoad(LD) && Cst) {
3569       // Determine any common known bits from the loaded constant pool value.
3570       Type *CstTy = Cst->getType();
3571       if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3572           !Op.getValueType().isScalableVector()) {
3573         // If its a vector splat, then we can (quickly) reuse the scalar path.
3574         // NOTE: We assume all elements match and none are UNDEF.
3575         if (CstTy->isVectorTy()) {
3576           if (const Constant *Splat = Cst->getSplatValue()) {
3577             Cst = Splat;
3578             CstTy = Cst->getType();
3579           }
3580         }
3581         // TODO - do we need to handle different bitwidths?
3582         if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3583           // Iterate across all vector elements finding common known bits.
3584           Known.One.setAllBits();
3585           Known.Zero.setAllBits();
3586           for (unsigned i = 0; i != NumElts; ++i) {
3587             if (!DemandedElts[i])
3588               continue;
3589             if (Constant *Elt = Cst->getAggregateElement(i)) {
3590               if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3591                 const APInt &Value = CInt->getValue();
3592                 Known.One &= Value;
3593                 Known.Zero &= ~Value;
3594                 continue;
3595               }
3596               if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3597                 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3598                 Known.One &= Value;
3599                 Known.Zero &= ~Value;
3600                 continue;
3601               }
3602             }
3603             Known.One.clearAllBits();
3604             Known.Zero.clearAllBits();
3605             break;
3606           }
3607         } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3608           if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3609             Known = KnownBits::makeConstant(CInt->getValue());
3610           } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3611             Known =
3612                 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3613           }
3614         }
3615       }
3616     } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
3617       // If this is a ZEXTLoad and we are looking at the loaded value.
3618       EVT VT = LD->getMemoryVT();
3619       unsigned MemBits = VT.getScalarSizeInBits();
3620       Known.Zero.setBitsFrom(MemBits);
3621     } else if (const MDNode *Ranges = LD->getRanges()) {
3622       EVT VT = LD->getValueType(0);
3623 
3624       // TODO: Handle for extending loads
3625       if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
3626         if (VT.isVector()) {
3627           // Handle truncation to the first demanded element.
3628           // TODO: Figure out which demanded elements are covered
3629           if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
3630             break;
3631 
3632           // Handle the case where a load has a vector type, but scalar memory
3633           // with an attached range.
3634           EVT MemVT = LD->getMemoryVT();
3635           KnownBits KnownFull(MemVT.getSizeInBits());
3636 
3637           computeKnownBitsFromRangeMetadata(*Ranges, KnownFull);
3638           Known = KnownFull.trunc(BitWidth);
3639         } else
3640           computeKnownBitsFromRangeMetadata(*Ranges, Known);
3641       }
3642     }
3643     break;
3644   }
3645   case ISD::ZERO_EXTEND_VECTOR_INREG: {
3646     if (Op.getValueType().isScalableVector())
3647       break;
3648     EVT InVT = Op.getOperand(0).getValueType();
3649     APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3650     Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3651     Known = Known.zext(BitWidth);
3652     break;
3653   }
3654   case ISD::ZERO_EXTEND: {
3655     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3656     Known = Known.zext(BitWidth);
3657     break;
3658   }
3659   case ISD::SIGN_EXTEND_VECTOR_INREG: {
3660     if (Op.getValueType().isScalableVector())
3661       break;
3662     EVT InVT = Op.getOperand(0).getValueType();
3663     APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3664     Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3665     // If the sign bit is known to be zero or one, then sext will extend
3666     // it to the top bits, else it will just zext.
3667     Known = Known.sext(BitWidth);
3668     break;
3669   }
3670   case ISD::SIGN_EXTEND: {
3671     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3672     // If the sign bit is known to be zero or one, then sext will extend
3673     // it to the top bits, else it will just zext.
3674     Known = Known.sext(BitWidth);
3675     break;
3676   }
3677   case ISD::ANY_EXTEND_VECTOR_INREG: {
3678     if (Op.getValueType().isScalableVector())
3679       break;
3680     EVT InVT = Op.getOperand(0).getValueType();
3681     APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3682     Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3683     Known = Known.anyext(BitWidth);
3684     break;
3685   }
3686   case ISD::ANY_EXTEND: {
3687     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3688     Known = Known.anyext(BitWidth);
3689     break;
3690   }
3691   case ISD::TRUNCATE: {
3692     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3693     Known = Known.trunc(BitWidth);
3694     break;
3695   }
3696   case ISD::AssertZext: {
3697     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3698     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3699     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3700     Known.Zero |= (~InMask);
3701     Known.One  &= (~Known.Zero);
3702     break;
3703   }
3704   case ISD::AssertAlign: {
3705     unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3706     assert(LogOfAlign != 0);
3707 
3708     // TODO: Should use maximum with source
3709     // If a node is guaranteed to be aligned, set low zero bits accordingly as
3710     // well as clearing one bits.
3711     Known.Zero.setLowBits(LogOfAlign);
3712     Known.One.clearLowBits(LogOfAlign);
3713     break;
3714   }
3715   case ISD::FGETSIGN:
3716     // All bits are zero except the low bit.
3717     Known.Zero.setBitsFrom(1);
3718     break;
3719   case ISD::ADD:
3720   case ISD::SUB: {
3721     SDNodeFlags Flags = Op.getNode()->getFlags();
3722     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3723     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3724     Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD,
3725                                         Flags.hasNoSignedWrap(), Known, Known2);
3726     break;
3727   }
3728   case ISD::USUBO:
3729   case ISD::SSUBO:
3730   case ISD::USUBO_CARRY:
3731   case ISD::SSUBO_CARRY:
3732     if (Op.getResNo() == 1) {
3733       // If we know the result of a setcc has the top bits zero, use this info.
3734       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3735               TargetLowering::ZeroOrOneBooleanContent &&
3736           BitWidth > 1)
3737         Known.Zero.setBitsFrom(1);
3738       break;
3739     }
3740     [[fallthrough]];
3741   case ISD::SUBC: {
3742     assert(Op.getResNo() == 0 &&
3743            "We only compute knownbits for the difference here.");
3744 
3745     // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
3746     KnownBits Borrow(1);
3747     if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
3748       Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3749       // Borrow has bit width 1
3750       Borrow = Borrow.trunc(1);
3751     } else {
3752       Borrow.setAllZero();
3753     }
3754 
3755     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3756     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3757     Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
3758     break;
3759   }
3760   case ISD::UADDO:
3761   case ISD::SADDO:
3762   case ISD::UADDO_CARRY:
3763   case ISD::SADDO_CARRY:
3764     if (Op.getResNo() == 1) {
3765       // If we know the result of a setcc has the top bits zero, use this info.
3766       if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3767               TargetLowering::ZeroOrOneBooleanContent &&
3768           BitWidth > 1)
3769         Known.Zero.setBitsFrom(1);
3770       break;
3771     }
3772     [[fallthrough]];
3773   case ISD::ADDC:
3774   case ISD::ADDE: {
3775     assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3776 
3777     // With ADDE and UADDO_CARRY, a carry bit may be added in.
3778     KnownBits Carry(1);
3779     if (Opcode == ISD::ADDE)
3780       // Can't track carry from glue, set carry to unknown.
3781       Carry.resetAll();
3782     else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
3783       Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3784       // Carry has bit width 1
3785       Carry = Carry.trunc(1);
3786     } else {
3787       Carry.setAllZero();
3788     }
3789 
3790     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3791     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3792     Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3793     break;
3794   }
3795   case ISD::UDIV: {
3796     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3797     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3798     Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());
3799     break;
3800   }
3801   case ISD::SDIV: {
3802     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3803     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3804     Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());
3805     break;
3806   }
3807   case ISD::SREM: {
3808     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3809     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3810     Known = KnownBits::srem(Known, Known2);
3811     break;
3812   }
3813   case ISD::UREM: {
3814     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3815     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3816     Known = KnownBits::urem(Known, Known2);
3817     break;
3818   }
3819   case ISD::EXTRACT_ELEMENT: {
3820     Known = computeKnownBits(Op.getOperand(0), Depth+1);
3821     const unsigned Index = Op.getConstantOperandVal(1);
3822     const unsigned EltBitWidth = Op.getValueSizeInBits();
3823 
3824     // Remove low part of known bits mask
3825     Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3826     Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3827 
3828     // Remove high part of known bit mask
3829     Known = Known.trunc(EltBitWidth);
3830     break;
3831   }
3832   case ISD::EXTRACT_VECTOR_ELT: {
3833     SDValue InVec = Op.getOperand(0);
3834     SDValue EltNo = Op.getOperand(1);
3835     EVT VecVT = InVec.getValueType();
3836     // computeKnownBits not yet implemented for scalable vectors.
3837     if (VecVT.isScalableVector())
3838       break;
3839     const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3840     const unsigned NumSrcElts = VecVT.getVectorNumElements();
3841 
3842     // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3843     // anything about the extended bits.
3844     if (BitWidth > EltBitWidth)
3845       Known = Known.trunc(EltBitWidth);
3846 
3847     // If we know the element index, just demand that vector element, else for
3848     // an unknown element index, ignore DemandedElts and demand them all.
3849     APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
3850     auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3851     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3852       DemandedSrcElts =
3853           APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3854 
3855     Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3856     if (BitWidth > EltBitWidth)
3857       Known = Known.anyext(BitWidth);
3858     break;
3859   }
3860   case ISD::INSERT_VECTOR_ELT: {
3861     if (Op.getValueType().isScalableVector())
3862       break;
3863 
3864     // If we know the element index, split the demand between the
3865     // source vector and the inserted element, otherwise assume we need
3866     // the original demanded vector elements and the value.
3867     SDValue InVec = Op.getOperand(0);
3868     SDValue InVal = Op.getOperand(1);
3869     SDValue EltNo = Op.getOperand(2);
3870     bool DemandedVal = true;
3871     APInt DemandedVecElts = DemandedElts;
3872     auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3873     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3874       unsigned EltIdx = CEltNo->getZExtValue();
3875       DemandedVal = !!DemandedElts[EltIdx];
3876       DemandedVecElts.clearBit(EltIdx);
3877     }
3878     Known.One.setAllBits();
3879     Known.Zero.setAllBits();
3880     if (DemandedVal) {
3881       Known2 = computeKnownBits(InVal, Depth + 1);
3882       Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));
3883     }
3884     if (!!DemandedVecElts) {
3885       Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3886       Known = Known.intersectWith(Known2);
3887     }
3888     break;
3889   }
3890   case ISD::BITREVERSE: {
3891     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3892     Known = Known2.reverseBits();
3893     break;
3894   }
3895   case ISD::BSWAP: {
3896     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3897     Known = Known2.byteSwap();
3898     break;
3899   }
3900   case ISD::ABS: {
3901     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3902     Known = Known2.abs();
3903     break;
3904   }
3905   case ISD::USUBSAT: {
3906     // The result of usubsat will never be larger than the LHS.
3907     Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3908     Known.Zero.setHighBits(Known2.countMinLeadingZeros());
3909     break;
3910   }
3911   case ISD::UMIN: {
3912     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3913     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3914     Known = KnownBits::umin(Known, Known2);
3915     break;
3916   }
3917   case ISD::UMAX: {
3918     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3919     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3920     Known = KnownBits::umax(Known, Known2);
3921     break;
3922   }
3923   case ISD::SMIN:
3924   case ISD::SMAX: {
3925     // If we have a clamp pattern, we know that the number of sign bits will be
3926     // the minimum of the clamp min/max range.
3927     bool IsMax = (Opcode == ISD::SMAX);
3928     ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3929     if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3930       if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3931         CstHigh =
3932             isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3933     if (CstLow && CstHigh) {
3934       if (!IsMax)
3935         std::swap(CstLow, CstHigh);
3936 
3937       const APInt &ValueLow = CstLow->getAPIntValue();
3938       const APInt &ValueHigh = CstHigh->getAPIntValue();
3939       if (ValueLow.sle(ValueHigh)) {
3940         unsigned LowSignBits = ValueLow.getNumSignBits();
3941         unsigned HighSignBits = ValueHigh.getNumSignBits();
3942         unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3943         if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3944           Known.One.setHighBits(MinSignBits);
3945           break;
3946         }
3947         if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3948           Known.Zero.setHighBits(MinSignBits);
3949           break;
3950         }
3951       }
3952     }
3953 
3954     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3955     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3956     if (IsMax)
3957       Known = KnownBits::smax(Known, Known2);
3958     else
3959       Known = KnownBits::smin(Known, Known2);
3960 
3961     // For SMAX, if CstLow is non-negative we know the result will be
3962     // non-negative and thus all sign bits are 0.
3963     // TODO: There's an equivalent of this for smin with negative constant for
3964     // known ones.
3965     if (IsMax && CstLow) {
3966       const APInt &ValueLow = CstLow->getAPIntValue();
3967       if (ValueLow.isNonNegative()) {
3968         unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3969         Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));
3970       }
3971     }
3972 
3973     break;
3974   }
3975   case ISD::FP_TO_UINT_SAT: {
3976     // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
3977     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3978     Known.Zero |= APInt::getBitsSetFrom(BitWidth, VT.getScalarSizeInBits());
3979     break;
3980   }
3981   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
3982     if (Op.getResNo() == 1) {
3983       // The boolean result conforms to getBooleanContents.
3984       // If we know the result of a setcc has the top bits zero, use this info.
3985       // We know that we have an integer-based boolean since these operations
3986       // are only available for integer.
3987       if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3988               TargetLowering::ZeroOrOneBooleanContent &&
3989           BitWidth > 1)
3990         Known.Zero.setBitsFrom(1);
3991       break;
3992     }
3993     [[fallthrough]];
3994   case ISD::ATOMIC_CMP_SWAP:
3995   case ISD::ATOMIC_SWAP:
3996   case ISD::ATOMIC_LOAD_ADD:
3997   case ISD::ATOMIC_LOAD_SUB:
3998   case ISD::ATOMIC_LOAD_AND:
3999   case ISD::ATOMIC_LOAD_CLR:
4000   case ISD::ATOMIC_LOAD_OR:
4001   case ISD::ATOMIC_LOAD_XOR:
4002   case ISD::ATOMIC_LOAD_NAND:
4003   case ISD::ATOMIC_LOAD_MIN:
4004   case ISD::ATOMIC_LOAD_MAX:
4005   case ISD::ATOMIC_LOAD_UMIN:
4006   case ISD::ATOMIC_LOAD_UMAX:
4007   case ISD::ATOMIC_LOAD: {
4008     unsigned MemBits =
4009         cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4010     // If we are looking at the loaded value.
4011     if (Op.getResNo() == 0) {
4012       if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4013         Known.Zero.setBitsFrom(MemBits);
4014     }
4015     break;
4016   }
4017   case ISD::FrameIndex:
4018   case ISD::TargetFrameIndex:
4019     TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
4020                                        Known, getMachineFunction());
4021     break;
4022 
4023   default:
4024     if (Opcode < ISD::BUILTIN_OP_END)
4025       break;
4026     [[fallthrough]];
4027   case ISD::INTRINSIC_WO_CHAIN:
4028   case ISD::INTRINSIC_W_CHAIN:
4029   case ISD::INTRINSIC_VOID:
4030     // TODO: Probably okay to remove after audit; here to reduce change size
4031     // in initial enablement patch for scalable vectors
4032     if (Op.getValueType().isScalableVector())
4033       break;
4034 
4035     // Allow the target to implement this method for its nodes.
4036     TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
4037     break;
4038   }
4039 
4040   assert(!Known.hasConflict() && "Bits known to be one AND zero?");
4041   return Known;
4042 }
4043 
4044 /// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4045 static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4046   switch (OR) {
4047   case ConstantRange::OverflowResult::MayOverflow:
4048     return SelectionDAG::OFK_Sometime;
4049   case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4050   case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4051     return SelectionDAG::OFK_Always;
4052   case ConstantRange::OverflowResult::NeverOverflows:
4053     return SelectionDAG::OFK_Never;
4054   }
4055   llvm_unreachable("Unknown OverflowResult");
4056 }
4057 
4058 SelectionDAG::OverflowKind
4059 SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4060   // X + 0 never overflow
4061   if (isNullConstant(N1))
4062     return OFK_Never;
4063 
4064   // If both operands each have at least two sign bits, the addition
4065   // cannot overflow.
4066   if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4067     return OFK_Never;
4068 
4069   // TODO: Add ConstantRange::signedAddMayOverflow handling.
4070   return OFK_Sometime;
4071 }
4072 
4073 SelectionDAG::OverflowKind
4074 SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4075   // X + 0 never overflow
4076   if (isNullConstant(N1))
4077     return OFK_Never;
4078 
4079   // mulhi + 1 never overflow
4080   KnownBits N1Known = computeKnownBits(N1);
4081   if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4082       N1Known.getMaxValue().ult(2))
4083     return OFK_Never;
4084 
4085   KnownBits N0Known = computeKnownBits(N0);
4086   if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4087       N0Known.getMaxValue().ult(2))
4088     return OFK_Never;
4089 
4090   // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4091   ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4092   ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4093   return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));
4094 }
4095 
4096 SelectionDAG::OverflowKind
4097 SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4098   // X - 0 never overflow
4099   if (isNullConstant(N1))
4100     return OFK_Never;
4101 
4102   // If both operands each have at least two sign bits, the subtraction
4103   // cannot overflow.
4104   if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4105     return OFK_Never;
4106 
4107   KnownBits N0Known = computeKnownBits(N0);
4108   KnownBits N1Known = computeKnownBits(N1);
4109   ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);
4110   ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);
4111   return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));
4112 }
4113 
4114 SelectionDAG::OverflowKind
4115 SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4116   // X - 0 never overflow
4117   if (isNullConstant(N1))
4118     return OFK_Never;
4119 
4120   KnownBits N0Known = computeKnownBits(N0);
4121   KnownBits N1Known = computeKnownBits(N1);
4122   ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4123   ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4124   return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));
4125 }
4126 
4127 SelectionDAG::OverflowKind
4128 SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4129   // X * 0 and X * 1 never overflow.
4130   if (isNullConstant(N1) || isOneConstant(N1))
4131     return OFK_Never;
4132 
4133   KnownBits N0Known = computeKnownBits(N0);
4134   KnownBits N1Known = computeKnownBits(N1);
4135   ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4136   ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4137   return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));
4138 }
4139 
4140 SelectionDAG::OverflowKind
4141 SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4142   // X * 0 and X * 1 never overflow.
4143   if (isNullConstant(N1) || isOneConstant(N1))
4144     return OFK_Never;
4145 
4146   // Get the size of the result.
4147   unsigned BitWidth = N0.getScalarValueSizeInBits();
4148 
4149   // Sum of the sign bits.
4150   unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);
4151 
4152   // If we have enough sign bits, then there's no overflow.
4153   if (SignBits > BitWidth + 1)
4154     return OFK_Never;
4155 
4156   if (SignBits == BitWidth + 1) {
4157     // The overflow occurs when the true multiplication of the
4158     // the operands is the minimum negative number.
4159     KnownBits N0Known = computeKnownBits(N0);
4160     KnownBits N1Known = computeKnownBits(N1);
4161     // If one of the operands is non-negative, then there's no
4162     // overflow.
4163     if (N0Known.isNonNegative() || N1Known.isNonNegative())
4164       return OFK_Never;
4165   }
4166 
4167   return OFK_Sometime;
4168 }
4169 
4170 bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
4171   if (Depth >= MaxRecursionDepth)
4172     return false; // Limit search depth.
4173 
4174   EVT OpVT = Val.getValueType();
4175   unsigned BitWidth = OpVT.getScalarSizeInBits();
4176 
4177   // Is the constant a known power of 2?
4178   if (ISD::matchUnaryPredicate(Val, [BitWidth](ConstantSDNode *C) {
4179         return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4180       }))
4181     return true;
4182 
4183   // A left-shift of a constant one will have exactly one bit set because
4184   // shifting the bit off the end is undefined.
4185   if (Val.getOpcode() == ISD::SHL) {
4186     auto *C = isConstOrConstSplat(Val.getOperand(0));
4187     if (C && C->getAPIntValue() == 1)
4188       return true;
4189     return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4190            isKnownNeverZero(Val, Depth);
4191   }
4192 
4193   // Similarly, a logical right-shift of a constant sign-bit will have exactly
4194   // one bit set.
4195   if (Val.getOpcode() == ISD::SRL) {
4196     auto *C = isConstOrConstSplat(Val.getOperand(0));
4197     if (C && C->getAPIntValue().isSignMask())
4198       return true;
4199     return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4200            isKnownNeverZero(Val, Depth);
4201   }
4202 
4203   if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4204     return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4205 
4206   // Are all operands of a build vector constant powers of two?
4207   if (Val.getOpcode() == ISD::BUILD_VECTOR)
4208     if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
4209           if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4210             return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4211           return false;
4212         }))
4213       return true;
4214 
4215   // Is the operand of a splat vector a constant power of two?
4216   if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4217     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
4218       if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
4219         return true;
4220 
4221   // vscale(power-of-two) is a power-of-two for some targets
4222   if (Val.getOpcode() == ISD::VSCALE &&
4223       getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4224       isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1))
4225     return true;
4226 
4227   if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4228       Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4229     return isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1) &&
4230            isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4231 
4232   if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4233     return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) &&
4234            isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1);
4235 
4236   if (Val.getOpcode() == ISD::AND) {
4237     // Looking for `x & -x` pattern:
4238     // If x == 0:
4239     //    x & -x -> 0
4240     // If x != 0:
4241     //    x & -x -> non-zero pow2
4242     // so if we find the pattern return whether we know `x` is non-zero.
4243     for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) {
4244       SDValue NegOp = Val.getOperand(OpIdx);
4245       if (NegOp.getOpcode() == ISD::SUB &&
4246           NegOp.getOperand(1) == Val.getOperand(1 - OpIdx) &&
4247           isNullOrNullSplat(NegOp.getOperand(0)))
4248         return isKnownNeverZero(Val.getOperand(1 - OpIdx), Depth);
4249     }
4250   }
4251 
4252   if (Val.getOpcode() == ISD::ZERO_EXTEND)
4253     return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4254 
4255   // More could be done here, though the above checks are enough
4256   // to handle some common cases.
4257   return false;
4258 }
4259 
4260 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4261   EVT VT = Op.getValueType();
4262 
4263   // Since the number of lanes in a scalable vector is unknown at compile time,
4264   // we track one bit which is implicitly broadcast to all lanes.  This means
4265   // that all lanes in a scalable vector are considered demanded.
4266   APInt DemandedElts = VT.isFixedLengthVector()
4267                            ? APInt::getAllOnes(VT.getVectorNumElements())
4268                            : APInt(1, 1);
4269   return ComputeNumSignBits(Op, DemandedElts, Depth);
4270 }
4271 
4272 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4273                                           unsigned Depth) const {
4274   EVT VT = Op.getValueType();
4275   assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4276   unsigned VTBits = VT.getScalarSizeInBits();
4277   unsigned NumElts = DemandedElts.getBitWidth();
4278   unsigned Tmp, Tmp2;
4279   unsigned FirstAnswer = 1;
4280 
4281   if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4282     const APInt &Val = C->getAPIntValue();
4283     return Val.getNumSignBits();
4284   }
4285 
4286   if (Depth >= MaxRecursionDepth)
4287     return 1;  // Limit search depth.
4288 
4289   if (!DemandedElts)
4290     return 1;  // No demanded elts, better to assume we don't know anything.
4291 
4292   unsigned Opcode = Op.getOpcode();
4293   switch (Opcode) {
4294   default: break;
4295   case ISD::AssertSext:
4296     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4297     return VTBits-Tmp+1;
4298   case ISD::AssertZext:
4299     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4300     return VTBits-Tmp;
4301   case ISD::MERGE_VALUES:
4302     return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,
4303                               Depth + 1);
4304   case ISD::SPLAT_VECTOR: {
4305     // Check if the sign bits of source go down as far as the truncated value.
4306     unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();
4307     unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4308     if (NumSrcSignBits > (NumSrcBits - VTBits))
4309       return NumSrcSignBits - (NumSrcBits - VTBits);
4310     break;
4311   }
4312   case ISD::BUILD_VECTOR:
4313     assert(!VT.isScalableVector());
4314     Tmp = VTBits;
4315     for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4316       if (!DemandedElts[i])
4317         continue;
4318 
4319       SDValue SrcOp = Op.getOperand(i);
4320       // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4321       // for constant nodes to ensure we only look at the sign bits.
4322       if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SrcOp)) {
4323         APInt T = C->getAPIntValue().trunc(VTBits);
4324         Tmp2 = T.getNumSignBits();
4325       } else {
4326         Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
4327 
4328         if (SrcOp.getValueSizeInBits() != VTBits) {
4329           assert(SrcOp.getValueSizeInBits() > VTBits &&
4330                  "Expected BUILD_VECTOR implicit truncation");
4331           unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4332           Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4333         }
4334       }
4335       Tmp = std::min(Tmp, Tmp2);
4336     }
4337     return Tmp;
4338 
4339   case ISD::VECTOR_SHUFFLE: {
4340     // Collect the minimum number of sign bits that are shared by every vector
4341     // element referenced by the shuffle.
4342     APInt DemandedLHS, DemandedRHS;
4343     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
4344     assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4345     if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
4346                                 DemandedLHS, DemandedRHS))
4347       return 1;
4348 
4349     Tmp = std::numeric_limits<unsigned>::max();
4350     if (!!DemandedLHS)
4351       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
4352     if (!!DemandedRHS) {
4353       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
4354       Tmp = std::min(Tmp, Tmp2);
4355     }
4356     // If we don't know anything, early out and try computeKnownBits fall-back.
4357     if (Tmp == 1)
4358       break;
4359     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4360     return Tmp;
4361   }
4362 
4363   case ISD::BITCAST: {
4364     if (VT.isScalableVector())
4365       break;
4366     SDValue N0 = Op.getOperand(0);
4367     EVT SrcVT = N0.getValueType();
4368     unsigned SrcBits = SrcVT.getScalarSizeInBits();
4369 
4370     // Ignore bitcasts from unsupported types..
4371     if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4372       break;
4373 
4374     // Fast handling of 'identity' bitcasts.
4375     if (VTBits == SrcBits)
4376       return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
4377 
4378     bool IsLE = getDataLayout().isLittleEndian();
4379 
4380     // Bitcast 'large element' scalar/vector to 'small element' vector.
4381     if ((SrcBits % VTBits) == 0) {
4382       assert(VT.isVector() && "Expected bitcast to vector");
4383 
4384       unsigned Scale = SrcBits / VTBits;
4385       APInt SrcDemandedElts =
4386           APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
4387 
4388       // Fast case - sign splat can be simply split across the small elements.
4389       Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
4390       if (Tmp == SrcBits)
4391         return VTBits;
4392 
4393       // Slow case - determine how far the sign extends into each sub-element.
4394       Tmp2 = VTBits;
4395       for (unsigned i = 0; i != NumElts; ++i)
4396         if (DemandedElts[i]) {
4397           unsigned SubOffset = i % Scale;
4398           SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4399           SubOffset = SubOffset * VTBits;
4400           if (Tmp <= SubOffset)
4401             return 1;
4402           Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4403         }
4404       return Tmp2;
4405     }
4406     break;
4407   }
4408 
4409   case ISD::FP_TO_SINT_SAT:
4410     // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4411     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4412     return VTBits - Tmp + 1;
4413   case ISD::SIGN_EXTEND:
4414     Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
4415     return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
4416   case ISD::SIGN_EXTEND_INREG:
4417     // Max of the input and what this extends.
4418     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4419     Tmp = VTBits-Tmp+1;
4420     Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4421     return std::max(Tmp, Tmp2);
4422   case ISD::SIGN_EXTEND_VECTOR_INREG: {
4423     if (VT.isScalableVector())
4424       break;
4425     SDValue Src = Op.getOperand(0);
4426     EVT SrcVT = Src.getValueType();
4427     APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
4428     Tmp = VTBits - SrcVT.getScalarSizeInBits();
4429     return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
4430   }
4431   case ISD::SRA:
4432     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4433     // SRA X, C -> adds C sign bits.
4434     if (const APInt *ShAmt =
4435             getValidMinimumShiftAmountConstant(Op, DemandedElts))
4436       Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
4437     return Tmp;
4438   case ISD::SHL:
4439     if (const APInt *ShAmt =
4440             getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
4441       // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4442       Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4443       if (ShAmt->ult(Tmp))
4444         return Tmp - ShAmt->getZExtValue();
4445     }
4446     break;
4447   case ISD::AND:
4448   case ISD::OR:
4449   case ISD::XOR:    // NOT is handled here.
4450     // Logical binary ops preserve the number of sign bits at the worst.
4451     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4452     if (Tmp != 1) {
4453       Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4454       FirstAnswer = std::min(Tmp, Tmp2);
4455       // We computed what we know about the sign bits as our first
4456       // answer. Now proceed to the generic code that uses
4457       // computeKnownBits, and pick whichever answer is better.
4458     }
4459     break;
4460 
4461   case ISD::SELECT:
4462   case ISD::VSELECT:
4463     Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4464     if (Tmp == 1) return 1;  // Early out.
4465     Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4466     return std::min(Tmp, Tmp2);
4467   case ISD::SELECT_CC:
4468     Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4469     if (Tmp == 1) return 1;  // Early out.
4470     Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
4471     return std::min(Tmp, Tmp2);
4472 
4473   case ISD::SMIN:
4474   case ISD::SMAX: {
4475     // If we have a clamp pattern, we know that the number of sign bits will be
4476     // the minimum of the clamp min/max range.
4477     bool IsMax = (Opcode == ISD::SMAX);
4478     ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4479     if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4480       if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4481         CstHigh =
4482             isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4483     if (CstLow && CstHigh) {
4484       if (!IsMax)
4485         std::swap(CstLow, CstHigh);
4486       if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
4487         Tmp = CstLow->getAPIntValue().getNumSignBits();
4488         Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4489         return std::min(Tmp, Tmp2);
4490       }
4491     }
4492 
4493     // Fallback - just get the minimum number of sign bits of the operands.
4494     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4495     if (Tmp == 1)
4496       return 1;  // Early out.
4497     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4498     return std::min(Tmp, Tmp2);
4499   }
4500   case ISD::UMIN:
4501   case ISD::UMAX:
4502     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4503     if (Tmp == 1)
4504       return 1;  // Early out.
4505     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4506     return std::min(Tmp, Tmp2);
4507   case ISD::SADDO:
4508   case ISD::UADDO:
4509   case ISD::SADDO_CARRY:
4510   case ISD::UADDO_CARRY:
4511   case ISD::SSUBO:
4512   case ISD::USUBO:
4513   case ISD::SSUBO_CARRY:
4514   case ISD::USUBO_CARRY:
4515   case ISD::SMULO:
4516   case ISD::UMULO:
4517     if (Op.getResNo() != 1)
4518       break;
4519     // The boolean result conforms to getBooleanContents.  Fall through.
4520     // If setcc returns 0/-1, all bits are sign bits.
4521     // We know that we have an integer-based boolean since these operations
4522     // are only available for integer.
4523     if (TLI->getBooleanContents(VT.isVector(), false) ==
4524         TargetLowering::ZeroOrNegativeOneBooleanContent)
4525       return VTBits;
4526     break;
4527   case ISD::SETCC:
4528   case ISD::SETCCCARRY:
4529   case ISD::STRICT_FSETCC:
4530   case ISD::STRICT_FSETCCS: {
4531     unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
4532     // If setcc returns 0/-1, all bits are sign bits.
4533     if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
4534         TargetLowering::ZeroOrNegativeOneBooleanContent)
4535       return VTBits;
4536     break;
4537   }
4538   case ISD::ROTL:
4539   case ISD::ROTR:
4540     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4541 
4542     // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
4543     if (Tmp == VTBits)
4544       return VTBits;
4545 
4546     if (ConstantSDNode *C =
4547             isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
4548       unsigned RotAmt = C->getAPIntValue().urem(VTBits);
4549 
4550       // Handle rotate right by N like a rotate left by 32-N.
4551       if (Opcode == ISD::ROTR)
4552         RotAmt = (VTBits - RotAmt) % VTBits;
4553 
4554       // If we aren't rotating out all of the known-in sign bits, return the
4555       // number that are left.  This handles rotl(sext(x), 1) for example.
4556       if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
4557     }
4558     break;
4559   case ISD::ADD:
4560   case ISD::ADDC:
4561     // Add can have at most one carry bit.  Thus we know that the output
4562     // is, at worst, one more bit than the inputs.
4563     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4564     if (Tmp == 1) return 1; // Early out.
4565 
4566     // Special case decrementing a value (ADD X, -1):
4567     if (ConstantSDNode *CRHS =
4568             isConstOrConstSplat(Op.getOperand(1), DemandedElts))
4569       if (CRHS->isAllOnes()) {
4570         KnownBits Known =
4571             computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4572 
4573         // If the input is known to be 0 or 1, the output is 0/-1, which is all
4574         // sign bits set.
4575         if ((Known.Zero | 1).isAllOnes())
4576           return VTBits;
4577 
4578         // If we are subtracting one from a positive number, there is no carry
4579         // out of the result.
4580         if (Known.isNonNegative())
4581           return Tmp;
4582       }
4583 
4584     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4585     if (Tmp2 == 1) return 1; // Early out.
4586     return std::min(Tmp, Tmp2) - 1;
4587   case ISD::SUB:
4588     Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4589     if (Tmp2 == 1) return 1; // Early out.
4590 
4591     // Handle NEG.
4592     if (ConstantSDNode *CLHS =
4593             isConstOrConstSplat(Op.getOperand(0), DemandedElts))
4594       if (CLHS->isZero()) {
4595         KnownBits Known =
4596             computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4597         // If the input is known to be 0 or 1, the output is 0/-1, which is all
4598         // sign bits set.
4599         if ((Known.Zero | 1).isAllOnes())
4600           return VTBits;
4601 
4602         // If the input is known to be positive (the sign bit is known clear),
4603         // the output of the NEG has the same number of sign bits as the input.
4604         if (Known.isNonNegative())
4605           return Tmp2;
4606 
4607         // Otherwise, we treat this like a SUB.
4608       }
4609 
4610     // Sub can have at most one carry bit.  Thus we know that the output
4611     // is, at worst, one more bit than the inputs.
4612     Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4613     if (Tmp == 1) return 1; // Early out.
4614     return std::min(Tmp, Tmp2) - 1;
4615   case ISD::MUL: {
4616     // The output of the Mul can be at most twice the valid bits in the inputs.
4617     unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4618     if (SignBitsOp0 == 1)
4619       break;
4620     unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4621     if (SignBitsOp1 == 1)
4622       break;
4623     unsigned OutValidBits =
4624         (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4625     return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4626   }
4627   case ISD::SREM:
4628     // The sign bit is the LHS's sign bit, except when the result of the
4629     // remainder is zero. The magnitude of the result should be less than or
4630     // equal to the magnitude of the LHS. Therefore, the result should have
4631     // at least as many sign bits as the left hand side.
4632     return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4633   case ISD::TRUNCATE: {
4634     // Check if the sign bits of source go down as far as the truncated value.
4635     unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4636     unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4637     if (NumSrcSignBits > (NumSrcBits - VTBits))
4638       return NumSrcSignBits - (NumSrcBits - VTBits);
4639     break;
4640   }
4641   case ISD::EXTRACT_ELEMENT: {
4642     if (VT.isScalableVector())
4643       break;
4644     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4645     const int BitWidth = Op.getValueSizeInBits();
4646     const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4647 
4648     // Get reverse index (starting from 1), Op1 value indexes elements from
4649     // little end. Sign starts at big end.
4650     const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4651 
4652     // If the sign portion ends in our element the subtraction gives correct
4653     // result. Otherwise it gives either negative or > bitwidth result
4654     return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth);
4655   }
4656   case ISD::INSERT_VECTOR_ELT: {
4657     if (VT.isScalableVector())
4658       break;
4659     // If we know the element index, split the demand between the
4660     // source vector and the inserted element, otherwise assume we need
4661     // the original demanded vector elements and the value.
4662     SDValue InVec = Op.getOperand(0);
4663     SDValue InVal = Op.getOperand(1);
4664     SDValue EltNo = Op.getOperand(2);
4665     bool DemandedVal = true;
4666     APInt DemandedVecElts = DemandedElts;
4667     auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4668     if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4669       unsigned EltIdx = CEltNo->getZExtValue();
4670       DemandedVal = !!DemandedElts[EltIdx];
4671       DemandedVecElts.clearBit(EltIdx);
4672     }
4673     Tmp = std::numeric_limits<unsigned>::max();
4674     if (DemandedVal) {
4675       // TODO - handle implicit truncation of inserted elements.
4676       if (InVal.getScalarValueSizeInBits() != VTBits)
4677         break;
4678       Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4679       Tmp = std::min(Tmp, Tmp2);
4680     }
4681     if (!!DemandedVecElts) {
4682       Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4683       Tmp = std::min(Tmp, Tmp2);
4684     }
4685     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4686     return Tmp;
4687   }
4688   case ISD::EXTRACT_VECTOR_ELT: {
4689     assert(!VT.isScalableVector());
4690     SDValue InVec = Op.getOperand(0);
4691     SDValue EltNo = Op.getOperand(1);
4692     EVT VecVT = InVec.getValueType();
4693     // ComputeNumSignBits not yet implemented for scalable vectors.
4694     if (VecVT.isScalableVector())
4695       break;
4696     const unsigned BitWidth = Op.getValueSizeInBits();
4697     const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4698     const unsigned NumSrcElts = VecVT.getVectorNumElements();
4699 
4700     // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4701     // anything about sign bits. But if the sizes match we can derive knowledge
4702     // about sign bits from the vector operand.
4703     if (BitWidth != EltBitWidth)
4704       break;
4705 
4706     // If we know the element index, just demand that vector element, else for
4707     // an unknown element index, ignore DemandedElts and demand them all.
4708     APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4709     auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4710     if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4711       DemandedSrcElts =
4712           APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4713 
4714     return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4715   }
4716   case ISD::EXTRACT_SUBVECTOR: {
4717     // Offset the demanded elts by the subvector index.
4718     SDValue Src = Op.getOperand(0);
4719     // Bail until we can represent demanded elements for scalable vectors.
4720     if (Src.getValueType().isScalableVector())
4721       break;
4722     uint64_t Idx = Op.getConstantOperandVal(1);
4723     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4724     APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
4725     return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4726   }
4727   case ISD::CONCAT_VECTORS: {
4728     if (VT.isScalableVector())
4729       break;
4730     // Determine the minimum number of sign bits across all demanded
4731     // elts of the input vectors. Early out if the result is already 1.
4732     Tmp = std::numeric_limits<unsigned>::max();
4733     EVT SubVectorVT = Op.getOperand(0).getValueType();
4734     unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4735     unsigned NumSubVectors = Op.getNumOperands();
4736     for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4737       APInt DemandedSub =
4738           DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4739       if (!DemandedSub)
4740         continue;
4741       Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4742       Tmp = std::min(Tmp, Tmp2);
4743     }
4744     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4745     return Tmp;
4746   }
4747   case ISD::INSERT_SUBVECTOR: {
4748     if (VT.isScalableVector())
4749       break;
4750     // Demand any elements from the subvector and the remainder from the src its
4751     // inserted into.
4752     SDValue Src = Op.getOperand(0);
4753     SDValue Sub = Op.getOperand(1);
4754     uint64_t Idx = Op.getConstantOperandVal(2);
4755     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4756     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4757     APInt DemandedSrcElts = DemandedElts;
4758     DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
4759 
4760     Tmp = std::numeric_limits<unsigned>::max();
4761     if (!!DemandedSubElts) {
4762       Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4763       if (Tmp == 1)
4764         return 1; // early-out
4765     }
4766     if (!!DemandedSrcElts) {
4767       Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4768       Tmp = std::min(Tmp, Tmp2);
4769     }
4770     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4771     return Tmp;
4772   }
4773   case ISD::LOAD: {
4774     LoadSDNode *LD = cast<LoadSDNode>(Op);
4775     if (const MDNode *Ranges = LD->getRanges()) {
4776       if (DemandedElts != 1)
4777         break;
4778 
4779       ConstantRange CR = getConstantRangeFromMetadata(*Ranges);
4780       if (VTBits > CR.getBitWidth()) {
4781         switch (LD->getExtensionType()) {
4782         case ISD::SEXTLOAD:
4783           CR = CR.signExtend(VTBits);
4784           break;
4785         case ISD::ZEXTLOAD:
4786           CR = CR.zeroExtend(VTBits);
4787           break;
4788         default:
4789           break;
4790         }
4791       }
4792 
4793       if (VTBits != CR.getBitWidth())
4794         break;
4795       return std::min(CR.getSignedMin().getNumSignBits(),
4796                       CR.getSignedMax().getNumSignBits());
4797     }
4798 
4799     break;
4800   }
4801   case ISD::ATOMIC_CMP_SWAP:
4802   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4803   case ISD::ATOMIC_SWAP:
4804   case ISD::ATOMIC_LOAD_ADD:
4805   case ISD::ATOMIC_LOAD_SUB:
4806   case ISD::ATOMIC_LOAD_AND:
4807   case ISD::ATOMIC_LOAD_CLR:
4808   case ISD::ATOMIC_LOAD_OR:
4809   case ISD::ATOMIC_LOAD_XOR:
4810   case ISD::ATOMIC_LOAD_NAND:
4811   case ISD::ATOMIC_LOAD_MIN:
4812   case ISD::ATOMIC_LOAD_MAX:
4813   case ISD::ATOMIC_LOAD_UMIN:
4814   case ISD::ATOMIC_LOAD_UMAX:
4815   case ISD::ATOMIC_LOAD: {
4816     Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4817     // If we are looking at the loaded value.
4818     if (Op.getResNo() == 0) {
4819       if (Tmp == VTBits)
4820         return 1; // early-out
4821       if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
4822         return VTBits - Tmp + 1;
4823       if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4824         return VTBits - Tmp;
4825     }
4826     break;
4827   }
4828   }
4829 
4830   // If we are looking at the loaded value of the SDNode.
4831   if (Op.getResNo() == 0) {
4832     // Handle LOADX separately here. EXTLOAD case will fallthrough.
4833     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4834       unsigned ExtType = LD->getExtensionType();
4835       switch (ExtType) {
4836       default: break;
4837       case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4838         Tmp = LD->getMemoryVT().getScalarSizeInBits();
4839         return VTBits - Tmp + 1;
4840       case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4841         Tmp = LD->getMemoryVT().getScalarSizeInBits();
4842         return VTBits - Tmp;
4843       case ISD::NON_EXTLOAD:
4844         if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4845           // We only need to handle vectors - computeKnownBits should handle
4846           // scalar cases.
4847           Type *CstTy = Cst->getType();
4848           if (CstTy->isVectorTy() && !VT.isScalableVector() &&
4849               (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
4850               VTBits == CstTy->getScalarSizeInBits()) {
4851             Tmp = VTBits;
4852             for (unsigned i = 0; i != NumElts; ++i) {
4853               if (!DemandedElts[i])
4854                 continue;
4855               if (Constant *Elt = Cst->getAggregateElement(i)) {
4856                 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4857                   const APInt &Value = CInt->getValue();
4858                   Tmp = std::min(Tmp, Value.getNumSignBits());
4859                   continue;
4860                 }
4861                 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4862                   APInt Value = CFP->getValueAPF().bitcastToAPInt();
4863                   Tmp = std::min(Tmp, Value.getNumSignBits());
4864                   continue;
4865                 }
4866               }
4867               // Unknown type. Conservatively assume no bits match sign bit.
4868               return 1;
4869             }
4870             return Tmp;
4871           }
4872         }
4873         break;
4874       }
4875     }
4876   }
4877 
4878   // Allow the target to implement this method for its nodes.
4879   if (Opcode >= ISD::BUILTIN_OP_END ||
4880       Opcode == ISD::INTRINSIC_WO_CHAIN ||
4881       Opcode == ISD::INTRINSIC_W_CHAIN ||
4882       Opcode == ISD::INTRINSIC_VOID) {
4883     // TODO: This can probably be removed once target code is audited.  This
4884     // is here purely to reduce patch size and review complexity.
4885     if (!VT.isScalableVector()) {
4886       unsigned NumBits =
4887         TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4888       if (NumBits > 1)
4889         FirstAnswer = std::max(FirstAnswer, NumBits);
4890     }
4891   }
4892 
4893   // Finally, if we can prove that the top bits of the result are 0's or 1's,
4894   // use this information.
4895   KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4896   return std::max(FirstAnswer, Known.countMinSignBits());
4897 }
4898 
4899 unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
4900                                                  unsigned Depth) const {
4901   unsigned SignBits = ComputeNumSignBits(Op, Depth);
4902   return Op.getScalarValueSizeInBits() - SignBits + 1;
4903 }
4904 
4905 unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
4906                                                  const APInt &DemandedElts,
4907                                                  unsigned Depth) const {
4908   unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
4909   return Op.getScalarValueSizeInBits() - SignBits + 1;
4910 }
4911 
4912 bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
4913                                                     unsigned Depth) const {
4914   // Early out for FREEZE.
4915   if (Op.getOpcode() == ISD::FREEZE)
4916     return true;
4917 
4918   // TODO: Assume we don't know anything for now.
4919   EVT VT = Op.getValueType();
4920   if (VT.isScalableVector())
4921     return false;
4922 
4923   APInt DemandedElts = VT.isVector()
4924                            ? APInt::getAllOnes(VT.getVectorNumElements())
4925                            : APInt(1, 1);
4926   return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4927 }
4928 
4929 bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
4930                                                     const APInt &DemandedElts,
4931                                                     bool PoisonOnly,
4932                                                     unsigned Depth) const {
4933   unsigned Opcode = Op.getOpcode();
4934 
4935   // Early out for FREEZE.
4936   if (Opcode == ISD::FREEZE)
4937     return true;
4938 
4939   if (Depth >= MaxRecursionDepth)
4940     return false; // Limit search depth.
4941 
4942   if (isIntOrFPConstant(Op))
4943     return true;
4944 
4945   switch (Opcode) {
4946   case ISD::VALUETYPE:
4947   case ISD::FrameIndex:
4948   case ISD::TargetFrameIndex:
4949     return true;
4950 
4951   case ISD::UNDEF:
4952     return PoisonOnly;
4953 
4954   case ISD::BUILD_VECTOR:
4955     // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
4956     // this shouldn't affect the result.
4957     for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
4958       if (!DemandedElts[i])
4959         continue;
4960       if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly,
4961                                             Depth + 1))
4962         return false;
4963     }
4964     return true;
4965 
4966     // TODO: Search for noundef attributes from library functions.
4967 
4968     // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
4969 
4970   default:
4971     // Allow the target to implement this method for its nodes.
4972     if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
4973         Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
4974       return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4975           Op, DemandedElts, *this, PoisonOnly, Depth);
4976     break;
4977   }
4978 
4979   // If Op can't create undef/poison and none of its operands are undef/poison
4980   // then Op is never undef/poison.
4981   // NOTE: TargetNodes should handle this in themselves in
4982   // isGuaranteedNotToBeUndefOrPoisonForTargetNode.
4983   return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
4984                                  Depth) &&
4985          all_of(Op->ops(), [&](SDValue V) {
4986            return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
4987          });
4988 }
4989 
4990 bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
4991                                           bool ConsiderFlags,
4992                                           unsigned Depth) const {
4993   // TODO: Assume we don't know anything for now.
4994   EVT VT = Op.getValueType();
4995   if (VT.isScalableVector())
4996     return true;
4997 
4998   APInt DemandedElts = VT.isVector()
4999                            ? APInt::getAllOnes(VT.getVectorNumElements())
5000                            : APInt(1, 1);
5001   return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5002                                 Depth);
5003 }
5004 
5005 bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5006                                           bool PoisonOnly, bool ConsiderFlags,
5007                                           unsigned Depth) const {
5008   // TODO: Assume we don't know anything for now.
5009   EVT VT = Op.getValueType();
5010   if (VT.isScalableVector())
5011     return true;
5012 
5013   unsigned Opcode = Op.getOpcode();
5014   switch (Opcode) {
5015   case ISD::FREEZE:
5016   case ISD::CONCAT_VECTORS:
5017   case ISD::INSERT_SUBVECTOR:
5018   case ISD::AND:
5019   case ISD::XOR:
5020   case ISD::ROTL:
5021   case ISD::ROTR:
5022   case ISD::FSHL:
5023   case ISD::FSHR:
5024   case ISD::BSWAP:
5025   case ISD::CTPOP:
5026   case ISD::BITREVERSE:
5027   case ISD::PARITY:
5028   case ISD::SIGN_EXTEND:
5029   case ISD::TRUNCATE:
5030   case ISD::SIGN_EXTEND_INREG:
5031   case ISD::SIGN_EXTEND_VECTOR_INREG:
5032   case ISD::ZERO_EXTEND_VECTOR_INREG:
5033   case ISD::BITCAST:
5034   case ISD::BUILD_VECTOR:
5035   case ISD::BUILD_PAIR:
5036     return false;
5037 
5038   // Matches hasPoisonGeneratingFlags().
5039   case ISD::ZERO_EXTEND:
5040     return ConsiderFlags && Op->getFlags().hasNonNeg();
5041 
5042   case ISD::ADD:
5043   case ISD::SUB:
5044   case ISD::MUL:
5045     // Matches hasPoisonGeneratingFlags().
5046     return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5047                              Op->getFlags().hasNoUnsignedWrap());
5048 
5049   case ISD::SHL:
5050     // If the max shift amount isn't in range, then the shift can create poison.
5051     if (!getValidMaximumShiftAmountConstant(Op, DemandedElts))
5052       return true;
5053 
5054     // Matches hasPoisonGeneratingFlags().
5055     return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5056                              Op->getFlags().hasNoUnsignedWrap());
5057 
5058   // Matches hasPoisonGeneratingFlags().
5059   case ISD::OR:
5060     return ConsiderFlags && Op->getFlags().hasDisjoint();
5061 
5062   case ISD::INSERT_VECTOR_ELT:{
5063     // Ensure that the element index is in bounds.
5064     EVT VecVT = Op.getOperand(0).getValueType();
5065     KnownBits KnownIdx = computeKnownBits(Op.getOperand(2), Depth + 1);
5066     return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
5067   }
5068 
5069   default:
5070     // Allow the target to implement this method for its nodes.
5071     if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5072         Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5073       return TLI->canCreateUndefOrPoisonForTargetNode(
5074           Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth);
5075     break;
5076   }
5077 
5078   // Be conservative and return true.
5079   return true;
5080 }
5081 
5082 bool SelectionDAG::isADDLike(SDValue Op) const {
5083   unsigned Opcode = Op.getOpcode();
5084   if (Opcode == ISD::OR)
5085     return Op->getFlags().hasDisjoint() ||
5086            haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));
5087   if (Opcode == ISD::XOR)
5088     return isMinSignedConstant(Op.getOperand(1));
5089   return false;
5090 }
5091 
5092 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5093   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
5094       !isa<ConstantSDNode>(Op.getOperand(1)))
5095     return false;
5096 
5097   if (Op.getOpcode() == ISD::OR &&
5098       !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
5099     return false;
5100 
5101   return true;
5102 }
5103 
5104 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5105   // If we're told that NaNs won't happen, assume they won't.
5106   if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5107     return true;
5108 
5109   if (Depth >= MaxRecursionDepth)
5110     return false; // Limit search depth.
5111 
5112   // If the value is a constant, we can obviously see if it is a NaN or not.
5113   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
5114     return !C->getValueAPF().isNaN() ||
5115            (SNaN && !C->getValueAPF().isSignaling());
5116   }
5117 
5118   unsigned Opcode = Op.getOpcode();
5119   switch (Opcode) {
5120   case ISD::FADD:
5121   case ISD::FSUB:
5122   case ISD::FMUL:
5123   case ISD::FDIV:
5124   case ISD::FREM:
5125   case ISD::FSIN:
5126   case ISD::FCOS:
5127   case ISD::FMA:
5128   case ISD::FMAD: {
5129     if (SNaN)
5130       return true;
5131     // TODO: Need isKnownNeverInfinity
5132     return false;
5133   }
5134   case ISD::FCANONICALIZE:
5135   case ISD::FEXP:
5136   case ISD::FEXP2:
5137   case ISD::FEXP10:
5138   case ISD::FTRUNC:
5139   case ISD::FFLOOR:
5140   case ISD::FCEIL:
5141   case ISD::FROUND:
5142   case ISD::FROUNDEVEN:
5143   case ISD::FRINT:
5144   case ISD::LRINT:
5145   case ISD::LLRINT:
5146   case ISD::FNEARBYINT:
5147   case ISD::FLDEXP: {
5148     if (SNaN)
5149       return true;
5150     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5151   }
5152   case ISD::FABS:
5153   case ISD::FNEG:
5154   case ISD::FCOPYSIGN: {
5155     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5156   }
5157   case ISD::SELECT:
5158     return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5159            isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5160   case ISD::FP_EXTEND:
5161   case ISD::FP_ROUND: {
5162     if (SNaN)
5163       return true;
5164     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5165   }
5166   case ISD::SINT_TO_FP:
5167   case ISD::UINT_TO_FP:
5168     return true;
5169   case ISD::FSQRT: // Need is known positive
5170   case ISD::FLOG:
5171   case ISD::FLOG2:
5172   case ISD::FLOG10:
5173   case ISD::FPOWI:
5174   case ISD::FPOW: {
5175     if (SNaN)
5176       return true;
5177     // TODO: Refine on operand
5178     return false;
5179   }
5180   case ISD::FMINNUM:
5181   case ISD::FMAXNUM: {
5182     // Only one needs to be known not-nan, since it will be returned if the
5183     // other ends up being one.
5184     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
5185            isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5186   }
5187   case ISD::FMINNUM_IEEE:
5188   case ISD::FMAXNUM_IEEE: {
5189     if (SNaN)
5190       return true;
5191     // This can return a NaN if either operand is an sNaN, or if both operands
5192     // are NaN.
5193     return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
5194             isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
5195            (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
5196             isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
5197   }
5198   case ISD::FMINIMUM:
5199   case ISD::FMAXIMUM: {
5200     // TODO: Does this quiet or return the origina NaN as-is?
5201     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5202            isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5203   }
5204   case ISD::EXTRACT_VECTOR_ELT: {
5205     return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5206   }
5207   case ISD::BUILD_VECTOR: {
5208     for (const SDValue &Opnd : Op->ops())
5209       if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
5210         return false;
5211     return true;
5212   }
5213   default:
5214     if (Opcode >= ISD::BUILTIN_OP_END ||
5215         Opcode == ISD::INTRINSIC_WO_CHAIN ||
5216         Opcode == ISD::INTRINSIC_W_CHAIN ||
5217         Opcode == ISD::INTRINSIC_VOID) {
5218       return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
5219     }
5220 
5221     return false;
5222   }
5223 }
5224 
5225 bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
5226   assert(Op.getValueType().isFloatingPoint() &&
5227          "Floating point type expected");
5228 
5229   // If the value is a constant, we can obviously see if it is a zero or not.
5230   return ISD::matchUnaryFpPredicate(
5231       Op, [](ConstantFPSDNode *C) { return !C->isZero(); });
5232 }
5233 
5234 bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
5235   if (Depth >= MaxRecursionDepth)
5236     return false; // Limit search depth.
5237 
5238   assert(!Op.getValueType().isFloatingPoint() &&
5239          "Floating point types unsupported - use isKnownNeverZeroFloat");
5240 
5241   // If the value is a constant, we can obviously see if it is a zero or not.
5242   if (ISD::matchUnaryPredicate(Op,
5243                                [](ConstantSDNode *C) { return !C->isZero(); }))
5244     return true;
5245 
5246   // TODO: Recognize more cases here. Most of the cases are also incomplete to
5247   // some degree.
5248   switch (Op.getOpcode()) {
5249   default:
5250     break;
5251 
5252   case ISD::OR:
5253     return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5254            isKnownNeverZero(Op.getOperand(0), Depth + 1);
5255 
5256   case ISD::VSELECT:
5257   case ISD::SELECT:
5258     return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5259            isKnownNeverZero(Op.getOperand(2), Depth + 1);
5260 
5261   case ISD::SHL: {
5262     if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5263       return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5264     KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5265     // 1 << X is never zero.
5266     if (ValKnown.One[0])
5267       return true;
5268     // If max shift cnt of known ones is non-zero, result is non-zero.
5269     APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5270     if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5271         !ValKnown.One.shl(MaxCnt).isZero())
5272       return true;
5273     break;
5274   }
5275   case ISD::UADDSAT:
5276   case ISD::UMAX:
5277     return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5278            isKnownNeverZero(Op.getOperand(0), Depth + 1);
5279 
5280     // TODO for smin/smax: If either operand is known negative/positive
5281     // respectively we don't need the other to be known at all.
5282   case ISD::SMAX:
5283   case ISD::SMIN:
5284   case ISD::UMIN:
5285     return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5286            isKnownNeverZero(Op.getOperand(0), Depth + 1);
5287 
5288   case ISD::ROTL:
5289   case ISD::ROTR:
5290   case ISD::BITREVERSE:
5291   case ISD::BSWAP:
5292   case ISD::CTPOP:
5293   case ISD::ABS:
5294     return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5295 
5296   case ISD::SRA:
5297   case ISD::SRL: {
5298     if (Op->getFlags().hasExact())
5299       return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5300     KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5301     if (ValKnown.isNegative())
5302       return true;
5303     // If max shift cnt of known ones is non-zero, result is non-zero.
5304     APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5305     if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5306         !ValKnown.One.lshr(MaxCnt).isZero())
5307       return true;
5308     break;
5309   }
5310   case ISD::UDIV:
5311   case ISD::SDIV:
5312     // div exact can only produce a zero if the dividend is zero.
5313     // TODO: For udiv this is also true if Op1 u<= Op0
5314     if (Op->getFlags().hasExact())
5315       return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5316     break;
5317 
5318   case ISD::ADD:
5319     if (Op->getFlags().hasNoUnsignedWrap())
5320       if (isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5321           isKnownNeverZero(Op.getOperand(0), Depth + 1))
5322         return true;
5323     // TODO: There are a lot more cases we can prove for add.
5324     break;
5325 
5326   case ISD::SUB: {
5327     if (isNullConstant(Op.getOperand(0)))
5328       return isKnownNeverZero(Op.getOperand(1), Depth + 1);
5329 
5330     std::optional<bool> ne =
5331         KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1),
5332                       computeKnownBits(Op.getOperand(1), Depth + 1));
5333     return ne && *ne;
5334   }
5335 
5336   case ISD::MUL:
5337     if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5338       if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5339           isKnownNeverZero(Op.getOperand(0), Depth + 1))
5340         return true;
5341     break;
5342 
5343   case ISD::ZERO_EXTEND:
5344   case ISD::SIGN_EXTEND:
5345     return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5346   }
5347 
5348   return computeKnownBits(Op, Depth).isNonZero();
5349 }
5350 
5351 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
5352   // Check the obvious case.
5353   if (A == B) return true;
5354 
5355   // For negative and positive zero.
5356   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
5357     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
5358       if (CA->isZero() && CB->isZero()) return true;
5359 
5360   // Otherwise they may not be equal.
5361   return false;
5362 }
5363 
5364 // Only bits set in Mask must be negated, other bits may be arbitrary.
5365 SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
5366   if (isBitwiseNot(V, AllowUndefs))
5367     return V.getOperand(0);
5368 
5369   // Handle any_extend (not (truncate X)) pattern, where Mask only sets
5370   // bits in the non-extended part.
5371   ConstantSDNode *MaskC = isConstOrConstSplat(Mask);
5372   if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5373     return SDValue();
5374   SDValue ExtArg = V.getOperand(0);
5375   if (ExtArg.getScalarValueSizeInBits() >=
5376           MaskC->getAPIntValue().getActiveBits() &&
5377       isBitwiseNot(ExtArg, AllowUndefs) &&
5378       ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&
5379       ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())
5380     return ExtArg.getOperand(0).getOperand(0);
5381   return SDValue();
5382 }
5383 
5384 static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
5385   // Match masked merge pattern (X & ~M) op (Y & M)
5386   // Including degenerate case (X & ~M) op M
5387   auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
5388                                       SDValue Other) {
5389     if (SDValue NotOperand =
5390             getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {
5391       if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
5392           NotOperand->getOpcode() == ISD::TRUNCATE)
5393         NotOperand = NotOperand->getOperand(0);
5394 
5395       if (Other == NotOperand)
5396         return true;
5397       if (Other->getOpcode() == ISD::AND)
5398         return NotOperand == Other->getOperand(0) ||
5399                NotOperand == Other->getOperand(1);
5400     }
5401     return false;
5402   };
5403 
5404   if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
5405     A = A->getOperand(0);
5406 
5407   if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
5408     B = B->getOperand(0);
5409 
5410   if (A->getOpcode() == ISD::AND)
5411     return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||
5412            MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);
5413   return false;
5414 }
5415 
5416 // FIXME: unify with llvm::haveNoCommonBitsSet.
5417 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
5418   assert(A.getValueType() == B.getValueType() &&
5419          "Values must have the same type");
5420   if (haveNoCommonBitsSetCommutative(A, B) ||
5421       haveNoCommonBitsSetCommutative(B, A))
5422     return true;
5423   return KnownBits::haveNoCommonBitsSet(computeKnownBits(A),
5424                                         computeKnownBits(B));
5425 }
5426 
5427 static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
5428                                SelectionDAG &DAG) {
5429   if (cast<ConstantSDNode>(Step)->isZero())
5430     return DAG.getConstant(0, DL, VT);
5431 
5432   return SDValue();
5433 }
5434 
5435 static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
5436                                 ArrayRef<SDValue> Ops,
5437                                 SelectionDAG &DAG) {
5438   int NumOps = Ops.size();
5439   assert(NumOps != 0 && "Can't build an empty vector!");
5440   assert(!VT.isScalableVector() &&
5441          "BUILD_VECTOR cannot be used with scalable types");
5442   assert(VT.getVectorNumElements() == (unsigned)NumOps &&
5443          "Incorrect element count in BUILD_VECTOR!");
5444 
5445   // BUILD_VECTOR of UNDEFs is UNDEF.
5446   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5447     return DAG.getUNDEF(VT);
5448 
5449   // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
5450   SDValue IdentitySrc;
5451   bool IsIdentity = true;
5452   for (int i = 0; i != NumOps; ++i) {
5453     if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5454         Ops[i].getOperand(0).getValueType() != VT ||
5455         (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
5456         !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
5457         Ops[i].getConstantOperandAPInt(1) != i) {
5458       IsIdentity = false;
5459       break;
5460     }
5461     IdentitySrc = Ops[i].getOperand(0);
5462   }
5463   if (IsIdentity)
5464     return IdentitySrc;
5465 
5466   return SDValue();
5467 }
5468 
5469 /// Try to simplify vector concatenation to an input value, undef, or build
5470 /// vector.
5471 static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
5472                                   ArrayRef<SDValue> Ops,
5473                                   SelectionDAG &DAG) {
5474   assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
5475   assert(llvm::all_of(Ops,
5476                       [Ops](SDValue Op) {
5477                         return Ops[0].getValueType() == Op.getValueType();
5478                       }) &&
5479          "Concatenation of vectors with inconsistent value types!");
5480   assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
5481              VT.getVectorElementCount() &&
5482          "Incorrect element count in vector concatenation!");
5483 
5484   if (Ops.size() == 1)
5485     return Ops[0];
5486 
5487   // Concat of UNDEFs is UNDEF.
5488   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5489     return DAG.getUNDEF(VT);
5490 
5491   // Scan the operands and look for extract operations from a single source
5492   // that correspond to insertion at the same location via this concatenation:
5493   // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
5494   SDValue IdentitySrc;
5495   bool IsIdentity = true;
5496   for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
5497     SDValue Op = Ops[i];
5498     unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
5499     if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5500         Op.getOperand(0).getValueType() != VT ||
5501         (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
5502         Op.getConstantOperandVal(1) != IdentityIndex) {
5503       IsIdentity = false;
5504       break;
5505     }
5506     assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
5507            "Unexpected identity source vector for concat of extracts");
5508     IdentitySrc = Op.getOperand(0);
5509   }
5510   if (IsIdentity) {
5511     assert(IdentitySrc && "Failed to set source vector of extracts");
5512     return IdentitySrc;
5513   }
5514 
5515   // The code below this point is only designed to work for fixed width
5516   // vectors, so we bail out for now.
5517   if (VT.isScalableVector())
5518     return SDValue();
5519 
5520   // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
5521   // simplified to one big BUILD_VECTOR.
5522   // FIXME: Add support for SCALAR_TO_VECTOR as well.
5523   EVT SVT = VT.getScalarType();
5524   SmallVector<SDValue, 16> Elts;
5525   for (SDValue Op : Ops) {
5526     EVT OpVT = Op.getValueType();
5527     if (Op.isUndef())
5528       Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
5529     else if (Op.getOpcode() == ISD::BUILD_VECTOR)
5530       Elts.append(Op->op_begin(), Op->op_end());
5531     else
5532       return SDValue();
5533   }
5534 
5535   // BUILD_VECTOR requires all inputs to be of the same type, find the
5536   // maximum type and extend them all.
5537   for (SDValue Op : Elts)
5538     SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
5539 
5540   if (SVT.bitsGT(VT.getScalarType())) {
5541     for (SDValue &Op : Elts) {
5542       if (Op.isUndef())
5543         Op = DAG.getUNDEF(SVT);
5544       else
5545         Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
5546                  ? DAG.getZExtOrTrunc(Op, DL, SVT)
5547                  : DAG.getSExtOrTrunc(Op, DL, SVT);
5548     }
5549   }
5550 
5551   SDValue V = DAG.getBuildVector(VT, DL, Elts);
5552   NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
5553   return V;
5554 }
5555 
5556 /// Gets or creates the specified node.
5557 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
5558   FoldingSetNodeID ID;
5559   AddNodeIDNode(ID, Opcode, getVTList(VT), std::nullopt);
5560   void *IP = nullptr;
5561   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5562     return SDValue(E, 0);
5563 
5564   auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5565                               getVTList(VT));
5566   CSEMap.InsertNode(N, IP);
5567 
5568   InsertNode(N);
5569   SDValue V = SDValue(N, 0);
5570   NewSDValueDbgMsg(V, "Creating new node: ", this);
5571   return V;
5572 }
5573 
5574 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5575                               SDValue N1) {
5576   SDNodeFlags Flags;
5577   if (Inserter)
5578     Flags = Inserter->getFlags();
5579   return getNode(Opcode, DL, VT, N1, Flags);
5580 }
5581 
5582 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5583                               SDValue N1, const SDNodeFlags Flags) {
5584   assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
5585 
5586   // Constant fold unary operations with a vector integer or float operand.
5587   switch (Opcode) {
5588   default:
5589     // FIXME: Entirely reasonable to perform folding of other unary
5590     // operations here as the need arises.
5591     break;
5592   case ISD::FNEG:
5593   case ISD::FABS:
5594   case ISD::FCEIL:
5595   case ISD::FTRUNC:
5596   case ISD::FFLOOR:
5597   case ISD::FP_EXTEND:
5598   case ISD::FP_TO_SINT:
5599   case ISD::FP_TO_UINT:
5600   case ISD::FP_TO_FP16:
5601   case ISD::FP_TO_BF16:
5602   case ISD::TRUNCATE:
5603   case ISD::ANY_EXTEND:
5604   case ISD::ZERO_EXTEND:
5605   case ISD::SIGN_EXTEND:
5606   case ISD::UINT_TO_FP:
5607   case ISD::SINT_TO_FP:
5608   case ISD::FP16_TO_FP:
5609   case ISD::BF16_TO_FP:
5610   case ISD::BITCAST:
5611   case ISD::ABS:
5612   case ISD::BITREVERSE:
5613   case ISD::BSWAP:
5614   case ISD::CTLZ:
5615   case ISD::CTLZ_ZERO_UNDEF:
5616   case ISD::CTTZ:
5617   case ISD::CTTZ_ZERO_UNDEF:
5618   case ISD::CTPOP:
5619   case ISD::STEP_VECTOR: {
5620     SDValue Ops = {N1};
5621     if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
5622       return Fold;
5623   }
5624   }
5625 
5626   unsigned OpOpcode = N1.getNode()->getOpcode();
5627   switch (Opcode) {
5628   case ISD::STEP_VECTOR:
5629     assert(VT.isScalableVector() &&
5630            "STEP_VECTOR can only be used with scalable types");
5631     assert(OpOpcode == ISD::TargetConstant &&
5632            VT.getVectorElementType() == N1.getValueType() &&
5633            "Unexpected step operand");
5634     break;
5635   case ISD::FREEZE:
5636     assert(VT == N1.getValueType() && "Unexpected VT!");
5637     if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false,
5638                                          /*Depth*/ 1))
5639       return N1;
5640     break;
5641   case ISD::TokenFactor:
5642   case ISD::MERGE_VALUES:
5643   case ISD::CONCAT_VECTORS:
5644     return N1;         // Factor, merge or concat of one node?  No need.
5645   case ISD::BUILD_VECTOR: {
5646     // Attempt to simplify BUILD_VECTOR.
5647     SDValue Ops[] = {N1};
5648     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5649       return V;
5650     break;
5651   }
5652   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
5653   case ISD::FP_EXTEND:
5654     assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
5655            "Invalid FP cast!");
5656     if (N1.getValueType() == VT) return N1;  // noop conversion.
5657     assert((!VT.isVector() || VT.getVectorElementCount() ==
5658                                   N1.getValueType().getVectorElementCount()) &&
5659            "Vector element count mismatch!");
5660     assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
5661     if (N1.isUndef())
5662       return getUNDEF(VT);
5663     break;
5664   case ISD::FP_TO_SINT:
5665   case ISD::FP_TO_UINT:
5666     if (N1.isUndef())
5667       return getUNDEF(VT);
5668     break;
5669   case ISD::SINT_TO_FP:
5670   case ISD::UINT_TO_FP:
5671     // [us]itofp(undef) = 0, because the result value is bounded.
5672     if (N1.isUndef())
5673       return getConstantFP(0.0, DL, VT);
5674     break;
5675   case ISD::SIGN_EXTEND:
5676     assert(VT.isInteger() && N1.getValueType().isInteger() &&
5677            "Invalid SIGN_EXTEND!");
5678     assert(VT.isVector() == N1.getValueType().isVector() &&
5679            "SIGN_EXTEND result type type should be vector iff the operand "
5680            "type is vector!");
5681     if (N1.getValueType() == VT) return N1;   // noop extension
5682     assert((!VT.isVector() || VT.getVectorElementCount() ==
5683                                   N1.getValueType().getVectorElementCount()) &&
5684            "Vector element count mismatch!");
5685     assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
5686     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
5687       return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5688     if (OpOpcode == ISD::UNDEF)
5689       // sext(undef) = 0, because the top bits will all be the same.
5690       return getConstant(0, DL, VT);
5691     break;
5692   case ISD::ZERO_EXTEND:
5693     assert(VT.isInteger() && N1.getValueType().isInteger() &&
5694            "Invalid ZERO_EXTEND!");
5695     assert(VT.isVector() == N1.getValueType().isVector() &&
5696            "ZERO_EXTEND result type type should be vector iff the operand "
5697            "type is vector!");
5698     if (N1.getValueType() == VT) return N1;   // noop extension
5699     assert((!VT.isVector() || VT.getVectorElementCount() ==
5700                                   N1.getValueType().getVectorElementCount()) &&
5701            "Vector element count mismatch!");
5702     assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
5703     if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
5704       return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0));
5705     if (OpOpcode == ISD::UNDEF)
5706       // zext(undef) = 0, because the top bits will be zero.
5707       return getConstant(0, DL, VT);
5708 
5709     // Skip unnecessary zext_inreg pattern:
5710     // (zext (trunc x)) -> x iff the upper bits are known zero.
5711     // TODO: Remove (zext (trunc (and x, c))) exception which some targets
5712     // use to recognise zext_inreg patterns.
5713     if (OpOpcode == ISD::TRUNCATE) {
5714       SDValue OpOp = N1.getOperand(0);
5715       if (OpOp.getValueType() == VT) {
5716         if (OpOp.getOpcode() != ISD::AND) {
5717           APInt HiBits = APInt::getBitsSetFrom(VT.getScalarSizeInBits(),
5718                                                N1.getScalarValueSizeInBits());
5719           if (MaskedValueIsZero(OpOp, HiBits)) {
5720             transferDbgValues(N1, OpOp);
5721             return OpOp;
5722           }
5723         }
5724       }
5725     }
5726     break;
5727   case ISD::ANY_EXTEND:
5728     assert(VT.isInteger() && N1.getValueType().isInteger() &&
5729            "Invalid ANY_EXTEND!");
5730     assert(VT.isVector() == N1.getValueType().isVector() &&
5731            "ANY_EXTEND result type type should be vector iff the operand "
5732            "type is vector!");
5733     if (N1.getValueType() == VT) return N1;   // noop extension
5734     assert((!VT.isVector() || VT.getVectorElementCount() ==
5735                                   N1.getValueType().getVectorElementCount()) &&
5736            "Vector element count mismatch!");
5737     assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
5738 
5739     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5740         OpOpcode == ISD::ANY_EXTEND)
5741       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
5742       return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5743     if (OpOpcode == ISD::UNDEF)
5744       return getUNDEF(VT);
5745 
5746     // (ext (trunc x)) -> x
5747     if (OpOpcode == ISD::TRUNCATE) {
5748       SDValue OpOp = N1.getOperand(0);
5749       if (OpOp.getValueType() == VT) {
5750         transferDbgValues(N1, OpOp);
5751         return OpOp;
5752       }
5753     }
5754     break;
5755   case ISD::TRUNCATE:
5756     assert(VT.isInteger() && N1.getValueType().isInteger() &&
5757            "Invalid TRUNCATE!");
5758     assert(VT.isVector() == N1.getValueType().isVector() &&
5759            "TRUNCATE result type type should be vector iff the operand "
5760            "type is vector!");
5761     if (N1.getValueType() == VT) return N1;   // noop truncate
5762     assert((!VT.isVector() || VT.getVectorElementCount() ==
5763                                   N1.getValueType().getVectorElementCount()) &&
5764            "Vector element count mismatch!");
5765     assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
5766     if (OpOpcode == ISD::TRUNCATE)
5767       return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5768     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5769         OpOpcode == ISD::ANY_EXTEND) {
5770       // If the source is smaller than the dest, we still need an extend.
5771       if (N1.getOperand(0).getValueType().getScalarType().bitsLT(
5772               VT.getScalarType()))
5773         return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5774       if (N1.getOperand(0).getValueType().bitsGT(VT))
5775         return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5776       return N1.getOperand(0);
5777     }
5778     if (OpOpcode == ISD::UNDEF)
5779       return getUNDEF(VT);
5780     if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
5781       return getVScale(DL, VT,
5782                        N1.getConstantOperandAPInt(0).trunc(VT.getSizeInBits()));
5783     break;
5784   case ISD::ANY_EXTEND_VECTOR_INREG:
5785   case ISD::ZERO_EXTEND_VECTOR_INREG:
5786   case ISD::SIGN_EXTEND_VECTOR_INREG:
5787     assert(VT.isVector() && "This DAG node is restricted to vector types.");
5788     assert(N1.getValueType().bitsLE(VT) &&
5789            "The input must be the same size or smaller than the result.");
5790     assert(VT.getVectorMinNumElements() <
5791                N1.getValueType().getVectorMinNumElements() &&
5792            "The destination vector type must have fewer lanes than the input.");
5793     break;
5794   case ISD::ABS:
5795     assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
5796     if (OpOpcode == ISD::UNDEF)
5797       return getConstant(0, DL, VT);
5798     break;
5799   case ISD::BSWAP:
5800     assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
5801     assert((VT.getScalarSizeInBits() % 16 == 0) &&
5802            "BSWAP types must be a multiple of 16 bits!");
5803     if (OpOpcode == ISD::UNDEF)
5804       return getUNDEF(VT);
5805     // bswap(bswap(X)) -> X.
5806     if (OpOpcode == ISD::BSWAP)
5807       return N1.getOperand(0);
5808     break;
5809   case ISD::BITREVERSE:
5810     assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
5811     if (OpOpcode == ISD::UNDEF)
5812       return getUNDEF(VT);
5813     break;
5814   case ISD::BITCAST:
5815     assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
5816            "Cannot BITCAST between types of different sizes!");
5817     if (VT == N1.getValueType()) return N1;   // noop conversion.
5818     if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5819       return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));
5820     if (OpOpcode == ISD::UNDEF)
5821       return getUNDEF(VT);
5822     break;
5823   case ISD::SCALAR_TO_VECTOR:
5824     assert(VT.isVector() && !N1.getValueType().isVector() &&
5825            (VT.getVectorElementType() == N1.getValueType() ||
5826             (VT.getVectorElementType().isInteger() &&
5827              N1.getValueType().isInteger() &&
5828              VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
5829            "Illegal SCALAR_TO_VECTOR node!");
5830     if (OpOpcode == ISD::UNDEF)
5831       return getUNDEF(VT);
5832     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5833     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5834         isa<ConstantSDNode>(N1.getOperand(1)) &&
5835         N1.getConstantOperandVal(1) == 0 &&
5836         N1.getOperand(0).getValueType() == VT)
5837       return N1.getOperand(0);
5838     break;
5839   case ISD::FNEG:
5840     // Negation of an unknown bag of bits is still completely undefined.
5841     if (OpOpcode == ISD::UNDEF)
5842       return getUNDEF(VT);
5843 
5844     if (OpOpcode == ISD::FNEG) // --X -> X
5845       return N1.getOperand(0);
5846     break;
5847   case ISD::FABS:
5848     if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5849       return getNode(ISD::FABS, DL, VT, N1.getOperand(0));
5850     break;
5851   case ISD::VSCALE:
5852     assert(VT == N1.getValueType() && "Unexpected VT!");
5853     break;
5854   case ISD::CTPOP:
5855     if (N1.getValueType().getScalarType() == MVT::i1)
5856       return N1;
5857     break;
5858   case ISD::CTLZ:
5859   case ISD::CTTZ:
5860     if (N1.getValueType().getScalarType() == MVT::i1)
5861       return getNOT(DL, N1, N1.getValueType());
5862     break;
5863   case ISD::VECREDUCE_ADD:
5864     if (N1.getValueType().getScalarType() == MVT::i1)
5865       return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);
5866     break;
5867   case ISD::VECREDUCE_SMIN:
5868   case ISD::VECREDUCE_UMAX:
5869     if (N1.getValueType().getScalarType() == MVT::i1)
5870       return getNode(ISD::VECREDUCE_OR, DL, VT, N1);
5871     break;
5872   case ISD::VECREDUCE_SMAX:
5873   case ISD::VECREDUCE_UMIN:
5874     if (N1.getValueType().getScalarType() == MVT::i1)
5875       return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
5876     break;
5877   }
5878 
5879   SDNode *N;
5880   SDVTList VTs = getVTList(VT);
5881   SDValue Ops[] = {N1};
5882   if (VT != MVT::Glue) { // Don't CSE glue producing nodes
5883     FoldingSetNodeID ID;
5884     AddNodeIDNode(ID, Opcode, VTs, Ops);
5885     void *IP = nullptr;
5886     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5887       E->intersectFlagsWith(Flags);
5888       return SDValue(E, 0);
5889     }
5890 
5891     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5892     N->setFlags(Flags);
5893     createOperands(N, Ops);
5894     CSEMap.InsertNode(N, IP);
5895   } else {
5896     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5897     createOperands(N, Ops);
5898   }
5899 
5900   InsertNode(N);
5901   SDValue V = SDValue(N, 0);
5902   NewSDValueDbgMsg(V, "Creating new node: ", this);
5903   return V;
5904 }
5905 
5906 static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5907                                       const APInt &C2) {
5908   switch (Opcode) {
5909   case ISD::ADD:  return C1 + C2;
5910   case ISD::SUB:  return C1 - C2;
5911   case ISD::MUL:  return C1 * C2;
5912   case ISD::AND:  return C1 & C2;
5913   case ISD::OR:   return C1 | C2;
5914   case ISD::XOR:  return C1 ^ C2;
5915   case ISD::SHL:  return C1 << C2;
5916   case ISD::SRL:  return C1.lshr(C2);
5917   case ISD::SRA:  return C1.ashr(C2);
5918   case ISD::ROTL: return C1.rotl(C2);
5919   case ISD::ROTR: return C1.rotr(C2);
5920   case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
5921   case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
5922   case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
5923   case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
5924   case ISD::SADDSAT: return C1.sadd_sat(C2);
5925   case ISD::UADDSAT: return C1.uadd_sat(C2);
5926   case ISD::SSUBSAT: return C1.ssub_sat(C2);
5927   case ISD::USUBSAT: return C1.usub_sat(C2);
5928   case ISD::SSHLSAT: return C1.sshl_sat(C2);
5929   case ISD::USHLSAT: return C1.ushl_sat(C2);
5930   case ISD::UDIV:
5931     if (!C2.getBoolValue())
5932       break;
5933     return C1.udiv(C2);
5934   case ISD::UREM:
5935     if (!C2.getBoolValue())
5936       break;
5937     return C1.urem(C2);
5938   case ISD::SDIV:
5939     if (!C2.getBoolValue())
5940       break;
5941     return C1.sdiv(C2);
5942   case ISD::SREM:
5943     if (!C2.getBoolValue())
5944       break;
5945     return C1.srem(C2);
5946   case ISD::MULHS: {
5947     unsigned FullWidth = C1.getBitWidth() * 2;
5948     APInt C1Ext = C1.sext(FullWidth);
5949     APInt C2Ext = C2.sext(FullWidth);
5950     return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5951   }
5952   case ISD::MULHU: {
5953     unsigned FullWidth = C1.getBitWidth() * 2;
5954     APInt C1Ext = C1.zext(FullWidth);
5955     APInt C2Ext = C2.zext(FullWidth);
5956     return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
5957   }
5958   case ISD::AVGFLOORS: {
5959     unsigned FullWidth = C1.getBitWidth() + 1;
5960     APInt C1Ext = C1.sext(FullWidth);
5961     APInt C2Ext = C2.sext(FullWidth);
5962     return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1);
5963   }
5964   case ISD::AVGFLOORU: {
5965     unsigned FullWidth = C1.getBitWidth() + 1;
5966     APInt C1Ext = C1.zext(FullWidth);
5967     APInt C2Ext = C2.zext(FullWidth);
5968     return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1);
5969   }
5970   case ISD::AVGCEILS: {
5971     unsigned FullWidth = C1.getBitWidth() + 1;
5972     APInt C1Ext = C1.sext(FullWidth);
5973     APInt C2Ext = C2.sext(FullWidth);
5974     return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1);
5975   }
5976   case ISD::AVGCEILU: {
5977     unsigned FullWidth = C1.getBitWidth() + 1;
5978     APInt C1Ext = C1.zext(FullWidth);
5979     APInt C2Ext = C2.zext(FullWidth);
5980     return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1);
5981   }
5982   case ISD::ABDS:
5983     return APIntOps::smax(C1, C2) - APIntOps::smin(C1, C2);
5984   case ISD::ABDU:
5985     return APIntOps::umax(C1, C2) - APIntOps::umin(C1, C2);
5986   }
5987   return std::nullopt;
5988 }
5989 
5990 // Handle constant folding with UNDEF.
5991 // TODO: Handle more cases.
5992 static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
5993                                                bool IsUndef1, const APInt &C2,
5994                                                bool IsUndef2) {
5995   if (!(IsUndef1 || IsUndef2))
5996     return FoldValue(Opcode, C1, C2);
5997 
5998   // Fold and(x, undef) -> 0
5999   // Fold mul(x, undef) -> 0
6000   if (Opcode == ISD::AND || Opcode == ISD::MUL)
6001     return APInt::getZero(C1.getBitWidth());
6002 
6003   return std::nullopt;
6004 }
6005 
6006 SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
6007                                        const GlobalAddressSDNode *GA,
6008                                        const SDNode *N2) {
6009   if (GA->getOpcode() != ISD::GlobalAddress)
6010     return SDValue();
6011   if (!TLI->isOffsetFoldingLegal(GA))
6012     return SDValue();
6013   auto *C2 = dyn_cast<ConstantSDNode>(N2);
6014   if (!C2)
6015     return SDValue();
6016   int64_t Offset = C2->getSExtValue();
6017   switch (Opcode) {
6018   case ISD::ADD: break;
6019   case ISD::SUB: Offset = -uint64_t(Offset); break;
6020   default: return SDValue();
6021   }
6022   return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
6023                           GA->getOffset() + uint64_t(Offset));
6024 }
6025 
6026 bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6027   switch (Opcode) {
6028   case ISD::SDIV:
6029   case ISD::UDIV:
6030   case ISD::SREM:
6031   case ISD::UREM: {
6032     // If a divisor is zero/undef or any element of a divisor vector is
6033     // zero/undef, the whole op is undef.
6034     assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6035     SDValue Divisor = Ops[1];
6036     if (Divisor.isUndef() || isNullConstant(Divisor))
6037       return true;
6038 
6039     return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
6040            llvm::any_of(Divisor->op_values(),
6041                         [](SDValue V) { return V.isUndef() ||
6042                                         isNullConstant(V); });
6043     // TODO: Handle signed overflow.
6044   }
6045   // TODO: Handle oversized shifts.
6046   default:
6047     return false;
6048   }
6049 }
6050 
6051 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
6052                                              EVT VT, ArrayRef<SDValue> Ops) {
6053   // If the opcode is a target-specific ISD node, there's nothing we can
6054   // do here and the operand rules may not line up with the below, so
6055   // bail early.
6056   // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6057   // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6058   // foldCONCAT_VECTORS in getNode before this is called.
6059   if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6060     return SDValue();
6061 
6062   unsigned NumOps = Ops.size();
6063   if (NumOps == 0)
6064     return SDValue();
6065 
6066   if (isUndef(Opcode, Ops))
6067     return getUNDEF(VT);
6068 
6069   // Handle unary special cases.
6070   if (NumOps == 1) {
6071     SDValue N1 = Ops[0];
6072 
6073     // Constant fold unary operations with an integer constant operand. Even
6074     // opaque constant will be folded, because the folding of unary operations
6075     // doesn't create new constants with different values. Nevertheless, the
6076     // opaque flag is preserved during folding to prevent future folding with
6077     // other constants.
6078     if (auto *C = dyn_cast<ConstantSDNode>(N1)) {
6079       const APInt &Val = C->getAPIntValue();
6080       switch (Opcode) {
6081       case ISD::SIGN_EXTEND:
6082         return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6083                            C->isTargetOpcode(), C->isOpaque());
6084       case ISD::TRUNCATE:
6085         if (C->isOpaque())
6086           break;
6087         [[fallthrough]];
6088       case ISD::ZERO_EXTEND:
6089         return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6090                            C->isTargetOpcode(), C->isOpaque());
6091       case ISD::ANY_EXTEND:
6092         // Some targets like RISCV prefer to sign extend some types.
6093         if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))
6094           return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6095                              C->isTargetOpcode(), C->isOpaque());
6096         return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6097                            C->isTargetOpcode(), C->isOpaque());
6098       case ISD::ABS:
6099         return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
6100                            C->isOpaque());
6101       case ISD::BITREVERSE:
6102         return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
6103                            C->isOpaque());
6104       case ISD::BSWAP:
6105         return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
6106                            C->isOpaque());
6107       case ISD::CTPOP:
6108         return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),
6109                            C->isOpaque());
6110       case ISD::CTLZ:
6111       case ISD::CTLZ_ZERO_UNDEF:
6112         return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),
6113                            C->isOpaque());
6114       case ISD::CTTZ:
6115       case ISD::CTTZ_ZERO_UNDEF:
6116         return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
6117                            C->isOpaque());
6118       case ISD::UINT_TO_FP:
6119       case ISD::SINT_TO_FP: {
6120         APFloat apf(EVTToAPFloatSemantics(VT),
6121                     APInt::getZero(VT.getSizeInBits()));
6122         (void)apf.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,
6123                                    APFloat::rmNearestTiesToEven);
6124         return getConstantFP(apf, DL, VT);
6125       }
6126       case ISD::FP16_TO_FP:
6127       case ISD::BF16_TO_FP: {
6128         bool Ignored;
6129         APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6130                                               : APFloat::BFloat(),
6131                     (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
6132 
6133         // This can return overflow, underflow, or inexact; we don't care.
6134         // FIXME need to be more flexible about rounding mode.
6135         (void)FPV.convert(EVTToAPFloatSemantics(VT),
6136                           APFloat::rmNearestTiesToEven, &Ignored);
6137         return getConstantFP(FPV, DL, VT);
6138       }
6139       case ISD::STEP_VECTOR:
6140         if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))
6141           return V;
6142         break;
6143       case ISD::BITCAST:
6144         if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
6145           return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6146         if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
6147           return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6148         if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
6149           return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6150         if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
6151           return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
6152         break;
6153       }
6154     }
6155 
6156     // Constant fold unary operations with a floating point constant operand.
6157     if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {
6158       APFloat V = C->getValueAPF(); // make copy
6159       switch (Opcode) {
6160       case ISD::FNEG:
6161         V.changeSign();
6162         return getConstantFP(V, DL, VT);
6163       case ISD::FABS:
6164         V.clearSign();
6165         return getConstantFP(V, DL, VT);
6166       case ISD::FCEIL: {
6167         APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
6168         if (fs == APFloat::opOK || fs == APFloat::opInexact)
6169           return getConstantFP(V, DL, VT);
6170         return SDValue();
6171       }
6172       case ISD::FTRUNC: {
6173         APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
6174         if (fs == APFloat::opOK || fs == APFloat::opInexact)
6175           return getConstantFP(V, DL, VT);
6176         return SDValue();
6177       }
6178       case ISD::FFLOOR: {
6179         APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
6180         if (fs == APFloat::opOK || fs == APFloat::opInexact)
6181           return getConstantFP(V, DL, VT);
6182         return SDValue();
6183       }
6184       case ISD::FP_EXTEND: {
6185         bool ignored;
6186         // This can return overflow, underflow, or inexact; we don't care.
6187         // FIXME need to be more flexible about rounding mode.
6188         (void)V.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
6189                         &ignored);
6190         return getConstantFP(V, DL, VT);
6191       }
6192       case ISD::FP_TO_SINT:
6193       case ISD::FP_TO_UINT: {
6194         bool ignored;
6195         APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6196         // FIXME need to be more flexible about rounding mode.
6197         APFloat::opStatus s =
6198             V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
6199         if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6200           break;
6201         return getConstant(IntVal, DL, VT);
6202       }
6203       case ISD::FP_TO_FP16:
6204       case ISD::FP_TO_BF16: {
6205         bool Ignored;
6206         // This can return overflow, underflow, or inexact; we don't care.
6207         // FIXME need to be more flexible about rounding mode.
6208         (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6209                                                   : APFloat::BFloat(),
6210                         APFloat::rmNearestTiesToEven, &Ignored);
6211         return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6212       }
6213       case ISD::BITCAST:
6214         if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
6215           return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6216                              VT);
6217         if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
6218           return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6219                              VT);
6220         if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
6221           return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6222                              VT);
6223         if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
6224           return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6225         break;
6226       }
6227     }
6228 
6229     // Early-out if we failed to constant fold a bitcast.
6230     if (Opcode == ISD::BITCAST)
6231       return SDValue();
6232   }
6233 
6234   // Handle binops special cases.
6235   if (NumOps == 2) {
6236     if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6237       return CFP;
6238 
6239     if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6240       if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6241         if (C1->isOpaque() || C2->isOpaque())
6242           return SDValue();
6243 
6244         std::optional<APInt> FoldAttempt =
6245             FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6246         if (!FoldAttempt)
6247           return SDValue();
6248 
6249         SDValue Folded = getConstant(*FoldAttempt, DL, VT);
6250         assert((!Folded || !VT.isVector()) &&
6251                "Can't fold vectors ops with scalar operands");
6252         return Folded;
6253       }
6254     }
6255 
6256     // fold (add Sym, c) -> Sym+c
6257     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[0]))
6258       return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());
6259     if (TLI->isCommutativeBinOp(Opcode))
6260       if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[1]))
6261         return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());
6262   }
6263 
6264   // This is for vector folding only from here on.
6265   if (!VT.isVector())
6266     return SDValue();
6267 
6268   ElementCount NumElts = VT.getVectorElementCount();
6269 
6270   // See if we can fold through bitcasted integer ops.
6271   if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6272       Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6273       Ops[0].getOpcode() == ISD::BITCAST &&
6274       Ops[1].getOpcode() == ISD::BITCAST) {
6275     SDValue N1 = peekThroughBitcasts(Ops[0]);
6276     SDValue N2 = peekThroughBitcasts(Ops[1]);
6277     auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6278     auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6279     EVT BVVT = N1.getValueType();
6280     if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
6281       bool IsLE = getDataLayout().isLittleEndian();
6282       unsigned EltBits = VT.getScalarSizeInBits();
6283       SmallVector<APInt> RawBits1, RawBits2;
6284       BitVector UndefElts1, UndefElts2;
6285       if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6286           BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6287         SmallVector<APInt> RawBits;
6288         for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6289           std::optional<APInt> Fold = FoldValueWithUndef(
6290               Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);
6291           if (!Fold)
6292             break;
6293           RawBits.push_back(*Fold);
6294         }
6295         if (RawBits.size() == NumElts.getFixedValue()) {
6296           // We have constant folded, but we need to cast this again back to
6297           // the original (possibly legalized) type.
6298           SmallVector<APInt> DstBits;
6299           BitVector DstUndefs;
6300           BuildVectorSDNode::recastRawBits(IsLE, BVVT.getScalarSizeInBits(),
6301                                            DstBits, RawBits, DstUndefs,
6302                                            BitVector(RawBits.size(), false));
6303           EVT BVEltVT = BV1->getOperand(0).getValueType();
6304           unsigned BVEltBits = BVEltVT.getSizeInBits();
6305           SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
6306           for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6307             if (DstUndefs[I])
6308               continue;
6309             Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);
6310           }
6311           return getBitcast(VT, getBuildVector(BVVT, DL, Ops));
6312         }
6313       }
6314     }
6315   }
6316 
6317   // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6318   //      (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6319   if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6320       Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6321     APInt RHSVal;
6322     if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) {
6323       APInt NewStep = Opcode == ISD::MUL
6324                           ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6325                           : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6326       return getStepVector(DL, VT, NewStep);
6327     }
6328   }
6329 
6330   auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6331     return !Op.getValueType().isVector() ||
6332            Op.getValueType().getVectorElementCount() == NumElts;
6333   };
6334 
6335   auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6336     return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6337            Op.getOpcode() == ISD::BUILD_VECTOR ||
6338            Op.getOpcode() == ISD::SPLAT_VECTOR;
6339   };
6340 
6341   // All operands must be vector types with the same number of elements as
6342   // the result type and must be either UNDEF or a build/splat vector
6343   // or UNDEF scalars.
6344   if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
6345       !llvm::all_of(Ops, IsScalarOrSameVectorSize))
6346     return SDValue();
6347 
6348   // If we are comparing vectors, then the result needs to be a i1 boolean that
6349   // is then extended back to the legal result type depending on how booleans
6350   // are represented.
6351   EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
6352   ISD::NodeType ExtendCode =
6353       (Opcode == ISD::SETCC && SVT != VT.getScalarType())
6354           ? TargetLowering::getExtendForContent(TLI->getBooleanContents(VT))
6355           : ISD::SIGN_EXTEND;
6356 
6357   // Find legal integer scalar type for constant promotion and
6358   // ensure that its scalar size is at least as large as source.
6359   EVT LegalSVT = VT.getScalarType();
6360   if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
6361     LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
6362     if (LegalSVT.bitsLT(VT.getScalarType()))
6363       return SDValue();
6364   }
6365 
6366   // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
6367   // only have one operand to check. For fixed-length vector types we may have
6368   // a combination of BUILD_VECTOR and SPLAT_VECTOR.
6369   unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
6370 
6371   // Constant fold each scalar lane separately.
6372   SmallVector<SDValue, 4> ScalarResults;
6373   for (unsigned I = 0; I != NumVectorElts; I++) {
6374     SmallVector<SDValue, 4> ScalarOps;
6375     for (SDValue Op : Ops) {
6376       EVT InSVT = Op.getValueType().getScalarType();
6377       if (Op.getOpcode() != ISD::BUILD_VECTOR &&
6378           Op.getOpcode() != ISD::SPLAT_VECTOR) {
6379         if (Op.isUndef())
6380           ScalarOps.push_back(getUNDEF(InSVT));
6381         else
6382           ScalarOps.push_back(Op);
6383         continue;
6384       }
6385 
6386       SDValue ScalarOp =
6387           Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
6388       EVT ScalarVT = ScalarOp.getValueType();
6389 
6390       // Build vector (integer) scalar operands may need implicit
6391       // truncation - do this before constant folding.
6392       if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
6393         // Don't create illegally-typed nodes unless they're constants or undef
6394         // - if we fail to constant fold we can't guarantee the (dead) nodes
6395         // we're creating will be cleaned up before being visited for
6396         // legalization.
6397         if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
6398             !isa<ConstantSDNode>(ScalarOp) &&
6399             TLI->getTypeAction(*getContext(), InSVT) !=
6400                 TargetLowering::TypeLegal)
6401           return SDValue();
6402         ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
6403       }
6404 
6405       ScalarOps.push_back(ScalarOp);
6406     }
6407 
6408     // Constant fold the scalar operands.
6409     SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps);
6410 
6411     // Legalize the (integer) scalar constant if necessary.
6412     if (LegalSVT != SVT)
6413       ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);
6414 
6415     // Scalar folding only succeeded if the result is a constant or UNDEF.
6416     if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
6417         ScalarResult.getOpcode() != ISD::ConstantFP)
6418       return SDValue();
6419     ScalarResults.push_back(ScalarResult);
6420   }
6421 
6422   SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
6423                                    : getBuildVector(VT, DL, ScalarResults);
6424   NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
6425   return V;
6426 }
6427 
6428 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
6429                                          EVT VT, ArrayRef<SDValue> Ops) {
6430   // TODO: Add support for unary/ternary fp opcodes.
6431   if (Ops.size() != 2)
6432     return SDValue();
6433 
6434   // TODO: We don't do any constant folding for strict FP opcodes here, but we
6435   //       should. That will require dealing with a potentially non-default
6436   //       rounding mode, checking the "opStatus" return value from the APFloat
6437   //       math calculations, and possibly other variations.
6438   SDValue N1 = Ops[0];
6439   SDValue N2 = Ops[1];
6440   ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
6441   ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
6442   if (N1CFP && N2CFP) {
6443     APFloat C1 = N1CFP->getValueAPF(); // make copy
6444     const APFloat &C2 = N2CFP->getValueAPF();
6445     switch (Opcode) {
6446     case ISD::FADD:
6447       C1.add(C2, APFloat::rmNearestTiesToEven);
6448       return getConstantFP(C1, DL, VT);
6449     case ISD::FSUB:
6450       C1.subtract(C2, APFloat::rmNearestTiesToEven);
6451       return getConstantFP(C1, DL, VT);
6452     case ISD::FMUL:
6453       C1.multiply(C2, APFloat::rmNearestTiesToEven);
6454       return getConstantFP(C1, DL, VT);
6455     case ISD::FDIV:
6456       C1.divide(C2, APFloat::rmNearestTiesToEven);
6457       return getConstantFP(C1, DL, VT);
6458     case ISD::FREM:
6459       C1.mod(C2);
6460       return getConstantFP(C1, DL, VT);
6461     case ISD::FCOPYSIGN:
6462       C1.copySign(C2);
6463       return getConstantFP(C1, DL, VT);
6464     case ISD::FMINNUM:
6465       return getConstantFP(minnum(C1, C2), DL, VT);
6466     case ISD::FMAXNUM:
6467       return getConstantFP(maxnum(C1, C2), DL, VT);
6468     case ISD::FMINIMUM:
6469       return getConstantFP(minimum(C1, C2), DL, VT);
6470     case ISD::FMAXIMUM:
6471       return getConstantFP(maximum(C1, C2), DL, VT);
6472     default: break;
6473     }
6474   }
6475   if (N1CFP && Opcode == ISD::FP_ROUND) {
6476     APFloat C1 = N1CFP->getValueAPF();    // make copy
6477     bool Unused;
6478     // This can return overflow, underflow, or inexact; we don't care.
6479     // FIXME need to be more flexible about rounding mode.
6480     (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
6481                       &Unused);
6482     return getConstantFP(C1, DL, VT);
6483   }
6484 
6485   switch (Opcode) {
6486   case ISD::FSUB:
6487     // -0.0 - undef --> undef (consistent with "fneg undef")
6488     if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
6489       if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
6490         return getUNDEF(VT);
6491     [[fallthrough]];
6492 
6493   case ISD::FADD:
6494   case ISD::FMUL:
6495   case ISD::FDIV:
6496   case ISD::FREM:
6497     // If both operands are undef, the result is undef. If 1 operand is undef,
6498     // the result is NaN. This should match the behavior of the IR optimizer.
6499     if (N1.isUndef() && N2.isUndef())
6500       return getUNDEF(VT);
6501     if (N1.isUndef() || N2.isUndef())
6502       return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
6503   }
6504   return SDValue();
6505 }
6506 
6507 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
6508   assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
6509 
6510   // There's no need to assert on a byte-aligned pointer. All pointers are at
6511   // least byte aligned.
6512   if (A == Align(1))
6513     return Val;
6514 
6515   FoldingSetNodeID ID;
6516   AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val});
6517   ID.AddInteger(A.value());
6518 
6519   void *IP = nullptr;
6520   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6521     return SDValue(E, 0);
6522 
6523   auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
6524                                          Val.getValueType(), A);
6525   createOperands(N, {Val});
6526 
6527   CSEMap.InsertNode(N, IP);
6528   InsertNode(N);
6529 
6530   SDValue V(N, 0);
6531   NewSDValueDbgMsg(V, "Creating new node: ", this);
6532   return V;
6533 }
6534 
6535 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6536                               SDValue N1, SDValue N2) {
6537   SDNodeFlags Flags;
6538   if (Inserter)
6539     Flags = Inserter->getFlags();
6540   return getNode(Opcode, DL, VT, N1, N2, Flags);
6541 }
6542 
6543 void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
6544                                                 SDValue &N2) const {
6545   if (!TLI->isCommutativeBinOp(Opcode))
6546     return;
6547 
6548   // Canonicalize:
6549   //   binop(const, nonconst) -> binop(nonconst, const)
6550   SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N1);
6551   SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N2);
6552   SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N1);
6553   SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N2);
6554   if ((N1C && !N2C) || (N1CFP && !N2CFP))
6555     std::swap(N1, N2);
6556 
6557   // Canonicalize:
6558   //  binop(splat(x), step_vector) -> binop(step_vector, splat(x))
6559   else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6560            N2.getOpcode() == ISD::STEP_VECTOR)
6561     std::swap(N1, N2);
6562 }
6563 
6564 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6565                               SDValue N1, SDValue N2, const SDNodeFlags Flags) {
6566   assert(N1.getOpcode() != ISD::DELETED_NODE &&
6567          N2.getOpcode() != ISD::DELETED_NODE &&
6568          "Operand is DELETED_NODE!");
6569 
6570   canonicalizeCommutativeBinop(Opcode, N1, N2);
6571 
6572   auto *N1C = dyn_cast<ConstantSDNode>(N1);
6573   auto *N2C = dyn_cast<ConstantSDNode>(N2);
6574 
6575   // Don't allow undefs in vector splats - we might be returning N2 when folding
6576   // to zero etc.
6577   ConstantSDNode *N2CV =
6578       isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
6579 
6580   switch (Opcode) {
6581   default: break;
6582   case ISD::TokenFactor:
6583     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
6584            N2.getValueType() == MVT::Other && "Invalid token factor!");
6585     // Fold trivial token factors.
6586     if (N1.getOpcode() == ISD::EntryToken) return N2;
6587     if (N2.getOpcode() == ISD::EntryToken) return N1;
6588     if (N1 == N2) return N1;
6589     break;
6590   case ISD::BUILD_VECTOR: {
6591     // Attempt to simplify BUILD_VECTOR.
6592     SDValue Ops[] = {N1, N2};
6593     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6594       return V;
6595     break;
6596   }
6597   case ISD::CONCAT_VECTORS: {
6598     SDValue Ops[] = {N1, N2};
6599     if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
6600       return V;
6601     break;
6602   }
6603   case ISD::AND:
6604     assert(VT.isInteger() && "This operator does not apply to FP types!");
6605     assert(N1.getValueType() == N2.getValueType() &&
6606            N1.getValueType() == VT && "Binary operator types must match!");
6607     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
6608     // worth handling here.
6609     if (N2CV && N2CV->isZero())
6610       return N2;
6611     if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
6612       return N1;
6613     break;
6614   case ISD::OR:
6615   case ISD::XOR:
6616   case ISD::ADD:
6617   case ISD::SUB:
6618     assert(VT.isInteger() && "This operator does not apply to FP types!");
6619     assert(N1.getValueType() == N2.getValueType() &&
6620            N1.getValueType() == VT && "Binary operator types must match!");
6621     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
6622     // it's worth handling here.
6623     if (N2CV && N2CV->isZero())
6624       return N1;
6625     if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
6626         VT.getVectorElementType() == MVT::i1)
6627       return getNode(ISD::XOR, DL, VT, N1, N2);
6628     break;
6629   case ISD::MUL:
6630     assert(VT.isInteger() && "This operator does not apply to FP types!");
6631     assert(N1.getValueType() == N2.getValueType() &&
6632            N1.getValueType() == VT && "Binary operator types must match!");
6633     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6634       return getNode(ISD::AND, DL, VT, N1, N2);
6635     if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6636       const APInt &MulImm = N1->getConstantOperandAPInt(0);
6637       const APInt &N2CImm = N2C->getAPIntValue();
6638       return getVScale(DL, VT, MulImm * N2CImm);
6639     }
6640     break;
6641   case ISD::UDIV:
6642   case ISD::UREM:
6643   case ISD::MULHU:
6644   case ISD::MULHS:
6645   case ISD::SDIV:
6646   case ISD::SREM:
6647   case ISD::SADDSAT:
6648   case ISD::SSUBSAT:
6649   case ISD::UADDSAT:
6650   case ISD::USUBSAT:
6651     assert(VT.isInteger() && "This operator does not apply to FP types!");
6652     assert(N1.getValueType() == N2.getValueType() &&
6653            N1.getValueType() == VT && "Binary operator types must match!");
6654     if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
6655       // fold (add_sat x, y) -> (or x, y) for bool types.
6656       if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
6657         return getNode(ISD::OR, DL, VT, N1, N2);
6658       // fold (sub_sat x, y) -> (and x, ~y) for bool types.
6659       if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
6660         return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
6661     }
6662     break;
6663   case ISD::ABDS:
6664   case ISD::ABDU:
6665     assert(VT.isInteger() && "This operator does not apply to FP types!");
6666     assert(N1.getValueType() == N2.getValueType() &&
6667            N1.getValueType() == VT && "Binary operator types must match!");
6668     break;
6669   case ISD::SMIN:
6670   case ISD::UMAX:
6671     assert(VT.isInteger() && "This operator does not apply to FP types!");
6672     assert(N1.getValueType() == N2.getValueType() &&
6673            N1.getValueType() == VT && "Binary operator types must match!");
6674     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6675       return getNode(ISD::OR, DL, VT, N1, N2);
6676     break;
6677   case ISD::SMAX:
6678   case ISD::UMIN:
6679     assert(VT.isInteger() && "This operator does not apply to FP types!");
6680     assert(N1.getValueType() == N2.getValueType() &&
6681            N1.getValueType() == VT && "Binary operator types must match!");
6682     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6683       return getNode(ISD::AND, DL, VT, N1, N2);
6684     break;
6685   case ISD::FADD:
6686   case ISD::FSUB:
6687   case ISD::FMUL:
6688   case ISD::FDIV:
6689   case ISD::FREM:
6690     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
6691     assert(N1.getValueType() == N2.getValueType() &&
6692            N1.getValueType() == VT && "Binary operator types must match!");
6693     if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
6694       return V;
6695     break;
6696   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
6697     assert(N1.getValueType() == VT &&
6698            N1.getValueType().isFloatingPoint() &&
6699            N2.getValueType().isFloatingPoint() &&
6700            "Invalid FCOPYSIGN!");
6701     break;
6702   case ISD::SHL:
6703     if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6704       const APInt &MulImm = N1->getConstantOperandAPInt(0);
6705       const APInt &ShiftImm = N2C->getAPIntValue();
6706       return getVScale(DL, VT, MulImm << ShiftImm);
6707     }
6708     [[fallthrough]];
6709   case ISD::SRA:
6710   case ISD::SRL:
6711     if (SDValue V = simplifyShift(N1, N2))
6712       return V;
6713     [[fallthrough]];
6714   case ISD::ROTL:
6715   case ISD::ROTR:
6716     assert(VT == N1.getValueType() &&
6717            "Shift operators return type must be the same as their first arg");
6718     assert(VT.isInteger() && N2.getValueType().isInteger() &&
6719            "Shifts only work on integers");
6720     assert((!VT.isVector() || VT == N2.getValueType()) &&
6721            "Vector shift amounts must be in the same as their first arg");
6722     // Verify that the shift amount VT is big enough to hold valid shift
6723     // amounts.  This catches things like trying to shift an i1024 value by an
6724     // i8, which is easy to fall into in generic code that uses
6725     // TLI.getShiftAmount().
6726     assert(N2.getValueType().getScalarSizeInBits() >=
6727                Log2_32_Ceil(VT.getScalarSizeInBits()) &&
6728            "Invalid use of small shift amount with oversized value!");
6729 
6730     // Always fold shifts of i1 values so the code generator doesn't need to
6731     // handle them.  Since we know the size of the shift has to be less than the
6732     // size of the value, the shift/rotate count is guaranteed to be zero.
6733     if (VT == MVT::i1)
6734       return N1;
6735     if (N2CV && N2CV->isZero())
6736       return N1;
6737     break;
6738   case ISD::FP_ROUND:
6739     assert(VT.isFloatingPoint() &&
6740            N1.getValueType().isFloatingPoint() &&
6741            VT.bitsLE(N1.getValueType()) &&
6742            N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
6743            "Invalid FP_ROUND!");
6744     if (N1.getValueType() == VT) return N1;  // noop conversion.
6745     break;
6746   case ISD::AssertSext:
6747   case ISD::AssertZext: {
6748     EVT EVT = cast<VTSDNode>(N2)->getVT();
6749     assert(VT == N1.getValueType() && "Not an inreg extend!");
6750     assert(VT.isInteger() && EVT.isInteger() &&
6751            "Cannot *_EXTEND_INREG FP types");
6752     assert(!EVT.isVector() &&
6753            "AssertSExt/AssertZExt type should be the vector element type "
6754            "rather than the vector type!");
6755     assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
6756     if (VT.getScalarType() == EVT) return N1; // noop assertion.
6757     break;
6758   }
6759   case ISD::SIGN_EXTEND_INREG: {
6760     EVT EVT = cast<VTSDNode>(N2)->getVT();
6761     assert(VT == N1.getValueType() && "Not an inreg extend!");
6762     assert(VT.isInteger() && EVT.isInteger() &&
6763            "Cannot *_EXTEND_INREG FP types");
6764     assert(EVT.isVector() == VT.isVector() &&
6765            "SIGN_EXTEND_INREG type should be vector iff the operand "
6766            "type is vector!");
6767     assert((!EVT.isVector() ||
6768             EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
6769            "Vector element counts must match in SIGN_EXTEND_INREG");
6770     assert(EVT.bitsLE(VT) && "Not extending!");
6771     if (EVT == VT) return N1;  // Not actually extending
6772 
6773     auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6774       unsigned FromBits = EVT.getScalarSizeInBits();
6775       Val <<= Val.getBitWidth() - FromBits;
6776       Val.ashrInPlace(Val.getBitWidth() - FromBits);
6777       return getConstant(Val, DL, ConstantVT);
6778     };
6779 
6780     if (N1C) {
6781       const APInt &Val = N1C->getAPIntValue();
6782       return SignExtendInReg(Val, VT);
6783     }
6784 
6785     if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
6786       SmallVector<SDValue, 8> Ops;
6787       llvm::EVT OpVT = N1.getOperand(0).getValueType();
6788       for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
6789         SDValue Op = N1.getOperand(i);
6790         if (Op.isUndef()) {
6791           Ops.push_back(getUNDEF(OpVT));
6792           continue;
6793         }
6794         ConstantSDNode *C = cast<ConstantSDNode>(Op);
6795         APInt Val = C->getAPIntValue();
6796         Ops.push_back(SignExtendInReg(Val, OpVT));
6797       }
6798       return getBuildVector(VT, DL, Ops);
6799     }
6800 
6801     if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6802         isa<ConstantSDNode>(N1.getOperand(0)))
6803       return getNode(
6804           ISD::SPLAT_VECTOR, DL, VT,
6805           SignExtendInReg(N1.getConstantOperandAPInt(0),
6806                           N1.getOperand(0).getValueType()));
6807     break;
6808   }
6809   case ISD::FP_TO_SINT_SAT:
6810   case ISD::FP_TO_UINT_SAT: {
6811     assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
6812            N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
6813     assert(N1.getValueType().isVector() == VT.isVector() &&
6814            "FP_TO_*INT_SAT type should be vector iff the operand type is "
6815            "vector!");
6816     assert((!VT.isVector() || VT.getVectorElementCount() ==
6817                                   N1.getValueType().getVectorElementCount()) &&
6818            "Vector element counts must match in FP_TO_*INT_SAT");
6819     assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
6820            "Type to saturate to must be a scalar.");
6821     assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
6822            "Not extending!");
6823     break;
6824   }
6825   case ISD::EXTRACT_VECTOR_ELT:
6826     assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
6827            "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
6828              element type of the vector.");
6829 
6830     // Extract from an undefined value or using an undefined index is undefined.
6831     if (N1.isUndef() || N2.isUndef())
6832       return getUNDEF(VT);
6833 
6834     // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
6835     // vectors. For scalable vectors we will provide appropriate support for
6836     // dealing with arbitrary indices.
6837     if (N2C && N1.getValueType().isFixedLengthVector() &&
6838         N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
6839       return getUNDEF(VT);
6840 
6841     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
6842     // expanding copies of large vectors from registers. This only works for
6843     // fixed length vectors, since we need to know the exact number of
6844     // elements.
6845     if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
6846         N1.getOperand(0).getValueType().isFixedLengthVector()) {
6847       unsigned Factor =
6848         N1.getOperand(0).getValueType().getVectorNumElements();
6849       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
6850                      N1.getOperand(N2C->getZExtValue() / Factor),
6851                      getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
6852     }
6853 
6854     // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
6855     // lowering is expanding large vector constants.
6856     if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
6857                 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
6858       assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
6859               N1.getValueType().isFixedLengthVector()) &&
6860              "BUILD_VECTOR used for scalable vectors");
6861       unsigned Index =
6862           N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
6863       SDValue Elt = N1.getOperand(Index);
6864 
6865       if (VT != Elt.getValueType())
6866         // If the vector element type is not legal, the BUILD_VECTOR operands
6867         // are promoted and implicitly truncated, and the result implicitly
6868         // extended. Make that explicit here.
6869         Elt = getAnyExtOrTrunc(Elt, DL, VT);
6870 
6871       return Elt;
6872     }
6873 
6874     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
6875     // operations are lowered to scalars.
6876     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
6877       // If the indices are the same, return the inserted element else
6878       // if the indices are known different, extract the element from
6879       // the original vector.
6880       SDValue N1Op2 = N1.getOperand(2);
6881       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
6882 
6883       if (N1Op2C && N2C) {
6884         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
6885           if (VT == N1.getOperand(1).getValueType())
6886             return N1.getOperand(1);
6887           if (VT.isFloatingPoint()) {
6888             assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
6889             return getFPExtendOrRound(N1.getOperand(1), DL, VT);
6890           }
6891           return getSExtOrTrunc(N1.getOperand(1), DL, VT);
6892         }
6893         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
6894       }
6895     }
6896 
6897     // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
6898     // when vector types are scalarized and v1iX is legal.
6899     // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
6900     // Here we are completely ignoring the extract element index (N2),
6901     // which is fine for fixed width vectors, since any index other than 0
6902     // is undefined anyway. However, this cannot be ignored for scalable
6903     // vectors - in theory we could support this, but we don't want to do this
6904     // without a profitability check.
6905     if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6906         N1.getValueType().isFixedLengthVector() &&
6907         N1.getValueType().getVectorNumElements() == 1) {
6908       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
6909                      N1.getOperand(1));
6910     }
6911     break;
6912   case ISD::EXTRACT_ELEMENT:
6913     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
6914     assert(!N1.getValueType().isVector() && !VT.isVector() &&
6915            (N1.getValueType().isInteger() == VT.isInteger()) &&
6916            N1.getValueType() != VT &&
6917            "Wrong types for EXTRACT_ELEMENT!");
6918 
6919     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
6920     // 64-bit integers into 32-bit parts.  Instead of building the extract of
6921     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
6922     if (N1.getOpcode() == ISD::BUILD_PAIR)
6923       return N1.getOperand(N2C->getZExtValue());
6924 
6925     // EXTRACT_ELEMENT of a constant int is also very common.
6926     if (N1C) {
6927       unsigned ElementSize = VT.getSizeInBits();
6928       unsigned Shift = ElementSize * N2C->getZExtValue();
6929       const APInt &Val = N1C->getAPIntValue();
6930       return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
6931     }
6932     break;
6933   case ISD::EXTRACT_SUBVECTOR: {
6934     EVT N1VT = N1.getValueType();
6935     assert(VT.isVector() && N1VT.isVector() &&
6936            "Extract subvector VTs must be vectors!");
6937     assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
6938            "Extract subvector VTs must have the same element type!");
6939     assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
6940            "Cannot extract a scalable vector from a fixed length vector!");
6941     assert((VT.isScalableVector() != N1VT.isScalableVector() ||
6942             VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
6943            "Extract subvector must be from larger vector to smaller vector!");
6944     assert(N2C && "Extract subvector index must be a constant");
6945     assert((VT.isScalableVector() != N1VT.isScalableVector() ||
6946             (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
6947                 N1VT.getVectorMinNumElements()) &&
6948            "Extract subvector overflow!");
6949     assert(N2C->getAPIntValue().getBitWidth() ==
6950                TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
6951            "Constant index for EXTRACT_SUBVECTOR has an invalid size");
6952 
6953     // Trivial extraction.
6954     if (VT == N1VT)
6955       return N1;
6956 
6957     // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
6958     if (N1.isUndef())
6959       return getUNDEF(VT);
6960 
6961     // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
6962     // the concat have the same type as the extract.
6963     if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
6964         VT == N1.getOperand(0).getValueType()) {
6965       unsigned Factor = VT.getVectorMinNumElements();
6966       return N1.getOperand(N2C->getZExtValue() / Factor);
6967     }
6968 
6969     // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
6970     // during shuffle legalization.
6971     if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
6972         VT == N1.getOperand(1).getValueType())
6973       return N1.getOperand(1);
6974     break;
6975   }
6976   }
6977 
6978   // Perform trivial constant folding.
6979   if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
6980     return SV;
6981 
6982   // Canonicalize an UNDEF to the RHS, even over a constant.
6983   if (N1.isUndef()) {
6984     if (TLI->isCommutativeBinOp(Opcode)) {
6985       std::swap(N1, N2);
6986     } else {
6987       switch (Opcode) {
6988       case ISD::SUB:
6989         return getUNDEF(VT);     // fold op(undef, arg2) -> undef
6990       case ISD::SIGN_EXTEND_INREG:
6991       case ISD::UDIV:
6992       case ISD::SDIV:
6993       case ISD::UREM:
6994       case ISD::SREM:
6995       case ISD::SSUBSAT:
6996       case ISD::USUBSAT:
6997         return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
6998       }
6999     }
7000   }
7001 
7002   // Fold a bunch of operators when the RHS is undef.
7003   if (N2.isUndef()) {
7004     switch (Opcode) {
7005     case ISD::XOR:
7006       if (N1.isUndef())
7007         // Handle undef ^ undef -> 0 special case. This is a common
7008         // idiom (misuse).
7009         return getConstant(0, DL, VT);
7010       [[fallthrough]];
7011     case ISD::ADD:
7012     case ISD::SUB:
7013     case ISD::UDIV:
7014     case ISD::SDIV:
7015     case ISD::UREM:
7016     case ISD::SREM:
7017       return getUNDEF(VT);       // fold op(arg1, undef) -> undef
7018     case ISD::MUL:
7019     case ISD::AND:
7020     case ISD::SSUBSAT:
7021     case ISD::USUBSAT:
7022       return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
7023     case ISD::OR:
7024     case ISD::SADDSAT:
7025     case ISD::UADDSAT:
7026       return getAllOnesConstant(DL, VT);
7027     }
7028   }
7029 
7030   // Memoize this node if possible.
7031   SDNode *N;
7032   SDVTList VTs = getVTList(VT);
7033   SDValue Ops[] = {N1, N2};
7034   if (VT != MVT::Glue) {
7035     FoldingSetNodeID ID;
7036     AddNodeIDNode(ID, Opcode, VTs, Ops);
7037     void *IP = nullptr;
7038     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7039       E->intersectFlagsWith(Flags);
7040       return SDValue(E, 0);
7041     }
7042 
7043     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7044     N->setFlags(Flags);
7045     createOperands(N, Ops);
7046     CSEMap.InsertNode(N, IP);
7047   } else {
7048     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7049     createOperands(N, Ops);
7050   }
7051 
7052   InsertNode(N);
7053   SDValue V = SDValue(N, 0);
7054   NewSDValueDbgMsg(V, "Creating new node: ", this);
7055   return V;
7056 }
7057 
7058 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7059                               SDValue N1, SDValue N2, SDValue N3) {
7060   SDNodeFlags Flags;
7061   if (Inserter)
7062     Flags = Inserter->getFlags();
7063   return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7064 }
7065 
7066 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7067                               SDValue N1, SDValue N2, SDValue N3,
7068                               const SDNodeFlags Flags) {
7069   assert(N1.getOpcode() != ISD::DELETED_NODE &&
7070          N2.getOpcode() != ISD::DELETED_NODE &&
7071          N3.getOpcode() != ISD::DELETED_NODE &&
7072          "Operand is DELETED_NODE!");
7073   // Perform various simplifications.
7074   switch (Opcode) {
7075   case ISD::FMA:
7076   case ISD::FMAD: {
7077     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7078     assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7079            N3.getValueType() == VT && "FMA types must match!");
7080     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
7081     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
7082     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
7083     if (N1CFP && N2CFP && N3CFP) {
7084       APFloat  V1 = N1CFP->getValueAPF();
7085       const APFloat &V2 = N2CFP->getValueAPF();
7086       const APFloat &V3 = N3CFP->getValueAPF();
7087       if (Opcode == ISD::FMAD) {
7088         V1.multiply(V2, APFloat::rmNearestTiesToEven);
7089         V1.add(V3, APFloat::rmNearestTiesToEven);
7090       } else
7091         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
7092       return getConstantFP(V1, DL, VT);
7093     }
7094     break;
7095   }
7096   case ISD::BUILD_VECTOR: {
7097     // Attempt to simplify BUILD_VECTOR.
7098     SDValue Ops[] = {N1, N2, N3};
7099     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7100       return V;
7101     break;
7102   }
7103   case ISD::CONCAT_VECTORS: {
7104     SDValue Ops[] = {N1, N2, N3};
7105     if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7106       return V;
7107     break;
7108   }
7109   case ISD::SETCC: {
7110     assert(VT.isInteger() && "SETCC result type must be an integer!");
7111     assert(N1.getValueType() == N2.getValueType() &&
7112            "SETCC operands must have the same type!");
7113     assert(VT.isVector() == N1.getValueType().isVector() &&
7114            "SETCC type should be vector iff the operand type is vector!");
7115     assert((!VT.isVector() || VT.getVectorElementCount() ==
7116                                   N1.getValueType().getVectorElementCount()) &&
7117            "SETCC vector element counts must match!");
7118     // Use FoldSetCC to simplify SETCC's.
7119     if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
7120       return V;
7121     // Vector constant folding.
7122     SDValue Ops[] = {N1, N2, N3};
7123     if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7124       NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
7125       return V;
7126     }
7127     break;
7128   }
7129   case ISD::SELECT:
7130   case ISD::VSELECT:
7131     if (SDValue V = simplifySelect(N1, N2, N3))
7132       return V;
7133     break;
7134   case ISD::VECTOR_SHUFFLE:
7135     llvm_unreachable("should use getVectorShuffle constructor!");
7136   case ISD::VECTOR_SPLICE: {
7137     if (cast<ConstantSDNode>(N3)->isZero())
7138       return N1;
7139     break;
7140   }
7141   case ISD::INSERT_VECTOR_ELT: {
7142     ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
7143     // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7144     // for scalable vectors where we will generate appropriate code to
7145     // deal with out-of-bounds cases correctly.
7146     if (N3C && N1.getValueType().isFixedLengthVector() &&
7147         N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
7148       return getUNDEF(VT);
7149 
7150     // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7151     if (N3.isUndef())
7152       return getUNDEF(VT);
7153 
7154     // If the inserted element is an UNDEF, just use the input vector.
7155     if (N2.isUndef())
7156       return N1;
7157 
7158     break;
7159   }
7160   case ISD::INSERT_SUBVECTOR: {
7161     // Inserting undef into undef is still undef.
7162     if (N1.isUndef() && N2.isUndef())
7163       return getUNDEF(VT);
7164 
7165     EVT N2VT = N2.getValueType();
7166     assert(VT == N1.getValueType() &&
7167            "Dest and insert subvector source types must match!");
7168     assert(VT.isVector() && N2VT.isVector() &&
7169            "Insert subvector VTs must be vectors!");
7170     assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
7171            "Insert subvector VTs must have the same element type!");
7172     assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7173            "Cannot insert a scalable vector into a fixed length vector!");
7174     assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7175             VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
7176            "Insert subvector must be from smaller vector to larger vector!");
7177     assert(isa<ConstantSDNode>(N3) &&
7178            "Insert subvector index must be constant");
7179     assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7180             (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7181                 VT.getVectorMinNumElements()) &&
7182            "Insert subvector overflow!");
7183     assert(N3->getAsAPIntVal().getBitWidth() ==
7184                TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7185            "Constant index for INSERT_SUBVECTOR has an invalid size");
7186 
7187     // Trivial insertion.
7188     if (VT == N2VT)
7189       return N2;
7190 
7191     // If this is an insert of an extracted vector into an undef vector, we
7192     // can just use the input to the extract.
7193     if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7194         N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
7195       return N2.getOperand(0);
7196     break;
7197   }
7198   case ISD::BITCAST:
7199     // Fold bit_convert nodes from a type to themselves.
7200     if (N1.getValueType() == VT)
7201       return N1;
7202     break;
7203   case ISD::VP_TRUNCATE:
7204   case ISD::VP_SIGN_EXTEND:
7205   case ISD::VP_ZERO_EXTEND:
7206     // Don't create noop casts.
7207     if (N1.getValueType() == VT)
7208       return N1;
7209     break;
7210   }
7211 
7212   // Memoize node if it doesn't produce a glue result.
7213   SDNode *N;
7214   SDVTList VTs = getVTList(VT);
7215   SDValue Ops[] = {N1, N2, N3};
7216   if (VT != MVT::Glue) {
7217     FoldingSetNodeID ID;
7218     AddNodeIDNode(ID, Opcode, VTs, Ops);
7219     void *IP = nullptr;
7220     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7221       E->intersectFlagsWith(Flags);
7222       return SDValue(E, 0);
7223     }
7224 
7225     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7226     N->setFlags(Flags);
7227     createOperands(N, Ops);
7228     CSEMap.InsertNode(N, IP);
7229   } else {
7230     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7231     createOperands(N, Ops);
7232   }
7233 
7234   InsertNode(N);
7235   SDValue V = SDValue(N, 0);
7236   NewSDValueDbgMsg(V, "Creating new node: ", this);
7237   return V;
7238 }
7239 
7240 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7241                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7242   SDValue Ops[] = { N1, N2, N3, N4 };
7243   return getNode(Opcode, DL, VT, Ops);
7244 }
7245 
7246 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7247                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7248                               SDValue N5) {
7249   SDValue Ops[] = { N1, N2, N3, N4, N5 };
7250   return getNode(Opcode, DL, VT, Ops);
7251 }
7252 
7253 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7254 /// the incoming stack arguments to be loaded from the stack.
7255 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
7256   SmallVector<SDValue, 8> ArgChains;
7257 
7258   // Include the original chain at the beginning of the list. When this is
7259   // used by target LowerCall hooks, this helps legalize find the
7260   // CALLSEQ_BEGIN node.
7261   ArgChains.push_back(Chain);
7262 
7263   // Add a chain value for each stack argument.
7264   for (SDNode *U : getEntryNode().getNode()->uses())
7265     if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))
7266       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
7267         if (FI->getIndex() < 0)
7268           ArgChains.push_back(SDValue(L, 1));
7269 
7270   // Build a tokenfactor for all the chains.
7271   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7272 }
7273 
7274 /// getMemsetValue - Vectorized representation of the memset value
7275 /// operand.
7276 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
7277                               const SDLoc &dl) {
7278   assert(!Value.isUndef());
7279 
7280   unsigned NumBits = VT.getScalarSizeInBits();
7281   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
7282     assert(C->getAPIntValue().getBitWidth() == 8);
7283     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
7284     if (VT.isInteger()) {
7285       bool IsOpaque = VT.getSizeInBits() > 64 ||
7286           !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
7287       return DAG.getConstant(Val, dl, VT, false, IsOpaque);
7288     }
7289     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
7290                              VT);
7291   }
7292 
7293   assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7294   EVT IntVT = VT.getScalarType();
7295   if (!IntVT.isInteger())
7296     IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
7297 
7298   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
7299   if (NumBits > 8) {
7300     // Use a multiplication with 0x010101... to extend the input to the
7301     // required length.
7302     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
7303     Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
7304                         DAG.getConstant(Magic, dl, IntVT));
7305   }
7306 
7307   if (VT != Value.getValueType() && !VT.isInteger())
7308     Value = DAG.getBitcast(VT.getScalarType(), Value);
7309   if (VT != Value.getValueType())
7310     Value = DAG.getSplatBuildVector(VT, dl, Value);
7311 
7312   return Value;
7313 }
7314 
7315 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
7316 /// used when a memcpy is turned into a memset when the source is a constant
7317 /// string ptr.
7318 static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
7319                                   const TargetLowering &TLI,
7320                                   const ConstantDataArraySlice &Slice) {
7321   // Handle vector with all elements zero.
7322   if (Slice.Array == nullptr) {
7323     if (VT.isInteger())
7324       return DAG.getConstant(0, dl, VT);
7325     if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7326       return DAG.getConstantFP(0.0, dl, VT);
7327     if (VT.isVector()) {
7328       unsigned NumElts = VT.getVectorNumElements();
7329       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
7330       return DAG.getNode(ISD::BITCAST, dl, VT,
7331                          DAG.getConstant(0, dl,
7332                                          EVT::getVectorVT(*DAG.getContext(),
7333                                                           EltVT, NumElts)));
7334     }
7335     llvm_unreachable("Expected type!");
7336   }
7337 
7338   assert(!VT.isVector() && "Can't handle vector type here!");
7339   unsigned NumVTBits = VT.getSizeInBits();
7340   unsigned NumVTBytes = NumVTBits / 8;
7341   unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
7342 
7343   APInt Val(NumVTBits, 0);
7344   if (DAG.getDataLayout().isLittleEndian()) {
7345     for (unsigned i = 0; i != NumBytes; ++i)
7346       Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
7347   } else {
7348     for (unsigned i = 0; i != NumBytes; ++i)
7349       Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7350   }
7351 
7352   // If the "cost" of materializing the integer immediate is less than the cost
7353   // of a load, then it is cost effective to turn the load into the immediate.
7354   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
7355   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
7356     return DAG.getConstant(Val, dl, VT);
7357   return SDValue();
7358 }
7359 
7360 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
7361                                            const SDLoc &DL,
7362                                            const SDNodeFlags Flags) {
7363   EVT VT = Base.getValueType();
7364   SDValue Index;
7365 
7366   if (Offset.isScalable())
7367     Index = getVScale(DL, Base.getValueType(),
7368                       APInt(Base.getValueSizeInBits().getFixedValue(),
7369                             Offset.getKnownMinValue()));
7370   else
7371     Index = getConstant(Offset.getFixedValue(), DL, VT);
7372 
7373   return getMemBasePlusOffset(Base, Index, DL, Flags);
7374 }
7375 
7376 SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
7377                                            const SDLoc &DL,
7378                                            const SDNodeFlags Flags) {
7379   assert(Offset.getValueType().isInteger());
7380   EVT BasePtrVT = Ptr.getValueType();
7381   return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
7382 }
7383 
7384 /// Returns true if memcpy source is constant data.
7385 static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
7386   uint64_t SrcDelta = 0;
7387   GlobalAddressSDNode *G = nullptr;
7388   if (Src.getOpcode() == ISD::GlobalAddress)
7389     G = cast<GlobalAddressSDNode>(Src);
7390   else if (Src.getOpcode() == ISD::ADD &&
7391            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
7392            Src.getOperand(1).getOpcode() == ISD::Constant) {
7393     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
7394     SrcDelta = Src.getConstantOperandVal(1);
7395   }
7396   if (!G)
7397     return false;
7398 
7399   return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
7400                                   SrcDelta + G->getOffset());
7401 }
7402 
7403 static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
7404                                       SelectionDAG &DAG) {
7405   // On Darwin, -Os means optimize for size without hurting performance, so
7406   // only really optimize for size when -Oz (MinSize) is used.
7407   if (MF.getTarget().getTargetTriple().isOSDarwin())
7408     return MF.getFunction().hasMinSize();
7409   return DAG.shouldOptForSize();
7410 }
7411 
7412 static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
7413                           SmallVector<SDValue, 32> &OutChains, unsigned From,
7414                           unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
7415                           SmallVector<SDValue, 16> &OutStoreChains) {
7416   assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
7417   assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
7418   SmallVector<SDValue, 16> GluedLoadChains;
7419   for (unsigned i = From; i < To; ++i) {
7420     OutChains.push_back(OutLoadChains[i]);
7421     GluedLoadChains.push_back(OutLoadChains[i]);
7422   }
7423 
7424   // Chain for all loads.
7425   SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7426                                   GluedLoadChains);
7427 
7428   for (unsigned i = From; i < To; ++i) {
7429     StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
7430     SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
7431                                   ST->getBasePtr(), ST->getMemoryVT(),
7432                                   ST->getMemOperand());
7433     OutChains.push_back(NewStore);
7434   }
7435 }
7436 
7437 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7438                                        SDValue Chain, SDValue Dst, SDValue Src,
7439                                        uint64_t Size, Align Alignment,
7440                                        bool isVol, bool AlwaysInline,
7441                                        MachinePointerInfo DstPtrInfo,
7442                                        MachinePointerInfo SrcPtrInfo,
7443                                        const AAMDNodes &AAInfo, AAResults *AA) {
7444   // Turn a memcpy of undef to nop.
7445   // FIXME: We need to honor volatile even is Src is undef.
7446   if (Src.isUndef())
7447     return Chain;
7448 
7449   // Expand memcpy to a series of load and store ops if the size operand falls
7450   // below a certain threshold.
7451   // TODO: In the AlwaysInline case, if the size is big then generate a loop
7452   // rather than maybe a humongous number of loads and stores.
7453   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7454   const DataLayout &DL = DAG.getDataLayout();
7455   LLVMContext &C = *DAG.getContext();
7456   std::vector<EVT> MemOps;
7457   bool DstAlignCanChange = false;
7458   MachineFunction &MF = DAG.getMachineFunction();
7459   MachineFrameInfo &MFI = MF.getFrameInfo();
7460   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7461   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7462   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7463     DstAlignCanChange = true;
7464   MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7465   if (!SrcAlign || Alignment > *SrcAlign)
7466     SrcAlign = Alignment;
7467   assert(SrcAlign && "SrcAlign must be set");
7468   ConstantDataArraySlice Slice;
7469   // If marked as volatile, perform a copy even when marked as constant.
7470   bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
7471   bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
7472   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
7473   const MemOp Op = isZeroConstant
7474                        ? MemOp::Set(Size, DstAlignCanChange, Alignment,
7475                                     /*IsZeroMemset*/ true, isVol)
7476                        : MemOp::Copy(Size, DstAlignCanChange, Alignment,
7477                                      *SrcAlign, isVol, CopyFromConstant);
7478   if (!TLI.findOptimalMemOpLowering(
7479           MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
7480           SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
7481     return SDValue();
7482 
7483   if (DstAlignCanChange) {
7484     Type *Ty = MemOps[0].getTypeForEVT(C);
7485     Align NewAlign = DL.getABITypeAlign(Ty);
7486 
7487     // Don't promote to an alignment that would require dynamic stack
7488     // realignment which may conflict with optimizations such as tail call
7489     // optimization.
7490     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7491     if (!TRI->hasStackRealignment(MF))
7492       while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7493         NewAlign = NewAlign.previous();
7494 
7495     if (NewAlign > Alignment) {
7496       // Give the stack frame object a larger alignment if needed.
7497       if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7498         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7499       Alignment = NewAlign;
7500     }
7501   }
7502 
7503   // Prepare AAInfo for loads/stores after lowering this memcpy.
7504   AAMDNodes NewAAInfo = AAInfo;
7505   NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7506 
7507   const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
7508   bool isConstant =
7509       AA && SrcVal &&
7510       AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
7511 
7512   MachineMemOperand::Flags MMOFlags =
7513       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7514   SmallVector<SDValue, 16> OutLoadChains;
7515   SmallVector<SDValue, 16> OutStoreChains;
7516   SmallVector<SDValue, 32> OutChains;
7517   unsigned NumMemOps = MemOps.size();
7518   uint64_t SrcOff = 0, DstOff = 0;
7519   for (unsigned i = 0; i != NumMemOps; ++i) {
7520     EVT VT = MemOps[i];
7521     unsigned VTSize = VT.getSizeInBits() / 8;
7522     SDValue Value, Store;
7523 
7524     if (VTSize > Size) {
7525       // Issuing an unaligned load / store pair  that overlaps with the previous
7526       // pair. Adjust the offset accordingly.
7527       assert(i == NumMemOps-1 && i != 0);
7528       SrcOff -= VTSize - Size;
7529       DstOff -= VTSize - Size;
7530     }
7531 
7532     if (CopyFromConstant &&
7533         (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
7534       // It's unlikely a store of a vector immediate can be done in a single
7535       // instruction. It would require a load from a constantpool first.
7536       // We only handle zero vectors here.
7537       // FIXME: Handle other cases where store of vector immediate is done in
7538       // a single instruction.
7539       ConstantDataArraySlice SubSlice;
7540       if (SrcOff < Slice.Length) {
7541         SubSlice = Slice;
7542         SubSlice.move(SrcOff);
7543       } else {
7544         // This is an out-of-bounds access and hence UB. Pretend we read zero.
7545         SubSlice.Array = nullptr;
7546         SubSlice.Offset = 0;
7547         SubSlice.Length = VTSize;
7548       }
7549       Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
7550       if (Value.getNode()) {
7551         Store = DAG.getStore(
7552             Chain, dl, Value,
7553             DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7554             DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7555         OutChains.push_back(Store);
7556       }
7557     }
7558 
7559     if (!Store.getNode()) {
7560       // The type might not be legal for the target.  This should only happen
7561       // if the type is smaller than a legal type, as on PPC, so the right
7562       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
7563       // to Load/Store if NVT==VT.
7564       // FIXME does the case above also need this?
7565       EVT NVT = TLI.getTypeToTransformTo(C, VT);
7566       assert(NVT.bitsGE(VT));
7567 
7568       bool isDereferenceable =
7569         SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7570       MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7571       if (isDereferenceable)
7572         SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7573       if (isConstant)
7574         SrcMMOFlags |= MachineMemOperand::MOInvariant;
7575 
7576       Value = DAG.getExtLoad(
7577           ISD::EXTLOAD, dl, NVT, Chain,
7578           DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7579           SrcPtrInfo.getWithOffset(SrcOff), VT,
7580           commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
7581       OutLoadChains.push_back(Value.getValue(1));
7582 
7583       Store = DAG.getTruncStore(
7584           Chain, dl, Value,
7585           DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7586           DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
7587       OutStoreChains.push_back(Store);
7588     }
7589     SrcOff += VTSize;
7590     DstOff += VTSize;
7591     Size -= VTSize;
7592   }
7593 
7594   unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
7595                                 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
7596   unsigned NumLdStInMemcpy = OutStoreChains.size();
7597 
7598   if (NumLdStInMemcpy) {
7599     // It may be that memcpy might be converted to memset if it's memcpy
7600     // of constants. In such a case, we won't have loads and stores, but
7601     // just stores. In the absence of loads, there is nothing to gang up.
7602     if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
7603       // If target does not care, just leave as it.
7604       for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7605         OutChains.push_back(OutLoadChains[i]);
7606         OutChains.push_back(OutStoreChains[i]);
7607       }
7608     } else {
7609       // Ld/St less than/equal limit set by target.
7610       if (NumLdStInMemcpy <= GluedLdStLimit) {
7611           chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7612                                         NumLdStInMemcpy, OutLoadChains,
7613                                         OutStoreChains);
7614       } else {
7615         unsigned NumberLdChain =  NumLdStInMemcpy / GluedLdStLimit;
7616         unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7617         unsigned GlueIter = 0;
7618 
7619         for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7620           unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7621           unsigned IndexTo   = NumLdStInMemcpy - GlueIter;
7622 
7623           chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
7624                                        OutLoadChains, OutStoreChains);
7625           GlueIter += GluedLdStLimit;
7626         }
7627 
7628         // Residual ld/st.
7629         if (RemainingLdStInMemcpy) {
7630           chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7631                                         RemainingLdStInMemcpy, OutLoadChains,
7632                                         OutStoreChains);
7633         }
7634       }
7635     }
7636   }
7637   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7638 }
7639 
7640 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
7641                                         SDValue Chain, SDValue Dst, SDValue Src,
7642                                         uint64_t Size, Align Alignment,
7643                                         bool isVol, bool AlwaysInline,
7644                                         MachinePointerInfo DstPtrInfo,
7645                                         MachinePointerInfo SrcPtrInfo,
7646                                         const AAMDNodes &AAInfo) {
7647   // Turn a memmove of undef to nop.
7648   // FIXME: We need to honor volatile even is Src is undef.
7649   if (Src.isUndef())
7650     return Chain;
7651 
7652   // Expand memmove to a series of load and store ops if the size operand falls
7653   // below a certain threshold.
7654   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7655   const DataLayout &DL = DAG.getDataLayout();
7656   LLVMContext &C = *DAG.getContext();
7657   std::vector<EVT> MemOps;
7658   bool DstAlignCanChange = false;
7659   MachineFunction &MF = DAG.getMachineFunction();
7660   MachineFrameInfo &MFI = MF.getFrameInfo();
7661   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7662   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7663   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7664     DstAlignCanChange = true;
7665   MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7666   if (!SrcAlign || Alignment > *SrcAlign)
7667     SrcAlign = Alignment;
7668   assert(SrcAlign && "SrcAlign must be set");
7669   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
7670   if (!TLI.findOptimalMemOpLowering(
7671           MemOps, Limit,
7672           MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
7673                       /*IsVolatile*/ true),
7674           DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
7675           MF.getFunction().getAttributes()))
7676     return SDValue();
7677 
7678   if (DstAlignCanChange) {
7679     Type *Ty = MemOps[0].getTypeForEVT(C);
7680     Align NewAlign = DL.getABITypeAlign(Ty);
7681 
7682     // Don't promote to an alignment that would require dynamic stack
7683     // realignment which may conflict with optimizations such as tail call
7684     // optimization.
7685     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7686     if (!TRI->hasStackRealignment(MF))
7687       while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7688         NewAlign = NewAlign.previous();
7689 
7690     if (NewAlign > Alignment) {
7691       // Give the stack frame object a larger alignment if needed.
7692       if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7693         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7694       Alignment = NewAlign;
7695     }
7696   }
7697 
7698   // Prepare AAInfo for loads/stores after lowering this memmove.
7699   AAMDNodes NewAAInfo = AAInfo;
7700   NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7701 
7702   MachineMemOperand::Flags MMOFlags =
7703       isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
7704   uint64_t SrcOff = 0, DstOff = 0;
7705   SmallVector<SDValue, 8> LoadValues;
7706   SmallVector<SDValue, 8> LoadChains;
7707   SmallVector<SDValue, 8> OutChains;
7708   unsigned NumMemOps = MemOps.size();
7709   for (unsigned i = 0; i < NumMemOps; i++) {
7710     EVT VT = MemOps[i];
7711     unsigned VTSize = VT.getSizeInBits() / 8;
7712     SDValue Value;
7713 
7714     bool isDereferenceable =
7715       SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7716     MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7717     if (isDereferenceable)
7718       SrcMMOFlags |= MachineMemOperand::MODereferenceable;
7719 
7720     Value = DAG.getLoad(
7721         VT, dl, Chain,
7722         DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7723         SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
7724     LoadValues.push_back(Value);
7725     LoadChains.push_back(Value.getValue(1));
7726     SrcOff += VTSize;
7727   }
7728   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7729   OutChains.clear();
7730   for (unsigned i = 0; i < NumMemOps; i++) {
7731     EVT VT = MemOps[i];
7732     unsigned VTSize = VT.getSizeInBits() / 8;
7733     SDValue Store;
7734 
7735     Store = DAG.getStore(
7736         Chain, dl, LoadValues[i],
7737         DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7738         DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7739     OutChains.push_back(Store);
7740     DstOff += VTSize;
7741   }
7742 
7743   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7744 }
7745 
7746 /// Lower the call to 'memset' intrinsic function into a series of store
7747 /// operations.
7748 ///
7749 /// \param DAG Selection DAG where lowered code is placed.
7750 /// \param dl Link to corresponding IR location.
7751 /// \param Chain Control flow dependency.
7752 /// \param Dst Pointer to destination memory location.
7753 /// \param Src Value of byte to write into the memory.
7754 /// \param Size Number of bytes to write.
7755 /// \param Alignment Alignment of the destination in bytes.
7756 /// \param isVol True if destination is volatile.
7757 /// \param AlwaysInline Makes sure no function call is generated.
7758 /// \param DstPtrInfo IR information on the memory pointer.
7759 /// \returns New head in the control flow, if lowering was successful, empty
7760 /// SDValue otherwise.
7761 ///
7762 /// The function tries to replace 'llvm.memset' intrinsic with several store
7763 /// operations and value calculation code. This is usually profitable for small
7764 /// memory size or when the semantic requires inlining.
7765 static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
7766                                SDValue Chain, SDValue Dst, SDValue Src,
7767                                uint64_t Size, Align Alignment, bool isVol,
7768                                bool AlwaysInline, MachinePointerInfo DstPtrInfo,
7769                                const AAMDNodes &AAInfo) {
7770   // Turn a memset of undef to nop.
7771   // FIXME: We need to honor volatile even is Src is undef.
7772   if (Src.isUndef())
7773     return Chain;
7774 
7775   // Expand memset to a series of load/store ops if the size operand
7776   // falls below a certain threshold.
7777   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7778   std::vector<EVT> MemOps;
7779   bool DstAlignCanChange = false;
7780   MachineFunction &MF = DAG.getMachineFunction();
7781   MachineFrameInfo &MFI = MF.getFrameInfo();
7782   bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7783   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7784   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7785     DstAlignCanChange = true;
7786   bool IsZeroVal = isNullConstant(Src);
7787   unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
7788 
7789   if (!TLI.findOptimalMemOpLowering(
7790           MemOps, Limit,
7791           MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
7792           DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
7793     return SDValue();
7794 
7795   if (DstAlignCanChange) {
7796     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
7797     const DataLayout &DL = DAG.getDataLayout();
7798     Align NewAlign = DL.getABITypeAlign(Ty);
7799 
7800     // Don't promote to an alignment that would require dynamic stack
7801     // realignment which may conflict with optimizations such as tail call
7802     // optimization.
7803     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
7804     if (!TRI->hasStackRealignment(MF))
7805       while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7806         NewAlign = NewAlign.previous();
7807 
7808     if (NewAlign > Alignment) {
7809       // Give the stack frame object a larger alignment if needed.
7810       if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7811         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7812       Alignment = NewAlign;
7813     }
7814   }
7815 
7816   SmallVector<SDValue, 8> OutChains;
7817   uint64_t DstOff = 0;
7818   unsigned NumMemOps = MemOps.size();
7819 
7820   // Find the largest store and generate the bit pattern for it.
7821   EVT LargestVT = MemOps[0];
7822   for (unsigned i = 1; i < NumMemOps; i++)
7823     if (MemOps[i].bitsGT(LargestVT))
7824       LargestVT = MemOps[i];
7825   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
7826 
7827   // Prepare AAInfo for loads/stores after lowering this memset.
7828   AAMDNodes NewAAInfo = AAInfo;
7829   NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7830 
7831   for (unsigned i = 0; i < NumMemOps; i++) {
7832     EVT VT = MemOps[i];
7833     unsigned VTSize = VT.getSizeInBits() / 8;
7834     if (VTSize > Size) {
7835       // Issuing an unaligned load / store pair  that overlaps with the previous
7836       // pair. Adjust the offset accordingly.
7837       assert(i == NumMemOps-1 && i != 0);
7838       DstOff -= VTSize - Size;
7839     }
7840 
7841     // If this store is smaller than the largest store see whether we can get
7842     // the smaller value for free with a truncate or extract vector element and
7843     // then store.
7844     SDValue Value = MemSetValue;
7845     if (VT.bitsLT(LargestVT)) {
7846       unsigned Index;
7847       unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
7848       EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);
7849       if (!LargestVT.isVector() && !VT.isVector() &&
7850           TLI.isTruncateFree(LargestVT, VT))
7851         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
7852       else if (LargestVT.isVector() && !VT.isVector() &&
7853                TLI.shallExtractConstSplatVectorElementToStore(
7854                    LargestVT.getTypeForEVT(*DAG.getContext()),
7855                    VT.getSizeInBits(), Index) &&
7856                TLI.isTypeLegal(SVT) &&
7857                LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
7858         // Target which can combine store(extractelement VectorTy, Idx) can get
7859         // the smaller value for free.
7860         SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);
7861         Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, TailValue,
7862                             DAG.getVectorIdxConstant(Index, dl));
7863       } else
7864         Value = getMemsetValue(Src, VT, DAG, dl);
7865     }
7866     assert(Value.getValueType() == VT && "Value with wrong type.");
7867     SDValue Store = DAG.getStore(
7868         Chain, dl, Value,
7869         DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7870         DstPtrInfo.getWithOffset(DstOff), Alignment,
7871         isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
7872         NewAAInfo);
7873     OutChains.push_back(Store);
7874     DstOff += VT.getSizeInBits() / 8;
7875     Size -= VTSize;
7876   }
7877 
7878   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7879 }
7880 
7881 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
7882                                             unsigned AS) {
7883   // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
7884   // pointer operands can be losslessly bitcasted to pointers of address space 0
7885   if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
7886     report_fatal_error("cannot lower memory intrinsic in address space " +
7887                        Twine(AS));
7888   }
7889 }
7890 
7891 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
7892                                 SDValue Src, SDValue Size, Align Alignment,
7893                                 bool isVol, bool AlwaysInline, bool isTailCall,
7894                                 MachinePointerInfo DstPtrInfo,
7895                                 MachinePointerInfo SrcPtrInfo,
7896                                 const AAMDNodes &AAInfo, AAResults *AA) {
7897   // Check to see if we should lower the memcpy to loads and stores first.
7898   // For cases within the target-specified limits, this is the best choice.
7899   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
7900   if (ConstantSize) {
7901     // Memcpy with size zero? Just return the original chain.
7902     if (ConstantSize->isZero())
7903       return Chain;
7904 
7905     SDValue Result = getMemcpyLoadsAndStores(
7906         *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
7907         isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7908     if (Result.getNode())
7909       return Result;
7910   }
7911 
7912   // Then check to see if we should lower the memcpy with target-specific
7913   // code. If the target chooses to do this, this is the next best.
7914   if (TSI) {
7915     SDValue Result = TSI->EmitTargetCodeForMemcpy(
7916         *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
7917         DstPtrInfo, SrcPtrInfo);
7918     if (Result.getNode())
7919       return Result;
7920   }
7921 
7922   // If we really need inline code and the target declined to provide it,
7923   // use a (potentially long) sequence of loads and stores.
7924   if (AlwaysInline) {
7925     assert(ConstantSize && "AlwaysInline requires a constant size!");
7926     return getMemcpyLoadsAndStores(
7927         *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
7928         isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7929   }
7930 
7931   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
7932   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
7933 
7934   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
7935   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
7936   // respect volatile, so they may do things like read or write memory
7937   // beyond the given memory regions. But fixing this isn't easy, and most
7938   // people don't care.
7939 
7940   // Emit a library call.
7941   TargetLowering::ArgListTy Args;
7942   TargetLowering::ArgListEntry Entry;
7943   Entry.Ty = PointerType::getUnqual(*getContext());
7944   Entry.Node = Dst; Args.push_back(Entry);
7945   Entry.Node = Src; Args.push_back(Entry);
7946 
7947   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7948   Entry.Node = Size; Args.push_back(Entry);
7949   // FIXME: pass in SDLoc
7950   TargetLowering::CallLoweringInfo CLI(*this);
7951   CLI.setDebugLoc(dl)
7952       .setChain(Chain)
7953       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
7954                     Dst.getValueType().getTypeForEVT(*getContext()),
7955                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
7956                                       TLI->getPointerTy(getDataLayout())),
7957                     std::move(Args))
7958       .setDiscardResult()
7959       .setTailCall(isTailCall);
7960 
7961   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
7962   return CallResult.second;
7963 }
7964 
7965 SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
7966                                       SDValue Dst, SDValue Src, SDValue Size,
7967                                       Type *SizeTy, unsigned ElemSz,
7968                                       bool isTailCall,
7969                                       MachinePointerInfo DstPtrInfo,
7970                                       MachinePointerInfo SrcPtrInfo) {
7971   // Emit a library call.
7972   TargetLowering::ArgListTy Args;
7973   TargetLowering::ArgListEntry Entry;
7974   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
7975   Entry.Node = Dst;
7976   Args.push_back(Entry);
7977 
7978   Entry.Node = Src;
7979   Args.push_back(Entry);
7980 
7981   Entry.Ty = SizeTy;
7982   Entry.Node = Size;
7983   Args.push_back(Entry);
7984 
7985   RTLIB::Libcall LibraryCall =
7986       RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
7987   if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
7988     report_fatal_error("Unsupported element size");
7989 
7990   TargetLowering::CallLoweringInfo CLI(*this);
7991   CLI.setDebugLoc(dl)
7992       .setChain(Chain)
7993       .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
7994                     Type::getVoidTy(*getContext()),
7995                     getExternalSymbol(TLI->getLibcallName(LibraryCall),
7996                                       TLI->getPointerTy(getDataLayout())),
7997                     std::move(Args))
7998       .setDiscardResult()
7999       .setTailCall(isTailCall);
8000 
8001   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8002   return CallResult.second;
8003 }
8004 
8005 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
8006                                  SDValue Src, SDValue Size, Align Alignment,
8007                                  bool isVol, bool isTailCall,
8008                                  MachinePointerInfo DstPtrInfo,
8009                                  MachinePointerInfo SrcPtrInfo,
8010                                  const AAMDNodes &AAInfo, AAResults *AA) {
8011   // Check to see if we should lower the memmove to loads and stores first.
8012   // For cases within the target-specified limits, this is the best choice.
8013   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8014   if (ConstantSize) {
8015     // Memmove with size zero? Just return the original chain.
8016     if (ConstantSize->isZero())
8017       return Chain;
8018 
8019     SDValue Result = getMemmoveLoadsAndStores(
8020         *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8021         isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
8022     if (Result.getNode())
8023       return Result;
8024   }
8025 
8026   // Then check to see if we should lower the memmove with target-specific
8027   // code. If the target chooses to do this, this is the next best.
8028   if (TSI) {
8029     SDValue Result =
8030         TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
8031                                       Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8032     if (Result.getNode())
8033       return Result;
8034   }
8035 
8036   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
8037   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
8038 
8039   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8040   // not be safe.  See memcpy above for more details.
8041 
8042   // Emit a library call.
8043   TargetLowering::ArgListTy Args;
8044   TargetLowering::ArgListEntry Entry;
8045   Entry.Ty = PointerType::getUnqual(*getContext());
8046   Entry.Node = Dst; Args.push_back(Entry);
8047   Entry.Node = Src; Args.push_back(Entry);
8048 
8049   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8050   Entry.Node = Size; Args.push_back(Entry);
8051   // FIXME:  pass in SDLoc
8052   TargetLowering::CallLoweringInfo CLI(*this);
8053   CLI.setDebugLoc(dl)
8054       .setChain(Chain)
8055       .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
8056                     Dst.getValueType().getTypeForEVT(*getContext()),
8057                     getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
8058                                       TLI->getPointerTy(getDataLayout())),
8059                     std::move(Args))
8060       .setDiscardResult()
8061       .setTailCall(isTailCall);
8062 
8063   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8064   return CallResult.second;
8065 }
8066 
8067 SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
8068                                        SDValue Dst, SDValue Src, SDValue Size,
8069                                        Type *SizeTy, unsigned ElemSz,
8070                                        bool isTailCall,
8071                                        MachinePointerInfo DstPtrInfo,
8072                                        MachinePointerInfo SrcPtrInfo) {
8073   // Emit a library call.
8074   TargetLowering::ArgListTy Args;
8075   TargetLowering::ArgListEntry Entry;
8076   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8077   Entry.Node = Dst;
8078   Args.push_back(Entry);
8079 
8080   Entry.Node = Src;
8081   Args.push_back(Entry);
8082 
8083   Entry.Ty = SizeTy;
8084   Entry.Node = Size;
8085   Args.push_back(Entry);
8086 
8087   RTLIB::Libcall LibraryCall =
8088       RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
8089   if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8090     report_fatal_error("Unsupported element size");
8091 
8092   TargetLowering::CallLoweringInfo CLI(*this);
8093   CLI.setDebugLoc(dl)
8094       .setChain(Chain)
8095       .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8096                     Type::getVoidTy(*getContext()),
8097                     getExternalSymbol(TLI->getLibcallName(LibraryCall),
8098                                       TLI->getPointerTy(getDataLayout())),
8099                     std::move(Args))
8100       .setDiscardResult()
8101       .setTailCall(isTailCall);
8102 
8103   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8104   return CallResult.second;
8105 }
8106 
8107 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
8108                                 SDValue Src, SDValue Size, Align Alignment,
8109                                 bool isVol, bool AlwaysInline, bool isTailCall,
8110                                 MachinePointerInfo DstPtrInfo,
8111                                 const AAMDNodes &AAInfo) {
8112   // Check to see if we should lower the memset to stores first.
8113   // For cases within the target-specified limits, this is the best choice.
8114   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8115   if (ConstantSize) {
8116     // Memset with size zero? Just return the original chain.
8117     if (ConstantSize->isZero())
8118       return Chain;
8119 
8120     SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8121                                      ConstantSize->getZExtValue(), Alignment,
8122                                      isVol, false, DstPtrInfo, AAInfo);
8123 
8124     if (Result.getNode())
8125       return Result;
8126   }
8127 
8128   // Then check to see if we should lower the memset with target-specific
8129   // code. If the target chooses to do this, this is the next best.
8130   if (TSI) {
8131     SDValue Result = TSI->EmitTargetCodeForMemset(
8132         *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8133     if (Result.getNode())
8134       return Result;
8135   }
8136 
8137   // If we really need inline code and the target declined to provide it,
8138   // use a (potentially long) sequence of loads and stores.
8139   if (AlwaysInline) {
8140     assert(ConstantSize && "AlwaysInline requires a constant size!");
8141     SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8142                                      ConstantSize->getZExtValue(), Alignment,
8143                                      isVol, true, DstPtrInfo, AAInfo);
8144     assert(Result &&
8145            "getMemsetStores must return a valid sequence when AlwaysInline");
8146     return Result;
8147   }
8148 
8149   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
8150 
8151   // Emit a library call.
8152   auto &Ctx = *getContext();
8153   const auto& DL = getDataLayout();
8154 
8155   TargetLowering::CallLoweringInfo CLI(*this);
8156   // FIXME: pass in SDLoc
8157   CLI.setDebugLoc(dl).setChain(Chain);
8158 
8159   const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);
8160 
8161   // Helper function to create an Entry from Node and Type.
8162   const auto CreateEntry = [](SDValue Node, Type *Ty) {
8163     TargetLowering::ArgListEntry Entry;
8164     Entry.Node = Node;
8165     Entry.Ty = Ty;
8166     return Entry;
8167   };
8168 
8169   // If zeroing out and bzero is present, use it.
8170   if (isNullConstant(Src) && BzeroName) {
8171     TargetLowering::ArgListTy Args;
8172     Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8173     Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8174     CLI.setLibCallee(
8175         TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),
8176         getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));
8177   } else {
8178     TargetLowering::ArgListTy Args;
8179     Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8180     Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8181     Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8182     CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
8183                      Dst.getValueType().getTypeForEVT(Ctx),
8184                      getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
8185                                        TLI->getPointerTy(DL)),
8186                      std::move(Args));
8187   }
8188 
8189   CLI.setDiscardResult().setTailCall(isTailCall);
8190 
8191   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8192   return CallResult.second;
8193 }
8194 
8195 SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
8196                                       SDValue Dst, SDValue Value, SDValue Size,
8197                                       Type *SizeTy, unsigned ElemSz,
8198                                       bool isTailCall,
8199                                       MachinePointerInfo DstPtrInfo) {
8200   // Emit a library call.
8201   TargetLowering::ArgListTy Args;
8202   TargetLowering::ArgListEntry Entry;
8203   Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8204   Entry.Node = Dst;
8205   Args.push_back(Entry);
8206 
8207   Entry.Ty = Type::getInt8Ty(*getContext());
8208   Entry.Node = Value;
8209   Args.push_back(Entry);
8210 
8211   Entry.Ty = SizeTy;
8212   Entry.Node = Size;
8213   Args.push_back(Entry);
8214 
8215   RTLIB::Libcall LibraryCall =
8216       RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
8217   if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8218     report_fatal_error("Unsupported element size");
8219 
8220   TargetLowering::CallLoweringInfo CLI(*this);
8221   CLI.setDebugLoc(dl)
8222       .setChain(Chain)
8223       .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8224                     Type::getVoidTy(*getContext()),
8225                     getExternalSymbol(TLI->getLibcallName(LibraryCall),
8226                                       TLI->getPointerTy(getDataLayout())),
8227                     std::move(Args))
8228       .setDiscardResult()
8229       .setTailCall(isTailCall);
8230 
8231   std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8232   return CallResult.second;
8233 }
8234 
8235 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8236                                 SDVTList VTList, ArrayRef<SDValue> Ops,
8237                                 MachineMemOperand *MMO) {
8238   FoldingSetNodeID ID;
8239   ID.AddInteger(MemVT.getRawBits());
8240   AddNodeIDNode(ID, Opcode, VTList, Ops);
8241   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8242   ID.AddInteger(MMO->getFlags());
8243   void* IP = nullptr;
8244   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8245     cast<AtomicSDNode>(E)->refineAlignment(MMO);
8246     return SDValue(E, 0);
8247   }
8248 
8249   auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8250                                     VTList, MemVT, MMO);
8251   createOperands(N, Ops);
8252 
8253   CSEMap.InsertNode(N, IP);
8254   InsertNode(N);
8255   return SDValue(N, 0);
8256 }
8257 
8258 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
8259                                        EVT MemVT, SDVTList VTs, SDValue Chain,
8260                                        SDValue Ptr, SDValue Cmp, SDValue Swp,
8261                                        MachineMemOperand *MMO) {
8262   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8263          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
8264   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8265 
8266   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8267   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8268 }
8269 
8270 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8271                                 SDValue Chain, SDValue Ptr, SDValue Val,
8272                                 MachineMemOperand *MMO) {
8273   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
8274           Opcode == ISD::ATOMIC_LOAD_SUB ||
8275           Opcode == ISD::ATOMIC_LOAD_AND ||
8276           Opcode == ISD::ATOMIC_LOAD_CLR ||
8277           Opcode == ISD::ATOMIC_LOAD_OR ||
8278           Opcode == ISD::ATOMIC_LOAD_XOR ||
8279           Opcode == ISD::ATOMIC_LOAD_NAND ||
8280           Opcode == ISD::ATOMIC_LOAD_MIN ||
8281           Opcode == ISD::ATOMIC_LOAD_MAX ||
8282           Opcode == ISD::ATOMIC_LOAD_UMIN ||
8283           Opcode == ISD::ATOMIC_LOAD_UMAX ||
8284           Opcode == ISD::ATOMIC_LOAD_FADD ||
8285           Opcode == ISD::ATOMIC_LOAD_FSUB ||
8286           Opcode == ISD::ATOMIC_LOAD_FMAX ||
8287           Opcode == ISD::ATOMIC_LOAD_FMIN ||
8288           Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
8289           Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
8290           Opcode == ISD::ATOMIC_SWAP ||
8291           Opcode == ISD::ATOMIC_STORE) &&
8292          "Invalid Atomic Op");
8293 
8294   EVT VT = Val.getValueType();
8295 
8296   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
8297                                                getVTList(VT, MVT::Other);
8298   SDValue Ops[] = {Chain, Ptr, Val};
8299   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8300 }
8301 
8302 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8303                                 EVT VT, SDValue Chain, SDValue Ptr,
8304                                 MachineMemOperand *MMO) {
8305   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
8306 
8307   SDVTList VTs = getVTList(VT, MVT::Other);
8308   SDValue Ops[] = {Chain, Ptr};
8309   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8310 }
8311 
8312 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
8313 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
8314   if (Ops.size() == 1)
8315     return Ops[0];
8316 
8317   SmallVector<EVT, 4> VTs;
8318   VTs.reserve(Ops.size());
8319   for (const SDValue &Op : Ops)
8320     VTs.push_back(Op.getValueType());
8321   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
8322 }
8323 
8324 SDValue SelectionDAG::getMemIntrinsicNode(
8325     unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
8326     EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
8327     MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
8328   if (!Size && MemVT.isScalableVector())
8329     Size = MemoryLocation::UnknownSize;
8330   else if (!Size)
8331     Size = MemVT.getStoreSize();
8332 
8333   MachineFunction &MF = getMachineFunction();
8334   MachineMemOperand *MMO =
8335       MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
8336 
8337   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
8338 }
8339 
8340 SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
8341                                           SDVTList VTList,
8342                                           ArrayRef<SDValue> Ops, EVT MemVT,
8343                                           MachineMemOperand *MMO) {
8344   assert((Opcode == ISD::INTRINSIC_VOID ||
8345           Opcode == ISD::INTRINSIC_W_CHAIN ||
8346           Opcode == ISD::PREFETCH ||
8347           (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
8348            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
8349          "Opcode is not a memory-accessing opcode!");
8350 
8351   // Memoize the node unless it returns a glue result.
8352   MemIntrinsicSDNode *N;
8353   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8354     FoldingSetNodeID ID;
8355     AddNodeIDNode(ID, Opcode, VTList, Ops);
8356     ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8357         Opcode, dl.getIROrder(), VTList, MemVT, MMO));
8358     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8359     ID.AddInteger(MMO->getFlags());
8360     ID.AddInteger(MemVT.getRawBits());
8361     void *IP = nullptr;
8362     if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8363       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
8364       return SDValue(E, 0);
8365     }
8366 
8367     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8368                                       VTList, MemVT, MMO);
8369     createOperands(N, Ops);
8370 
8371   CSEMap.InsertNode(N, IP);
8372   } else {
8373     N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8374                                       VTList, MemVT, MMO);
8375     createOperands(N, Ops);
8376   }
8377   InsertNode(N);
8378   SDValue V(N, 0);
8379   NewSDValueDbgMsg(V, "Creating new node: ", this);
8380   return V;
8381 }
8382 
8383 SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
8384                                       SDValue Chain, int FrameIndex,
8385                                       int64_t Size, int64_t Offset) {
8386   const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
8387   const auto VTs = getVTList(MVT::Other);
8388   SDValue Ops[2] = {
8389       Chain,
8390       getFrameIndex(FrameIndex,
8391                     getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
8392                     true)};
8393 
8394   FoldingSetNodeID ID;
8395   AddNodeIDNode(ID, Opcode, VTs, Ops);
8396   ID.AddInteger(FrameIndex);
8397   ID.AddInteger(Size);
8398   ID.AddInteger(Offset);
8399   void *IP = nullptr;
8400   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8401     return SDValue(E, 0);
8402 
8403   LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
8404       Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
8405   createOperands(N, Ops);
8406   CSEMap.InsertNode(N, IP);
8407   InsertNode(N);
8408   SDValue V(N, 0);
8409   NewSDValueDbgMsg(V, "Creating new node: ", this);
8410   return V;
8411 }
8412 
8413 SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
8414                                          uint64_t Guid, uint64_t Index,
8415                                          uint32_t Attr) {
8416   const unsigned Opcode = ISD::PSEUDO_PROBE;
8417   const auto VTs = getVTList(MVT::Other);
8418   SDValue Ops[] = {Chain};
8419   FoldingSetNodeID ID;
8420   AddNodeIDNode(ID, Opcode, VTs, Ops);
8421   ID.AddInteger(Guid);
8422   ID.AddInteger(Index);
8423   void *IP = nullptr;
8424   if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
8425     return SDValue(E, 0);
8426 
8427   auto *N = newSDNode<PseudoProbeSDNode>(
8428       Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
8429   createOperands(N, Ops);
8430   CSEMap.InsertNode(N, IP);
8431   InsertNode(N);
8432   SDValue V(N, 0);
8433   NewSDValueDbgMsg(V, "Creating new node: ", this);
8434   return V;
8435 }
8436 
8437 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8438 /// MachinePointerInfo record from it.  This is particularly useful because the
8439 /// code generator has many cases where it doesn't bother passing in a
8440 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8441 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8442                                            SelectionDAG &DAG, SDValue Ptr,
8443                                            int64_t Offset = 0) {
8444   // If this is FI+Offset, we can model it.
8445   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
8446     return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
8447                                              FI->getIndex(), Offset);
8448 
8449   // If this is (FI+Offset1)+Offset2, we can model it.
8450   if (Ptr.getOpcode() != ISD::ADD ||
8451       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
8452       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
8453     return Info;
8454 
8455   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
8456   return MachinePointerInfo::getFixedStack(
8457       DAG.getMachineFunction(), FI,
8458       Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
8459 }
8460 
8461 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8462 /// MachinePointerInfo record from it.  This is particularly useful because the
8463 /// code generator has many cases where it doesn't bother passing in a
8464 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8465 static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
8466                                            SelectionDAG &DAG, SDValue Ptr,
8467                                            SDValue OffsetOp) {
8468   // If the 'Offset' value isn't a constant, we can't handle this.
8469   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
8470     return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
8471   if (OffsetOp.isUndef())
8472     return InferPointerInfo(Info, DAG, Ptr);
8473   return Info;
8474 }
8475 
8476 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8477                               EVT VT, const SDLoc &dl, SDValue Chain,
8478                               SDValue Ptr, SDValue Offset,
8479                               MachinePointerInfo PtrInfo, EVT MemVT,
8480                               Align Alignment,
8481                               MachineMemOperand::Flags MMOFlags,
8482                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
8483   assert(Chain.getValueType() == MVT::Other &&
8484         "Invalid chain type");
8485 
8486   MMOFlags |= MachineMemOperand::MOLoad;
8487   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8488   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8489   // clients.
8490   if (PtrInfo.V.isNull())
8491     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8492 
8493   uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
8494   MachineFunction &MF = getMachineFunction();
8495   MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8496                                                    Alignment, AAInfo, Ranges);
8497   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
8498 }
8499 
8500 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
8501                               EVT VT, const SDLoc &dl, SDValue Chain,
8502                               SDValue Ptr, SDValue Offset, EVT MemVT,
8503                               MachineMemOperand *MMO) {
8504   if (VT == MemVT) {
8505     ExtType = ISD::NON_EXTLOAD;
8506   } else if (ExtType == ISD::NON_EXTLOAD) {
8507     assert(VT == MemVT && "Non-extending load from different memory type!");
8508   } else {
8509     // Extending load.
8510     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
8511            "Should only be an extending load, not truncating!");
8512     assert(VT.isInteger() == MemVT.isInteger() &&
8513            "Cannot convert from FP to Int or Int -> FP!");
8514     assert(VT.isVector() == MemVT.isVector() &&
8515            "Cannot use an ext load to convert to or from a vector!");
8516     assert((!VT.isVector() ||
8517             VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
8518            "Cannot use an ext load to change the number of vector elements!");
8519   }
8520 
8521   bool Indexed = AM != ISD::UNINDEXED;
8522   assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8523 
8524   SDVTList VTs = Indexed ?
8525     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
8526   SDValue Ops[] = { Chain, Ptr, Offset };
8527   FoldingSetNodeID ID;
8528   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
8529   ID.AddInteger(MemVT.getRawBits());
8530   ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
8531       dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
8532   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8533   ID.AddInteger(MMO->getFlags());
8534   void *IP = nullptr;
8535   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8536     cast<LoadSDNode>(E)->refineAlignment(MMO);
8537     return SDValue(E, 0);
8538   }
8539   auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8540                                   ExtType, MemVT, MMO);
8541   createOperands(N, Ops);
8542 
8543   CSEMap.InsertNode(N, IP);
8544   InsertNode(N);
8545   SDValue V(N, 0);
8546   NewSDValueDbgMsg(V, "Creating new node: ", this);
8547   return V;
8548 }
8549 
8550 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8551                               SDValue Ptr, MachinePointerInfo PtrInfo,
8552                               MaybeAlign Alignment,
8553                               MachineMemOperand::Flags MMOFlags,
8554                               const AAMDNodes &AAInfo, const MDNode *Ranges) {
8555   SDValue Undef = getUNDEF(Ptr.getValueType());
8556   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8557                  PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
8558 }
8559 
8560 SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
8561                               SDValue Ptr, MachineMemOperand *MMO) {
8562   SDValue Undef = getUNDEF(Ptr.getValueType());
8563   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8564                  VT, MMO);
8565 }
8566 
8567 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8568                                  EVT VT, SDValue Chain, SDValue Ptr,
8569                                  MachinePointerInfo PtrInfo, EVT MemVT,
8570                                  MaybeAlign Alignment,
8571                                  MachineMemOperand::Flags MMOFlags,
8572                                  const AAMDNodes &AAInfo) {
8573   SDValue Undef = getUNDEF(Ptr.getValueType());
8574   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
8575                  MemVT, Alignment, MMOFlags, AAInfo);
8576 }
8577 
8578 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
8579                                  EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
8580                                  MachineMemOperand *MMO) {
8581   SDValue Undef = getUNDEF(Ptr.getValueType());
8582   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
8583                  MemVT, MMO);
8584 }
8585 
8586 SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
8587                                      SDValue Base, SDValue Offset,
8588                                      ISD::MemIndexedMode AM) {
8589   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
8590   assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8591   // Don't propagate the invariant or dereferenceable flags.
8592   auto MMOFlags =
8593       LD->getMemOperand()->getFlags() &
8594       ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8595   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8596                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
8597                  LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
8598 }
8599 
8600 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8601                                SDValue Ptr, MachinePointerInfo PtrInfo,
8602                                Align Alignment,
8603                                MachineMemOperand::Flags MMOFlags,
8604                                const AAMDNodes &AAInfo) {
8605   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8606 
8607   MMOFlags |= MachineMemOperand::MOStore;
8608   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8609 
8610   if (PtrInfo.V.isNull())
8611     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8612 
8613   MachineFunction &MF = getMachineFunction();
8614   uint64_t Size =
8615       MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
8616   MachineMemOperand *MMO =
8617       MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
8618   return getStore(Chain, dl, Val, Ptr, MMO);
8619 }
8620 
8621 SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8622                                SDValue Ptr, MachineMemOperand *MMO) {
8623   assert(Chain.getValueType() == MVT::Other &&
8624         "Invalid chain type");
8625   EVT VT = Val.getValueType();
8626   SDVTList VTs = getVTList(MVT::Other);
8627   SDValue Undef = getUNDEF(Ptr.getValueType());
8628   SDValue Ops[] = { Chain, Val, Ptr, Undef };
8629   FoldingSetNodeID ID;
8630   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8631   ID.AddInteger(VT.getRawBits());
8632   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8633       dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
8634   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8635   ID.AddInteger(MMO->getFlags());
8636   void *IP = nullptr;
8637   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8638     cast<StoreSDNode>(E)->refineAlignment(MMO);
8639     return SDValue(E, 0);
8640   }
8641   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8642                                    ISD::UNINDEXED, false, VT, MMO);
8643   createOperands(N, Ops);
8644 
8645   CSEMap.InsertNode(N, IP);
8646   InsertNode(N);
8647   SDValue V(N, 0);
8648   NewSDValueDbgMsg(V, "Creating new node: ", this);
8649   return V;
8650 }
8651 
8652 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8653                                     SDValue Ptr, MachinePointerInfo PtrInfo,
8654                                     EVT SVT, Align Alignment,
8655                                     MachineMemOperand::Flags MMOFlags,
8656                                     const AAMDNodes &AAInfo) {
8657   assert(Chain.getValueType() == MVT::Other &&
8658         "Invalid chain type");
8659 
8660   MMOFlags |= MachineMemOperand::MOStore;
8661   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8662 
8663   if (PtrInfo.V.isNull())
8664     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8665 
8666   MachineFunction &MF = getMachineFunction();
8667   MachineMemOperand *MMO = MF.getMachineMemOperand(
8668       PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
8669       Alignment, AAInfo);
8670   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
8671 }
8672 
8673 SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
8674                                     SDValue Ptr, EVT SVT,
8675                                     MachineMemOperand *MMO) {
8676   EVT VT = Val.getValueType();
8677 
8678   assert(Chain.getValueType() == MVT::Other &&
8679         "Invalid chain type");
8680   if (VT == SVT)
8681     return getStore(Chain, dl, Val, Ptr, MMO);
8682 
8683   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8684          "Should only be a truncating store, not extending!");
8685   assert(VT.isInteger() == SVT.isInteger() &&
8686          "Can't do FP-INT conversion!");
8687   assert(VT.isVector() == SVT.isVector() &&
8688          "Cannot use trunc store to convert to or from a vector!");
8689   assert((!VT.isVector() ||
8690           VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8691          "Cannot use trunc store to change the number of vector elements!");
8692 
8693   SDVTList VTs = getVTList(MVT::Other);
8694   SDValue Undef = getUNDEF(Ptr.getValueType());
8695   SDValue Ops[] = { Chain, Val, Ptr, Undef };
8696   FoldingSetNodeID ID;
8697   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8698   ID.AddInteger(SVT.getRawBits());
8699   ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8700       dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
8701   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8702   ID.AddInteger(MMO->getFlags());
8703   void *IP = nullptr;
8704   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8705     cast<StoreSDNode>(E)->refineAlignment(MMO);
8706     return SDValue(E, 0);
8707   }
8708   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8709                                    ISD::UNINDEXED, true, SVT, MMO);
8710   createOperands(N, Ops);
8711 
8712   CSEMap.InsertNode(N, IP);
8713   InsertNode(N);
8714   SDValue V(N, 0);
8715   NewSDValueDbgMsg(V, "Creating new node: ", this);
8716   return V;
8717 }
8718 
8719 SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
8720                                       SDValue Base, SDValue Offset,
8721                                       ISD::MemIndexedMode AM) {
8722   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
8723   assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
8724   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8725   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
8726   FoldingSetNodeID ID;
8727   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8728   ID.AddInteger(ST->getMemoryVT().getRawBits());
8729   ID.AddInteger(ST->getRawSubclassData());
8730   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
8731   ID.AddInteger(ST->getMemOperand()->getFlags());
8732   void *IP = nullptr;
8733   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8734     return SDValue(E, 0);
8735 
8736   auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8737                                    ST->isTruncatingStore(), ST->getMemoryVT(),
8738                                    ST->getMemOperand());
8739   createOperands(N, Ops);
8740 
8741   CSEMap.InsertNode(N, IP);
8742   InsertNode(N);
8743   SDValue V(N, 0);
8744   NewSDValueDbgMsg(V, "Creating new node: ", this);
8745   return V;
8746 }
8747 
8748 SDValue SelectionDAG::getLoadVP(
8749     ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
8750     SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
8751     MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
8752     MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
8753     const MDNode *Ranges, bool IsExpanding) {
8754   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8755 
8756   MMOFlags |= MachineMemOperand::MOLoad;
8757   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8758   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8759   // clients.
8760   if (PtrInfo.V.isNull())
8761     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8762 
8763   uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
8764   MachineFunction &MF = getMachineFunction();
8765   MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8766                                                    Alignment, AAInfo, Ranges);
8767   return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
8768                    MMO, IsExpanding);
8769 }
8770 
8771 SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
8772                                 ISD::LoadExtType ExtType, EVT VT,
8773                                 const SDLoc &dl, SDValue Chain, SDValue Ptr,
8774                                 SDValue Offset, SDValue Mask, SDValue EVL,
8775                                 EVT MemVT, MachineMemOperand *MMO,
8776                                 bool IsExpanding) {
8777   bool Indexed = AM != ISD::UNINDEXED;
8778   assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8779 
8780   SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
8781                          : getVTList(VT, MVT::Other);
8782   SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
8783   FoldingSetNodeID ID;
8784   AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops);
8785   ID.AddInteger(MemVT.getRawBits());
8786   ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
8787       dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
8788   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8789   ID.AddInteger(MMO->getFlags());
8790   void *IP = nullptr;
8791   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8792     cast<VPLoadSDNode>(E)->refineAlignment(MMO);
8793     return SDValue(E, 0);
8794   }
8795   auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8796                                     ExtType, IsExpanding, MemVT, MMO);
8797   createOperands(N, Ops);
8798 
8799   CSEMap.InsertNode(N, IP);
8800   InsertNode(N);
8801   SDValue V(N, 0);
8802   NewSDValueDbgMsg(V, "Creating new node: ", this);
8803   return V;
8804 }
8805 
8806 SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8807                                 SDValue Ptr, SDValue Mask, SDValue EVL,
8808                                 MachinePointerInfo PtrInfo,
8809                                 MaybeAlign Alignment,
8810                                 MachineMemOperand::Flags MMOFlags,
8811                                 const AAMDNodes &AAInfo, const MDNode *Ranges,
8812                                 bool IsExpanding) {
8813   SDValue Undef = getUNDEF(Ptr.getValueType());
8814   return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8815                    Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
8816                    IsExpanding);
8817 }
8818 
8819 SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
8820                                 SDValue Ptr, SDValue Mask, SDValue EVL,
8821                                 MachineMemOperand *MMO, bool IsExpanding) {
8822   SDValue Undef = getUNDEF(Ptr.getValueType());
8823   return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8824                    Mask, EVL, VT, MMO, IsExpanding);
8825 }
8826 
8827 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8828                                    EVT VT, SDValue Chain, SDValue Ptr,
8829                                    SDValue Mask, SDValue EVL,
8830                                    MachinePointerInfo PtrInfo, EVT MemVT,
8831                                    MaybeAlign Alignment,
8832                                    MachineMemOperand::Flags MMOFlags,
8833                                    const AAMDNodes &AAInfo, bool IsExpanding) {
8834   SDValue Undef = getUNDEF(Ptr.getValueType());
8835   return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8836                    EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,
8837                    IsExpanding);
8838 }
8839 
8840 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
8841                                    EVT VT, SDValue Chain, SDValue Ptr,
8842                                    SDValue Mask, SDValue EVL, EVT MemVT,
8843                                    MachineMemOperand *MMO, bool IsExpanding) {
8844   SDValue Undef = getUNDEF(Ptr.getValueType());
8845   return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8846                    EVL, MemVT, MMO, IsExpanding);
8847 }
8848 
8849 SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
8850                                        SDValue Base, SDValue Offset,
8851                                        ISD::MemIndexedMode AM) {
8852   auto *LD = cast<VPLoadSDNode>(OrigLoad);
8853   assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8854   // Don't propagate the invariant or dereferenceable flags.
8855   auto MMOFlags =
8856       LD->getMemOperand()->getFlags() &
8857       ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
8858   return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8859                    LD->getChain(), Base, Offset, LD->getMask(),
8860                    LD->getVectorLength(), LD->getPointerInfo(),
8861                    LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
8862                    nullptr, LD->isExpandingLoad());
8863 }
8864 
8865 SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
8866                                  SDValue Ptr, SDValue Offset, SDValue Mask,
8867                                  SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
8868                                  ISD::MemIndexedMode AM, bool IsTruncating,
8869                                  bool IsCompressing) {
8870   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8871   bool Indexed = AM != ISD::UNINDEXED;
8872   assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
8873   SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
8874                          : getVTList(MVT::Other);
8875   SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
8876   FoldingSetNodeID ID;
8877   AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8878   ID.AddInteger(MemVT.getRawBits());
8879   ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
8880       dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
8881   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8882   ID.AddInteger(MMO->getFlags());
8883   void *IP = nullptr;
8884   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8885     cast<VPStoreSDNode>(E)->refineAlignment(MMO);
8886     return SDValue(E, 0);
8887   }
8888   auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8889                                      IsTruncating, IsCompressing, MemVT, MMO);
8890   createOperands(N, Ops);
8891 
8892   CSEMap.InsertNode(N, IP);
8893   InsertNode(N);
8894   SDValue V(N, 0);
8895   NewSDValueDbgMsg(V, "Creating new node: ", this);
8896   return V;
8897 }
8898 
8899 SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
8900                                       SDValue Val, SDValue Ptr, SDValue Mask,
8901                                       SDValue EVL, MachinePointerInfo PtrInfo,
8902                                       EVT SVT, Align Alignment,
8903                                       MachineMemOperand::Flags MMOFlags,
8904                                       const AAMDNodes &AAInfo,
8905                                       bool IsCompressing) {
8906   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8907 
8908   MMOFlags |= MachineMemOperand::MOStore;
8909   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8910 
8911   if (PtrInfo.V.isNull())
8912     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8913 
8914   MachineFunction &MF = getMachineFunction();
8915   MachineMemOperand *MMO = MF.getMachineMemOperand(
8916       PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
8917       Alignment, AAInfo);
8918   return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
8919                          IsCompressing);
8920 }
8921 
8922 SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
8923                                       SDValue Val, SDValue Ptr, SDValue Mask,
8924                                       SDValue EVL, EVT SVT,
8925                                       MachineMemOperand *MMO,
8926                                       bool IsCompressing) {
8927   EVT VT = Val.getValueType();
8928 
8929   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8930   if (VT == SVT)
8931     return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask,
8932                       EVL, VT, MMO, ISD::UNINDEXED,
8933                       /*IsTruncating*/ false, IsCompressing);
8934 
8935   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
8936          "Should only be a truncating store, not extending!");
8937   assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
8938   assert(VT.isVector() == SVT.isVector() &&
8939          "Cannot use trunc store to convert to or from a vector!");
8940   assert((!VT.isVector() ||
8941           VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
8942          "Cannot use trunc store to change the number of vector elements!");
8943 
8944   SDVTList VTs = getVTList(MVT::Other);
8945   SDValue Undef = getUNDEF(Ptr.getValueType());
8946   SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
8947   FoldingSetNodeID ID;
8948   AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8949   ID.AddInteger(SVT.getRawBits());
8950   ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
8951       dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
8952   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8953   ID.AddInteger(MMO->getFlags());
8954   void *IP = nullptr;
8955   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8956     cast<VPStoreSDNode>(E)->refineAlignment(MMO);
8957     return SDValue(E, 0);
8958   }
8959   auto *N =
8960       newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8961                                ISD::UNINDEXED, true, IsCompressing, SVT, MMO);
8962   createOperands(N, Ops);
8963 
8964   CSEMap.InsertNode(N, IP);
8965   InsertNode(N);
8966   SDValue V(N, 0);
8967   NewSDValueDbgMsg(V, "Creating new node: ", this);
8968   return V;
8969 }
8970 
8971 SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
8972                                         SDValue Base, SDValue Offset,
8973                                         ISD::MemIndexedMode AM) {
8974   auto *ST = cast<VPStoreSDNode>(OrigStore);
8975   assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
8976   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8977   SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
8978                    Offset,         ST->getMask(),  ST->getVectorLength()};
8979   FoldingSetNodeID ID;
8980   AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8981   ID.AddInteger(ST->getMemoryVT().getRawBits());
8982   ID.AddInteger(ST->getRawSubclassData());
8983   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
8984   ID.AddInteger(ST->getMemOperand()->getFlags());
8985   void *IP = nullptr;
8986   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8987     return SDValue(E, 0);
8988 
8989   auto *N = newSDNode<VPStoreSDNode>(
8990       dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(),
8991       ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
8992   createOperands(N, Ops);
8993 
8994   CSEMap.InsertNode(N, IP);
8995   InsertNode(N);
8996   SDValue V(N, 0);
8997   NewSDValueDbgMsg(V, "Creating new node: ", this);
8998   return V;
8999 }
9000 
9001 SDValue SelectionDAG::getStridedLoadVP(
9002     ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9003     SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9004     SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
9005     MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9006     const MDNode *Ranges, bool IsExpanding) {
9007   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9008 
9009   MMOFlags |= MachineMemOperand::MOLoad;
9010   assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9011   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9012   // clients.
9013   if (PtrInfo.V.isNull())
9014     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
9015 
9016   uint64_t Size = MemoryLocation::UnknownSize;
9017   MachineFunction &MF = getMachineFunction();
9018   MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
9019                                                    Alignment, AAInfo, Ranges);
9020   return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask,
9021                           EVL, MemVT, MMO, IsExpanding);
9022 }
9023 
9024 SDValue SelectionDAG::getStridedLoadVP(
9025     ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9026     SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9027     SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9028   bool Indexed = AM != ISD::UNINDEXED;
9029   assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9030 
9031   SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9032   SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9033                          : getVTList(VT, MVT::Other);
9034   FoldingSetNodeID ID;
9035   AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
9036   ID.AddInteger(VT.getRawBits());
9037   ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9038       DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9039   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9040 
9041   void *IP = nullptr;
9042   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9043     cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9044     return SDValue(E, 0);
9045   }
9046 
9047   auto *N =
9048       newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,
9049                                      ExtType, IsExpanding, MemVT, MMO);
9050   createOperands(N, Ops);
9051   CSEMap.InsertNode(N, IP);
9052   InsertNode(N);
9053   SDValue V(N, 0);
9054   NewSDValueDbgMsg(V, "Creating new node: ", this);
9055   return V;
9056 }
9057 
9058 SDValue SelectionDAG::getStridedLoadVP(
9059     EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride,
9060     SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, MaybeAlign Alignment,
9061     MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9062     const MDNode *Ranges, bool IsExpanding) {
9063   SDValue Undef = getUNDEF(Ptr.getValueType());
9064   return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9065                           Undef, Stride, Mask, EVL, PtrInfo, VT, Alignment,
9066                           MMOFlags, AAInfo, Ranges, IsExpanding);
9067 }
9068 
9069 SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
9070                                        SDValue Ptr, SDValue Stride,
9071                                        SDValue Mask, SDValue EVL,
9072                                        MachineMemOperand *MMO,
9073                                        bool IsExpanding) {
9074   SDValue Undef = getUNDEF(Ptr.getValueType());
9075   return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9076                           Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9077 }
9078 
9079 SDValue SelectionDAG::getExtStridedLoadVP(
9080     ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9081     SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL,
9082     MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment,
9083     MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9084     bool IsExpanding) {
9085   SDValue Undef = getUNDEF(Ptr.getValueType());
9086   return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9087                           Stride, Mask, EVL, PtrInfo, MemVT, Alignment,
9088                           MMOFlags, AAInfo, nullptr, IsExpanding);
9089 }
9090 
9091 SDValue SelectionDAG::getExtStridedLoadVP(
9092     ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9093     SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9094     MachineMemOperand *MMO, bool IsExpanding) {
9095   SDValue Undef = getUNDEF(Ptr.getValueType());
9096   return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9097                           Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9098 }
9099 
9100 SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL,
9101                                               SDValue Base, SDValue Offset,
9102                                               ISD::MemIndexedMode AM) {
9103   auto *SLD = cast<VPStridedLoadSDNode>(OrigLoad);
9104   assert(SLD->getOffset().isUndef() &&
9105          "Strided load is already a indexed load!");
9106   // Don't propagate the invariant or dereferenceable flags.
9107   auto MMOFlags =
9108       SLD->getMemOperand()->getFlags() &
9109       ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
9110   return getStridedLoadVP(
9111       AM, SLD->getExtensionType(), OrigLoad.getValueType(), DL, SLD->getChain(),
9112       Base, Offset, SLD->getStride(), SLD->getMask(), SLD->getVectorLength(),
9113       SLD->getPointerInfo(), SLD->getMemoryVT(), SLD->getAlign(), MMOFlags,
9114       SLD->getAAInfo(), nullptr, SLD->isExpandingLoad());
9115 }
9116 
9117 SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
9118                                         SDValue Val, SDValue Ptr,
9119                                         SDValue Offset, SDValue Stride,
9120                                         SDValue Mask, SDValue EVL, EVT MemVT,
9121                                         MachineMemOperand *MMO,
9122                                         ISD::MemIndexedMode AM,
9123                                         bool IsTruncating, bool IsCompressing) {
9124   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9125   bool Indexed = AM != ISD::UNINDEXED;
9126   assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9127   SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9128                          : getVTList(MVT::Other);
9129   SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9130   FoldingSetNodeID ID;
9131   AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9132   ID.AddInteger(MemVT.getRawBits());
9133   ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9134       DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9135   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9136   void *IP = nullptr;
9137   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9138     cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9139     return SDValue(E, 0);
9140   }
9141   auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9142                                             VTs, AM, IsTruncating,
9143                                             IsCompressing, MemVT, MMO);
9144   createOperands(N, Ops);
9145 
9146   CSEMap.InsertNode(N, IP);
9147   InsertNode(N);
9148   SDValue V(N, 0);
9149   NewSDValueDbgMsg(V, "Creating new node: ", this);
9150   return V;
9151 }
9152 
9153 SDValue SelectionDAG::getTruncStridedStoreVP(
9154     SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride,
9155     SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT,
9156     Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9157     bool IsCompressing) {
9158   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9159 
9160   MMOFlags |= MachineMemOperand::MOStore;
9161   assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9162 
9163   if (PtrInfo.V.isNull())
9164     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
9165 
9166   MachineFunction &MF = getMachineFunction();
9167   MachineMemOperand *MMO = MF.getMachineMemOperand(
9168       PtrInfo, MMOFlags, MemoryLocation::UnknownSize, Alignment, AAInfo);
9169   return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT,
9170                                 MMO, IsCompressing);
9171 }
9172 
9173 SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
9174                                              SDValue Val, SDValue Ptr,
9175                                              SDValue Stride, SDValue Mask,
9176                                              SDValue EVL, EVT SVT,
9177                                              MachineMemOperand *MMO,
9178                                              bool IsCompressing) {
9179   EVT VT = Val.getValueType();
9180 
9181   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9182   if (VT == SVT)
9183     return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()),
9184                              Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED,
9185                              /*IsTruncating*/ false, IsCompressing);
9186 
9187   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9188          "Should only be a truncating store, not extending!");
9189   assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9190   assert(VT.isVector() == SVT.isVector() &&
9191          "Cannot use trunc store to convert to or from a vector!");
9192   assert((!VT.isVector() ||
9193           VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9194          "Cannot use trunc store to change the number of vector elements!");
9195 
9196   SDVTList VTs = getVTList(MVT::Other);
9197   SDValue Undef = getUNDEF(Ptr.getValueType());
9198   SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9199   FoldingSetNodeID ID;
9200   AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9201   ID.AddInteger(SVT.getRawBits());
9202   ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9203       DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9204   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9205   void *IP = nullptr;
9206   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9207     cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9208     return SDValue(E, 0);
9209   }
9210   auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9211                                             VTs, ISD::UNINDEXED, true,
9212                                             IsCompressing, SVT, MMO);
9213   createOperands(N, Ops);
9214 
9215   CSEMap.InsertNode(N, IP);
9216   InsertNode(N);
9217   SDValue V(N, 0);
9218   NewSDValueDbgMsg(V, "Creating new node: ", this);
9219   return V;
9220 }
9221 
9222 SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore,
9223                                                const SDLoc &DL, SDValue Base,
9224                                                SDValue Offset,
9225                                                ISD::MemIndexedMode AM) {
9226   auto *SST = cast<VPStridedStoreSDNode>(OrigStore);
9227   assert(SST->getOffset().isUndef() &&
9228          "Strided store is already an indexed store!");
9229   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9230   SDValue Ops[] = {
9231       SST->getChain(), SST->getValue(),       Base, Offset, SST->getStride(),
9232       SST->getMask(),  SST->getVectorLength()};
9233   FoldingSetNodeID ID;
9234   AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9235   ID.AddInteger(SST->getMemoryVT().getRawBits());
9236   ID.AddInteger(SST->getRawSubclassData());
9237   ID.AddInteger(SST->getPointerInfo().getAddrSpace());
9238   void *IP = nullptr;
9239   if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9240     return SDValue(E, 0);
9241 
9242   auto *N = newSDNode<VPStridedStoreSDNode>(
9243       DL.getIROrder(), DL.getDebugLoc(), VTs, AM, SST->isTruncatingStore(),
9244       SST->isCompressingStore(), SST->getMemoryVT(), SST->getMemOperand());
9245   createOperands(N, Ops);
9246 
9247   CSEMap.InsertNode(N, IP);
9248   InsertNode(N);
9249   SDValue V(N, 0);
9250   NewSDValueDbgMsg(V, "Creating new node: ", this);
9251   return V;
9252 }
9253 
9254 SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9255                                   ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
9256                                   ISD::MemIndexType IndexType) {
9257   assert(Ops.size() == 6 && "Incompatible number of operands");
9258 
9259   FoldingSetNodeID ID;
9260   AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops);
9261   ID.AddInteger(VT.getRawBits());
9262   ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9263       dl.getIROrder(), VTs, VT, MMO, IndexType));
9264   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9265   ID.AddInteger(MMO->getFlags());
9266   void *IP = nullptr;
9267   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9268     cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9269     return SDValue(E, 0);
9270   }
9271 
9272   auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9273                                       VT, MMO, IndexType);
9274   createOperands(N, Ops);
9275 
9276   assert(N->getMask().getValueType().getVectorElementCount() ==
9277              N->getValueType(0).getVectorElementCount() &&
9278          "Vector width mismatch between mask and data");
9279   assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9280              N->getValueType(0).getVectorElementCount().isScalable() &&
9281          "Scalable flags of index and data do not match");
9282   assert(ElementCount::isKnownGE(
9283              N->getIndex().getValueType().getVectorElementCount(),
9284              N->getValueType(0).getVectorElementCount()) &&
9285          "Vector width mismatch between index and data");
9286   assert(isa<ConstantSDNode>(N->getScale()) &&
9287          N->getScale()->getAsAPIntVal().isPowerOf2() &&
9288          "Scale should be a constant power of 2");
9289 
9290   CSEMap.InsertNode(N, IP);
9291   InsertNode(N);
9292   SDValue V(N, 0);
9293   NewSDValueDbgMsg(V, "Creating new node: ", this);
9294   return V;
9295 }
9296 
9297 SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
9298                                    ArrayRef<SDValue> Ops,
9299                                    MachineMemOperand *MMO,
9300                                    ISD::MemIndexType IndexType) {
9301   assert(Ops.size() == 7 && "Incompatible number of operands");
9302 
9303   FoldingSetNodeID ID;
9304   AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops);
9305   ID.AddInteger(VT.getRawBits());
9306   ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9307       dl.getIROrder(), VTs, VT, MMO, IndexType));
9308   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9309   ID.AddInteger(MMO->getFlags());
9310   void *IP = nullptr;
9311   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9312     cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9313     return SDValue(E, 0);
9314   }
9315   auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9316                                        VT, MMO, IndexType);
9317   createOperands(N, Ops);
9318 
9319   assert(N->getMask().getValueType().getVectorElementCount() ==
9320              N->getValue().getValueType().getVectorElementCount() &&
9321          "Vector width mismatch between mask and data");
9322   assert(
9323       N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9324           N->getValue().getValueType().getVectorElementCount().isScalable() &&
9325       "Scalable flags of index and data do not match");
9326   assert(ElementCount::isKnownGE(
9327              N->getIndex().getValueType().getVectorElementCount(),
9328              N->getValue().getValueType().getVectorElementCount()) &&
9329          "Vector width mismatch between index and data");
9330   assert(isa<ConstantSDNode>(N->getScale()) &&
9331          N->getScale()->getAsAPIntVal().isPowerOf2() &&
9332          "Scale should be a constant power of 2");
9333 
9334   CSEMap.InsertNode(N, IP);
9335   InsertNode(N);
9336   SDValue V(N, 0);
9337   NewSDValueDbgMsg(V, "Creating new node: ", this);
9338   return V;
9339 }
9340 
9341 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
9342                                     SDValue Base, SDValue Offset, SDValue Mask,
9343                                     SDValue PassThru, EVT MemVT,
9344                                     MachineMemOperand *MMO,
9345                                     ISD::MemIndexedMode AM,
9346                                     ISD::LoadExtType ExtTy, bool isExpanding) {
9347   bool Indexed = AM != ISD::UNINDEXED;
9348   assert((Indexed || Offset.isUndef()) &&
9349          "Unindexed masked load with an offset!");
9350   SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9351                          : getVTList(VT, MVT::Other);
9352   SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9353   FoldingSetNodeID ID;
9354   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
9355   ID.AddInteger(MemVT.getRawBits());
9356   ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9357       dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9358   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9359   ID.AddInteger(MMO->getFlags());
9360   void *IP = nullptr;
9361   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9362     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9363     return SDValue(E, 0);
9364   }
9365   auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9366                                         AM, ExtTy, isExpanding, MemVT, MMO);
9367   createOperands(N, Ops);
9368 
9369   CSEMap.InsertNode(N, IP);
9370   InsertNode(N);
9371   SDValue V(N, 0);
9372   NewSDValueDbgMsg(V, "Creating new node: ", this);
9373   return V;
9374 }
9375 
9376 SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
9377                                            SDValue Base, SDValue Offset,
9378                                            ISD::MemIndexedMode AM) {
9379   MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
9380   assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9381   return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
9382                        Offset, LD->getMask(), LD->getPassThru(),
9383                        LD->getMemoryVT(), LD->getMemOperand(), AM,
9384                        LD->getExtensionType(), LD->isExpandingLoad());
9385 }
9386 
9387 SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
9388                                      SDValue Val, SDValue Base, SDValue Offset,
9389                                      SDValue Mask, EVT MemVT,
9390                                      MachineMemOperand *MMO,
9391                                      ISD::MemIndexedMode AM, bool IsTruncating,
9392                                      bool IsCompressing) {
9393   assert(Chain.getValueType() == MVT::Other &&
9394         "Invalid chain type");
9395   bool Indexed = AM != ISD::UNINDEXED;
9396   assert((Indexed || Offset.isUndef()) &&
9397          "Unindexed masked store with an offset!");
9398   SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9399                          : getVTList(MVT::Other);
9400   SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
9401   FoldingSetNodeID ID;
9402   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
9403   ID.AddInteger(MemVT.getRawBits());
9404   ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9405       dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9406   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9407   ID.AddInteger(MMO->getFlags());
9408   void *IP = nullptr;
9409   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9410     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
9411     return SDValue(E, 0);
9412   }
9413   auto *N =
9414       newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9415                                    IsTruncating, IsCompressing, MemVT, MMO);
9416   createOperands(N, Ops);
9417 
9418   CSEMap.InsertNode(N, IP);
9419   InsertNode(N);
9420   SDValue V(N, 0);
9421   NewSDValueDbgMsg(V, "Creating new node: ", this);
9422   return V;
9423 }
9424 
9425 SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
9426                                             SDValue Base, SDValue Offset,
9427                                             ISD::MemIndexedMode AM) {
9428   MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
9429   assert(ST->getOffset().isUndef() &&
9430          "Masked store is already a indexed store!");
9431   return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
9432                         ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
9433                         AM, ST->isTruncatingStore(), ST->isCompressingStore());
9434 }
9435 
9436 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9437                                       ArrayRef<SDValue> Ops,
9438                                       MachineMemOperand *MMO,
9439                                       ISD::MemIndexType IndexType,
9440                                       ISD::LoadExtType ExtTy) {
9441   assert(Ops.size() == 6 && "Incompatible number of operands");
9442 
9443   FoldingSetNodeID ID;
9444   AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
9445   ID.AddInteger(MemVT.getRawBits());
9446   ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9447       dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
9448   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9449   ID.AddInteger(MMO->getFlags());
9450   void *IP = nullptr;
9451   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9452     cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
9453     return SDValue(E, 0);
9454   }
9455 
9456   auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9457                                           VTs, MemVT, MMO, IndexType, ExtTy);
9458   createOperands(N, Ops);
9459 
9460   assert(N->getPassThru().getValueType() == N->getValueType(0) &&
9461          "Incompatible type of the PassThru value in MaskedGatherSDNode");
9462   assert(N->getMask().getValueType().getVectorElementCount() ==
9463              N->getValueType(0).getVectorElementCount() &&
9464          "Vector width mismatch between mask and data");
9465   assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9466              N->getValueType(0).getVectorElementCount().isScalable() &&
9467          "Scalable flags of index and data do not match");
9468   assert(ElementCount::isKnownGE(
9469              N->getIndex().getValueType().getVectorElementCount(),
9470              N->getValueType(0).getVectorElementCount()) &&
9471          "Vector width mismatch between index and data");
9472   assert(isa<ConstantSDNode>(N->getScale()) &&
9473          N->getScale()->getAsAPIntVal().isPowerOf2() &&
9474          "Scale should be a constant power of 2");
9475 
9476   CSEMap.InsertNode(N, IP);
9477   InsertNode(N);
9478   SDValue V(N, 0);
9479   NewSDValueDbgMsg(V, "Creating new node: ", this);
9480   return V;
9481 }
9482 
9483 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
9484                                        ArrayRef<SDValue> Ops,
9485                                        MachineMemOperand *MMO,
9486                                        ISD::MemIndexType IndexType,
9487                                        bool IsTrunc) {
9488   assert(Ops.size() == 6 && "Incompatible number of operands");
9489 
9490   FoldingSetNodeID ID;
9491   AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
9492   ID.AddInteger(MemVT.getRawBits());
9493   ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9494       dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
9495   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9496   ID.AddInteger(MMO->getFlags());
9497   void *IP = nullptr;
9498   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9499     cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
9500     return SDValue(E, 0);
9501   }
9502 
9503   auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9504                                            VTs, MemVT, MMO, IndexType, IsTrunc);
9505   createOperands(N, Ops);
9506 
9507   assert(N->getMask().getValueType().getVectorElementCount() ==
9508              N->getValue().getValueType().getVectorElementCount() &&
9509          "Vector width mismatch between mask and data");
9510   assert(
9511       N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9512           N->getValue().getValueType().getVectorElementCount().isScalable() &&
9513       "Scalable flags of index and data do not match");
9514   assert(ElementCount::isKnownGE(
9515              N->getIndex().getValueType().getVectorElementCount(),
9516              N->getValue().getValueType().getVectorElementCount()) &&
9517          "Vector width mismatch between index and data");
9518   assert(isa<ConstantSDNode>(N->getScale()) &&
9519          N->getScale()->getAsAPIntVal().isPowerOf2() &&
9520          "Scale should be a constant power of 2");
9521 
9522   CSEMap.InsertNode(N, IP);
9523   InsertNode(N);
9524   SDValue V(N, 0);
9525   NewSDValueDbgMsg(V, "Creating new node: ", this);
9526   return V;
9527 }
9528 
9529 SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9530                                   EVT MemVT, MachineMemOperand *MMO) {
9531   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9532   SDVTList VTs = getVTList(MVT::Other);
9533   SDValue Ops[] = {Chain, Ptr};
9534   FoldingSetNodeID ID;
9535   AddNodeIDNode(ID, ISD::GET_FPENV_MEM, VTs, Ops);
9536   ID.AddInteger(MemVT.getRawBits());
9537   ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9538       ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9539   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9540   ID.AddInteger(MMO->getFlags());
9541   void *IP = nullptr;
9542   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9543     return SDValue(E, 0);
9544 
9545   auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),
9546                                            dl.getDebugLoc(), VTs, MemVT, MMO);
9547   createOperands(N, Ops);
9548 
9549   CSEMap.InsertNode(N, IP);
9550   InsertNode(N);
9551   SDValue V(N, 0);
9552   NewSDValueDbgMsg(V, "Creating new node: ", this);
9553   return V;
9554 }
9555 
9556 SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
9557                                   EVT MemVT, MachineMemOperand *MMO) {
9558   assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9559   SDVTList VTs = getVTList(MVT::Other);
9560   SDValue Ops[] = {Chain, Ptr};
9561   FoldingSetNodeID ID;
9562   AddNodeIDNode(ID, ISD::SET_FPENV_MEM, VTs, Ops);
9563   ID.AddInteger(MemVT.getRawBits());
9564   ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9565       ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9566   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9567   ID.AddInteger(MMO->getFlags());
9568   void *IP = nullptr;
9569   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9570     return SDValue(E, 0);
9571 
9572   auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),
9573                                            dl.getDebugLoc(), VTs, MemVT, MMO);
9574   createOperands(N, Ops);
9575 
9576   CSEMap.InsertNode(N, IP);
9577   InsertNode(N);
9578   SDValue V(N, 0);
9579   NewSDValueDbgMsg(V, "Creating new node: ", this);
9580   return V;
9581 }
9582 
9583 SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
9584   // select undef, T, F --> T (if T is a constant), otherwise F
9585   // select, ?, undef, F --> F
9586   // select, ?, T, undef --> T
9587   if (Cond.isUndef())
9588     return isConstantValueOfAnyType(T) ? T : F;
9589   if (T.isUndef())
9590     return F;
9591   if (F.isUndef())
9592     return T;
9593 
9594   // select true, T, F --> T
9595   // select false, T, F --> F
9596   if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
9597     return CondC->isZero() ? F : T;
9598 
9599   // TODO: This should simplify VSELECT with non-zero constant condition using
9600   // something like this (but check boolean contents to be complete?):
9601   if (ConstantSDNode *CondC = isConstOrConstSplat(Cond, /*AllowUndefs*/ false,
9602                                                   /*AllowTruncation*/ true))
9603     if (CondC->isZero())
9604       return F;
9605 
9606   // select ?, T, T --> T
9607   if (T == F)
9608     return T;
9609 
9610   return SDValue();
9611 }
9612 
9613 SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
9614   // shift undef, Y --> 0 (can always assume that the undef value is 0)
9615   if (X.isUndef())
9616     return getConstant(0, SDLoc(X.getNode()), X.getValueType());
9617   // shift X, undef --> undef (because it may shift by the bitwidth)
9618   if (Y.isUndef())
9619     return getUNDEF(X.getValueType());
9620 
9621   // shift 0, Y --> 0
9622   // shift X, 0 --> X
9623   if (isNullOrNullSplat(X) || isNullOrNullSplat(Y))
9624     return X;
9625 
9626   // shift X, C >= bitwidth(X) --> undef
9627   // All vector elements must be too big (or undef) to avoid partial undefs.
9628   auto isShiftTooBig = [X](ConstantSDNode *Val) {
9629     return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
9630   };
9631   if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
9632     return getUNDEF(X.getValueType());
9633 
9634   return SDValue();
9635 }
9636 
9637 SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
9638                                       SDNodeFlags Flags) {
9639   // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
9640   // (an undef operand can be chosen to be Nan/Inf), then the result of this
9641   // operation is poison. That result can be relaxed to undef.
9642   ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
9643   ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
9644   bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9645                 (YC && YC->getValueAPF().isNaN());
9646   bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9647                 (YC && YC->getValueAPF().isInfinity());
9648 
9649   if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
9650     return getUNDEF(X.getValueType());
9651 
9652   if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
9653     return getUNDEF(X.getValueType());
9654 
9655   if (!YC)
9656     return SDValue();
9657 
9658   // X + -0.0 --> X
9659   if (Opcode == ISD::FADD)
9660     if (YC->getValueAPF().isNegZero())
9661       return X;
9662 
9663   // X - +0.0 --> X
9664   if (Opcode == ISD::FSUB)
9665     if (YC->getValueAPF().isPosZero())
9666       return X;
9667 
9668   // X * 1.0 --> X
9669   // X / 1.0 --> X
9670   if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
9671     if (YC->getValueAPF().isExactlyValue(1.0))
9672       return X;
9673 
9674   // X * 0.0 --> 0.0
9675   if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
9676     if (YC->getValueAPF().isZero())
9677       return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
9678 
9679   return SDValue();
9680 }
9681 
9682 SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
9683                                SDValue Ptr, SDValue SV, unsigned Align) {
9684   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
9685   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
9686 }
9687 
9688 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9689                               ArrayRef<SDUse> Ops) {
9690   switch (Ops.size()) {
9691   case 0: return getNode(Opcode, DL, VT);
9692   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
9693   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
9694   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
9695   default: break;
9696   }
9697 
9698   // Copy from an SDUse array into an SDValue array for use with
9699   // the regular getNode logic.
9700   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
9701   return getNode(Opcode, DL, VT, NewOps);
9702 }
9703 
9704 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9705                               ArrayRef<SDValue> Ops) {
9706   SDNodeFlags Flags;
9707   if (Inserter)
9708     Flags = Inserter->getFlags();
9709   return getNode(Opcode, DL, VT, Ops, Flags);
9710 }
9711 
9712 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9713                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9714   unsigned NumOps = Ops.size();
9715   switch (NumOps) {
9716   case 0: return getNode(Opcode, DL, VT);
9717   case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
9718   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
9719   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
9720   default: break;
9721   }
9722 
9723 #ifndef NDEBUG
9724   for (const auto &Op : Ops)
9725     assert(Op.getOpcode() != ISD::DELETED_NODE &&
9726            "Operand is DELETED_NODE!");
9727 #endif
9728 
9729   switch (Opcode) {
9730   default: break;
9731   case ISD::BUILD_VECTOR:
9732     // Attempt to simplify BUILD_VECTOR.
9733     if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
9734       return V;
9735     break;
9736   case ISD::CONCAT_VECTORS:
9737     if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
9738       return V;
9739     break;
9740   case ISD::SELECT_CC:
9741     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
9742     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
9743            "LHS and RHS of condition must have same type!");
9744     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9745            "True and False arms of SelectCC must have same type!");
9746     assert(Ops[2].getValueType() == VT &&
9747            "select_cc node must be of same type as true and false value!");
9748     assert((!Ops[0].getValueType().isVector() ||
9749             Ops[0].getValueType().getVectorElementCount() ==
9750                 VT.getVectorElementCount()) &&
9751            "Expected select_cc with vector result to have the same sized "
9752            "comparison type!");
9753     break;
9754   case ISD::BR_CC:
9755     assert(NumOps == 5 && "BR_CC takes 5 operands!");
9756     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9757            "LHS/RHS of comparison should match types!");
9758     break;
9759   case ISD::VP_ADD:
9760   case ISD::VP_SUB:
9761     // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
9762     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9763       Opcode = ISD::VP_XOR;
9764     break;
9765   case ISD::VP_MUL:
9766     // If it is VP_MUL mask operation then turn it to VP_AND
9767     if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9768       Opcode = ISD::VP_AND;
9769     break;
9770   case ISD::VP_REDUCE_MUL:
9771     // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
9772     if (VT == MVT::i1)
9773       Opcode = ISD::VP_REDUCE_AND;
9774     break;
9775   case ISD::VP_REDUCE_ADD:
9776     // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
9777     if (VT == MVT::i1)
9778       Opcode = ISD::VP_REDUCE_XOR;
9779     break;
9780   case ISD::VP_REDUCE_SMAX:
9781   case ISD::VP_REDUCE_UMIN:
9782     // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
9783     // VP_REDUCE_AND.
9784     if (VT == MVT::i1)
9785       Opcode = ISD::VP_REDUCE_AND;
9786     break;
9787   case ISD::VP_REDUCE_SMIN:
9788   case ISD::VP_REDUCE_UMAX:
9789     // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
9790     // VP_REDUCE_OR.
9791     if (VT == MVT::i1)
9792       Opcode = ISD::VP_REDUCE_OR;
9793     break;
9794   }
9795 
9796   // Memoize nodes.
9797   SDNode *N;
9798   SDVTList VTs = getVTList(VT);
9799 
9800   if (VT != MVT::Glue) {
9801     FoldingSetNodeID ID;
9802     AddNodeIDNode(ID, Opcode, VTs, Ops);
9803     void *IP = nullptr;
9804 
9805     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9806       return SDValue(E, 0);
9807 
9808     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9809     createOperands(N, Ops);
9810 
9811     CSEMap.InsertNode(N, IP);
9812   } else {
9813     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9814     createOperands(N, Ops);
9815   }
9816 
9817   N->setFlags(Flags);
9818   InsertNode(N);
9819   SDValue V(N, 0);
9820   NewSDValueDbgMsg(V, "Creating new node: ", this);
9821   return V;
9822 }
9823 
9824 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9825                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
9826   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
9827 }
9828 
9829 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9830                               ArrayRef<SDValue> Ops) {
9831   SDNodeFlags Flags;
9832   if (Inserter)
9833     Flags = Inserter->getFlags();
9834   return getNode(Opcode, DL, VTList, Ops, Flags);
9835 }
9836 
9837 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9838                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9839   if (VTList.NumVTs == 1)
9840     return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
9841 
9842 #ifndef NDEBUG
9843   for (const auto &Op : Ops)
9844     assert(Op.getOpcode() != ISD::DELETED_NODE &&
9845            "Operand is DELETED_NODE!");
9846 #endif
9847 
9848   switch (Opcode) {
9849   case ISD::SADDO:
9850   case ISD::UADDO:
9851   case ISD::SSUBO:
9852   case ISD::USUBO: {
9853     assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9854            "Invalid add/sub overflow op!");
9855     assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9856            Ops[0].getValueType() == Ops[1].getValueType() &&
9857            Ops[0].getValueType() == VTList.VTs[0] &&
9858            "Binary operator types must match!");
9859     SDValue N1 = Ops[0], N2 = Ops[1];
9860     canonicalizeCommutativeBinop(Opcode, N1, N2);
9861 
9862     // (X +- 0) -> X with zero-overflow.
9863     ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,
9864                                                /*AllowTruncation*/ true);
9865     if (N2CV && N2CV->isZero()) {
9866       SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
9867       return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
9868     }
9869 
9870     if (VTList.VTs[0].isVector() &&
9871         VTList.VTs[0].getVectorElementType() == MVT::i1 &&
9872         VTList.VTs[1].getVectorElementType() == MVT::i1) {
9873       SDValue F1 = getFreeze(N1);
9874       SDValue F2 = getFreeze(N2);
9875       // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
9876       if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
9877         return getNode(ISD::MERGE_VALUES, DL, VTList,
9878                        {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9879                         getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
9880                        Flags);
9881       // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
9882       if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
9883         SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
9884         return getNode(ISD::MERGE_VALUES, DL, VTList,
9885                        {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9886                         getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
9887                        Flags);
9888       }
9889     }
9890     break;
9891   }
9892   case ISD::SMUL_LOHI:
9893   case ISD::UMUL_LOHI: {
9894     assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
9895     assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
9896            VTList.VTs[0] == Ops[0].getValueType() &&
9897            VTList.VTs[0] == Ops[1].getValueType() &&
9898            "Binary operator types must match!");
9899     // Constant fold.
9900     ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Ops[0]);
9901     ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ops[1]);
9902     if (LHS && RHS) {
9903       unsigned Width = VTList.VTs[0].getScalarSizeInBits();
9904       unsigned OutWidth = Width * 2;
9905       APInt Val = LHS->getAPIntValue();
9906       APInt Mul = RHS->getAPIntValue();
9907       if (Opcode == ISD::SMUL_LOHI) {
9908         Val = Val.sext(OutWidth);
9909         Mul = Mul.sext(OutWidth);
9910       } else {
9911         Val = Val.zext(OutWidth);
9912         Mul = Mul.zext(OutWidth);
9913       }
9914       Val *= Mul;
9915 
9916       SDValue Hi =
9917           getConstant(Val.extractBits(Width, Width), DL, VTList.VTs[0]);
9918       SDValue Lo = getConstant(Val.trunc(Width), DL, VTList.VTs[0]);
9919       return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags);
9920     }
9921     break;
9922   }
9923   case ISD::FFREXP: {
9924     assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
9925     assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
9926            VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
9927 
9928     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Ops[0])) {
9929       int FrexpExp;
9930       APFloat FrexpMant =
9931           frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);
9932       SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);
9933       SDValue Result1 =
9934           getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]);
9935       return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);
9936     }
9937 
9938     break;
9939   }
9940   case ISD::STRICT_FP_EXTEND:
9941     assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9942            "Invalid STRICT_FP_EXTEND!");
9943     assert(VTList.VTs[0].isFloatingPoint() &&
9944            Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
9945     assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9946            "STRICT_FP_EXTEND result type should be vector iff the operand "
9947            "type is vector!");
9948     assert((!VTList.VTs[0].isVector() ||
9949             VTList.VTs[0].getVectorElementCount() ==
9950                 Ops[1].getValueType().getVectorElementCount()) &&
9951            "Vector element count mismatch!");
9952     assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
9953            "Invalid fpext node, dst <= src!");
9954     break;
9955   case ISD::STRICT_FP_ROUND:
9956     assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
9957     assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9958            "STRICT_FP_ROUND result type should be vector iff the operand "
9959            "type is vector!");
9960     assert((!VTList.VTs[0].isVector() ||
9961             VTList.VTs[0].getVectorElementCount() ==
9962                 Ops[1].getValueType().getVectorElementCount()) &&
9963            "Vector element count mismatch!");
9964     assert(VTList.VTs[0].isFloatingPoint() &&
9965            Ops[1].getValueType().isFloatingPoint() &&
9966            VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
9967            isa<ConstantSDNode>(Ops[2]) &&
9968            (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
9969            "Invalid STRICT_FP_ROUND!");
9970     break;
9971 #if 0
9972   // FIXME: figure out how to safely handle things like
9973   // int foo(int x) { return 1 << (x & 255); }
9974   // int bar() { return foo(256); }
9975   case ISD::SRA_PARTS:
9976   case ISD::SRL_PARTS:
9977   case ISD::SHL_PARTS:
9978     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
9979         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
9980       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9981     else if (N3.getOpcode() == ISD::AND)
9982       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
9983         // If the and is only masking out bits that cannot effect the shift,
9984         // eliminate the and.
9985         unsigned NumBits = VT.getScalarSizeInBits()*2;
9986         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
9987           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9988       }
9989     break;
9990 #endif
9991   }
9992 
9993   // Memoize the node unless it returns a glue result.
9994   SDNode *N;
9995   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9996     FoldingSetNodeID ID;
9997     AddNodeIDNode(ID, Opcode, VTList, Ops);
9998     void *IP = nullptr;
9999     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
10000       return SDValue(E, 0);
10001 
10002     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10003     createOperands(N, Ops);
10004     CSEMap.InsertNode(N, IP);
10005   } else {
10006     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10007     createOperands(N, Ops);
10008   }
10009 
10010   N->setFlags(Flags);
10011   InsertNode(N);
10012   SDValue V(N, 0);
10013   NewSDValueDbgMsg(V, "Creating new node: ", this);
10014   return V;
10015 }
10016 
10017 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10018                               SDVTList VTList) {
10019   return getNode(Opcode, DL, VTList, std::nullopt);
10020 }
10021 
10022 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10023                               SDValue N1) {
10024   SDValue Ops[] = { N1 };
10025   return getNode(Opcode, DL, VTList, Ops);
10026 }
10027 
10028 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10029                               SDValue N1, SDValue N2) {
10030   SDValue Ops[] = { N1, N2 };
10031   return getNode(Opcode, DL, VTList, Ops);
10032 }
10033 
10034 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10035                               SDValue N1, SDValue N2, SDValue N3) {
10036   SDValue Ops[] = { N1, N2, N3 };
10037   return getNode(Opcode, DL, VTList, Ops);
10038 }
10039 
10040 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10041                               SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10042   SDValue Ops[] = { N1, N2, N3, N4 };
10043   return getNode(Opcode, DL, VTList, Ops);
10044 }
10045 
10046 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10047                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10048                               SDValue N5) {
10049   SDValue Ops[] = { N1, N2, N3, N4, N5 };
10050   return getNode(Opcode, DL, VTList, Ops);
10051 }
10052 
10053 SDVTList SelectionDAG::getVTList(EVT VT) {
10054   return makeVTList(SDNode::getValueTypeList(VT), 1);
10055 }
10056 
10057 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
10058   FoldingSetNodeID ID;
10059   ID.AddInteger(2U);
10060   ID.AddInteger(VT1.getRawBits());
10061   ID.AddInteger(VT2.getRawBits());
10062 
10063   void *IP = nullptr;
10064   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10065   if (!Result) {
10066     EVT *Array = Allocator.Allocate<EVT>(2);
10067     Array[0] = VT1;
10068     Array[1] = VT2;
10069     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10070     VTListMap.InsertNode(Result, IP);
10071   }
10072   return Result->getSDVTList();
10073 }
10074 
10075 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
10076   FoldingSetNodeID ID;
10077   ID.AddInteger(3U);
10078   ID.AddInteger(VT1.getRawBits());
10079   ID.AddInteger(VT2.getRawBits());
10080   ID.AddInteger(VT3.getRawBits());
10081 
10082   void *IP = nullptr;
10083   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10084   if (!Result) {
10085     EVT *Array = Allocator.Allocate<EVT>(3);
10086     Array[0] = VT1;
10087     Array[1] = VT2;
10088     Array[2] = VT3;
10089     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10090     VTListMap.InsertNode(Result, IP);
10091   }
10092   return Result->getSDVTList();
10093 }
10094 
10095 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
10096   FoldingSetNodeID ID;
10097   ID.AddInteger(4U);
10098   ID.AddInteger(VT1.getRawBits());
10099   ID.AddInteger(VT2.getRawBits());
10100   ID.AddInteger(VT3.getRawBits());
10101   ID.AddInteger(VT4.getRawBits());
10102 
10103   void *IP = nullptr;
10104   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10105   if (!Result) {
10106     EVT *Array = Allocator.Allocate<EVT>(4);
10107     Array[0] = VT1;
10108     Array[1] = VT2;
10109     Array[2] = VT3;
10110     Array[3] = VT4;
10111     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10112     VTListMap.InsertNode(Result, IP);
10113   }
10114   return Result->getSDVTList();
10115 }
10116 
10117 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
10118   unsigned NumVTs = VTs.size();
10119   FoldingSetNodeID ID;
10120   ID.AddInteger(NumVTs);
10121   for (unsigned index = 0; index < NumVTs; index++) {
10122     ID.AddInteger(VTs[index].getRawBits());
10123   }
10124 
10125   void *IP = nullptr;
10126   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10127   if (!Result) {
10128     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
10129     llvm::copy(VTs, Array);
10130     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10131     VTListMap.InsertNode(Result, IP);
10132   }
10133   return Result->getSDVTList();
10134 }
10135 
10136 
10137 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10138 /// specified operands.  If the resultant node already exists in the DAG,
10139 /// this does not modify the specified node, instead it returns the node that
10140 /// already exists.  If the resultant node does not exist in the DAG, the
10141 /// input node is returned.  As a degenerate case, if you specify the same
10142 /// input operands as the node already has, the input node is returned.
10143 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
10144   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10145 
10146   // Check to see if there is no change.
10147   if (Op == N->getOperand(0)) return N;
10148 
10149   // See if the modified node already exists.
10150   void *InsertPos = nullptr;
10151   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10152     return Existing;
10153 
10154   // Nope it doesn't.  Remove the node from its current place in the maps.
10155   if (InsertPos)
10156     if (!RemoveNodeFromCSEMaps(N))
10157       InsertPos = nullptr;
10158 
10159   // Now we update the operands.
10160   N->OperandList[0].set(Op);
10161 
10162   updateDivergence(N);
10163   // If this gets put into a CSE map, add it.
10164   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10165   return N;
10166 }
10167 
10168 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
10169   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10170 
10171   // Check to see if there is no change.
10172   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
10173     return N;   // No operands changed, just return the input node.
10174 
10175   // See if the modified node already exists.
10176   void *InsertPos = nullptr;
10177   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10178     return Existing;
10179 
10180   // Nope it doesn't.  Remove the node from its current place in the maps.
10181   if (InsertPos)
10182     if (!RemoveNodeFromCSEMaps(N))
10183       InsertPos = nullptr;
10184 
10185   // Now we update the operands.
10186   if (N->OperandList[0] != Op1)
10187     N->OperandList[0].set(Op1);
10188   if (N->OperandList[1] != Op2)
10189     N->OperandList[1].set(Op2);
10190 
10191   updateDivergence(N);
10192   // If this gets put into a CSE map, add it.
10193   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10194   return N;
10195 }
10196 
10197 SDNode *SelectionDAG::
10198 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
10199   SDValue Ops[] = { Op1, Op2, Op3 };
10200   return UpdateNodeOperands(N, Ops);
10201 }
10202 
10203 SDNode *SelectionDAG::
10204 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10205                    SDValue Op3, SDValue Op4) {
10206   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10207   return UpdateNodeOperands(N, Ops);
10208 }
10209 
10210 SDNode *SelectionDAG::
10211 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
10212                    SDValue Op3, SDValue Op4, SDValue Op5) {
10213   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10214   return UpdateNodeOperands(N, Ops);
10215 }
10216 
10217 SDNode *SelectionDAG::
10218 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
10219   unsigned NumOps = Ops.size();
10220   assert(N->getNumOperands() == NumOps &&
10221          "Update with wrong number of operands");
10222 
10223   // If no operands changed just return the input node.
10224   if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
10225     return N;
10226 
10227   // See if the modified node already exists.
10228   void *InsertPos = nullptr;
10229   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10230     return Existing;
10231 
10232   // Nope it doesn't.  Remove the node from its current place in the maps.
10233   if (InsertPos)
10234     if (!RemoveNodeFromCSEMaps(N))
10235       InsertPos = nullptr;
10236 
10237   // Now we update the operands.
10238   for (unsigned i = 0; i != NumOps; ++i)
10239     if (N->OperandList[i] != Ops[i])
10240       N->OperandList[i].set(Ops[i]);
10241 
10242   updateDivergence(N);
10243   // If this gets put into a CSE map, add it.
10244   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10245   return N;
10246 }
10247 
10248 /// DropOperands - Release the operands and set this node to have
10249 /// zero operands.
10250 void SDNode::DropOperands() {
10251   // Unlike the code in MorphNodeTo that does this, we don't need to
10252   // watch for dead nodes here.
10253   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10254     SDUse &Use = *I++;
10255     Use.set(SDValue());
10256   }
10257 }
10258 
10259 void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
10260                                   ArrayRef<MachineMemOperand *> NewMemRefs) {
10261   if (NewMemRefs.empty()) {
10262     N->clearMemRefs();
10263     return;
10264   }
10265 
10266   // Check if we can avoid allocating by storing a single reference directly.
10267   if (NewMemRefs.size() == 1) {
10268     N->MemRefs = NewMemRefs[0];
10269     N->NumMemRefs = 1;
10270     return;
10271   }
10272 
10273   MachineMemOperand **MemRefsBuffer =
10274       Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
10275   llvm::copy(NewMemRefs, MemRefsBuffer);
10276   N->MemRefs = MemRefsBuffer;
10277   N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10278 }
10279 
10280 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10281 /// machine opcode.
10282 ///
10283 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10284                                    EVT VT) {
10285   SDVTList VTs = getVTList(VT);
10286   return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10287 }
10288 
10289 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10290                                    EVT VT, SDValue Op1) {
10291   SDVTList VTs = getVTList(VT);
10292   SDValue Ops[] = { Op1 };
10293   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10294 }
10295 
10296 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10297                                    EVT VT, SDValue Op1,
10298                                    SDValue Op2) {
10299   SDVTList VTs = getVTList(VT);
10300   SDValue Ops[] = { Op1, Op2 };
10301   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10302 }
10303 
10304 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10305                                    EVT VT, SDValue Op1,
10306                                    SDValue Op2, SDValue Op3) {
10307   SDVTList VTs = getVTList(VT);
10308   SDValue Ops[] = { Op1, Op2, Op3 };
10309   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10310 }
10311 
10312 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10313                                    EVT VT, ArrayRef<SDValue> Ops) {
10314   SDVTList VTs = getVTList(VT);
10315   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10316 }
10317 
10318 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10319                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10320   SDVTList VTs = getVTList(VT1, VT2);
10321   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10322 }
10323 
10324 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10325                                    EVT VT1, EVT VT2) {
10326   SDVTList VTs = getVTList(VT1, VT2);
10327   return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10328 }
10329 
10330 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10331                                    EVT VT1, EVT VT2, EVT VT3,
10332                                    ArrayRef<SDValue> Ops) {
10333   SDVTList VTs = getVTList(VT1, VT2, VT3);
10334   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10335 }
10336 
10337 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10338                                    EVT VT1, EVT VT2,
10339                                    SDValue Op1, SDValue Op2) {
10340   SDVTList VTs = getVTList(VT1, VT2);
10341   SDValue Ops[] = { Op1, Op2 };
10342   return SelectNodeTo(N, MachineOpc, VTs, Ops);
10343 }
10344 
10345 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
10346                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
10347   SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
10348   // Reset the NodeID to -1.
10349   New->setNodeId(-1);
10350   if (New != N) {
10351     ReplaceAllUsesWith(N, New);
10352     RemoveDeadNode(N);
10353   }
10354   return New;
10355 }
10356 
10357 /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
10358 /// the line number information on the merged node since it is not possible to
10359 /// preserve the information that operation is associated with multiple lines.
10360 /// This will make the debugger working better at -O0, were there is a higher
10361 /// probability having other instructions associated with that line.
10362 ///
10363 /// For IROrder, we keep the smaller of the two
10364 SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
10365   DebugLoc NLoc = N->getDebugLoc();
10366   if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
10367     N->setDebugLoc(DebugLoc());
10368   }
10369   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
10370   N->setIROrder(Order);
10371   return N;
10372 }
10373 
10374 /// MorphNodeTo - This *mutates* the specified node to have the specified
10375 /// return type, opcode, and operands.
10376 ///
10377 /// Note that MorphNodeTo returns the resultant node.  If there is already a
10378 /// node of the specified opcode and operands, it returns that node instead of
10379 /// the current one.  Note that the SDLoc need not be the same.
10380 ///
10381 /// Using MorphNodeTo is faster than creating a new node and swapping it in
10382 /// with ReplaceAllUsesWith both because it often avoids allocating a new
10383 /// node, and because it doesn't require CSE recalculation for any of
10384 /// the node's users.
10385 ///
10386 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
10387 /// As a consequence it isn't appropriate to use from within the DAG combiner or
10388 /// the legalizer which maintain worklists that would need to be updated when
10389 /// deleting things.
10390 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
10391                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
10392   // If an identical node already exists, use it.
10393   void *IP = nullptr;
10394   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
10395     FoldingSetNodeID ID;
10396     AddNodeIDNode(ID, Opc, VTs, Ops);
10397     if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
10398       return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
10399   }
10400 
10401   if (!RemoveNodeFromCSEMaps(N))
10402     IP = nullptr;
10403 
10404   // Start the morphing.
10405   N->NodeType = Opc;
10406   N->ValueList = VTs.VTs;
10407   N->NumValues = VTs.NumVTs;
10408 
10409   // Clear the operands list, updating used nodes to remove this from their
10410   // use list.  Keep track of any operands that become dead as a result.
10411   SmallPtrSet<SDNode*, 16> DeadNodeSet;
10412   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
10413     SDUse &Use = *I++;
10414     SDNode *Used = Use.getNode();
10415     Use.set(SDValue());
10416     if (Used->use_empty())
10417       DeadNodeSet.insert(Used);
10418   }
10419 
10420   // For MachineNode, initialize the memory references information.
10421   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
10422     MN->clearMemRefs();
10423 
10424   // Swap for an appropriately sized array from the recycler.
10425   removeOperands(N);
10426   createOperands(N, Ops);
10427 
10428   // Delete any nodes that are still dead after adding the uses for the
10429   // new operands.
10430   if (!DeadNodeSet.empty()) {
10431     SmallVector<SDNode *, 16> DeadNodes;
10432     for (SDNode *N : DeadNodeSet)
10433       if (N->use_empty())
10434         DeadNodes.push_back(N);
10435     RemoveDeadNodes(DeadNodes);
10436   }
10437 
10438   if (IP)
10439     CSEMap.InsertNode(N, IP);   // Memoize the new node.
10440   return N;
10441 }
10442 
10443 SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
10444   unsigned OrigOpc = Node->getOpcode();
10445   unsigned NewOpc;
10446   switch (OrigOpc) {
10447   default:
10448     llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
10449 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
10450   case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10451 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
10452   case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10453 #include "llvm/IR/ConstrainedOps.def"
10454   }
10455 
10456   assert(Node->getNumValues() == 2 && "Unexpected number of results!");
10457 
10458   // We're taking this node out of the chain, so we need to re-link things.
10459   SDValue InputChain = Node->getOperand(0);
10460   SDValue OutputChain = SDValue(Node, 1);
10461   ReplaceAllUsesOfValueWith(OutputChain, InputChain);
10462 
10463   SmallVector<SDValue, 3> Ops;
10464   for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10465     Ops.push_back(Node->getOperand(i));
10466 
10467   SDVTList VTs = getVTList(Node->getValueType(0));
10468   SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
10469 
10470   // MorphNodeTo can operate in two ways: if an existing node with the
10471   // specified operands exists, it can just return it.  Otherwise, it
10472   // updates the node in place to have the requested operands.
10473   if (Res == Node) {
10474     // If we updated the node in place, reset the node ID.  To the isel,
10475     // this should be just like a newly allocated machine node.
10476     Res->setNodeId(-1);
10477   } else {
10478     ReplaceAllUsesWith(Node, Res);
10479     RemoveDeadNode(Node);
10480   }
10481 
10482   return Res;
10483 }
10484 
10485 /// getMachineNode - These are used for target selectors to create a new node
10486 /// with specified return type(s), MachineInstr opcode, and operands.
10487 ///
10488 /// Note that getMachineNode returns the resultant node.  If there is already a
10489 /// node of the specified opcode and operands, it returns that node instead of
10490 /// the current one.
10491 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10492                                             EVT VT) {
10493   SDVTList VTs = getVTList(VT);
10494   return getMachineNode(Opcode, dl, VTs, std::nullopt);
10495 }
10496 
10497 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10498                                             EVT VT, SDValue Op1) {
10499   SDVTList VTs = getVTList(VT);
10500   SDValue Ops[] = { Op1 };
10501   return getMachineNode(Opcode, dl, VTs, Ops);
10502 }
10503 
10504 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10505                                             EVT VT, SDValue Op1, SDValue Op2) {
10506   SDVTList VTs = getVTList(VT);
10507   SDValue Ops[] = { Op1, Op2 };
10508   return getMachineNode(Opcode, dl, VTs, Ops);
10509 }
10510 
10511 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10512                                             EVT VT, SDValue Op1, SDValue Op2,
10513                                             SDValue Op3) {
10514   SDVTList VTs = getVTList(VT);
10515   SDValue Ops[] = { Op1, Op2, Op3 };
10516   return getMachineNode(Opcode, dl, VTs, Ops);
10517 }
10518 
10519 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10520                                             EVT VT, ArrayRef<SDValue> Ops) {
10521   SDVTList VTs = getVTList(VT);
10522   return getMachineNode(Opcode, dl, VTs, Ops);
10523 }
10524 
10525 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10526                                             EVT VT1, EVT VT2, SDValue Op1,
10527                                             SDValue Op2) {
10528   SDVTList VTs = getVTList(VT1, VT2);
10529   SDValue Ops[] = { Op1, Op2 };
10530   return getMachineNode(Opcode, dl, VTs, Ops);
10531 }
10532 
10533 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10534                                             EVT VT1, EVT VT2, SDValue Op1,
10535                                             SDValue Op2, SDValue Op3) {
10536   SDVTList VTs = getVTList(VT1, VT2);
10537   SDValue Ops[] = { Op1, Op2, Op3 };
10538   return getMachineNode(Opcode, dl, VTs, Ops);
10539 }
10540 
10541 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10542                                             EVT VT1, EVT VT2,
10543                                             ArrayRef<SDValue> Ops) {
10544   SDVTList VTs = getVTList(VT1, VT2);
10545   return getMachineNode(Opcode, dl, VTs, Ops);
10546 }
10547 
10548 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10549                                             EVT VT1, EVT VT2, EVT VT3,
10550                                             SDValue Op1, SDValue Op2) {
10551   SDVTList VTs = getVTList(VT1, VT2, VT3);
10552   SDValue Ops[] = { Op1, Op2 };
10553   return getMachineNode(Opcode, dl, VTs, Ops);
10554 }
10555 
10556 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10557                                             EVT VT1, EVT VT2, EVT VT3,
10558                                             SDValue Op1, SDValue Op2,
10559                                             SDValue Op3) {
10560   SDVTList VTs = getVTList(VT1, VT2, VT3);
10561   SDValue Ops[] = { Op1, Op2, Op3 };
10562   return getMachineNode(Opcode, dl, VTs, Ops);
10563 }
10564 
10565 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10566                                             EVT VT1, EVT VT2, EVT VT3,
10567                                             ArrayRef<SDValue> Ops) {
10568   SDVTList VTs = getVTList(VT1, VT2, VT3);
10569   return getMachineNode(Opcode, dl, VTs, Ops);
10570 }
10571 
10572 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
10573                                             ArrayRef<EVT> ResultTys,
10574                                             ArrayRef<SDValue> Ops) {
10575   SDVTList VTs = getVTList(ResultTys);
10576   return getMachineNode(Opcode, dl, VTs, Ops);
10577 }
10578 
10579 MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
10580                                             SDVTList VTs,
10581                                             ArrayRef<SDValue> Ops) {
10582   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
10583   MachineSDNode *N;
10584   void *IP = nullptr;
10585 
10586   if (DoCSE) {
10587     FoldingSetNodeID ID;
10588     AddNodeIDNode(ID, ~Opcode, VTs, Ops);
10589     IP = nullptr;
10590     if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
10591       return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
10592     }
10593   }
10594 
10595   // Allocate a new MachineSDNode.
10596   N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
10597   createOperands(N, Ops);
10598 
10599   if (DoCSE)
10600     CSEMap.InsertNode(N, IP);
10601 
10602   InsertNode(N);
10603   NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
10604   return N;
10605 }
10606 
10607 /// getTargetExtractSubreg - A convenience function for creating
10608 /// TargetOpcode::EXTRACT_SUBREG nodes.
10609 SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10610                                              SDValue Operand) {
10611   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10612   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
10613                                   VT, Operand, SRIdxVal);
10614   return SDValue(Subreg, 0);
10615 }
10616 
10617 /// getTargetInsertSubreg - A convenience function for creating
10618 /// TargetOpcode::INSERT_SUBREG nodes.
10619 SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
10620                                             SDValue Operand, SDValue Subreg) {
10621   SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10622   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
10623                                   VT, Operand, Subreg, SRIdxVal);
10624   return SDValue(Result, 0);
10625 }
10626 
10627 /// getNodeIfExists - Get the specified node if it's already available, or
10628 /// else return NULL.
10629 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10630                                       ArrayRef<SDValue> Ops) {
10631   SDNodeFlags Flags;
10632   if (Inserter)
10633     Flags = Inserter->getFlags();
10634   return getNodeIfExists(Opcode, VTList, Ops, Flags);
10635 }
10636 
10637 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
10638                                       ArrayRef<SDValue> Ops,
10639                                       const SDNodeFlags Flags) {
10640   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10641     FoldingSetNodeID ID;
10642     AddNodeIDNode(ID, Opcode, VTList, Ops);
10643     void *IP = nullptr;
10644     if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
10645       E->intersectFlagsWith(Flags);
10646       return E;
10647     }
10648   }
10649   return nullptr;
10650 }
10651 
10652 /// doesNodeExist - Check if a node exists without modifying its flags.
10653 bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
10654                                  ArrayRef<SDValue> Ops) {
10655   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10656     FoldingSetNodeID ID;
10657     AddNodeIDNode(ID, Opcode, VTList, Ops);
10658     void *IP = nullptr;
10659     if (FindNodeOrInsertPos(ID, SDLoc(), IP))
10660       return true;
10661   }
10662   return false;
10663 }
10664 
10665 /// getDbgValue - Creates a SDDbgValue node.
10666 ///
10667 /// SDNode
10668 SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
10669                                       SDNode *N, unsigned R, bool IsIndirect,
10670                                       const DebugLoc &DL, unsigned O) {
10671   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10672          "Expected inlined-at fields to agree");
10673   return new (DbgInfo->getAlloc())
10674       SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
10675                  {}, IsIndirect, DL, O,
10676                  /*IsVariadic=*/false);
10677 }
10678 
10679 /// Constant
10680 SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
10681                                               DIExpression *Expr,
10682                                               const Value *C,
10683                                               const DebugLoc &DL, unsigned O) {
10684   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10685          "Expected inlined-at fields to agree");
10686   return new (DbgInfo->getAlloc())
10687       SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
10688                  /*IsIndirect=*/false, DL, O,
10689                  /*IsVariadic=*/false);
10690 }
10691 
10692 /// FrameIndex
10693 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10694                                                 DIExpression *Expr, unsigned FI,
10695                                                 bool IsIndirect,
10696                                                 const DebugLoc &DL,
10697                                                 unsigned O) {
10698   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10699          "Expected inlined-at fields to agree");
10700   return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
10701 }
10702 
10703 /// FrameIndex with dependencies
10704 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
10705                                                 DIExpression *Expr, unsigned FI,
10706                                                 ArrayRef<SDNode *> Dependencies,
10707                                                 bool IsIndirect,
10708                                                 const DebugLoc &DL,
10709                                                 unsigned O) {
10710   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10711          "Expected inlined-at fields to agree");
10712   return new (DbgInfo->getAlloc())
10713       SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
10714                  Dependencies, IsIndirect, DL, O,
10715                  /*IsVariadic=*/false);
10716 }
10717 
10718 /// VReg
10719 SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
10720                                           unsigned VReg, bool IsIndirect,
10721                                           const DebugLoc &DL, unsigned O) {
10722   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10723          "Expected inlined-at fields to agree");
10724   return new (DbgInfo->getAlloc())
10725       SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
10726                  {}, IsIndirect, DL, O,
10727                  /*IsVariadic=*/false);
10728 }
10729 
10730 SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
10731                                           ArrayRef<SDDbgOperand> Locs,
10732                                           ArrayRef<SDNode *> Dependencies,
10733                                           bool IsIndirect, const DebugLoc &DL,
10734                                           unsigned O, bool IsVariadic) {
10735   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10736          "Expected inlined-at fields to agree");
10737   return new (DbgInfo->getAlloc())
10738       SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
10739                  DL, O, IsVariadic);
10740 }
10741 
10742 void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
10743                                      unsigned OffsetInBits, unsigned SizeInBits,
10744                                      bool InvalidateDbg) {
10745   SDNode *FromNode = From.getNode();
10746   SDNode *ToNode = To.getNode();
10747   assert(FromNode && ToNode && "Can't modify dbg values");
10748 
10749   // PR35338
10750   // TODO: assert(From != To && "Redundant dbg value transfer");
10751   // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
10752   if (From == To || FromNode == ToNode)
10753     return;
10754 
10755   if (!FromNode->getHasDebugValue())
10756     return;
10757 
10758   SDDbgOperand FromLocOp =
10759       SDDbgOperand::fromNode(From.getNode(), From.getResNo());
10760   SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo());
10761 
10762   SmallVector<SDDbgValue *, 2> ClonedDVs;
10763   for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
10764     if (Dbg->isInvalidated())
10765       continue;
10766 
10767     // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
10768 
10769     // Create a new location ops vector that is equal to the old vector, but
10770     // with each instance of FromLocOp replaced with ToLocOp.
10771     bool Changed = false;
10772     auto NewLocOps = Dbg->copyLocationOps();
10773     std::replace_if(
10774         NewLocOps.begin(), NewLocOps.end(),
10775         [&Changed, FromLocOp](const SDDbgOperand &Op) {
10776           bool Match = Op == FromLocOp;
10777           Changed |= Match;
10778           return Match;
10779         },
10780         ToLocOp);
10781     // Ignore this SDDbgValue if we didn't find a matching location.
10782     if (!Changed)
10783       continue;
10784 
10785     DIVariable *Var = Dbg->getVariable();
10786     auto *Expr = Dbg->getExpression();
10787     // If a fragment is requested, update the expression.
10788     if (SizeInBits) {
10789       // When splitting a larger (e.g., sign-extended) value whose
10790       // lower bits are described with an SDDbgValue, do not attempt
10791       // to transfer the SDDbgValue to the upper bits.
10792       if (auto FI = Expr->getFragmentInfo())
10793         if (OffsetInBits + SizeInBits > FI->SizeInBits)
10794           continue;
10795       auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
10796                                                              SizeInBits);
10797       if (!Fragment)
10798         continue;
10799       Expr = *Fragment;
10800     }
10801 
10802     auto AdditionalDependencies = Dbg->getAdditionalDependencies();
10803     // Clone the SDDbgValue and move it to To.
10804     SDDbgValue *Clone = getDbgValueList(
10805         Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
10806         Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
10807         Dbg->isVariadic());
10808     ClonedDVs.push_back(Clone);
10809 
10810     if (InvalidateDbg) {
10811       // Invalidate value and indicate the SDDbgValue should not be emitted.
10812       Dbg->setIsInvalidated();
10813       Dbg->setIsEmitted();
10814     }
10815   }
10816 
10817   for (SDDbgValue *Dbg : ClonedDVs) {
10818     assert(is_contained(Dbg->getSDNodes(), ToNode) &&
10819            "Transferred DbgValues should depend on the new SDNode");
10820     AddDbgValue(Dbg, false);
10821   }
10822 }
10823 
10824 void SelectionDAG::salvageDebugInfo(SDNode &N) {
10825   if (!N.getHasDebugValue())
10826     return;
10827 
10828   SmallVector<SDDbgValue *, 2> ClonedDVs;
10829   for (auto *DV : GetDbgValues(&N)) {
10830     if (DV->isInvalidated())
10831       continue;
10832     switch (N.getOpcode()) {
10833     default:
10834       break;
10835     case ISD::ADD: {
10836       SDValue N0 = N.getOperand(0);
10837       SDValue N1 = N.getOperand(1);
10838       if (!isa<ConstantSDNode>(N0)) {
10839         bool RHSConstant = isa<ConstantSDNode>(N1);
10840         uint64_t Offset;
10841         if (RHSConstant)
10842           Offset = N.getConstantOperandVal(1);
10843         // We are not allowed to turn indirect debug values variadic, so
10844         // don't salvage those.
10845         if (!RHSConstant && DV->isIndirect())
10846           continue;
10847 
10848         // Rewrite an ADD constant node into a DIExpression. Since we are
10849         // performing arithmetic to compute the variable's *value* in the
10850         // DIExpression, we need to mark the expression with a
10851         // DW_OP_stack_value.
10852         auto *DIExpr = DV->getExpression();
10853         auto NewLocOps = DV->copyLocationOps();
10854         bool Changed = false;
10855         size_t OrigLocOpsSize = NewLocOps.size();
10856         for (size_t i = 0; i < OrigLocOpsSize; ++i) {
10857           // We're not given a ResNo to compare against because the whole
10858           // node is going away. We know that any ISD::ADD only has one
10859           // result, so we can assume any node match is using the result.
10860           if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10861               NewLocOps[i].getSDNode() != &N)
10862             continue;
10863           NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10864           if (RHSConstant) {
10865             SmallVector<uint64_t, 3> ExprOps;
10866             DIExpression::appendOffset(ExprOps, Offset);
10867             DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
10868           } else {
10869             // Convert to a variadic expression (if not already).
10870             // convertToVariadicExpression() returns a const pointer, so we use
10871             // a temporary const variable here.
10872             const auto *TmpDIExpr =
10873                 DIExpression::convertToVariadicExpression(DIExpr);
10874             SmallVector<uint64_t, 3> ExprOps;
10875             ExprOps.push_back(dwarf::DW_OP_LLVM_arg);
10876             ExprOps.push_back(NewLocOps.size());
10877             ExprOps.push_back(dwarf::DW_OP_plus);
10878             SDDbgOperand RHS =
10879                 SDDbgOperand::fromNode(N1.getNode(), N1.getResNo());
10880             NewLocOps.push_back(RHS);
10881             DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);
10882           }
10883           Changed = true;
10884         }
10885         (void)Changed;
10886         assert(Changed && "Salvage target doesn't use N");
10887 
10888         bool IsVariadic =
10889             DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
10890 
10891         auto AdditionalDependencies = DV->getAdditionalDependencies();
10892         SDDbgValue *Clone = getDbgValueList(
10893             DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
10894             DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
10895         ClonedDVs.push_back(Clone);
10896         DV->setIsInvalidated();
10897         DV->setIsEmitted();
10898         LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
10899                    N0.getNode()->dumprFull(this);
10900                    dbgs() << " into " << *DIExpr << '\n');
10901       }
10902       break;
10903     }
10904     case ISD::TRUNCATE: {
10905       SDValue N0 = N.getOperand(0);
10906       TypeSize FromSize = N0.getValueSizeInBits();
10907       TypeSize ToSize = N.getValueSizeInBits(0);
10908 
10909       DIExpression *DbgExpression = DV->getExpression();
10910       auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);
10911       auto NewLocOps = DV->copyLocationOps();
10912       bool Changed = false;
10913       for (size_t i = 0; i < NewLocOps.size(); ++i) {
10914         if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10915             NewLocOps[i].getSDNode() != &N)
10916           continue;
10917 
10918         NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10919         DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);
10920         Changed = true;
10921       }
10922       assert(Changed && "Salvage target doesn't use N");
10923       (void)Changed;
10924 
10925       SDDbgValue *Clone =
10926           getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,
10927                           DV->getAdditionalDependencies(), DV->isIndirect(),
10928                           DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
10929 
10930       ClonedDVs.push_back(Clone);
10931       DV->setIsInvalidated();
10932       DV->setIsEmitted();
10933       LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
10934                  dbgs() << " into " << *DbgExpression << '\n');
10935       break;
10936     }
10937     }
10938   }
10939 
10940   for (SDDbgValue *Dbg : ClonedDVs) {
10941     assert(!Dbg->getSDNodes().empty() &&
10942            "Salvaged DbgValue should depend on a new SDNode");
10943     AddDbgValue(Dbg, false);
10944   }
10945 }
10946 
10947 /// Creates a SDDbgLabel node.
10948 SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
10949                                       const DebugLoc &DL, unsigned O) {
10950   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
10951          "Expected inlined-at fields to agree");
10952   return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
10953 }
10954 
10955 namespace {
10956 
10957 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
10958 /// pointed to by a use iterator is deleted, increment the use iterator
10959 /// so that it doesn't dangle.
10960 ///
10961 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
10962   SDNode::use_iterator &UI;
10963   SDNode::use_iterator &UE;
10964 
10965   void NodeDeleted(SDNode *N, SDNode *E) override {
10966     // Increment the iterator as needed.
10967     while (UI != UE && N == *UI)
10968       ++UI;
10969   }
10970 
10971 public:
10972   RAUWUpdateListener(SelectionDAG &d,
10973                      SDNode::use_iterator &ui,
10974                      SDNode::use_iterator &ue)
10975     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
10976 };
10977 
10978 } // end anonymous namespace
10979 
10980 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
10981 /// This can cause recursive merging of nodes in the DAG.
10982 ///
10983 /// This version assumes From has a single result value.
10984 ///
10985 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
10986   SDNode *From = FromN.getNode();
10987   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
10988          "Cannot replace with this method!");
10989   assert(From != To.getNode() && "Cannot replace uses of with self");
10990 
10991   // Preserve Debug Values
10992   transferDbgValues(FromN, To);
10993   // Preserve extra info.
10994   copyExtraInfo(From, To.getNode());
10995 
10996   // Iterate over all the existing uses of From. New uses will be added
10997   // to the beginning of the use list, which we avoid visiting.
10998   // This specifically avoids visiting uses of From that arise while the
10999   // replacement is happening, because any such uses would be the result
11000   // of CSE: If an existing node looks like From after one of its operands
11001   // is replaced by To, we don't want to replace of all its users with To
11002   // too. See PR3018 for more info.
11003   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11004   RAUWUpdateListener Listener(*this, UI, UE);
11005   while (UI != UE) {
11006     SDNode *User = *UI;
11007 
11008     // This node is about to morph, remove its old self from the CSE maps.
11009     RemoveNodeFromCSEMaps(User);
11010 
11011     // A user can appear in a use list multiple times, and when this
11012     // happens the uses are usually next to each other in the list.
11013     // To help reduce the number of CSE recomputations, process all
11014     // the uses of this user that we can find this way.
11015     do {
11016       SDUse &Use = UI.getUse();
11017       ++UI;
11018       Use.set(To);
11019       if (To->isDivergent() != From->isDivergent())
11020         updateDivergence(User);
11021     } while (UI != UE && *UI == User);
11022     // Now that we have modified User, add it back to the CSE maps.  If it
11023     // already exists there, recursively merge the results together.
11024     AddModifiedNodeToCSEMaps(User);
11025   }
11026 
11027   // If we just RAUW'd the root, take note.
11028   if (FromN == getRoot())
11029     setRoot(To);
11030 }
11031 
11032 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11033 /// This can cause recursive merging of nodes in the DAG.
11034 ///
11035 /// This version assumes that for each value of From, there is a
11036 /// corresponding value in To in the same position with the same type.
11037 ///
11038 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
11039 #ifndef NDEBUG
11040   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11041     assert((!From->hasAnyUseOfValue(i) ||
11042             From->getValueType(i) == To->getValueType(i)) &&
11043            "Cannot use this version of ReplaceAllUsesWith!");
11044 #endif
11045 
11046   // Handle the trivial case.
11047   if (From == To)
11048     return;
11049 
11050   // Preserve Debug Info. Only do this if there's a use.
11051   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11052     if (From->hasAnyUseOfValue(i)) {
11053       assert((i < To->getNumValues()) && "Invalid To location");
11054       transferDbgValues(SDValue(From, i), SDValue(To, i));
11055     }
11056   // Preserve extra info.
11057   copyExtraInfo(From, To);
11058 
11059   // Iterate over just the existing users of From. See the comments in
11060   // the ReplaceAllUsesWith above.
11061   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11062   RAUWUpdateListener Listener(*this, UI, UE);
11063   while (UI != UE) {
11064     SDNode *User = *UI;
11065 
11066     // This node is about to morph, remove its old self from the CSE maps.
11067     RemoveNodeFromCSEMaps(User);
11068 
11069     // A user can appear in a use list multiple times, and when this
11070     // happens the uses are usually next to each other in the list.
11071     // To help reduce the number of CSE recomputations, process all
11072     // the uses of this user that we can find this way.
11073     do {
11074       SDUse &Use = UI.getUse();
11075       ++UI;
11076       Use.setNode(To);
11077       if (To->isDivergent() != From->isDivergent())
11078         updateDivergence(User);
11079     } while (UI != UE && *UI == User);
11080 
11081     // Now that we have modified User, add it back to the CSE maps.  If it
11082     // already exists there, recursively merge the results together.
11083     AddModifiedNodeToCSEMaps(User);
11084   }
11085 
11086   // If we just RAUW'd the root, take note.
11087   if (From == getRoot().getNode())
11088     setRoot(SDValue(To, getRoot().getResNo()));
11089 }
11090 
11091 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11092 /// This can cause recursive merging of nodes in the DAG.
11093 ///
11094 /// This version can replace From with any result values.  To must match the
11095 /// number and types of values returned by From.
11096 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
11097   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
11098     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
11099 
11100   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11101     // Preserve Debug Info.
11102     transferDbgValues(SDValue(From, i), To[i]);
11103     // Preserve extra info.
11104     copyExtraInfo(From, To[i].getNode());
11105   }
11106 
11107   // Iterate over just the existing users of From. See the comments in
11108   // the ReplaceAllUsesWith above.
11109   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11110   RAUWUpdateListener Listener(*this, UI, UE);
11111   while (UI != UE) {
11112     SDNode *User = *UI;
11113 
11114     // This node is about to morph, remove its old self from the CSE maps.
11115     RemoveNodeFromCSEMaps(User);
11116 
11117     // A user can appear in a use list multiple times, and when this happens the
11118     // uses are usually next to each other in the list.  To help reduce the
11119     // number of CSE and divergence recomputations, process all the uses of this
11120     // user that we can find this way.
11121     bool To_IsDivergent = false;
11122     do {
11123       SDUse &Use = UI.getUse();
11124       const SDValue &ToOp = To[Use.getResNo()];
11125       ++UI;
11126       Use.set(ToOp);
11127       To_IsDivergent |= ToOp->isDivergent();
11128     } while (UI != UE && *UI == User);
11129 
11130     if (To_IsDivergent != From->isDivergent())
11131       updateDivergence(User);
11132 
11133     // Now that we have modified User, add it back to the CSE maps.  If it
11134     // already exists there, recursively merge the results together.
11135     AddModifiedNodeToCSEMaps(User);
11136   }
11137 
11138   // If we just RAUW'd the root, take note.
11139   if (From == getRoot().getNode())
11140     setRoot(SDValue(To[getRoot().getResNo()]));
11141 }
11142 
11143 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11144 /// uses of other values produced by From.getNode() alone.  The Deleted
11145 /// vector is handled the same way as for ReplaceAllUsesWith.
11146 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
11147   // Handle the really simple, really trivial case efficiently.
11148   if (From == To) return;
11149 
11150   // Handle the simple, trivial, case efficiently.
11151   if (From.getNode()->getNumValues() == 1) {
11152     ReplaceAllUsesWith(From, To);
11153     return;
11154   }
11155 
11156   // Preserve Debug Info.
11157   transferDbgValues(From, To);
11158   copyExtraInfo(From.getNode(), To.getNode());
11159 
11160   // Iterate over just the existing users of From. See the comments in
11161   // the ReplaceAllUsesWith above.
11162   SDNode::use_iterator UI = From.getNode()->use_begin(),
11163                        UE = From.getNode()->use_end();
11164   RAUWUpdateListener Listener(*this, UI, UE);
11165   while (UI != UE) {
11166     SDNode *User = *UI;
11167     bool UserRemovedFromCSEMaps = false;
11168 
11169     // A user can appear in a use list multiple times, and when this
11170     // happens the uses are usually next to each other in the list.
11171     // To help reduce the number of CSE recomputations, process all
11172     // the uses of this user that we can find this way.
11173     do {
11174       SDUse &Use = UI.getUse();
11175 
11176       // Skip uses of different values from the same node.
11177       if (Use.getResNo() != From.getResNo()) {
11178         ++UI;
11179         continue;
11180       }
11181 
11182       // If this node hasn't been modified yet, it's still in the CSE maps,
11183       // so remove its old self from the CSE maps.
11184       if (!UserRemovedFromCSEMaps) {
11185         RemoveNodeFromCSEMaps(User);
11186         UserRemovedFromCSEMaps = true;
11187       }
11188 
11189       ++UI;
11190       Use.set(To);
11191       if (To->isDivergent() != From->isDivergent())
11192         updateDivergence(User);
11193     } while (UI != UE && *UI == User);
11194     // We are iterating over all uses of the From node, so if a use
11195     // doesn't use the specific value, no changes are made.
11196     if (!UserRemovedFromCSEMaps)
11197       continue;
11198 
11199     // Now that we have modified User, add it back to the CSE maps.  If it
11200     // already exists there, recursively merge the results together.
11201     AddModifiedNodeToCSEMaps(User);
11202   }
11203 
11204   // If we just RAUW'd the root, take note.
11205   if (From == getRoot())
11206     setRoot(To);
11207 }
11208 
11209 namespace {
11210 
11211 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11212 /// to record information about a use.
11213 struct UseMemo {
11214   SDNode *User;
11215   unsigned Index;
11216   SDUse *Use;
11217 };
11218 
11219 /// operator< - Sort Memos by User.
11220 bool operator<(const UseMemo &L, const UseMemo &R) {
11221   return (intptr_t)L.User < (intptr_t)R.User;
11222 }
11223 
11224 /// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11225 /// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11226 /// the node already has been taken care of recursively.
11227 class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11228   SmallVector<UseMemo, 4> &Uses;
11229 
11230   void NodeDeleted(SDNode *N, SDNode *E) override {
11231     for (UseMemo &Memo : Uses)
11232       if (Memo.User == N)
11233         Memo.User = nullptr;
11234   }
11235 
11236 public:
11237   RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses)
11238       : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11239 };
11240 
11241 } // end anonymous namespace
11242 
11243 bool SelectionDAG::calculateDivergence(SDNode *N) {
11244   if (TLI->isSDNodeAlwaysUniform(N)) {
11245     assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11246            "Conflicting divergence information!");
11247     return false;
11248   }
11249   if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11250     return true;
11251   for (const auto &Op : N->ops()) {
11252     if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
11253       return true;
11254   }
11255   return false;
11256 }
11257 
11258 void SelectionDAG::updateDivergence(SDNode *N) {
11259   SmallVector<SDNode *, 16> Worklist(1, N);
11260   do {
11261     N = Worklist.pop_back_val();
11262     bool IsDivergent = calculateDivergence(N);
11263     if (N->SDNodeBits.IsDivergent != IsDivergent) {
11264       N->SDNodeBits.IsDivergent = IsDivergent;
11265       llvm::append_range(Worklist, N->uses());
11266     }
11267   } while (!Worklist.empty());
11268 }
11269 
11270 void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11271   DenseMap<SDNode *, unsigned> Degree;
11272   Order.reserve(AllNodes.size());
11273   for (auto &N : allnodes()) {
11274     unsigned NOps = N.getNumOperands();
11275     Degree[&N] = NOps;
11276     if (0 == NOps)
11277       Order.push_back(&N);
11278   }
11279   for (size_t I = 0; I != Order.size(); ++I) {
11280     SDNode *N = Order[I];
11281     for (auto *U : N->uses()) {
11282       unsigned &UnsortedOps = Degree[U];
11283       if (0 == --UnsortedOps)
11284         Order.push_back(U);
11285     }
11286   }
11287 }
11288 
11289 #ifndef NDEBUG
11290 void SelectionDAG::VerifyDAGDivergence() {
11291   std::vector<SDNode *> TopoOrder;
11292   CreateTopologicalOrder(TopoOrder);
11293   for (auto *N : TopoOrder) {
11294     assert(calculateDivergence(N) == N->isDivergent() &&
11295            "Divergence bit inconsistency detected");
11296   }
11297 }
11298 #endif
11299 
11300 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11301 /// uses of other values produced by From.getNode() alone.  The same value
11302 /// may appear in both the From and To list.  The Deleted vector is
11303 /// handled the same way as for ReplaceAllUsesWith.
11304 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
11305                                               const SDValue *To,
11306                                               unsigned Num){
11307   // Handle the simple, trivial case efficiently.
11308   if (Num == 1)
11309     return ReplaceAllUsesOfValueWith(*From, *To);
11310 
11311   transferDbgValues(*From, *To);
11312   copyExtraInfo(From->getNode(), To->getNode());
11313 
11314   // Read up all the uses and make records of them. This helps
11315   // processing new uses that are introduced during the
11316   // replacement process.
11317   SmallVector<UseMemo, 4> Uses;
11318   for (unsigned i = 0; i != Num; ++i) {
11319     unsigned FromResNo = From[i].getResNo();
11320     SDNode *FromNode = From[i].getNode();
11321     for (SDNode::use_iterator UI = FromNode->use_begin(),
11322          E = FromNode->use_end(); UI != E; ++UI) {
11323       SDUse &Use = UI.getUse();
11324       if (Use.getResNo() == FromResNo) {
11325         UseMemo Memo = { *UI, i, &Use };
11326         Uses.push_back(Memo);
11327       }
11328     }
11329   }
11330 
11331   // Sort the uses, so that all the uses from a given User are together.
11332   llvm::sort(Uses);
11333   RAUOVWUpdateListener Listener(*this, Uses);
11334 
11335   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
11336        UseIndex != UseIndexEnd; ) {
11337     // We know that this user uses some value of From.  If it is the right
11338     // value, update it.
11339     SDNode *User = Uses[UseIndex].User;
11340     // If the node has been deleted by recursive CSE updates when updating
11341     // another node, then just skip this entry.
11342     if (User == nullptr) {
11343       ++UseIndex;
11344       continue;
11345     }
11346 
11347     // This node is about to morph, remove its old self from the CSE maps.
11348     RemoveNodeFromCSEMaps(User);
11349 
11350     // The Uses array is sorted, so all the uses for a given User
11351     // are next to each other in the list.
11352     // To help reduce the number of CSE recomputations, process all
11353     // the uses of this user that we can find this way.
11354     do {
11355       unsigned i = Uses[UseIndex].Index;
11356       SDUse &Use = *Uses[UseIndex].Use;
11357       ++UseIndex;
11358 
11359       Use.set(To[i]);
11360     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
11361 
11362     // Now that we have modified User, add it back to the CSE maps.  If it
11363     // already exists there, recursively merge the results together.
11364     AddModifiedNodeToCSEMaps(User);
11365   }
11366 }
11367 
11368 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
11369 /// based on their topological order. It returns the maximum id and a vector
11370 /// of the SDNodes* in assigned order by reference.
11371 unsigned SelectionDAG::AssignTopologicalOrder() {
11372   unsigned DAGSize = 0;
11373 
11374   // SortedPos tracks the progress of the algorithm. Nodes before it are
11375   // sorted, nodes after it are unsorted. When the algorithm completes
11376   // it is at the end of the list.
11377   allnodes_iterator SortedPos = allnodes_begin();
11378 
11379   // Visit all the nodes. Move nodes with no operands to the front of
11380   // the list immediately. Annotate nodes that do have operands with their
11381   // operand count. Before we do this, the Node Id fields of the nodes
11382   // may contain arbitrary values. After, the Node Id fields for nodes
11383   // before SortedPos will contain the topological sort index, and the
11384   // Node Id fields for nodes At SortedPos and after will contain the
11385   // count of outstanding operands.
11386   for (SDNode &N : llvm::make_early_inc_range(allnodes())) {
11387     checkForCycles(&N, this);
11388     unsigned Degree = N.getNumOperands();
11389     if (Degree == 0) {
11390       // A node with no uses, add it to the result array immediately.
11391       N.setNodeId(DAGSize++);
11392       allnodes_iterator Q(&N);
11393       if (Q != SortedPos)
11394         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
11395       assert(SortedPos != AllNodes.end() && "Overran node list");
11396       ++SortedPos;
11397     } else {
11398       // Temporarily use the Node Id as scratch space for the degree count.
11399       N.setNodeId(Degree);
11400     }
11401   }
11402 
11403   // Visit all the nodes. As we iterate, move nodes into sorted order,
11404   // such that by the time the end is reached all nodes will be sorted.
11405   for (SDNode &Node : allnodes()) {
11406     SDNode *N = &Node;
11407     checkForCycles(N, this);
11408     // N is in sorted position, so all its uses have one less operand
11409     // that needs to be sorted.
11410     for (SDNode *P : N->uses()) {
11411       unsigned Degree = P->getNodeId();
11412       assert(Degree != 0 && "Invalid node degree");
11413       --Degree;
11414       if (Degree == 0) {
11415         // All of P's operands are sorted, so P may sorted now.
11416         P->setNodeId(DAGSize++);
11417         if (P->getIterator() != SortedPos)
11418           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
11419         assert(SortedPos != AllNodes.end() && "Overran node list");
11420         ++SortedPos;
11421       } else {
11422         // Update P's outstanding operand count.
11423         P->setNodeId(Degree);
11424       }
11425     }
11426     if (Node.getIterator() == SortedPos) {
11427 #ifndef NDEBUG
11428       allnodes_iterator I(N);
11429       SDNode *S = &*++I;
11430       dbgs() << "Overran sorted position:\n";
11431       S->dumprFull(this); dbgs() << "\n";
11432       dbgs() << "Checking if this is due to cycles\n";
11433       checkForCycles(this, true);
11434 #endif
11435       llvm_unreachable(nullptr);
11436     }
11437   }
11438 
11439   assert(SortedPos == AllNodes.end() &&
11440          "Topological sort incomplete!");
11441   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
11442          "First node in topological sort is not the entry token!");
11443   assert(AllNodes.front().getNodeId() == 0 &&
11444          "First node in topological sort has non-zero id!");
11445   assert(AllNodes.front().getNumOperands() == 0 &&
11446          "First node in topological sort has operands!");
11447   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
11448          "Last node in topologic sort has unexpected id!");
11449   assert(AllNodes.back().use_empty() &&
11450          "Last node in topologic sort has users!");
11451   assert(DAGSize == allnodes_size() && "Node count mismatch!");
11452   return DAGSize;
11453 }
11454 
11455 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
11456 /// value is produced by SD.
11457 void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
11458   for (SDNode *SD : DB->getSDNodes()) {
11459     if (!SD)
11460       continue;
11461     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
11462     SD->setHasDebugValue(true);
11463   }
11464   DbgInfo->add(DB, isParameter);
11465 }
11466 
11467 void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
11468 
11469 SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
11470                                                    SDValue NewMemOpChain) {
11471   assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
11472   assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
11473   // The new memory operation must have the same position as the old load in
11474   // terms of memory dependency. Create a TokenFactor for the old load and new
11475   // memory operation and update uses of the old load's output chain to use that
11476   // TokenFactor.
11477   if (OldChain == NewMemOpChain || OldChain.use_empty())
11478     return NewMemOpChain;
11479 
11480   SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
11481                                 OldChain, NewMemOpChain);
11482   ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
11483   UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
11484   return TokenFactor;
11485 }
11486 
11487 SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
11488                                                    SDValue NewMemOp) {
11489   assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
11490   SDValue OldChain = SDValue(OldLoad, 1);
11491   SDValue NewMemOpChain = NewMemOp.getValue(1);
11492   return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
11493 }
11494 
11495 SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
11496                                                      Function **OutFunction) {
11497   assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
11498 
11499   auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
11500   auto *Module = MF->getFunction().getParent();
11501   auto *Function = Module->getFunction(Symbol);
11502 
11503   if (OutFunction != nullptr)
11504       *OutFunction = Function;
11505 
11506   if (Function != nullptr) {
11507     auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
11508     return getGlobalAddress(Function, SDLoc(Op), PtrTy);
11509   }
11510 
11511   std::string ErrorStr;
11512   raw_string_ostream ErrorFormatter(ErrorStr);
11513   ErrorFormatter << "Undefined external symbol ";
11514   ErrorFormatter << '"' << Symbol << '"';
11515   report_fatal_error(Twine(ErrorFormatter.str()));
11516 }
11517 
11518 //===----------------------------------------------------------------------===//
11519 //                              SDNode Class
11520 //===----------------------------------------------------------------------===//
11521 
11522 bool llvm::isNullConstant(SDValue V) {
11523   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11524   return Const != nullptr && Const->isZero();
11525 }
11526 
11527 bool llvm::isNullFPConstant(SDValue V) {
11528   ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
11529   return Const != nullptr && Const->isZero() && !Const->isNegative();
11530 }
11531 
11532 bool llvm::isAllOnesConstant(SDValue V) {
11533   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11534   return Const != nullptr && Const->isAllOnes();
11535 }
11536 
11537 bool llvm::isOneConstant(SDValue V) {
11538   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11539   return Const != nullptr && Const->isOne();
11540 }
11541 
11542 bool llvm::isMinSignedConstant(SDValue V) {
11543   ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11544   return Const != nullptr && Const->isMinSignedValue();
11545 }
11546 
11547 bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
11548                              unsigned OperandNo) {
11549   // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
11550   // TODO: Target-specific opcodes could be added.
11551   if (auto *Const = isConstOrConstSplat(V)) {
11552     switch (Opcode) {
11553     case ISD::ADD:
11554     case ISD::OR:
11555     case ISD::XOR:
11556     case ISD::UMAX:
11557       return Const->isZero();
11558     case ISD::MUL:
11559       return Const->isOne();
11560     case ISD::AND:
11561     case ISD::UMIN:
11562       return Const->isAllOnes();
11563     case ISD::SMAX:
11564       return Const->isMinSignedValue();
11565     case ISD::SMIN:
11566       return Const->isMaxSignedValue();
11567     case ISD::SUB:
11568     case ISD::SHL:
11569     case ISD::SRA:
11570     case ISD::SRL:
11571       return OperandNo == 1 && Const->isZero();
11572     case ISD::UDIV:
11573     case ISD::SDIV:
11574       return OperandNo == 1 && Const->isOne();
11575     }
11576   } else if (auto *ConstFP = isConstOrConstSplatFP(V)) {
11577     switch (Opcode) {
11578     case ISD::FADD:
11579       return ConstFP->isZero() &&
11580              (Flags.hasNoSignedZeros() || ConstFP->isNegative());
11581     case ISD::FSUB:
11582       return OperandNo == 1 && ConstFP->isZero() &&
11583              (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11584     case ISD::FMUL:
11585       return ConstFP->isExactlyValue(1.0);
11586     case ISD::FDIV:
11587       return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
11588     case ISD::FMINNUM:
11589     case ISD::FMAXNUM: {
11590       // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
11591       EVT VT = V.getValueType();
11592       const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT);
11593       APFloat NeutralAF = !Flags.hasNoNaNs()
11594                               ? APFloat::getQNaN(Semantics)
11595                               : !Flags.hasNoInfs()
11596                                     ? APFloat::getInf(Semantics)
11597                                     : APFloat::getLargest(Semantics);
11598       if (Opcode == ISD::FMAXNUM)
11599         NeutralAF.changeSign();
11600 
11601       return ConstFP->isExactlyValue(NeutralAF);
11602     }
11603     }
11604   }
11605   return false;
11606 }
11607 
11608 SDValue llvm::peekThroughBitcasts(SDValue V) {
11609   while (V.getOpcode() == ISD::BITCAST)
11610     V = V.getOperand(0);
11611   return V;
11612 }
11613 
11614 SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
11615   while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
11616     V = V.getOperand(0);
11617   return V;
11618 }
11619 
11620 SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
11621   while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
11622     V = V.getOperand(0);
11623   return V;
11624 }
11625 
11626 SDValue llvm::peekThroughTruncates(SDValue V) {
11627   while (V.getOpcode() == ISD::TRUNCATE)
11628     V = V.getOperand(0);
11629   return V;
11630 }
11631 
11632 bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
11633   if (V.getOpcode() != ISD::XOR)
11634     return false;
11635   V = peekThroughBitcasts(V.getOperand(1));
11636   unsigned NumBits = V.getScalarValueSizeInBits();
11637   ConstantSDNode *C =
11638       isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
11639   return C && (C->getAPIntValue().countr_one() >= NumBits);
11640 }
11641 
11642 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
11643                                           bool AllowTruncation) {
11644   EVT VT = N.getValueType();
11645   APInt DemandedElts = VT.isFixedLengthVector()
11646                            ? APInt::getAllOnes(VT.getVectorMinNumElements())
11647                            : APInt(1, 1);
11648   return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
11649 }
11650 
11651 ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
11652                                           bool AllowUndefs,
11653                                           bool AllowTruncation) {
11654   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
11655     return CN;
11656 
11657   // SplatVectors can truncate their operands. Ignore that case here unless
11658   // AllowTruncation is set.
11659   if (N->getOpcode() == ISD::SPLAT_VECTOR) {
11660     EVT VecEltVT = N->getValueType(0).getVectorElementType();
11661     if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
11662       EVT CVT = CN->getValueType(0);
11663       assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
11664       if (AllowTruncation || CVT == VecEltVT)
11665         return CN;
11666     }
11667   }
11668 
11669   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11670     BitVector UndefElements;
11671     ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
11672 
11673     // BuildVectors can truncate their operands. Ignore that case here unless
11674     // AllowTruncation is set.
11675     // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11676     if (CN && (UndefElements.none() || AllowUndefs)) {
11677       EVT CVT = CN->getValueType(0);
11678       EVT NSVT = N.getValueType().getScalarType();
11679       assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
11680       if (AllowTruncation || (CVT == NSVT))
11681         return CN;
11682     }
11683   }
11684 
11685   return nullptr;
11686 }
11687 
11688 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
11689   EVT VT = N.getValueType();
11690   APInt DemandedElts = VT.isFixedLengthVector()
11691                            ? APInt::getAllOnes(VT.getVectorMinNumElements())
11692                            : APInt(1, 1);
11693   return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
11694 }
11695 
11696 ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
11697                                               const APInt &DemandedElts,
11698                                               bool AllowUndefs) {
11699   if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
11700     return CN;
11701 
11702   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11703     BitVector UndefElements;
11704     ConstantFPSDNode *CN =
11705         BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
11706     // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11707     if (CN && (UndefElements.none() || AllowUndefs))
11708       return CN;
11709   }
11710 
11711   if (N.getOpcode() == ISD::SPLAT_VECTOR)
11712     if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
11713       return CN;
11714 
11715   return nullptr;
11716 }
11717 
11718 bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
11719   // TODO: may want to use peekThroughBitcast() here.
11720   ConstantSDNode *C =
11721       isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
11722   return C && C->isZero();
11723 }
11724 
11725 bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
11726   ConstantSDNode *C =
11727       isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
11728   return C && C->isOne();
11729 }
11730 
11731 bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
11732   N = peekThroughBitcasts(N);
11733   unsigned BitWidth = N.getScalarValueSizeInBits();
11734   ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
11735   return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
11736 }
11737 
11738 HandleSDNode::~HandleSDNode() {
11739   DropOperands();
11740 }
11741 
11742 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
11743                                          const DebugLoc &DL,
11744                                          const GlobalValue *GA, EVT VT,
11745                                          int64_t o, unsigned TF)
11746     : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
11747   TheGlobal = GA;
11748 }
11749 
11750 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
11751                                          EVT VT, unsigned SrcAS,
11752                                          unsigned DestAS)
11753     : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
11754       SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
11755 
11756 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
11757                      SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
11758     : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
11759   MemSDNodeBits.IsVolatile = MMO->isVolatile();
11760   MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
11761   MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
11762   MemSDNodeBits.IsInvariant = MMO->isInvariant();
11763 
11764   // We check here that the size of the memory operand fits within the size of
11765   // the MMO. This is because the MMO might indicate only a possible address
11766   // range instead of specifying the affected memory addresses precisely.
11767   // TODO: Make MachineMemOperands aware of scalable vectors.
11768   assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() &&
11769          "Size mismatch!");
11770 }
11771 
11772 /// Profile - Gather unique data for the node.
11773 ///
11774 void SDNode::Profile(FoldingSetNodeID &ID) const {
11775   AddNodeIDNode(ID, this);
11776 }
11777 
11778 namespace {
11779 
11780   struct EVTArray {
11781     std::vector<EVT> VTs;
11782 
11783     EVTArray() {
11784       VTs.reserve(MVT::VALUETYPE_SIZE);
11785       for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
11786         VTs.push_back(MVT((MVT::SimpleValueType)i));
11787     }
11788   };
11789 
11790 } // end anonymous namespace
11791 
11792 /// getValueTypeList - Return a pointer to the specified value type.
11793 ///
11794 const EVT *SDNode::getValueTypeList(EVT VT) {
11795   static std::set<EVT, EVT::compareRawBits> EVTs;
11796   static EVTArray SimpleVTArray;
11797   static sys::SmartMutex<true> VTMutex;
11798 
11799   if (VT.isExtended()) {
11800     sys::SmartScopedLock<true> Lock(VTMutex);
11801     return &(*EVTs.insert(VT).first);
11802   }
11803   assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
11804   return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
11805 }
11806 
11807 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
11808 /// indicated value.  This method ignores uses of other values defined by this
11809 /// operation.
11810 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
11811   assert(Value < getNumValues() && "Bad value!");
11812 
11813   // TODO: Only iterate over uses of a given value of the node
11814   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
11815     if (UI.getUse().getResNo() == Value) {
11816       if (NUses == 0)
11817         return false;
11818       --NUses;
11819     }
11820   }
11821 
11822   // Found exactly the right number of uses?
11823   return NUses == 0;
11824 }
11825 
11826 /// hasAnyUseOfValue - Return true if there are any use of the indicated
11827 /// value. This method ignores uses of other values defined by this operation.
11828 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
11829   assert(Value < getNumValues() && "Bad value!");
11830 
11831   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
11832     if (UI.getUse().getResNo() == Value)
11833       return true;
11834 
11835   return false;
11836 }
11837 
11838 /// isOnlyUserOf - Return true if this node is the only use of N.
11839 bool SDNode::isOnlyUserOf(const SDNode *N) const {
11840   bool Seen = false;
11841   for (const SDNode *User : N->uses()) {
11842     if (User == this)
11843       Seen = true;
11844     else
11845       return false;
11846   }
11847 
11848   return Seen;
11849 }
11850 
11851 /// Return true if the only users of N are contained in Nodes.
11852 bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
11853   bool Seen = false;
11854   for (const SDNode *User : N->uses()) {
11855     if (llvm::is_contained(Nodes, User))
11856       Seen = true;
11857     else
11858       return false;
11859   }
11860 
11861   return Seen;
11862 }
11863 
11864 /// isOperand - Return true if this node is an operand of N.
11865 bool SDValue::isOperandOf(const SDNode *N) const {
11866   return is_contained(N->op_values(), *this);
11867 }
11868 
11869 bool SDNode::isOperandOf(const SDNode *N) const {
11870   return any_of(N->op_values(),
11871                 [this](SDValue Op) { return this == Op.getNode(); });
11872 }
11873 
11874 /// reachesChainWithoutSideEffects - Return true if this operand (which must
11875 /// be a chain) reaches the specified operand without crossing any
11876 /// side-effecting instructions on any chain path.  In practice, this looks
11877 /// through token factors and non-volatile loads.  In order to remain efficient,
11878 /// this only looks a couple of nodes in, it does not do an exhaustive search.
11879 ///
11880 /// Note that we only need to examine chains when we're searching for
11881 /// side-effects; SelectionDAG requires that all side-effects are represented
11882 /// by chains, even if another operand would force a specific ordering. This
11883 /// constraint is necessary to allow transformations like splitting loads.
11884 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
11885                                              unsigned Depth) const {
11886   if (*this == Dest) return true;
11887 
11888   // Don't search too deeply, we just want to be able to see through
11889   // TokenFactor's etc.
11890   if (Depth == 0) return false;
11891 
11892   // If this is a token factor, all inputs to the TF happen in parallel.
11893   if (getOpcode() == ISD::TokenFactor) {
11894     // First, try a shallow search.
11895     if (is_contained((*this)->ops(), Dest)) {
11896       // We found the chain we want as an operand of this TokenFactor.
11897       // Essentially, we reach the chain without side-effects if we could
11898       // serialize the TokenFactor into a simple chain of operations with
11899       // Dest as the last operation. This is automatically true if the
11900       // chain has one use: there are no other ordering constraints.
11901       // If the chain has more than one use, we give up: some other
11902       // use of Dest might force a side-effect between Dest and the current
11903       // node.
11904       if (Dest.hasOneUse())
11905         return true;
11906     }
11907     // Next, try a deep search: check whether every operand of the TokenFactor
11908     // reaches Dest.
11909     return llvm::all_of((*this)->ops(), [=](SDValue Op) {
11910       return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
11911     });
11912   }
11913 
11914   // Loads don't have side effects, look through them.
11915   if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
11916     if (Ld->isUnordered())
11917       return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
11918   }
11919   return false;
11920 }
11921 
11922 bool SDNode::hasPredecessor(const SDNode *N) const {
11923   SmallPtrSet<const SDNode *, 32> Visited;
11924   SmallVector<const SDNode *, 16> Worklist;
11925   Worklist.push_back(this);
11926   return hasPredecessorHelper(N, Visited, Worklist);
11927 }
11928 
11929 void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
11930   this->Flags.intersectWith(Flags);
11931 }
11932 
11933 SDValue
11934 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
11935                                   ArrayRef<ISD::NodeType> CandidateBinOps,
11936                                   bool AllowPartials) {
11937   // The pattern must end in an extract from index 0.
11938   if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
11939       !isNullConstant(Extract->getOperand(1)))
11940     return SDValue();
11941 
11942   // Match against one of the candidate binary ops.
11943   SDValue Op = Extract->getOperand(0);
11944   if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
11945         return Op.getOpcode() == unsigned(BinOp);
11946       }))
11947     return SDValue();
11948 
11949   // Floating-point reductions may require relaxed constraints on the final step
11950   // of the reduction because they may reorder intermediate operations.
11951   unsigned CandidateBinOp = Op.getOpcode();
11952   if (Op.getValueType().isFloatingPoint()) {
11953     SDNodeFlags Flags = Op->getFlags();
11954     switch (CandidateBinOp) {
11955     case ISD::FADD:
11956       if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
11957         return SDValue();
11958       break;
11959     default:
11960       llvm_unreachable("Unhandled FP opcode for binop reduction");
11961     }
11962   }
11963 
11964   // Matching failed - attempt to see if we did enough stages that a partial
11965   // reduction from a subvector is possible.
11966   auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
11967     if (!AllowPartials || !Op)
11968       return SDValue();
11969     EVT OpVT = Op.getValueType();
11970     EVT OpSVT = OpVT.getScalarType();
11971     EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
11972     if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
11973       return SDValue();
11974     BinOp = (ISD::NodeType)CandidateBinOp;
11975     return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
11976                    getVectorIdxConstant(0, SDLoc(Op)));
11977   };
11978 
11979   // At each stage, we're looking for something that looks like:
11980   // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
11981   //                    <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
11982   //                               i32 undef, i32 undef, i32 undef, i32 undef>
11983   // %a = binop <8 x i32> %op, %s
11984   // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
11985   // we expect something like:
11986   // <4,5,6,7,u,u,u,u>
11987   // <2,3,u,u,u,u,u,u>
11988   // <1,u,u,u,u,u,u,u>
11989   // While a partial reduction match would be:
11990   // <2,3,u,u,u,u,u,u>
11991   // <1,u,u,u,u,u,u,u>
11992   unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
11993   SDValue PrevOp;
11994   for (unsigned i = 0; i < Stages; ++i) {
11995     unsigned MaskEnd = (1 << i);
11996 
11997     if (Op.getOpcode() != CandidateBinOp)
11998       return PartialReduction(PrevOp, MaskEnd);
11999 
12000     SDValue Op0 = Op.getOperand(0);
12001     SDValue Op1 = Op.getOperand(1);
12002 
12003     ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
12004     if (Shuffle) {
12005       Op = Op1;
12006     } else {
12007       Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
12008       Op = Op0;
12009     }
12010 
12011     // The first operand of the shuffle should be the same as the other operand
12012     // of the binop.
12013     if (!Shuffle || Shuffle->getOperand(0) != Op)
12014       return PartialReduction(PrevOp, MaskEnd);
12015 
12016     // Verify the shuffle has the expected (at this stage of the pyramid) mask.
12017     for (int Index = 0; Index < (int)MaskEnd; ++Index)
12018       if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
12019         return PartialReduction(PrevOp, MaskEnd);
12020 
12021     PrevOp = Op;
12022   }
12023 
12024   // Handle subvector reductions, which tend to appear after the shuffle
12025   // reduction stages.
12026   while (Op.getOpcode() == CandidateBinOp) {
12027     unsigned NumElts = Op.getValueType().getVectorNumElements();
12028     SDValue Op0 = Op.getOperand(0);
12029     SDValue Op1 = Op.getOperand(1);
12030     if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12031         Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12032         Op0.getOperand(0) != Op1.getOperand(0))
12033       break;
12034     SDValue Src = Op0.getOperand(0);
12035     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12036     if (NumSrcElts != (2 * NumElts))
12037       break;
12038     if (!(Op0.getConstantOperandAPInt(1) == 0 &&
12039           Op1.getConstantOperandAPInt(1) == NumElts) &&
12040         !(Op1.getConstantOperandAPInt(1) == 0 &&
12041           Op0.getConstantOperandAPInt(1) == NumElts))
12042       break;
12043     Op = Src;
12044   }
12045 
12046   BinOp = (ISD::NodeType)CandidateBinOp;
12047   return Op;
12048 }
12049 
12050 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
12051   EVT VT = N->getValueType(0);
12052   EVT EltVT = VT.getVectorElementType();
12053   unsigned NE = VT.getVectorNumElements();
12054 
12055   SDLoc dl(N);
12056 
12057   // If ResNE is 0, fully unroll the vector op.
12058   if (ResNE == 0)
12059     ResNE = NE;
12060   else if (NE > ResNE)
12061     NE = ResNE;
12062 
12063   if (N->getNumValues() == 2) {
12064     SmallVector<SDValue, 8> Scalars0, Scalars1;
12065     SmallVector<SDValue, 4> Operands(N->getNumOperands());
12066     EVT VT1 = N->getValueType(1);
12067     EVT EltVT1 = VT1.getVectorElementType();
12068 
12069     unsigned i;
12070     for (i = 0; i != NE; ++i) {
12071       for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12072         SDValue Operand = N->getOperand(j);
12073         EVT OperandVT = Operand.getValueType();
12074 
12075         // A vector operand; extract a single element.
12076         EVT OperandEltVT = OperandVT.getVectorElementType();
12077         Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12078                               Operand, getVectorIdxConstant(i, dl));
12079       }
12080 
12081       SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);
12082       Scalars0.push_back(EltOp);
12083       Scalars1.push_back(EltOp.getValue(1));
12084     }
12085 
12086     SDValue Vec0 = getBuildVector(VT, dl, Scalars0);
12087     SDValue Vec1 = getBuildVector(VT1, dl, Scalars1);
12088     return getMergeValues({Vec0, Vec1}, dl);
12089   }
12090 
12091   assert(N->getNumValues() == 1 &&
12092          "Can't unroll a vector with multiple results!");
12093 
12094   SmallVector<SDValue, 8> Scalars;
12095   SmallVector<SDValue, 4> Operands(N->getNumOperands());
12096 
12097   unsigned i;
12098   for (i= 0; i != NE; ++i) {
12099     for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12100       SDValue Operand = N->getOperand(j);
12101       EVT OperandVT = Operand.getValueType();
12102       if (OperandVT.isVector()) {
12103         // A vector operand; extract a single element.
12104         EVT OperandEltVT = OperandVT.getVectorElementType();
12105         Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12106                               Operand, getVectorIdxConstant(i, dl));
12107       } else {
12108         // A scalar operand; just use it as is.
12109         Operands[j] = Operand;
12110       }
12111     }
12112 
12113     switch (N->getOpcode()) {
12114     default: {
12115       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
12116                                 N->getFlags()));
12117       break;
12118     }
12119     case ISD::VSELECT:
12120       Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
12121       break;
12122     case ISD::SHL:
12123     case ISD::SRA:
12124     case ISD::SRL:
12125     case ISD::ROTL:
12126     case ISD::ROTR:
12127       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
12128                                getShiftAmountOperand(Operands[0].getValueType(),
12129                                                      Operands[1])));
12130       break;
12131     case ISD::SIGN_EXTEND_INREG: {
12132       EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
12133       Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
12134                                 Operands[0],
12135                                 getValueType(ExtVT)));
12136     }
12137     }
12138   }
12139 
12140   for (; i < ResNE; ++i)
12141     Scalars.push_back(getUNDEF(EltVT));
12142 
12143   EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
12144   return getBuildVector(VecVT, dl, Scalars);
12145 }
12146 
12147 std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12148     SDNode *N, unsigned ResNE) {
12149   unsigned Opcode = N->getOpcode();
12150   assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12151           Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12152           Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12153          "Expected an overflow opcode");
12154 
12155   EVT ResVT = N->getValueType(0);
12156   EVT OvVT = N->getValueType(1);
12157   EVT ResEltVT = ResVT.getVectorElementType();
12158   EVT OvEltVT = OvVT.getVectorElementType();
12159   SDLoc dl(N);
12160 
12161   // If ResNE is 0, fully unroll the vector op.
12162   unsigned NE = ResVT.getVectorNumElements();
12163   if (ResNE == 0)
12164     ResNE = NE;
12165   else if (NE > ResNE)
12166     NE = ResNE;
12167 
12168   SmallVector<SDValue, 8> LHSScalars;
12169   SmallVector<SDValue, 8> RHSScalars;
12170   ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
12171   ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
12172 
12173   EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
12174   SDVTList VTs = getVTList(ResEltVT, SVT);
12175   SmallVector<SDValue, 8> ResScalars;
12176   SmallVector<SDValue, 8> OvScalars;
12177   for (unsigned i = 0; i < NE; ++i) {
12178     SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12179     SDValue Ov =
12180         getSelect(dl, OvEltVT, Res.getValue(1),
12181                   getBoolConstant(true, dl, OvEltVT, ResVT),
12182                   getConstant(0, dl, OvEltVT));
12183 
12184     ResScalars.push_back(Res);
12185     OvScalars.push_back(Ov);
12186   }
12187 
12188   ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
12189   OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
12190 
12191   EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
12192   EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
12193   return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
12194                         getBuildVector(NewOvVT, dl, OvScalars));
12195 }
12196 
12197 bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
12198                                                   LoadSDNode *Base,
12199                                                   unsigned Bytes,
12200                                                   int Dist) const {
12201   if (LD->isVolatile() || Base->isVolatile())
12202     return false;
12203   // TODO: probably too restrictive for atomics, revisit
12204   if (!LD->isSimple())
12205     return false;
12206   if (LD->isIndexed() || Base->isIndexed())
12207     return false;
12208   if (LD->getChain() != Base->getChain())
12209     return false;
12210   EVT VT = LD->getMemoryVT();
12211   if (VT.getSizeInBits() / 8 != Bytes)
12212     return false;
12213 
12214   auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
12215   auto LocDecomp = BaseIndexOffset::match(LD, *this);
12216 
12217   int64_t Offset = 0;
12218   if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
12219     return (Dist * (int64_t)Bytes == Offset);
12220   return false;
12221 }
12222 
12223 /// InferPtrAlignment - Infer alignment of a load / store address. Return
12224 /// std::nullopt if it cannot be inferred.
12225 MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
12226   // If this is a GlobalAddress + cst, return the alignment.
12227   const GlobalValue *GV = nullptr;
12228   int64_t GVOffset = 0;
12229   if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
12230     unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12231     KnownBits Known(PtrWidth);
12232     llvm::computeKnownBits(GV, Known, getDataLayout());
12233     unsigned AlignBits = Known.countMinTrailingZeros();
12234     if (AlignBits)
12235       return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
12236   }
12237 
12238   // If this is a direct reference to a stack slot, use information about the
12239   // stack slot's alignment.
12240   int FrameIdx = INT_MIN;
12241   int64_t FrameOffset = 0;
12242   if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
12243     FrameIdx = FI->getIndex();
12244   } else if (isBaseWithConstantOffset(Ptr) &&
12245              isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
12246     // Handle FI+Cst
12247     FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
12248     FrameOffset = Ptr.getConstantOperandVal(1);
12249   }
12250 
12251   if (FrameIdx != INT_MIN) {
12252     const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
12253     return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
12254   }
12255 
12256   return std::nullopt;
12257 }
12258 
12259 /// Split the scalar node with EXTRACT_ELEMENT using the provided
12260 /// VTs and return the low/high part.
12261 std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12262                                                       const SDLoc &DL,
12263                                                       const EVT &LoVT,
12264                                                       const EVT &HiVT) {
12265   assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12266          "Split node must be a scalar type");
12267   SDValue Lo =
12268       getNode(ISD::EXTRACT_ELEMENT, DL, LoVT, N, getIntPtrConstant(0, DL));
12269   SDValue Hi =
12270       getNode(ISD::EXTRACT_ELEMENT, DL, HiVT, N, getIntPtrConstant(1, DL));
12271   return std::make_pair(Lo, Hi);
12272 }
12273 
12274 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12275 /// which is split (or expanded) into two not necessarily identical pieces.
12276 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12277   // Currently all types are split in half.
12278   EVT LoVT, HiVT;
12279   if (!VT.isVector())
12280     LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
12281   else
12282     LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
12283 
12284   return std::make_pair(LoVT, HiVT);
12285 }
12286 
12287 /// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12288 /// type, dependent on an enveloping VT that has been split into two identical
12289 /// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12290 std::pair<EVT, EVT>
12291 SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
12292                                        bool *HiIsEmpty) const {
12293   EVT EltTp = VT.getVectorElementType();
12294   // Examples:
12295   //   custom VL=8  with enveloping VL=8/8 yields 8/0 (hi empty)
12296   //   custom VL=9  with enveloping VL=8/8 yields 8/1
12297   //   custom VL=10 with enveloping VL=8/8 yields 8/2
12298   //   etc.
12299   ElementCount VTNumElts = VT.getVectorElementCount();
12300   ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12301   assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12302          "Mixing fixed width and scalable vectors when enveloping a type");
12303   EVT LoVT, HiVT;
12304   if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12305     LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12306     HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
12307     *HiIsEmpty = false;
12308   } else {
12309     // Flag that hi type has zero storage size, but return split envelop type
12310     // (this would be easier if vector types with zero elements were allowed).
12311     LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
12312     HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12313     *HiIsEmpty = true;
12314   }
12315   return std::make_pair(LoVT, HiVT);
12316 }
12317 
12318 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12319 /// low/high part.
12320 std::pair<SDValue, SDValue>
12321 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
12322                           const EVT &HiVT) {
12323   assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
12324          LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
12325          "Splitting vector with an invalid mixture of fixed and scalable "
12326          "vector types");
12327   assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
12328              N.getValueType().getVectorMinNumElements() &&
12329          "More vector elements requested than available!");
12330   SDValue Lo, Hi;
12331   Lo =
12332       getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL));
12333   // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
12334   // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
12335   // IDX with the runtime scaling factor of the result vector type. For
12336   // fixed-width result vectors, that runtime scaling factor is 1.
12337   Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
12338                getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
12339   return std::make_pair(Lo, Hi);
12340 }
12341 
12342 std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
12343                                                    const SDLoc &DL) {
12344   // Split the vector length parameter.
12345   // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
12346   EVT VT = N.getValueType();
12347   assert(VecVT.getVectorElementCount().isKnownEven() &&
12348          "Expecting the mask to be an evenly-sized vector");
12349   unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
12350   SDValue HalfNumElts =
12351       VecVT.isFixedLengthVector()
12352           ? getConstant(HalfMinNumElts, DL, VT)
12353           : getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
12354   SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);
12355   SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);
12356   return std::make_pair(Lo, Hi);
12357 }
12358 
12359 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
12360 SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
12361   EVT VT = N.getValueType();
12362   EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
12363                                 NextPowerOf2(VT.getVectorNumElements()));
12364   return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
12365                  getVectorIdxConstant(0, DL));
12366 }
12367 
12368 void SelectionDAG::ExtractVectorElements(SDValue Op,
12369                                          SmallVectorImpl<SDValue> &Args,
12370                                          unsigned Start, unsigned Count,
12371                                          EVT EltVT) {
12372   EVT VT = Op.getValueType();
12373   if (Count == 0)
12374     Count = VT.getVectorNumElements();
12375   if (EltVT == EVT())
12376     EltVT = VT.getVectorElementType();
12377   SDLoc SL(Op);
12378   for (unsigned i = Start, e = Start + Count; i != e; ++i) {
12379     Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
12380                            getVectorIdxConstant(i, SL)));
12381   }
12382 }
12383 
12384 // getAddressSpace - Return the address space this GlobalAddress belongs to.
12385 unsigned GlobalAddressSDNode::getAddressSpace() const {
12386   return getGlobal()->getType()->getAddressSpace();
12387 }
12388 
12389 Type *ConstantPoolSDNode::getType() const {
12390   if (isMachineConstantPoolEntry())
12391     return Val.MachineCPVal->getType();
12392   return Val.ConstVal->getType();
12393 }
12394 
12395 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
12396                                         unsigned &SplatBitSize,
12397                                         bool &HasAnyUndefs,
12398                                         unsigned MinSplatBits,
12399                                         bool IsBigEndian) const {
12400   EVT VT = getValueType(0);
12401   assert(VT.isVector() && "Expected a vector type");
12402   unsigned VecWidth = VT.getSizeInBits();
12403   if (MinSplatBits > VecWidth)
12404     return false;
12405 
12406   // FIXME: The widths are based on this node's type, but build vectors can
12407   // truncate their operands.
12408   SplatValue = APInt(VecWidth, 0);
12409   SplatUndef = APInt(VecWidth, 0);
12410 
12411   // Get the bits. Bits with undefined values (when the corresponding element
12412   // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
12413   // in SplatValue. If any of the values are not constant, give up and return
12414   // false.
12415   unsigned int NumOps = getNumOperands();
12416   assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
12417   unsigned EltWidth = VT.getScalarSizeInBits();
12418 
12419   for (unsigned j = 0; j < NumOps; ++j) {
12420     unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12421     SDValue OpVal = getOperand(i);
12422     unsigned BitPos = j * EltWidth;
12423 
12424     if (OpVal.isUndef())
12425       SplatUndef.setBits(BitPos, BitPos + EltWidth);
12426     else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
12427       SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
12428     else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
12429       SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
12430     else
12431       return false;
12432   }
12433 
12434   // The build_vector is all constants or undefs. Find the smallest element
12435   // size that splats the vector.
12436   HasAnyUndefs = (SplatUndef != 0);
12437 
12438   // FIXME: This does not work for vectors with elements less than 8 bits.
12439   while (VecWidth > 8) {
12440     // If we can't split in half, stop here.
12441     if (VecWidth & 1)
12442       break;
12443 
12444     unsigned HalfSize = VecWidth / 2;
12445     APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
12446     APInt LowValue = SplatValue.extractBits(HalfSize, 0);
12447     APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
12448     APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
12449 
12450     // If the two halves do not match (ignoring undef bits), stop here.
12451     if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12452         MinSplatBits > HalfSize)
12453       break;
12454 
12455     SplatValue = HighValue | LowValue;
12456     SplatUndef = HighUndef & LowUndef;
12457 
12458     VecWidth = HalfSize;
12459   }
12460 
12461   // FIXME: The loop above only tries to split in halves. But if the input
12462   // vector for example is <3 x i16> it wouldn't be able to detect a
12463   // SplatBitSize of 16. No idea if that is a design flaw currently limiting
12464   // optimizations. I guess that back in the days when this helper was created
12465   // vectors normally was power-of-2 sized.
12466 
12467   SplatBitSize = VecWidth;
12468   return true;
12469 }
12470 
12471 SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
12472                                          BitVector *UndefElements) const {
12473   unsigned NumOps = getNumOperands();
12474   if (UndefElements) {
12475     UndefElements->clear();
12476     UndefElements->resize(NumOps);
12477   }
12478   assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12479   if (!DemandedElts)
12480     return SDValue();
12481   SDValue Splatted;
12482   for (unsigned i = 0; i != NumOps; ++i) {
12483     if (!DemandedElts[i])
12484       continue;
12485     SDValue Op = getOperand(i);
12486     if (Op.isUndef()) {
12487       if (UndefElements)
12488         (*UndefElements)[i] = true;
12489     } else if (!Splatted) {
12490       Splatted = Op;
12491     } else if (Splatted != Op) {
12492       return SDValue();
12493     }
12494   }
12495 
12496   if (!Splatted) {
12497     unsigned FirstDemandedIdx = DemandedElts.countr_zero();
12498     assert(getOperand(FirstDemandedIdx).isUndef() &&
12499            "Can only have a splat without a constant for all undefs.");
12500     return getOperand(FirstDemandedIdx);
12501   }
12502 
12503   return Splatted;
12504 }
12505 
12506 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
12507   APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12508   return getSplatValue(DemandedElts, UndefElements);
12509 }
12510 
12511 bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
12512                                             SmallVectorImpl<SDValue> &Sequence,
12513                                             BitVector *UndefElements) const {
12514   unsigned NumOps = getNumOperands();
12515   Sequence.clear();
12516   if (UndefElements) {
12517     UndefElements->clear();
12518     UndefElements->resize(NumOps);
12519   }
12520   assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12521   if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
12522     return false;
12523 
12524   // Set the undefs even if we don't find a sequence (like getSplatValue).
12525   if (UndefElements)
12526     for (unsigned I = 0; I != NumOps; ++I)
12527       if (DemandedElts[I] && getOperand(I).isUndef())
12528         (*UndefElements)[I] = true;
12529 
12530   // Iteratively widen the sequence length looking for repetitions.
12531   for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12532     Sequence.append(SeqLen, SDValue());
12533     for (unsigned I = 0; I != NumOps; ++I) {
12534       if (!DemandedElts[I])
12535         continue;
12536       SDValue &SeqOp = Sequence[I % SeqLen];
12537       SDValue Op = getOperand(I);
12538       if (Op.isUndef()) {
12539         if (!SeqOp)
12540           SeqOp = Op;
12541         continue;
12542       }
12543       if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
12544         Sequence.clear();
12545         break;
12546       }
12547       SeqOp = Op;
12548     }
12549     if (!Sequence.empty())
12550       return true;
12551   }
12552 
12553   assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
12554   return false;
12555 }
12556 
12557 bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
12558                                             BitVector *UndefElements) const {
12559   APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12560   return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
12561 }
12562 
12563 ConstantSDNode *
12564 BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
12565                                         BitVector *UndefElements) const {
12566   return dyn_cast_or_null<ConstantSDNode>(
12567       getSplatValue(DemandedElts, UndefElements));
12568 }
12569 
12570 ConstantSDNode *
12571 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
12572   return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
12573 }
12574 
12575 ConstantFPSDNode *
12576 BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
12577                                           BitVector *UndefElements) const {
12578   return dyn_cast_or_null<ConstantFPSDNode>(
12579       getSplatValue(DemandedElts, UndefElements));
12580 }
12581 
12582 ConstantFPSDNode *
12583 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
12584   return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
12585 }
12586 
12587 int32_t
12588 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
12589                                                    uint32_t BitWidth) const {
12590   if (ConstantFPSDNode *CN =
12591           dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
12592     bool IsExact;
12593     APSInt IntVal(BitWidth);
12594     const APFloat &APF = CN->getValueAPF();
12595     if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
12596             APFloat::opOK ||
12597         !IsExact)
12598       return -1;
12599 
12600     return IntVal.exactLogBase2();
12601   }
12602   return -1;
12603 }
12604 
12605 bool BuildVectorSDNode::getConstantRawBits(
12606     bool IsLittleEndian, unsigned DstEltSizeInBits,
12607     SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
12608   // Early-out if this contains anything but Undef/Constant/ConstantFP.
12609   if (!isConstant())
12610     return false;
12611 
12612   unsigned NumSrcOps = getNumOperands();
12613   unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();
12614   assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12615          "Invalid bitcast scale");
12616 
12617   // Extract raw src bits.
12618   SmallVector<APInt> SrcBitElements(NumSrcOps,
12619                                     APInt::getZero(SrcEltSizeInBits));
12620   BitVector SrcUndeElements(NumSrcOps, false);
12621 
12622   for (unsigned I = 0; I != NumSrcOps; ++I) {
12623     SDValue Op = getOperand(I);
12624     if (Op.isUndef()) {
12625       SrcUndeElements.set(I);
12626       continue;
12627     }
12628     auto *CInt = dyn_cast<ConstantSDNode>(Op);
12629     auto *CFP = dyn_cast<ConstantFPSDNode>(Op);
12630     assert((CInt || CFP) && "Unknown constant");
12631     SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
12632                              : CFP->getValueAPF().bitcastToAPInt();
12633   }
12634 
12635   // Recast to dst width.
12636   recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
12637                 SrcBitElements, UndefElements, SrcUndeElements);
12638   return true;
12639 }
12640 
12641 void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
12642                                       unsigned DstEltSizeInBits,
12643                                       SmallVectorImpl<APInt> &DstBitElements,
12644                                       ArrayRef<APInt> SrcBitElements,
12645                                       BitVector &DstUndefElements,
12646                                       const BitVector &SrcUndefElements) {
12647   unsigned NumSrcOps = SrcBitElements.size();
12648   unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12649   assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12650          "Invalid bitcast scale");
12651   assert(NumSrcOps == SrcUndefElements.size() &&
12652          "Vector size mismatch");
12653 
12654   unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
12655   DstUndefElements.clear();
12656   DstUndefElements.resize(NumDstOps, false);
12657   DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));
12658 
12659   // Concatenate src elements constant bits together into dst element.
12660   if (SrcEltSizeInBits <= DstEltSizeInBits) {
12661     unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
12662     for (unsigned I = 0; I != NumDstOps; ++I) {
12663       DstUndefElements.set(I);
12664       APInt &DstBits = DstBitElements[I];
12665       for (unsigned J = 0; J != Scale; ++J) {
12666         unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12667         if (SrcUndefElements[Idx])
12668           continue;
12669         DstUndefElements.reset(I);
12670         const APInt &SrcBits = SrcBitElements[Idx];
12671         assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
12672                "Illegal constant bitwidths");
12673         DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);
12674       }
12675     }
12676     return;
12677   }
12678 
12679   // Split src element constant bits into dst elements.
12680   unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
12681   for (unsigned I = 0; I != NumSrcOps; ++I) {
12682     if (SrcUndefElements[I]) {
12683       DstUndefElements.set(I * Scale, (I + 1) * Scale);
12684       continue;
12685     }
12686     const APInt &SrcBits = SrcBitElements[I];
12687     for (unsigned J = 0; J != Scale; ++J) {
12688       unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12689       APInt &DstBits = DstBitElements[Idx];
12690       DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
12691     }
12692   }
12693 }
12694 
12695 bool BuildVectorSDNode::isConstant() const {
12696   for (const SDValue &Op : op_values()) {
12697     unsigned Opc = Op.getOpcode();
12698     if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
12699       return false;
12700   }
12701   return true;
12702 }
12703 
12704 std::optional<std::pair<APInt, APInt>>
12705 BuildVectorSDNode::isConstantSequence() const {
12706   unsigned NumOps = getNumOperands();
12707   if (NumOps < 2)
12708     return std::nullopt;
12709 
12710   if (!isa<ConstantSDNode>(getOperand(0)) ||
12711       !isa<ConstantSDNode>(getOperand(1)))
12712     return std::nullopt;
12713 
12714   unsigned EltSize = getValueType(0).getScalarSizeInBits();
12715   APInt Start = getConstantOperandAPInt(0).trunc(EltSize);
12716   APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start;
12717 
12718   if (Stride.isZero())
12719     return std::nullopt;
12720 
12721   for (unsigned i = 2; i < NumOps; ++i) {
12722     if (!isa<ConstantSDNode>(getOperand(i)))
12723       return std::nullopt;
12724 
12725     APInt Val = getConstantOperandAPInt(i).trunc(EltSize);
12726     if (Val != (Start + (Stride * i)))
12727       return std::nullopt;
12728   }
12729 
12730   return std::make_pair(Start, Stride);
12731 }
12732 
12733 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
12734   // Find the first non-undef value in the shuffle mask.
12735   unsigned i, e;
12736   for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
12737     /* search */;
12738 
12739   // If all elements are undefined, this shuffle can be considered a splat
12740   // (although it should eventually get simplified away completely).
12741   if (i == e)
12742     return true;
12743 
12744   // Make sure all remaining elements are either undef or the same as the first
12745   // non-undef value.
12746   for (int Idx = Mask[i]; i != e; ++i)
12747     if (Mask[i] >= 0 && Mask[i] != Idx)
12748       return false;
12749   return true;
12750 }
12751 
12752 // Returns the SDNode if it is a constant integer BuildVector
12753 // or constant integer.
12754 SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
12755   if (isa<ConstantSDNode>(N))
12756     return N.getNode();
12757   if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
12758     return N.getNode();
12759   // Treat a GlobalAddress supporting constant offset folding as a
12760   // constant integer.
12761   if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
12762     if (GA->getOpcode() == ISD::GlobalAddress &&
12763         TLI->isOffsetFoldingLegal(GA))
12764       return GA;
12765   if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12766       isa<ConstantSDNode>(N.getOperand(0)))
12767     return N.getNode();
12768   return nullptr;
12769 }
12770 
12771 // Returns the SDNode if it is a constant float BuildVector
12772 // or constant float.
12773 SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
12774   if (isa<ConstantFPSDNode>(N))
12775     return N.getNode();
12776 
12777   if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
12778     return N.getNode();
12779 
12780   if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12781       isa<ConstantFPSDNode>(N.getOperand(0)))
12782     return N.getNode();
12783 
12784   return nullptr;
12785 }
12786 
12787 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
12788   assert(!Node->OperandList && "Node already has operands");
12789   assert(SDNode::getMaxNumOperands() >= Vals.size() &&
12790          "too many operands to fit into SDNode");
12791   SDUse *Ops = OperandRecycler.allocate(
12792       ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
12793 
12794   bool IsDivergent = false;
12795   for (unsigned I = 0; I != Vals.size(); ++I) {
12796     Ops[I].setUser(Node);
12797     Ops[I].setInitial(Vals[I]);
12798     if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
12799       IsDivergent |= Ops[I].getNode()->isDivergent();
12800   }
12801   Node->NumOperands = Vals.size();
12802   Node->OperandList = Ops;
12803   if (!TLI->isSDNodeAlwaysUniform(Node)) {
12804     IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
12805     Node->SDNodeBits.IsDivergent = IsDivergent;
12806   }
12807   checkForCycles(Node);
12808 }
12809 
12810 SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
12811                                      SmallVectorImpl<SDValue> &Vals) {
12812   size_t Limit = SDNode::getMaxNumOperands();
12813   while (Vals.size() > Limit) {
12814     unsigned SliceIdx = Vals.size() - Limit;
12815     auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
12816     SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
12817     Vals.erase(Vals.begin() + SliceIdx, Vals.end());
12818     Vals.emplace_back(NewTF);
12819   }
12820   return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
12821 }
12822 
12823 SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
12824                                         EVT VT, SDNodeFlags Flags) {
12825   switch (Opcode) {
12826   default:
12827     return SDValue();
12828   case ISD::ADD:
12829   case ISD::OR:
12830   case ISD::XOR:
12831   case ISD::UMAX:
12832     return getConstant(0, DL, VT);
12833   case ISD::MUL:
12834     return getConstant(1, DL, VT);
12835   case ISD::AND:
12836   case ISD::UMIN:
12837     return getAllOnesConstant(DL, VT);
12838   case ISD::SMAX:
12839     return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT);
12840   case ISD::SMIN:
12841     return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT);
12842   case ISD::FADD:
12843     return getConstantFP(-0.0, DL, VT);
12844   case ISD::FMUL:
12845     return getConstantFP(1.0, DL, VT);
12846   case ISD::FMINNUM:
12847   case ISD::FMAXNUM: {
12848     // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12849     const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12850     APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
12851                         !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
12852                         APFloat::getLargest(Semantics);
12853     if (Opcode == ISD::FMAXNUM)
12854       NeutralAF.changeSign();
12855 
12856     return getConstantFP(NeutralAF, DL, VT);
12857   }
12858   case ISD::FMINIMUM:
12859   case ISD::FMAXIMUM: {
12860     // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
12861     const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12862     APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
12863                                            : APFloat::getLargest(Semantics);
12864     if (Opcode == ISD::FMAXIMUM)
12865       NeutralAF.changeSign();
12866 
12867     return getConstantFP(NeutralAF, DL, VT);
12868   }
12869 
12870   }
12871 }
12872 
12873 /// Helper used to make a call to a library function that has one argument of
12874 /// pointer type.
12875 ///
12876 /// Such functions include 'fegetmode', 'fesetenv' and some others, which are
12877 /// used to get or set floating-point state. They have one argument of pointer
12878 /// type, which points to the memory region containing bits of the
12879 /// floating-point state. The value returned by such function is ignored in the
12880 /// created call.
12881 ///
12882 /// \param LibFunc Reference to library function (value of RTLIB::Libcall).
12883 /// \param Ptr Pointer used to save/load state.
12884 /// \param InChain Ingoing token chain.
12885 /// \returns Outgoing chain token.
12886 SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
12887                                             SDValue InChain,
12888                                             const SDLoc &DLoc) {
12889   assert(InChain.getValueType() == MVT::Other && "Expected token chain");
12890   TargetLowering::ArgListTy Args;
12891   TargetLowering::ArgListEntry Entry;
12892   Entry.Node = Ptr;
12893   Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext());
12894   Args.push_back(Entry);
12895   RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
12896   SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),
12897                                      TLI->getPointerTy(getDataLayout()));
12898   TargetLowering::CallLoweringInfo CLI(*this);
12899   CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
12900       TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee,
12901       std::move(Args));
12902   return TLI->LowerCallTo(CLI).second;
12903 }
12904 
12905 void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
12906   assert(From && To && "Invalid SDNode; empty source SDValue?");
12907   auto I = SDEI.find(From);
12908   if (I == SDEI.end())
12909     return;
12910 
12911   // Use of operator[] on the DenseMap may cause an insertion, which invalidates
12912   // the iterator, hence the need to make a copy to prevent a use-after-free.
12913   NodeExtraInfo NEI = I->second;
12914   if (LLVM_LIKELY(!NEI.PCSections)) {
12915     // No deep copy required for the types of extra info set.
12916     //
12917     // FIXME: Investigate if other types of extra info also need deep copy. This
12918     // depends on the types of nodes they can be attached to: if some extra info
12919     // is only ever attached to nodes where a replacement To node is always the
12920     // node where later use and propagation of the extra info has the intended
12921     // semantics, no deep copy is required.
12922     SDEI[To] = std::move(NEI);
12923     return;
12924   }
12925 
12926   // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
12927   // through the replacement of From with To. Otherwise, replacements of a node
12928   // (From) with more complex nodes (To and its operands) may result in lost
12929   // extra info where the root node (To) is insignificant in further propagating
12930   // and using extra info when further lowering to MIR.
12931   //
12932   // In the first step pre-populate the visited set with the nodes reachable
12933   // from the old From node. This avoids copying NodeExtraInfo to parts of the
12934   // DAG that is not new and should be left untouched.
12935   SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
12936   DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
12937   auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
12938     if (MaxDepth == 0) {
12939       // Remember this node in case we need to increase MaxDepth and continue
12940       // populating FromReach from this node.
12941       Leafs.emplace_back(N);
12942       return;
12943     }
12944     if (!FromReach.insert(N).second)
12945       return;
12946     for (const SDValue &Op : N->op_values())
12947       Self(Self, Op.getNode(), MaxDepth - 1);
12948   };
12949 
12950   // Copy extra info to To and all its transitive operands (that are new).
12951   SmallPtrSet<const SDNode *, 8> Visited;
12952   auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
12953     if (FromReach.contains(N))
12954       return true;
12955     if (!Visited.insert(N).second)
12956       return true;
12957     if (getEntryNode().getNode() == N)
12958       return false;
12959     for (const SDValue &Op : N->op_values()) {
12960       if (!Self(Self, Op.getNode()))
12961         return false;
12962     }
12963     // Copy only if entry node was not reached.
12964     SDEI[N] = NEI;
12965     return true;
12966   };
12967 
12968   // We first try with a lower MaxDepth, assuming that the path to common
12969   // operands between From and To is relatively short. This significantly
12970   // improves performance in the common case. The initial MaxDepth is big
12971   // enough to avoid retry in the common case; the last MaxDepth is large
12972   // enough to avoid having to use the fallback below (and protects from
12973   // potential stack exhaustion from recursion).
12974   for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
12975        PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
12976     // StartFrom is the previous (or initial) set of leafs reachable at the
12977     // previous maximum depth.
12978     SmallVector<const SDNode *> StartFrom;
12979     std::swap(StartFrom, Leafs);
12980     for (const SDNode *N : StartFrom)
12981       VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
12982     if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
12983       return;
12984     // This should happen very rarely (reached the entry node).
12985     LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
12986     assert(!Leafs.empty());
12987   }
12988 
12989   // This should not happen - but if it did, that means the subgraph reachable
12990   // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
12991   // could not visit all reachable common operands. Consequently, we were able
12992   // to reach the entry node.
12993   errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
12994   assert(false && "From subgraph too complex - increase max. MaxDepth?");
12995   // Best-effort fallback if assertions disabled.
12996   SDEI[To] = std::move(NEI);
12997 }
12998 
12999 #ifndef NDEBUG
13000 static void checkForCyclesHelper(const SDNode *N,
13001                                  SmallPtrSetImpl<const SDNode*> &Visited,
13002                                  SmallPtrSetImpl<const SDNode*> &Checked,
13003                                  const llvm::SelectionDAG *DAG) {
13004   // If this node has already been checked, don't check it again.
13005   if (Checked.count(N))
13006     return;
13007 
13008   // If a node has already been visited on this depth-first walk, reject it as
13009   // a cycle.
13010   if (!Visited.insert(N).second) {
13011     errs() << "Detected cycle in SelectionDAG\n";
13012     dbgs() << "Offending node:\n";
13013     N->dumprFull(DAG); dbgs() << "\n";
13014     abort();
13015   }
13016 
13017   for (const SDValue &Op : N->op_values())
13018     checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
13019 
13020   Checked.insert(N);
13021   Visited.erase(N);
13022 }
13023 #endif
13024 
13025 void llvm::checkForCycles(const llvm::SDNode *N,
13026                           const llvm::SelectionDAG *DAG,
13027                           bool force) {
13028 #ifndef NDEBUG
13029   bool check = force;
13030 #ifdef EXPENSIVE_CHECKS
13031   check = true;
13032 #endif  // EXPENSIVE_CHECKS
13033   if (check) {
13034     assert(N && "Checking nonexistent SDNode");
13035     SmallPtrSet<const SDNode*, 32> visited;
13036     SmallPtrSet<const SDNode*, 32> checked;
13037     checkForCyclesHelper(N, visited, checked, DAG);
13038   }
13039 #endif  // !NDEBUG
13040 }
13041 
13042 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13043   checkForCycles(DAG->getRoot().getNode(), DAG, force);
13044 }
13045