xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp (revision 924226fba12cc9a228c73b956e1b7fa24c60b055)
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/CodeGen/TargetLowering.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/CodeGen/CallingConvLower.h"
16 #include "llvm/CodeGen/MachineFrameInfo.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/MachineJumpTableInfo.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/SelectionDAG.h"
21 #include "llvm/CodeGen/TargetRegisterInfo.h"
22 #include "llvm/CodeGen/TargetSubtargetInfo.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalVariable.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/Support/DivisionByConstantInfo.h"
30 #include "llvm/Support/ErrorHandling.h"
31 #include "llvm/Support/KnownBits.h"
32 #include "llvm/Support/MathExtras.h"
33 #include "llvm/Target/TargetLoweringObjectFile.h"
34 #include "llvm/Target/TargetMachine.h"
35 #include <cctype>
36 using namespace llvm;
37 
38 /// NOTE: The TargetMachine owns TLOF.
39 TargetLowering::TargetLowering(const TargetMachine &tm)
40     : TargetLoweringBase(tm) {}
41 
42 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43   return nullptr;
44 }
45 
46 bool TargetLowering::isPositionIndependent() const {
47   return getTargetMachine().isPositionIndependent();
48 }
49 
50 /// Check whether a given call node is in tail position within its function. If
51 /// so, it sets Chain to the input chain of the tail call.
52 bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
53                                           SDValue &Chain) const {
54   const Function &F = DAG.getMachineFunction().getFunction();
55 
56   // First, check if tail calls have been disabled in this function.
57   if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
58     return false;
59 
60   // Conservatively require the attributes of the call to match those of
61   // the return. Ignore following attributes because they don't affect the
62   // call sequence.
63   AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
64   for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65                            Attribute::DereferenceableOrNull, Attribute::NoAlias,
66                            Attribute::NonNull, Attribute::NoUndef})
67     CallerAttrs.removeAttribute(Attr);
68 
69   if (CallerAttrs.hasAttributes())
70     return false;
71 
72   // It's not safe to eliminate the sign / zero extension of the return value.
73   if (CallerAttrs.contains(Attribute::ZExt) ||
74       CallerAttrs.contains(Attribute::SExt))
75     return false;
76 
77   // Check if the only use is a function return node.
78   return isUsedByReturnOnly(Node, Chain);
79 }
80 
81 bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
82     const uint32_t *CallerPreservedMask,
83     const SmallVectorImpl<CCValAssign> &ArgLocs,
84     const SmallVectorImpl<SDValue> &OutVals) const {
85   for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
86     const CCValAssign &ArgLoc = ArgLocs[I];
87     if (!ArgLoc.isRegLoc())
88       continue;
89     MCRegister Reg = ArgLoc.getLocReg();
90     // Only look at callee saved registers.
91     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
92       continue;
93     // Check that we pass the value used for the caller.
94     // (We look for a CopyFromReg reading a virtual register that is used
95     //  for the function live-in value of register Reg)
96     SDValue Value = OutVals[I];
97     if (Value->getOpcode() != ISD::CopyFromReg)
98       return false;
99     Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
100     if (MRI.getLiveInPhysReg(ArgReg) != Reg)
101       return false;
102   }
103   return true;
104 }
105 
106 /// Set CallLoweringInfo attribute flags based on a call instruction
107 /// and called function attributes.
108 void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
109                                                      unsigned ArgIdx) {
110   IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
111   IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
112   IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
113   IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
114   IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
115   IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
116   IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
117   IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
118   IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
119   IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
120   IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
121   IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
122   Alignment = Call->getParamStackAlign(ArgIdx);
123   IndirectType = nullptr;
124   assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&
125          "multiple ABI attributes?");
126   if (IsByVal) {
127     IndirectType = Call->getParamByValType(ArgIdx);
128     if (!Alignment)
129       Alignment = Call->getParamAlign(ArgIdx);
130   }
131   if (IsPreallocated)
132     IndirectType = Call->getParamPreallocatedType(ArgIdx);
133   if (IsInAlloca)
134     IndirectType = Call->getParamInAllocaType(ArgIdx);
135 }
136 
137 /// Generate a libcall taking the given operands as arguments and returning a
138 /// result of type RetVT.
139 std::pair<SDValue, SDValue>
140 TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
141                             ArrayRef<SDValue> Ops,
142                             MakeLibCallOptions CallOptions,
143                             const SDLoc &dl,
144                             SDValue InChain) const {
145   if (!InChain)
146     InChain = DAG.getEntryNode();
147 
148   TargetLowering::ArgListTy Args;
149   Args.reserve(Ops.size());
150 
151   TargetLowering::ArgListEntry Entry;
152   for (unsigned i = 0; i < Ops.size(); ++i) {
153     SDValue NewOp = Ops[i];
154     Entry.Node = NewOp;
155     Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
156     Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
157                                                  CallOptions.IsSExt);
158     Entry.IsZExt = !Entry.IsSExt;
159 
160     if (CallOptions.IsSoften &&
161         !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
162       Entry.IsSExt = Entry.IsZExt = false;
163     }
164     Args.push_back(Entry);
165   }
166 
167   if (LC == RTLIB::UNKNOWN_LIBCALL)
168     report_fatal_error("Unsupported library call operation!");
169   SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
170                                          getPointerTy(DAG.getDataLayout()));
171 
172   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
173   TargetLowering::CallLoweringInfo CLI(DAG);
174   bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
175   bool zeroExtend = !signExtend;
176 
177   if (CallOptions.IsSoften &&
178       !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
179     signExtend = zeroExtend = false;
180   }
181 
182   CLI.setDebugLoc(dl)
183       .setChain(InChain)
184       .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
185       .setNoReturn(CallOptions.DoesNotReturn)
186       .setDiscardResult(!CallOptions.IsReturnValueUsed)
187       .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
188       .setSExtResult(signExtend)
189       .setZExtResult(zeroExtend);
190   return LowerCallTo(CLI);
191 }
192 
193 bool TargetLowering::findOptimalMemOpLowering(
194     std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
195     unsigned SrcAS, const AttributeList &FuncAttributes) const {
196   if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
197     return false;
198 
199   EVT VT = getOptimalMemOpType(Op, FuncAttributes);
200 
201   if (VT == MVT::Other) {
202     // Use the largest integer type whose alignment constraints are satisfied.
203     // We only need to check DstAlign here as SrcAlign is always greater or
204     // equal to DstAlign (or zero).
205     VT = MVT::i64;
206     if (Op.isFixedDstAlign())
207       while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
208              !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
209         VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
210     assert(VT.isInteger());
211 
212     // Find the largest legal integer type.
213     MVT LVT = MVT::i64;
214     while (!isTypeLegal(LVT))
215       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
216     assert(LVT.isInteger());
217 
218     // If the type we've chosen is larger than the largest legal integer type
219     // then use that instead.
220     if (VT.bitsGT(LVT))
221       VT = LVT;
222   }
223 
224   unsigned NumMemOps = 0;
225   uint64_t Size = Op.size();
226   while (Size) {
227     unsigned VTSize = VT.getSizeInBits() / 8;
228     while (VTSize > Size) {
229       // For now, only use non-vector load / store's for the left-over pieces.
230       EVT NewVT = VT;
231       unsigned NewVTSize;
232 
233       bool Found = false;
234       if (VT.isVector() || VT.isFloatingPoint()) {
235         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
236         if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
237             isSafeMemOpType(NewVT.getSimpleVT()))
238           Found = true;
239         else if (NewVT == MVT::i64 &&
240                  isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
241                  isSafeMemOpType(MVT::f64)) {
242           // i64 is usually not legal on 32-bit targets, but f64 may be.
243           NewVT = MVT::f64;
244           Found = true;
245         }
246       }
247 
248       if (!Found) {
249         do {
250           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
251           if (NewVT == MVT::i8)
252             break;
253         } while (!isSafeMemOpType(NewVT.getSimpleVT()));
254       }
255       NewVTSize = NewVT.getSizeInBits() / 8;
256 
257       // If the new VT cannot cover all of the remaining bits, then consider
258       // issuing a (or a pair of) unaligned and overlapping load / store.
259       bool Fast;
260       if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
261           allowsMisalignedMemoryAccesses(
262               VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
263               MachineMemOperand::MONone, &Fast) &&
264           Fast)
265         VTSize = Size;
266       else {
267         VT = NewVT;
268         VTSize = NewVTSize;
269       }
270     }
271 
272     if (++NumMemOps > Limit)
273       return false;
274 
275     MemOps.push_back(VT);
276     Size -= VTSize;
277   }
278 
279   return true;
280 }
281 
282 /// Soften the operands of a comparison. This code is shared among BR_CC,
283 /// SELECT_CC, and SETCC handlers.
284 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
285                                          SDValue &NewLHS, SDValue &NewRHS,
286                                          ISD::CondCode &CCCode,
287                                          const SDLoc &dl, const SDValue OldLHS,
288                                          const SDValue OldRHS) const {
289   SDValue Chain;
290   return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
291                              OldRHS, Chain);
292 }
293 
294 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
295                                          SDValue &NewLHS, SDValue &NewRHS,
296                                          ISD::CondCode &CCCode,
297                                          const SDLoc &dl, const SDValue OldLHS,
298                                          const SDValue OldRHS,
299                                          SDValue &Chain,
300                                          bool IsSignaling) const {
301   // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
302   // not supporting it. We can update this code when libgcc provides such
303   // functions.
304 
305   assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
306          && "Unsupported setcc type!");
307 
308   // Expand into one or more soft-fp libcall(s).
309   RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
310   bool ShouldInvertCC = false;
311   switch (CCCode) {
312   case ISD::SETEQ:
313   case ISD::SETOEQ:
314     LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
315           (VT == MVT::f64) ? RTLIB::OEQ_F64 :
316           (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
317     break;
318   case ISD::SETNE:
319   case ISD::SETUNE:
320     LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
321           (VT == MVT::f64) ? RTLIB::UNE_F64 :
322           (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
323     break;
324   case ISD::SETGE:
325   case ISD::SETOGE:
326     LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
327           (VT == MVT::f64) ? RTLIB::OGE_F64 :
328           (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
329     break;
330   case ISD::SETLT:
331   case ISD::SETOLT:
332     LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
333           (VT == MVT::f64) ? RTLIB::OLT_F64 :
334           (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
335     break;
336   case ISD::SETLE:
337   case ISD::SETOLE:
338     LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
339           (VT == MVT::f64) ? RTLIB::OLE_F64 :
340           (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
341     break;
342   case ISD::SETGT:
343   case ISD::SETOGT:
344     LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
345           (VT == MVT::f64) ? RTLIB::OGT_F64 :
346           (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
347     break;
348   case ISD::SETO:
349     ShouldInvertCC = true;
350     LLVM_FALLTHROUGH;
351   case ISD::SETUO:
352     LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
353           (VT == MVT::f64) ? RTLIB::UO_F64 :
354           (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
355     break;
356   case ISD::SETONE:
357     // SETONE = O && UNE
358     ShouldInvertCC = true;
359     LLVM_FALLTHROUGH;
360   case ISD::SETUEQ:
361     LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
362           (VT == MVT::f64) ? RTLIB::UO_F64 :
363           (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
364     LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
365           (VT == MVT::f64) ? RTLIB::OEQ_F64 :
366           (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
367     break;
368   default:
369     // Invert CC for unordered comparisons
370     ShouldInvertCC = true;
371     switch (CCCode) {
372     case ISD::SETULT:
373       LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
374             (VT == MVT::f64) ? RTLIB::OGE_F64 :
375             (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
376       break;
377     case ISD::SETULE:
378       LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
379             (VT == MVT::f64) ? RTLIB::OGT_F64 :
380             (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
381       break;
382     case ISD::SETUGT:
383       LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
384             (VT == MVT::f64) ? RTLIB::OLE_F64 :
385             (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
386       break;
387     case ISD::SETUGE:
388       LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
389             (VT == MVT::f64) ? RTLIB::OLT_F64 :
390             (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
391       break;
392     default: llvm_unreachable("Do not know how to soften this setcc!");
393     }
394   }
395 
396   // Use the target specific return value for comparions lib calls.
397   EVT RetVT = getCmpLibcallReturnType();
398   SDValue Ops[2] = {NewLHS, NewRHS};
399   TargetLowering::MakeLibCallOptions CallOptions;
400   EVT OpsVT[2] = { OldLHS.getValueType(),
401                    OldRHS.getValueType() };
402   CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
403   auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
404   NewLHS = Call.first;
405   NewRHS = DAG.getConstant(0, dl, RetVT);
406 
407   CCCode = getCmpLibcallCC(LC1);
408   if (ShouldInvertCC) {
409     assert(RetVT.isInteger());
410     CCCode = getSetCCInverse(CCCode, RetVT);
411   }
412 
413   if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
414     // Update Chain.
415     Chain = Call.second;
416   } else {
417     EVT SetCCVT =
418         getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
419     SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
420     auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
421     CCCode = getCmpLibcallCC(LC2);
422     if (ShouldInvertCC)
423       CCCode = getSetCCInverse(CCCode, RetVT);
424     NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
425     if (Chain)
426       Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
427                           Call2.second);
428     NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
429                          Tmp.getValueType(), Tmp, NewLHS);
430     NewRHS = SDValue();
431   }
432 }
433 
434 /// Return the entry encoding for a jump table in the current function. The
435 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
436 unsigned TargetLowering::getJumpTableEncoding() const {
437   // In non-pic modes, just use the address of a block.
438   if (!isPositionIndependent())
439     return MachineJumpTableInfo::EK_BlockAddress;
440 
441   // In PIC mode, if the target supports a GPRel32 directive, use it.
442   if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
443     return MachineJumpTableInfo::EK_GPRel32BlockAddress;
444 
445   // Otherwise, use a label difference.
446   return MachineJumpTableInfo::EK_LabelDifference32;
447 }
448 
449 SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
450                                                  SelectionDAG &DAG) const {
451   // If our PIC model is GP relative, use the global offset table as the base.
452   unsigned JTEncoding = getJumpTableEncoding();
453 
454   if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
455       (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
456     return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
457 
458   return Table;
459 }
460 
461 /// This returns the relocation base for the given PIC jumptable, the same as
462 /// getPICJumpTableRelocBase, but as an MCExpr.
463 const MCExpr *
464 TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
465                                              unsigned JTI,MCContext &Ctx) const{
466   // The normal PIC reloc base is the label at the start of the jump table.
467   return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
468 }
469 
470 bool
471 TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
472   const TargetMachine &TM = getTargetMachine();
473   const GlobalValue *GV = GA->getGlobal();
474 
475   // If the address is not even local to this DSO we will have to load it from
476   // a got and then add the offset.
477   if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
478     return false;
479 
480   // If the code is position independent we will have to add a base register.
481   if (isPositionIndependent())
482     return false;
483 
484   // Otherwise we can do it.
485   return true;
486 }
487 
488 //===----------------------------------------------------------------------===//
489 //  Optimization Methods
490 //===----------------------------------------------------------------------===//
491 
492 /// If the specified instruction has a constant integer operand and there are
493 /// bits set in that constant that are not demanded, then clear those bits and
494 /// return true.
495 bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
496                                             const APInt &DemandedBits,
497                                             const APInt &DemandedElts,
498                                             TargetLoweringOpt &TLO) const {
499   SDLoc DL(Op);
500   unsigned Opcode = Op.getOpcode();
501 
502   // Do target-specific constant optimization.
503   if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
504     return TLO.New.getNode();
505 
506   // FIXME: ISD::SELECT, ISD::SELECT_CC
507   switch (Opcode) {
508   default:
509     break;
510   case ISD::XOR:
511   case ISD::AND:
512   case ISD::OR: {
513     auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
514     if (!Op1C || Op1C->isOpaque())
515       return false;
516 
517     // If this is a 'not' op, don't touch it because that's a canonical form.
518     const APInt &C = Op1C->getAPIntValue();
519     if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
520       return false;
521 
522     if (!C.isSubsetOf(DemandedBits)) {
523       EVT VT = Op.getValueType();
524       SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
525       SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
526       return TLO.CombineTo(Op, NewOp);
527     }
528 
529     break;
530   }
531   }
532 
533   return false;
534 }
535 
536 bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
537                                             const APInt &DemandedBits,
538                                             TargetLoweringOpt &TLO) const {
539   EVT VT = Op.getValueType();
540   APInt DemandedElts = VT.isVector()
541                            ? APInt::getAllOnes(VT.getVectorNumElements())
542                            : APInt(1, 1);
543   return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
544 }
545 
546 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
547 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
548 /// generalized for targets with other types of implicit widening casts.
549 bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
550                                       const APInt &Demanded,
551                                       TargetLoweringOpt &TLO) const {
552   assert(Op.getNumOperands() == 2 &&
553          "ShrinkDemandedOp only supports binary operators!");
554   assert(Op.getNode()->getNumValues() == 1 &&
555          "ShrinkDemandedOp only supports nodes with one result!");
556 
557   SelectionDAG &DAG = TLO.DAG;
558   SDLoc dl(Op);
559 
560   // Early return, as this function cannot handle vector types.
561   if (Op.getValueType().isVector())
562     return false;
563 
564   // Don't do this if the node has another user, which may require the
565   // full value.
566   if (!Op.getNode()->hasOneUse())
567     return false;
568 
569   // Search for the smallest integer type with free casts to and from
570   // Op's type. For expedience, just check power-of-2 integer types.
571   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
572   unsigned DemandedSize = Demanded.getActiveBits();
573   unsigned SmallVTBits = DemandedSize;
574   if (!isPowerOf2_32(SmallVTBits))
575     SmallVTBits = NextPowerOf2(SmallVTBits);
576   for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
577     EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
578     if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
579         TLI.isZExtFree(SmallVT, Op.getValueType())) {
580       // We found a type with free casts.
581       SDValue X = DAG.getNode(
582           Op.getOpcode(), dl, SmallVT,
583           DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
584           DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
585       assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
586       SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
587       return TLO.CombineTo(Op, Z);
588     }
589   }
590   return false;
591 }
592 
593 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
594                                           DAGCombinerInfo &DCI) const {
595   SelectionDAG &DAG = DCI.DAG;
596   TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
597                         !DCI.isBeforeLegalizeOps());
598   KnownBits Known;
599 
600   bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
601   if (Simplified) {
602     DCI.AddToWorklist(Op.getNode());
603     DCI.CommitTargetLoweringOpt(TLO);
604   }
605   return Simplified;
606 }
607 
608 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
609                                           const APInt &DemandedElts,
610                                           DAGCombinerInfo &DCI) const {
611   SelectionDAG &DAG = DCI.DAG;
612   TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
613                         !DCI.isBeforeLegalizeOps());
614   KnownBits Known;
615 
616   bool Simplified =
617       SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
618   if (Simplified) {
619     DCI.AddToWorklist(Op.getNode());
620     DCI.CommitTargetLoweringOpt(TLO);
621   }
622   return Simplified;
623 }
624 
625 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
626                                           KnownBits &Known,
627                                           TargetLoweringOpt &TLO,
628                                           unsigned Depth,
629                                           bool AssumeSingleUse) const {
630   EVT VT = Op.getValueType();
631 
632   // TODO: We can probably do more work on calculating the known bits and
633   // simplifying the operations for scalable vectors, but for now we just
634   // bail out.
635   if (VT.isScalableVector()) {
636     // Pretend we don't know anything for now.
637     Known = KnownBits(DemandedBits.getBitWidth());
638     return false;
639   }
640 
641   APInt DemandedElts = VT.isVector()
642                            ? APInt::getAllOnes(VT.getVectorNumElements())
643                            : APInt(1, 1);
644   return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
645                               AssumeSingleUse);
646 }
647 
648 // TODO: Can we merge SelectionDAG::GetDemandedBits into this?
649 // TODO: Under what circumstances can we create nodes? Constant folding?
650 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
651     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
652     SelectionDAG &DAG, unsigned Depth) const {
653   // Limit search depth.
654   if (Depth >= SelectionDAG::MaxRecursionDepth)
655     return SDValue();
656 
657   // Ignore UNDEFs.
658   if (Op.isUndef())
659     return SDValue();
660 
661   // Not demanding any bits/elts from Op.
662   if (DemandedBits == 0 || DemandedElts == 0)
663     return DAG.getUNDEF(Op.getValueType());
664 
665   bool IsLE = DAG.getDataLayout().isLittleEndian();
666   unsigned NumElts = DemandedElts.getBitWidth();
667   unsigned BitWidth = DemandedBits.getBitWidth();
668   KnownBits LHSKnown, RHSKnown;
669   switch (Op.getOpcode()) {
670   case ISD::BITCAST: {
671     SDValue Src = peekThroughBitcasts(Op.getOperand(0));
672     EVT SrcVT = Src.getValueType();
673     EVT DstVT = Op.getValueType();
674     if (SrcVT == DstVT)
675       return Src;
676 
677     unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
678     unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
679     if (NumSrcEltBits == NumDstEltBits)
680       if (SDValue V = SimplifyMultipleUseDemandedBits(
681               Src, DemandedBits, DemandedElts, DAG, Depth + 1))
682         return DAG.getBitcast(DstVT, V);
683 
684     if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
685       unsigned Scale = NumDstEltBits / NumSrcEltBits;
686       unsigned NumSrcElts = SrcVT.getVectorNumElements();
687       APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
688       APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
689       for (unsigned i = 0; i != Scale; ++i) {
690         unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
691         unsigned BitOffset = EltOffset * NumSrcEltBits;
692         APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
693         if (!Sub.isZero()) {
694           DemandedSrcBits |= Sub;
695           for (unsigned j = 0; j != NumElts; ++j)
696             if (DemandedElts[j])
697               DemandedSrcElts.setBit((j * Scale) + i);
698         }
699       }
700 
701       if (SDValue V = SimplifyMultipleUseDemandedBits(
702               Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
703         return DAG.getBitcast(DstVT, V);
704     }
705 
706     // TODO - bigendian once we have test coverage.
707     if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
708       unsigned Scale = NumSrcEltBits / NumDstEltBits;
709       unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
710       APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
711       APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
712       for (unsigned i = 0; i != NumElts; ++i)
713         if (DemandedElts[i]) {
714           unsigned Offset = (i % Scale) * NumDstEltBits;
715           DemandedSrcBits.insertBits(DemandedBits, Offset);
716           DemandedSrcElts.setBit(i / Scale);
717         }
718 
719       if (SDValue V = SimplifyMultipleUseDemandedBits(
720               Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
721         return DAG.getBitcast(DstVT, V);
722     }
723 
724     break;
725   }
726   case ISD::AND: {
727     LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
728     RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
729 
730     // If all of the demanded bits are known 1 on one side, return the other.
731     // These bits cannot contribute to the result of the 'and' in this
732     // context.
733     if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
734       return Op.getOperand(0);
735     if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
736       return Op.getOperand(1);
737     break;
738   }
739   case ISD::OR: {
740     LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
741     RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
742 
743     // If all of the demanded bits are known zero on one side, return the
744     // other.  These bits cannot contribute to the result of the 'or' in this
745     // context.
746     if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
747       return Op.getOperand(0);
748     if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
749       return Op.getOperand(1);
750     break;
751   }
752   case ISD::XOR: {
753     LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
754     RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
755 
756     // If all of the demanded bits are known zero on one side, return the
757     // other.
758     if (DemandedBits.isSubsetOf(RHSKnown.Zero))
759       return Op.getOperand(0);
760     if (DemandedBits.isSubsetOf(LHSKnown.Zero))
761       return Op.getOperand(1);
762     break;
763   }
764   case ISD::SHL: {
765     // If we are only demanding sign bits then we can use the shift source
766     // directly.
767     if (const APInt *MaxSA =
768             DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
769       SDValue Op0 = Op.getOperand(0);
770       unsigned ShAmt = MaxSA->getZExtValue();
771       unsigned NumSignBits =
772           DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
773       unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
774       if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
775         return Op0;
776     }
777     break;
778   }
779   case ISD::SETCC: {
780     SDValue Op0 = Op.getOperand(0);
781     SDValue Op1 = Op.getOperand(1);
782     ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
783     // If (1) we only need the sign-bit, (2) the setcc operands are the same
784     // width as the setcc result, and (3) the result of a setcc conforms to 0 or
785     // -1, we may be able to bypass the setcc.
786     if (DemandedBits.isSignMask() &&
787         Op0.getScalarValueSizeInBits() == BitWidth &&
788         getBooleanContents(Op0.getValueType()) ==
789             BooleanContent::ZeroOrNegativeOneBooleanContent) {
790       // If we're testing X < 0, then this compare isn't needed - just use X!
791       // FIXME: We're limiting to integer types here, but this should also work
792       // if we don't care about FP signed-zero. The use of SETLT with FP means
793       // that we don't care about NaNs.
794       if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
795           (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
796         return Op0;
797     }
798     break;
799   }
800   case ISD::SIGN_EXTEND_INREG: {
801     // If none of the extended bits are demanded, eliminate the sextinreg.
802     SDValue Op0 = Op.getOperand(0);
803     EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
804     unsigned ExBits = ExVT.getScalarSizeInBits();
805     if (DemandedBits.getActiveBits() <= ExBits)
806       return Op0;
807     // If the input is already sign extended, just drop the extension.
808     unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
809     if (NumSignBits >= (BitWidth - ExBits + 1))
810       return Op0;
811     break;
812   }
813   case ISD::ANY_EXTEND_VECTOR_INREG:
814   case ISD::SIGN_EXTEND_VECTOR_INREG:
815   case ISD::ZERO_EXTEND_VECTOR_INREG: {
816     // If we only want the lowest element and none of extended bits, then we can
817     // return the bitcasted source vector.
818     SDValue Src = Op.getOperand(0);
819     EVT SrcVT = Src.getValueType();
820     EVT DstVT = Op.getValueType();
821     if (IsLE && DemandedElts == 1 &&
822         DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
823         DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
824       return DAG.getBitcast(DstVT, Src);
825     }
826     break;
827   }
828   case ISD::INSERT_VECTOR_ELT: {
829     // If we don't demand the inserted element, return the base vector.
830     SDValue Vec = Op.getOperand(0);
831     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
832     EVT VecVT = Vec.getValueType();
833     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
834         !DemandedElts[CIdx->getZExtValue()])
835       return Vec;
836     break;
837   }
838   case ISD::INSERT_SUBVECTOR: {
839     SDValue Vec = Op.getOperand(0);
840     SDValue Sub = Op.getOperand(1);
841     uint64_t Idx = Op.getConstantOperandVal(2);
842     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
843     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
844     // If we don't demand the inserted subvector, return the base vector.
845     if (DemandedSubElts == 0)
846       return Vec;
847     // If this simply widens the lowest subvector, see if we can do it earlier.
848     if (Idx == 0 && Vec.isUndef()) {
849       if (SDValue NewSub = SimplifyMultipleUseDemandedBits(
850               Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))
851         return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
852                            Op.getOperand(0), NewSub, Op.getOperand(2));
853     }
854     break;
855   }
856   case ISD::VECTOR_SHUFFLE: {
857     ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
858 
859     // If all the demanded elts are from one operand and are inline,
860     // then we can use the operand directly.
861     bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
862     for (unsigned i = 0; i != NumElts; ++i) {
863       int M = ShuffleMask[i];
864       if (M < 0 || !DemandedElts[i])
865         continue;
866       AllUndef = false;
867       IdentityLHS &= (M == (int)i);
868       IdentityRHS &= ((M - NumElts) == i);
869     }
870 
871     if (AllUndef)
872       return DAG.getUNDEF(Op.getValueType());
873     if (IdentityLHS)
874       return Op.getOperand(0);
875     if (IdentityRHS)
876       return Op.getOperand(1);
877     break;
878   }
879   default:
880     if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
881       if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
882               Op, DemandedBits, DemandedElts, DAG, Depth))
883         return V;
884     break;
885   }
886   return SDValue();
887 }
888 
889 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
890     SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
891     unsigned Depth) const {
892   EVT VT = Op.getValueType();
893   APInt DemandedElts = VT.isVector()
894                            ? APInt::getAllOnes(VT.getVectorNumElements())
895                            : APInt(1, 1);
896   return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
897                                          Depth);
898 }
899 
900 SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
901     SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
902     unsigned Depth) const {
903   APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
904   return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
905                                          Depth);
906 }
907 
908 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
909 /// result of Op are ever used downstream. If we can use this information to
910 /// simplify Op, create a new simplified DAG node and return true, returning the
911 /// original and new nodes in Old and New. Otherwise, analyze the expression and
912 /// return a mask of Known bits for the expression (used to simplify the
913 /// caller).  The Known bits may only be accurate for those bits in the
914 /// OriginalDemandedBits and OriginalDemandedElts.
915 bool TargetLowering::SimplifyDemandedBits(
916     SDValue Op, const APInt &OriginalDemandedBits,
917     const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
918     unsigned Depth, bool AssumeSingleUse) const {
919   unsigned BitWidth = OriginalDemandedBits.getBitWidth();
920   assert(Op.getScalarValueSizeInBits() == BitWidth &&
921          "Mask size mismatches value type size!");
922 
923   // Don't know anything.
924   Known = KnownBits(BitWidth);
925 
926   // TODO: We can probably do more work on calculating the known bits and
927   // simplifying the operations for scalable vectors, but for now we just
928   // bail out.
929   if (Op.getValueType().isScalableVector())
930     return false;
931 
932   bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
933   unsigned NumElts = OriginalDemandedElts.getBitWidth();
934   assert((!Op.getValueType().isVector() ||
935           NumElts == Op.getValueType().getVectorNumElements()) &&
936          "Unexpected vector size");
937 
938   APInt DemandedBits = OriginalDemandedBits;
939   APInt DemandedElts = OriginalDemandedElts;
940   SDLoc dl(Op);
941   auto &DL = TLO.DAG.getDataLayout();
942 
943   // Undef operand.
944   if (Op.isUndef())
945     return false;
946 
947   if (Op.getOpcode() == ISD::Constant) {
948     // We know all of the bits for a constant!
949     Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
950     return false;
951   }
952 
953   if (Op.getOpcode() == ISD::ConstantFP) {
954     // We know all of the bits for a floating point constant!
955     Known = KnownBits::makeConstant(
956         cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
957     return false;
958   }
959 
960   // Other users may use these bits.
961   EVT VT = Op.getValueType();
962   if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
963     if (Depth != 0) {
964       // If not at the root, Just compute the Known bits to
965       // simplify things downstream.
966       Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
967       return false;
968     }
969     // If this is the root being simplified, allow it to have multiple uses,
970     // just set the DemandedBits/Elts to all bits.
971     DemandedBits = APInt::getAllOnes(BitWidth);
972     DemandedElts = APInt::getAllOnes(NumElts);
973   } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
974     // Not demanding any bits/elts from Op.
975     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
976   } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
977     // Limit search depth.
978     return false;
979   }
980 
981   KnownBits Known2;
982   switch (Op.getOpcode()) {
983   case ISD::TargetConstant:
984     llvm_unreachable("Can't simplify this node");
985   case ISD::SCALAR_TO_VECTOR: {
986     if (!DemandedElts[0])
987       return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
988 
989     KnownBits SrcKnown;
990     SDValue Src = Op.getOperand(0);
991     unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
992     APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
993     if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
994       return true;
995 
996     // Upper elements are undef, so only get the knownbits if we just demand
997     // the bottom element.
998     if (DemandedElts == 1)
999       Known = SrcKnown.anyextOrTrunc(BitWidth);
1000     break;
1001   }
1002   case ISD::BUILD_VECTOR:
1003     // Collect the known bits that are shared by every demanded element.
1004     // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1005     Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1006     return false; // Don't fall through, will infinitely loop.
1007   case ISD::LOAD: {
1008     auto *LD = cast<LoadSDNode>(Op);
1009     if (getTargetConstantFromLoad(LD)) {
1010       Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1011       return false; // Don't fall through, will infinitely loop.
1012     }
1013     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
1014       // If this is a ZEXTLoad and we are looking at the loaded value.
1015       EVT MemVT = LD->getMemoryVT();
1016       unsigned MemBits = MemVT.getScalarSizeInBits();
1017       Known.Zero.setBitsFrom(MemBits);
1018       return false; // Don't fall through, will infinitely loop.
1019     }
1020     break;
1021   }
1022   case ISD::INSERT_VECTOR_ELT: {
1023     SDValue Vec = Op.getOperand(0);
1024     SDValue Scl = Op.getOperand(1);
1025     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1026     EVT VecVT = Vec.getValueType();
1027 
1028     // If index isn't constant, assume we need all vector elements AND the
1029     // inserted element.
1030     APInt DemandedVecElts(DemandedElts);
1031     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1032       unsigned Idx = CIdx->getZExtValue();
1033       DemandedVecElts.clearBit(Idx);
1034 
1035       // Inserted element is not required.
1036       if (!DemandedElts[Idx])
1037         return TLO.CombineTo(Op, Vec);
1038     }
1039 
1040     KnownBits KnownScl;
1041     unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1042     APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1043     if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1044       return true;
1045 
1046     Known = KnownScl.anyextOrTrunc(BitWidth);
1047 
1048     KnownBits KnownVec;
1049     if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1050                              Depth + 1))
1051       return true;
1052 
1053     if (!!DemandedVecElts)
1054       Known = KnownBits::commonBits(Known, KnownVec);
1055 
1056     return false;
1057   }
1058   case ISD::INSERT_SUBVECTOR: {
1059     // Demand any elements from the subvector and the remainder from the src its
1060     // inserted into.
1061     SDValue Src = Op.getOperand(0);
1062     SDValue Sub = Op.getOperand(1);
1063     uint64_t Idx = Op.getConstantOperandVal(2);
1064     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1065     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1066     APInt DemandedSrcElts = DemandedElts;
1067     DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
1068 
1069     KnownBits KnownSub, KnownSrc;
1070     if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1071                              Depth + 1))
1072       return true;
1073     if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1074                              Depth + 1))
1075       return true;
1076 
1077     Known.Zero.setAllBits();
1078     Known.One.setAllBits();
1079     if (!!DemandedSubElts)
1080       Known = KnownBits::commonBits(Known, KnownSub);
1081     if (!!DemandedSrcElts)
1082       Known = KnownBits::commonBits(Known, KnownSrc);
1083 
1084     // Attempt to avoid multi-use src if we don't need anything from it.
1085     if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1086         !DemandedSrcElts.isAllOnes()) {
1087       SDValue NewSub = SimplifyMultipleUseDemandedBits(
1088           Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1089       SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1090           Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1091       if (NewSub || NewSrc) {
1092         NewSub = NewSub ? NewSub : Sub;
1093         NewSrc = NewSrc ? NewSrc : Src;
1094         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1095                                         Op.getOperand(2));
1096         return TLO.CombineTo(Op, NewOp);
1097       }
1098     }
1099     break;
1100   }
1101   case ISD::EXTRACT_SUBVECTOR: {
1102     // Offset the demanded elts by the subvector index.
1103     SDValue Src = Op.getOperand(0);
1104     if (Src.getValueType().isScalableVector())
1105       break;
1106     uint64_t Idx = Op.getConstantOperandVal(1);
1107     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1108     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1109 
1110     if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1111                              Depth + 1))
1112       return true;
1113 
1114     // Attempt to avoid multi-use src if we don't need anything from it.
1115     if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1116       SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1117           Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1118       if (DemandedSrc) {
1119         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1120                                         Op.getOperand(1));
1121         return TLO.CombineTo(Op, NewOp);
1122       }
1123     }
1124     break;
1125   }
1126   case ISD::CONCAT_VECTORS: {
1127     Known.Zero.setAllBits();
1128     Known.One.setAllBits();
1129     EVT SubVT = Op.getOperand(0).getValueType();
1130     unsigned NumSubVecs = Op.getNumOperands();
1131     unsigned NumSubElts = SubVT.getVectorNumElements();
1132     for (unsigned i = 0; i != NumSubVecs; ++i) {
1133       APInt DemandedSubElts =
1134           DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1135       if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1136                                Known2, TLO, Depth + 1))
1137         return true;
1138       // Known bits are shared by every demanded subvector element.
1139       if (!!DemandedSubElts)
1140         Known = KnownBits::commonBits(Known, Known2);
1141     }
1142     break;
1143   }
1144   case ISD::VECTOR_SHUFFLE: {
1145     ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1146 
1147     // Collect demanded elements from shuffle operands..
1148     APInt DemandedLHS(NumElts, 0);
1149     APInt DemandedRHS(NumElts, 0);
1150     for (unsigned i = 0; i != NumElts; ++i) {
1151       if (!DemandedElts[i])
1152         continue;
1153       int M = ShuffleMask[i];
1154       if (M < 0) {
1155         // For UNDEF elements, we don't know anything about the common state of
1156         // the shuffle result.
1157         DemandedLHS.clearAllBits();
1158         DemandedRHS.clearAllBits();
1159         break;
1160       }
1161       assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1162       if (M < (int)NumElts)
1163         DemandedLHS.setBit(M);
1164       else
1165         DemandedRHS.setBit(M - NumElts);
1166     }
1167 
1168     if (!!DemandedLHS || !!DemandedRHS) {
1169       SDValue Op0 = Op.getOperand(0);
1170       SDValue Op1 = Op.getOperand(1);
1171 
1172       Known.Zero.setAllBits();
1173       Known.One.setAllBits();
1174       if (!!DemandedLHS) {
1175         if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1176                                  Depth + 1))
1177           return true;
1178         Known = KnownBits::commonBits(Known, Known2);
1179       }
1180       if (!!DemandedRHS) {
1181         if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1182                                  Depth + 1))
1183           return true;
1184         Known = KnownBits::commonBits(Known, Known2);
1185       }
1186 
1187       // Attempt to avoid multi-use ops if we don't need anything from them.
1188       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1189           Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1190       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1191           Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1192       if (DemandedOp0 || DemandedOp1) {
1193         Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1194         Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1195         SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1196         return TLO.CombineTo(Op, NewOp);
1197       }
1198     }
1199     break;
1200   }
1201   case ISD::AND: {
1202     SDValue Op0 = Op.getOperand(0);
1203     SDValue Op1 = Op.getOperand(1);
1204 
1205     // If the RHS is a constant, check to see if the LHS would be zero without
1206     // using the bits from the RHS.  Below, we use knowledge about the RHS to
1207     // simplify the LHS, here we're using information from the LHS to simplify
1208     // the RHS.
1209     if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1210       // Do not increment Depth here; that can cause an infinite loop.
1211       KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1212       // If the LHS already has zeros where RHSC does, this 'and' is dead.
1213       if ((LHSKnown.Zero & DemandedBits) ==
1214           (~RHSC->getAPIntValue() & DemandedBits))
1215         return TLO.CombineTo(Op, Op0);
1216 
1217       // If any of the set bits in the RHS are known zero on the LHS, shrink
1218       // the constant.
1219       if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1220                                  DemandedElts, TLO))
1221         return true;
1222 
1223       // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1224       // constant, but if this 'and' is only clearing bits that were just set by
1225       // the xor, then this 'and' can be eliminated by shrinking the mask of
1226       // the xor. For example, for a 32-bit X:
1227       // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1228       if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1229           LHSKnown.One == ~RHSC->getAPIntValue()) {
1230         SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1231         return TLO.CombineTo(Op, Xor);
1232       }
1233     }
1234 
1235     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1236                              Depth + 1))
1237       return true;
1238     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1239     if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1240                              Known2, TLO, Depth + 1))
1241       return true;
1242     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1243 
1244     // Attempt to avoid multi-use ops if we don't need anything from them.
1245     if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1246       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1247           Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1248       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1249           Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1250       if (DemandedOp0 || DemandedOp1) {
1251         Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1252         Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1253         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1254         return TLO.CombineTo(Op, NewOp);
1255       }
1256     }
1257 
1258     // If all of the demanded bits are known one on one side, return the other.
1259     // These bits cannot contribute to the result of the 'and'.
1260     if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1261       return TLO.CombineTo(Op, Op0);
1262     if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1263       return TLO.CombineTo(Op, Op1);
1264     // If all of the demanded bits in the inputs are known zeros, return zero.
1265     if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1266       return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1267     // If the RHS is a constant, see if we can simplify it.
1268     if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1269                                TLO))
1270       return true;
1271     // If the operation can be done in a smaller type, do so.
1272     if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1273       return true;
1274 
1275     Known &= Known2;
1276     break;
1277   }
1278   case ISD::OR: {
1279     SDValue Op0 = Op.getOperand(0);
1280     SDValue Op1 = Op.getOperand(1);
1281 
1282     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1283                              Depth + 1))
1284       return true;
1285     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1286     if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1287                              Known2, TLO, Depth + 1))
1288       return true;
1289     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1290 
1291     // Attempt to avoid multi-use ops if we don't need anything from them.
1292     if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1293       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1294           Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1295       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1296           Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1297       if (DemandedOp0 || DemandedOp1) {
1298         Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1299         Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1300         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1301         return TLO.CombineTo(Op, NewOp);
1302       }
1303     }
1304 
1305     // If all of the demanded bits are known zero on one side, return the other.
1306     // These bits cannot contribute to the result of the 'or'.
1307     if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1308       return TLO.CombineTo(Op, Op0);
1309     if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1310       return TLO.CombineTo(Op, Op1);
1311     // If the RHS is a constant, see if we can simplify it.
1312     if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1313       return true;
1314     // If the operation can be done in a smaller type, do so.
1315     if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1316       return true;
1317 
1318     Known |= Known2;
1319     break;
1320   }
1321   case ISD::XOR: {
1322     SDValue Op0 = Op.getOperand(0);
1323     SDValue Op1 = Op.getOperand(1);
1324 
1325     if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1326                              Depth + 1))
1327       return true;
1328     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1329     if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1330                              Depth + 1))
1331       return true;
1332     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1333 
1334     // Attempt to avoid multi-use ops if we don't need anything from them.
1335     if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1336       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1337           Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1338       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1339           Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1340       if (DemandedOp0 || DemandedOp1) {
1341         Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1342         Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1343         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1344         return TLO.CombineTo(Op, NewOp);
1345       }
1346     }
1347 
1348     // If all of the demanded bits are known zero on one side, return the other.
1349     // These bits cannot contribute to the result of the 'xor'.
1350     if (DemandedBits.isSubsetOf(Known.Zero))
1351       return TLO.CombineTo(Op, Op0);
1352     if (DemandedBits.isSubsetOf(Known2.Zero))
1353       return TLO.CombineTo(Op, Op1);
1354     // If the operation can be done in a smaller type, do so.
1355     if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1356       return true;
1357 
1358     // If all of the unknown bits are known to be zero on one side or the other
1359     // turn this into an *inclusive* or.
1360     //    e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1361     if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1362       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1363 
1364     ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1365     if (C) {
1366       // If one side is a constant, and all of the set bits in the constant are
1367       // also known set on the other side, turn this into an AND, as we know
1368       // the bits will be cleared.
1369       //    e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1370       // NB: it is okay if more bits are known than are requested
1371       if (C->getAPIntValue() == Known2.One) {
1372         SDValue ANDC =
1373             TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1374         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1375       }
1376 
1377       // If the RHS is a constant, see if we can change it. Don't alter a -1
1378       // constant because that's a 'not' op, and that is better for combining
1379       // and codegen.
1380       if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1381         // We're flipping all demanded bits. Flip the undemanded bits too.
1382         SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1383         return TLO.CombineTo(Op, New);
1384       }
1385     }
1386 
1387     // If we can't turn this into a 'not', try to shrink the constant.
1388     if (!C || !C->isAllOnes())
1389       if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1390         return true;
1391 
1392     Known ^= Known2;
1393     break;
1394   }
1395   case ISD::SELECT:
1396     if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1397                              Depth + 1))
1398       return true;
1399     if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1400                              Depth + 1))
1401       return true;
1402     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1403     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1404 
1405     // If the operands are constants, see if we can simplify them.
1406     if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1407       return true;
1408 
1409     // Only known if known in both the LHS and RHS.
1410     Known = KnownBits::commonBits(Known, Known2);
1411     break;
1412   case ISD::SELECT_CC:
1413     if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1414                              Depth + 1))
1415       return true;
1416     if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1417                              Depth + 1))
1418       return true;
1419     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1420     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1421 
1422     // If the operands are constants, see if we can simplify them.
1423     if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1424       return true;
1425 
1426     // Only known if known in both the LHS and RHS.
1427     Known = KnownBits::commonBits(Known, Known2);
1428     break;
1429   case ISD::SETCC: {
1430     SDValue Op0 = Op.getOperand(0);
1431     SDValue Op1 = Op.getOperand(1);
1432     ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1433     // If (1) we only need the sign-bit, (2) the setcc operands are the same
1434     // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1435     // -1, we may be able to bypass the setcc.
1436     if (DemandedBits.isSignMask() &&
1437         Op0.getScalarValueSizeInBits() == BitWidth &&
1438         getBooleanContents(Op0.getValueType()) ==
1439             BooleanContent::ZeroOrNegativeOneBooleanContent) {
1440       // If we're testing X < 0, then this compare isn't needed - just use X!
1441       // FIXME: We're limiting to integer types here, but this should also work
1442       // if we don't care about FP signed-zero. The use of SETLT with FP means
1443       // that we don't care about NaNs.
1444       if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1445           (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1446         return TLO.CombineTo(Op, Op0);
1447 
1448       // TODO: Should we check for other forms of sign-bit comparisons?
1449       // Examples: X <= -1, X >= 0
1450     }
1451     if (getBooleanContents(Op0.getValueType()) ==
1452             TargetLowering::ZeroOrOneBooleanContent &&
1453         BitWidth > 1)
1454       Known.Zero.setBitsFrom(1);
1455     break;
1456   }
1457   case ISD::SHL: {
1458     SDValue Op0 = Op.getOperand(0);
1459     SDValue Op1 = Op.getOperand(1);
1460     EVT ShiftVT = Op1.getValueType();
1461 
1462     if (const APInt *SA =
1463             TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1464       unsigned ShAmt = SA->getZExtValue();
1465       if (ShAmt == 0)
1466         return TLO.CombineTo(Op, Op0);
1467 
1468       // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1469       // single shift.  We can do this if the bottom bits (which are shifted
1470       // out) are never demanded.
1471       // TODO - support non-uniform vector amounts.
1472       if (Op0.getOpcode() == ISD::SRL) {
1473         if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1474           if (const APInt *SA2 =
1475                   TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1476             unsigned C1 = SA2->getZExtValue();
1477             unsigned Opc = ISD::SHL;
1478             int Diff = ShAmt - C1;
1479             if (Diff < 0) {
1480               Diff = -Diff;
1481               Opc = ISD::SRL;
1482             }
1483             SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1484             return TLO.CombineTo(
1485                 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1486           }
1487         }
1488       }
1489 
1490       // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1491       // are not demanded. This will likely allow the anyext to be folded away.
1492       // TODO - support non-uniform vector amounts.
1493       if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1494         SDValue InnerOp = Op0.getOperand(0);
1495         EVT InnerVT = InnerOp.getValueType();
1496         unsigned InnerBits = InnerVT.getScalarSizeInBits();
1497         if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1498             isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1499           EVT ShTy = getShiftAmountTy(InnerVT, DL);
1500           if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1501             ShTy = InnerVT;
1502           SDValue NarrowShl =
1503               TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1504                               TLO.DAG.getConstant(ShAmt, dl, ShTy));
1505           return TLO.CombineTo(
1506               Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1507         }
1508 
1509         // Repeat the SHL optimization above in cases where an extension
1510         // intervenes: (shl (anyext (shr x, c1)), c2) to
1511         // (shl (anyext x), c2-c1).  This requires that the bottom c1 bits
1512         // aren't demanded (as above) and that the shifted upper c1 bits of
1513         // x aren't demanded.
1514         // TODO - support non-uniform vector amounts.
1515         if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1516             InnerOp.hasOneUse()) {
1517           if (const APInt *SA2 =
1518                   TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1519             unsigned InnerShAmt = SA2->getZExtValue();
1520             if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1521                 DemandedBits.getActiveBits() <=
1522                     (InnerBits - InnerShAmt + ShAmt) &&
1523                 DemandedBits.countTrailingZeros() >= ShAmt) {
1524               SDValue NewSA =
1525                   TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1526               SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1527                                                InnerOp.getOperand(0));
1528               return TLO.CombineTo(
1529                   Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1530             }
1531           }
1532         }
1533       }
1534 
1535       APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1536       if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1537                                Depth + 1))
1538         return true;
1539       assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1540       Known.Zero <<= ShAmt;
1541       Known.One <<= ShAmt;
1542       // low bits known zero.
1543       Known.Zero.setLowBits(ShAmt);
1544 
1545       // Try shrinking the operation as long as the shift amount will still be
1546       // in range.
1547       if ((ShAmt < DemandedBits.getActiveBits()) &&
1548           ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1549         return true;
1550     }
1551 
1552     // If we are only demanding sign bits then we can use the shift source
1553     // directly.
1554     if (const APInt *MaxSA =
1555             TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1556       unsigned ShAmt = MaxSA->getZExtValue();
1557       unsigned NumSignBits =
1558           TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1559       unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1560       if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1561         return TLO.CombineTo(Op, Op0);
1562     }
1563     break;
1564   }
1565   case ISD::SRL: {
1566     SDValue Op0 = Op.getOperand(0);
1567     SDValue Op1 = Op.getOperand(1);
1568     EVT ShiftVT = Op1.getValueType();
1569 
1570     if (const APInt *SA =
1571             TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1572       unsigned ShAmt = SA->getZExtValue();
1573       if (ShAmt == 0)
1574         return TLO.CombineTo(Op, Op0);
1575 
1576       // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1577       // single shift.  We can do this if the top bits (which are shifted out)
1578       // are never demanded.
1579       // TODO - support non-uniform vector amounts.
1580       if (Op0.getOpcode() == ISD::SHL) {
1581         if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1582           if (const APInt *SA2 =
1583                   TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1584             unsigned C1 = SA2->getZExtValue();
1585             unsigned Opc = ISD::SRL;
1586             int Diff = ShAmt - C1;
1587             if (Diff < 0) {
1588               Diff = -Diff;
1589               Opc = ISD::SHL;
1590             }
1591             SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1592             return TLO.CombineTo(
1593                 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1594           }
1595         }
1596       }
1597 
1598       APInt InDemandedMask = (DemandedBits << ShAmt);
1599 
1600       // If the shift is exact, then it does demand the low bits (and knows that
1601       // they are zero).
1602       if (Op->getFlags().hasExact())
1603         InDemandedMask.setLowBits(ShAmt);
1604 
1605       // Compute the new bits that are at the top now.
1606       if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1607                                Depth + 1))
1608         return true;
1609       assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1610       Known.Zero.lshrInPlace(ShAmt);
1611       Known.One.lshrInPlace(ShAmt);
1612       // High bits known zero.
1613       Known.Zero.setHighBits(ShAmt);
1614     }
1615     break;
1616   }
1617   case ISD::SRA: {
1618     SDValue Op0 = Op.getOperand(0);
1619     SDValue Op1 = Op.getOperand(1);
1620     EVT ShiftVT = Op1.getValueType();
1621 
1622     // If we only want bits that already match the signbit then we don't need
1623     // to shift.
1624     unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1625     if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1626         NumHiDemandedBits)
1627       return TLO.CombineTo(Op, Op0);
1628 
1629     // If this is an arithmetic shift right and only the low-bit is set, we can
1630     // always convert this into a logical shr, even if the shift amount is
1631     // variable.  The low bit of the shift cannot be an input sign bit unless
1632     // the shift amount is >= the size of the datatype, which is undefined.
1633     if (DemandedBits.isOne())
1634       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1635 
1636     if (const APInt *SA =
1637             TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1638       unsigned ShAmt = SA->getZExtValue();
1639       if (ShAmt == 0)
1640         return TLO.CombineTo(Op, Op0);
1641 
1642       APInt InDemandedMask = (DemandedBits << ShAmt);
1643 
1644       // If the shift is exact, then it does demand the low bits (and knows that
1645       // they are zero).
1646       if (Op->getFlags().hasExact())
1647         InDemandedMask.setLowBits(ShAmt);
1648 
1649       // If any of the demanded bits are produced by the sign extension, we also
1650       // demand the input sign bit.
1651       if (DemandedBits.countLeadingZeros() < ShAmt)
1652         InDemandedMask.setSignBit();
1653 
1654       if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1655                                Depth + 1))
1656         return true;
1657       assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1658       Known.Zero.lshrInPlace(ShAmt);
1659       Known.One.lshrInPlace(ShAmt);
1660 
1661       // If the input sign bit is known to be zero, or if none of the top bits
1662       // are demanded, turn this into an unsigned shift right.
1663       if (Known.Zero[BitWidth - ShAmt - 1] ||
1664           DemandedBits.countLeadingZeros() >= ShAmt) {
1665         SDNodeFlags Flags;
1666         Flags.setExact(Op->getFlags().hasExact());
1667         return TLO.CombineTo(
1668             Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1669       }
1670 
1671       int Log2 = DemandedBits.exactLogBase2();
1672       if (Log2 >= 0) {
1673         // The bit must come from the sign.
1674         SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1675         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1676       }
1677 
1678       if (Known.One[BitWidth - ShAmt - 1])
1679         // New bits are known one.
1680         Known.One.setHighBits(ShAmt);
1681 
1682       // Attempt to avoid multi-use ops if we don't need anything from them.
1683       if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1684         SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1685             Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1686         if (DemandedOp0) {
1687           SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1688           return TLO.CombineTo(Op, NewOp);
1689         }
1690       }
1691     }
1692     break;
1693   }
1694   case ISD::FSHL:
1695   case ISD::FSHR: {
1696     SDValue Op0 = Op.getOperand(0);
1697     SDValue Op1 = Op.getOperand(1);
1698     SDValue Op2 = Op.getOperand(2);
1699     bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1700 
1701     if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1702       unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1703 
1704       // For fshl, 0-shift returns the 1st arg.
1705       // For fshr, 0-shift returns the 2nd arg.
1706       if (Amt == 0) {
1707         if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1708                                  Known, TLO, Depth + 1))
1709           return true;
1710         break;
1711       }
1712 
1713       // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1714       // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1715       APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1716       APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1717       if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1718                                Depth + 1))
1719         return true;
1720       if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1721                                Depth + 1))
1722         return true;
1723 
1724       Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1725       Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1726       Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1727       Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1728       Known.One |= Known2.One;
1729       Known.Zero |= Known2.Zero;
1730     }
1731 
1732     // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1733     if (isPowerOf2_32(BitWidth)) {
1734       APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1735       if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1736                                Known2, TLO, Depth + 1))
1737         return true;
1738     }
1739     break;
1740   }
1741   case ISD::ROTL:
1742   case ISD::ROTR: {
1743     SDValue Op0 = Op.getOperand(0);
1744     SDValue Op1 = Op.getOperand(1);
1745     bool IsROTL = (Op.getOpcode() == ISD::ROTL);
1746 
1747     // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1748     if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1749       return TLO.CombineTo(Op, Op0);
1750 
1751     if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
1752       unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1753       unsigned RevAmt = BitWidth - Amt;
1754 
1755       // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
1756       // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
1757       APInt Demanded0 = DemandedBits.rotr(IsROTL ? Amt : RevAmt);
1758       if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1759                                Depth + 1))
1760         return true;
1761 
1762       // rot*(x, 0) --> x
1763       if (Amt == 0)
1764         return TLO.CombineTo(Op, Op0);
1765 
1766       // See if we don't demand either half of the rotated bits.
1767       if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) &&
1768           DemandedBits.countTrailingZeros() >= (IsROTL ? Amt : RevAmt)) {
1769         Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType());
1770         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1));
1771       }
1772       if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
1773           DemandedBits.countLeadingZeros() >= (IsROTL ? RevAmt : Amt)) {
1774         Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType());
1775         return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1776       }
1777     }
1778 
1779     // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1780     if (isPowerOf2_32(BitWidth)) {
1781       APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1782       if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1783                                Depth + 1))
1784         return true;
1785     }
1786     break;
1787   }
1788   case ISD::UMIN: {
1789     // Check if one arg is always less than (or equal) to the other arg.
1790     SDValue Op0 = Op.getOperand(0);
1791     SDValue Op1 = Op.getOperand(1);
1792     KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1793     KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1794     Known = KnownBits::umin(Known0, Known1);
1795     if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1796       return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1797     if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1798       return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1799     break;
1800   }
1801   case ISD::UMAX: {
1802     // Check if one arg is always greater than (or equal) to the other arg.
1803     SDValue Op0 = Op.getOperand(0);
1804     SDValue Op1 = Op.getOperand(1);
1805     KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1806     KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1807     Known = KnownBits::umax(Known0, Known1);
1808     if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1809       return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1810     if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1811       return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1812     break;
1813   }
1814   case ISD::BITREVERSE: {
1815     SDValue Src = Op.getOperand(0);
1816     APInt DemandedSrcBits = DemandedBits.reverseBits();
1817     if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1818                              Depth + 1))
1819       return true;
1820     Known.One = Known2.One.reverseBits();
1821     Known.Zero = Known2.Zero.reverseBits();
1822     break;
1823   }
1824   case ISD::BSWAP: {
1825     SDValue Src = Op.getOperand(0);
1826 
1827     // If the only bits demanded come from one byte of the bswap result,
1828     // just shift the input byte into position to eliminate the bswap.
1829     unsigned NLZ = DemandedBits.countLeadingZeros();
1830     unsigned NTZ = DemandedBits.countTrailingZeros();
1831 
1832     // Round NTZ down to the next byte.  If we have 11 trailing zeros, then
1833     // we need all the bits down to bit 8.  Likewise, round NLZ.  If we
1834     // have 14 leading zeros, round to 8.
1835     NLZ = alignDown(NLZ, 8);
1836     NTZ = alignDown(NTZ, 8);
1837     // If we need exactly one byte, we can do this transformation.
1838     if (BitWidth - NLZ - NTZ == 8) {
1839       // Replace this with either a left or right shift to get the byte into
1840       // the right place.
1841       unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
1842       if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) {
1843         EVT ShiftAmtTy = getShiftAmountTy(VT, DL);
1844         unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
1845         SDValue ShAmt = TLO.DAG.getConstant(ShiftAmount, dl, ShiftAmtTy);
1846         SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt);
1847         return TLO.CombineTo(Op, NewOp);
1848       }
1849     }
1850 
1851     APInt DemandedSrcBits = DemandedBits.byteSwap();
1852     if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1853                              Depth + 1))
1854       return true;
1855     Known.One = Known2.One.byteSwap();
1856     Known.Zero = Known2.Zero.byteSwap();
1857     break;
1858   }
1859   case ISD::CTPOP: {
1860     // If only 1 bit is demanded, replace with PARITY as long as we're before
1861     // op legalization.
1862     // FIXME: Limit to scalars for now.
1863     if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
1864       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1865                                                Op.getOperand(0)));
1866 
1867     Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1868     break;
1869   }
1870   case ISD::SIGN_EXTEND_INREG: {
1871     SDValue Op0 = Op.getOperand(0);
1872     EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1873     unsigned ExVTBits = ExVT.getScalarSizeInBits();
1874 
1875     // If we only care about the highest bit, don't bother shifting right.
1876     if (DemandedBits.isSignMask()) {
1877       unsigned MinSignedBits =
1878           TLO.DAG.ComputeMaxSignificantBits(Op0, DemandedElts, Depth + 1);
1879       bool AlreadySignExtended = ExVTBits >= MinSignedBits;
1880       // However if the input is already sign extended we expect the sign
1881       // extension to be dropped altogether later and do not simplify.
1882       if (!AlreadySignExtended) {
1883         // Compute the correct shift amount type, which must be getShiftAmountTy
1884         // for scalar types after legalization.
1885         SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ExVTBits, dl,
1886                                                getShiftAmountTy(VT, DL));
1887         return TLO.CombineTo(Op,
1888                              TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1889       }
1890     }
1891 
1892     // If none of the extended bits are demanded, eliminate the sextinreg.
1893     if (DemandedBits.getActiveBits() <= ExVTBits)
1894       return TLO.CombineTo(Op, Op0);
1895 
1896     APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1897 
1898     // Since the sign extended bits are demanded, we know that the sign
1899     // bit is demanded.
1900     InputDemandedBits.setBit(ExVTBits - 1);
1901 
1902     if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1903       return true;
1904     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1905 
1906     // If the sign bit of the input is known set or clear, then we know the
1907     // top bits of the result.
1908 
1909     // If the input sign bit is known zero, convert this into a zero extension.
1910     if (Known.Zero[ExVTBits - 1])
1911       return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1912 
1913     APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1914     if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1915       Known.One.setBitsFrom(ExVTBits);
1916       Known.Zero &= Mask;
1917     } else { // Input sign bit unknown
1918       Known.Zero &= Mask;
1919       Known.One &= Mask;
1920     }
1921     break;
1922   }
1923   case ISD::BUILD_PAIR: {
1924     EVT HalfVT = Op.getOperand(0).getValueType();
1925     unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1926 
1927     APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1928     APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1929 
1930     KnownBits KnownLo, KnownHi;
1931 
1932     if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1933       return true;
1934 
1935     if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1936       return true;
1937 
1938     Known.Zero = KnownLo.Zero.zext(BitWidth) |
1939                  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1940 
1941     Known.One = KnownLo.One.zext(BitWidth) |
1942                 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1943     break;
1944   }
1945   case ISD::ZERO_EXTEND:
1946   case ISD::ZERO_EXTEND_VECTOR_INREG: {
1947     SDValue Src = Op.getOperand(0);
1948     EVT SrcVT = Src.getValueType();
1949     unsigned InBits = SrcVT.getScalarSizeInBits();
1950     unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1951     bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1952 
1953     // If none of the top bits are demanded, convert this into an any_extend.
1954     if (DemandedBits.getActiveBits() <= InBits) {
1955       // If we only need the non-extended bits of the bottom element
1956       // then we can just bitcast to the result.
1957       if (IsLE && IsVecInReg && DemandedElts == 1 &&
1958           VT.getSizeInBits() == SrcVT.getSizeInBits())
1959         return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1960 
1961       unsigned Opc =
1962           IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1963       if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1964         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1965     }
1966 
1967     APInt InDemandedBits = DemandedBits.trunc(InBits);
1968     APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1969     if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1970                              Depth + 1))
1971       return true;
1972     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1973     assert(Known.getBitWidth() == InBits && "Src width has changed?");
1974     Known = Known.zext(BitWidth);
1975 
1976     // Attempt to avoid multi-use ops if we don't need anything from them.
1977     if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1978             Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1979       return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1980     break;
1981   }
1982   case ISD::SIGN_EXTEND:
1983   case ISD::SIGN_EXTEND_VECTOR_INREG: {
1984     SDValue Src = Op.getOperand(0);
1985     EVT SrcVT = Src.getValueType();
1986     unsigned InBits = SrcVT.getScalarSizeInBits();
1987     unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1988     bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1989 
1990     // If none of the top bits are demanded, convert this into an any_extend.
1991     if (DemandedBits.getActiveBits() <= InBits) {
1992       // If we only need the non-extended bits of the bottom element
1993       // then we can just bitcast to the result.
1994       if (IsLE && IsVecInReg && DemandedElts == 1 &&
1995           VT.getSizeInBits() == SrcVT.getSizeInBits())
1996         return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1997 
1998       unsigned Opc =
1999           IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2000       if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2001         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2002     }
2003 
2004     APInt InDemandedBits = DemandedBits.trunc(InBits);
2005     APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
2006 
2007     // Since some of the sign extended bits are demanded, we know that the sign
2008     // bit is demanded.
2009     InDemandedBits.setBit(InBits - 1);
2010 
2011     if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2012                              Depth + 1))
2013       return true;
2014     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2015     assert(Known.getBitWidth() == InBits && "Src width has changed?");
2016 
2017     // If the sign bit is known one, the top bits match.
2018     Known = Known.sext(BitWidth);
2019 
2020     // If the sign bit is known zero, convert this to a zero extend.
2021     if (Known.isNonNegative()) {
2022       unsigned Opc =
2023           IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2024       if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2025         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2026     }
2027 
2028     // Attempt to avoid multi-use ops if we don't need anything from them.
2029     if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2030             Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2031       return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2032     break;
2033   }
2034   case ISD::ANY_EXTEND:
2035   case ISD::ANY_EXTEND_VECTOR_INREG: {
2036     SDValue Src = Op.getOperand(0);
2037     EVT SrcVT = Src.getValueType();
2038     unsigned InBits = SrcVT.getScalarSizeInBits();
2039     unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2040     bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2041 
2042     // If we only need the bottom element then we can just bitcast.
2043     // TODO: Handle ANY_EXTEND?
2044     if (IsLE && IsVecInReg && DemandedElts == 1 &&
2045         VT.getSizeInBits() == SrcVT.getSizeInBits())
2046       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2047 
2048     APInt InDemandedBits = DemandedBits.trunc(InBits);
2049     APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
2050     if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2051                              Depth + 1))
2052       return true;
2053     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2054     assert(Known.getBitWidth() == InBits && "Src width has changed?");
2055     Known = Known.anyext(BitWidth);
2056 
2057     // Attempt to avoid multi-use ops if we don't need anything from them.
2058     if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2059             Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2060       return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2061     break;
2062   }
2063   case ISD::TRUNCATE: {
2064     SDValue Src = Op.getOperand(0);
2065 
2066     // Simplify the input, using demanded bit information, and compute the known
2067     // zero/one bits live out.
2068     unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2069     APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2070     if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2071                              Depth + 1))
2072       return true;
2073     Known = Known.trunc(BitWidth);
2074 
2075     // Attempt to avoid multi-use ops if we don't need anything from them.
2076     if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2077             Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2078       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2079 
2080     // If the input is only used by this truncate, see if we can shrink it based
2081     // on the known demanded bits.
2082     if (Src.getNode()->hasOneUse()) {
2083       switch (Src.getOpcode()) {
2084       default:
2085         break;
2086       case ISD::SRL:
2087         // Shrink SRL by a constant if none of the high bits shifted in are
2088         // demanded.
2089         if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2090           // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2091           // undesirable.
2092           break;
2093 
2094         const APInt *ShAmtC =
2095             TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2096         if (!ShAmtC || ShAmtC->uge(BitWidth))
2097           break;
2098         uint64_t ShVal = ShAmtC->getZExtValue();
2099 
2100         APInt HighBits =
2101             APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2102         HighBits.lshrInPlace(ShVal);
2103         HighBits = HighBits.trunc(BitWidth);
2104 
2105         if (!(HighBits & DemandedBits)) {
2106           // None of the shifted in bits are needed.  Add a truncate of the
2107           // shift input, then shift it.
2108           SDValue NewShAmt = TLO.DAG.getConstant(
2109               ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2110           SDValue NewTrunc =
2111               TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2112           return TLO.CombineTo(
2113               Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2114         }
2115         break;
2116       }
2117     }
2118 
2119     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2120     break;
2121   }
2122   case ISD::AssertZext: {
2123     // AssertZext demands all of the high bits, plus any of the low bits
2124     // demanded by its users.
2125     EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2126     APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2127     if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2128                              TLO, Depth + 1))
2129       return true;
2130     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2131 
2132     Known.Zero |= ~InMask;
2133     break;
2134   }
2135   case ISD::EXTRACT_VECTOR_ELT: {
2136     SDValue Src = Op.getOperand(0);
2137     SDValue Idx = Op.getOperand(1);
2138     ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2139     unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2140 
2141     if (SrcEltCnt.isScalable())
2142       return false;
2143 
2144     // Demand the bits from every vector element without a constant index.
2145     unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2146     APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
2147     if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2148       if (CIdx->getAPIntValue().ult(NumSrcElts))
2149         DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2150 
2151     // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2152     // anything about the extended bits.
2153     APInt DemandedSrcBits = DemandedBits;
2154     if (BitWidth > EltBitWidth)
2155       DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2156 
2157     if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2158                              Depth + 1))
2159       return true;
2160 
2161     // Attempt to avoid multi-use ops if we don't need anything from them.
2162     if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2163       if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2164               Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2165         SDValue NewOp =
2166             TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2167         return TLO.CombineTo(Op, NewOp);
2168       }
2169     }
2170 
2171     Known = Known2;
2172     if (BitWidth > EltBitWidth)
2173       Known = Known.anyext(BitWidth);
2174     break;
2175   }
2176   case ISD::BITCAST: {
2177     SDValue Src = Op.getOperand(0);
2178     EVT SrcVT = Src.getValueType();
2179     unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2180 
2181     // If this is an FP->Int bitcast and if the sign bit is the only
2182     // thing demanded, turn this into a FGETSIGN.
2183     if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2184         DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2185         SrcVT.isFloatingPoint()) {
2186       bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2187       bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2188       if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2189           SrcVT != MVT::f128) {
2190         // Cannot eliminate/lower SHL for f128 yet.
2191         EVT Ty = OpVTLegal ? VT : MVT::i32;
2192         // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2193         // place.  We expect the SHL to be eliminated by other optimizations.
2194         SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2195         unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2196         if (!OpVTLegal && OpVTSizeInBits > 32)
2197           Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2198         unsigned ShVal = Op.getValueSizeInBits() - 1;
2199         SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2200         return TLO.CombineTo(Op,
2201                              TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2202       }
2203     }
2204 
2205     // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2206     // Demand the elt/bit if any of the original elts/bits are demanded.
2207     if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2208       unsigned Scale = BitWidth / NumSrcEltBits;
2209       unsigned NumSrcElts = SrcVT.getVectorNumElements();
2210       APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2211       APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2212       for (unsigned i = 0; i != Scale; ++i) {
2213         unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2214         unsigned BitOffset = EltOffset * NumSrcEltBits;
2215         APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
2216         if (!Sub.isZero()) {
2217           DemandedSrcBits |= Sub;
2218           for (unsigned j = 0; j != NumElts; ++j)
2219             if (DemandedElts[j])
2220               DemandedSrcElts.setBit((j * Scale) + i);
2221         }
2222       }
2223 
2224       APInt KnownSrcUndef, KnownSrcZero;
2225       if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2226                                      KnownSrcZero, TLO, Depth + 1))
2227         return true;
2228 
2229       KnownBits KnownSrcBits;
2230       if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2231                                KnownSrcBits, TLO, Depth + 1))
2232         return true;
2233     } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2234       // TODO - bigendian once we have test coverage.
2235       unsigned Scale = NumSrcEltBits / BitWidth;
2236       unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2237       APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2238       APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2239       for (unsigned i = 0; i != NumElts; ++i)
2240         if (DemandedElts[i]) {
2241           unsigned Offset = (i % Scale) * BitWidth;
2242           DemandedSrcBits.insertBits(DemandedBits, Offset);
2243           DemandedSrcElts.setBit(i / Scale);
2244         }
2245 
2246       if (SrcVT.isVector()) {
2247         APInt KnownSrcUndef, KnownSrcZero;
2248         if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2249                                        KnownSrcZero, TLO, Depth + 1))
2250           return true;
2251       }
2252 
2253       KnownBits KnownSrcBits;
2254       if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2255                                KnownSrcBits, TLO, Depth + 1))
2256         return true;
2257     }
2258 
2259     // If this is a bitcast, let computeKnownBits handle it.  Only do this on a
2260     // recursive call where Known may be useful to the caller.
2261     if (Depth > 0) {
2262       Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2263       return false;
2264     }
2265     break;
2266   }
2267   case ISD::MUL:
2268     // 'Quadratic Reciprocity': mul(x,x) -> 0 if we're only demanding bit[1]
2269     if (DemandedBits == 2 && Op.getOperand(0) == Op.getOperand(1))
2270       return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
2271     LLVM_FALLTHROUGH;
2272   case ISD::ADD:
2273   case ISD::SUB: {
2274     // Add, Sub, and Mul don't demand any bits in positions beyond that
2275     // of the highest bit demanded of them.
2276     SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2277     SDNodeFlags Flags = Op.getNode()->getFlags();
2278     unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2279     APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2280     if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2281                              Depth + 1) ||
2282         SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2283                              Depth + 1) ||
2284         // See if the operation should be performed at a smaller bit width.
2285         ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2286       if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2287         // Disable the nsw and nuw flags. We can no longer guarantee that we
2288         // won't wrap after simplification.
2289         Flags.setNoSignedWrap(false);
2290         Flags.setNoUnsignedWrap(false);
2291         SDValue NewOp =
2292             TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2293         return TLO.CombineTo(Op, NewOp);
2294       }
2295       return true;
2296     }
2297 
2298     // Attempt to avoid multi-use ops if we don't need anything from them.
2299     if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2300       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2301           Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2302       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2303           Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2304       if (DemandedOp0 || DemandedOp1) {
2305         Flags.setNoSignedWrap(false);
2306         Flags.setNoUnsignedWrap(false);
2307         Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2308         Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2309         SDValue NewOp =
2310             TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2311         return TLO.CombineTo(Op, NewOp);
2312       }
2313     }
2314 
2315     // If we have a constant operand, we may be able to turn it into -1 if we
2316     // do not demand the high bits. This can make the constant smaller to
2317     // encode, allow more general folding, or match specialized instruction
2318     // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2319     // is probably not useful (and could be detrimental).
2320     ConstantSDNode *C = isConstOrConstSplat(Op1);
2321     APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2322     if (C && !C->isAllOnes() && !C->isOne() &&
2323         (C->getAPIntValue() | HighMask).isAllOnes()) {
2324       SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2325       // Disable the nsw and nuw flags. We can no longer guarantee that we
2326       // won't wrap after simplification.
2327       Flags.setNoSignedWrap(false);
2328       Flags.setNoUnsignedWrap(false);
2329       SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2330       return TLO.CombineTo(Op, NewOp);
2331     }
2332 
2333     LLVM_FALLTHROUGH;
2334   }
2335   default:
2336     if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2337       if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2338                                             Known, TLO, Depth))
2339         return true;
2340       break;
2341     }
2342 
2343     // Just use computeKnownBits to compute output bits.
2344     Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2345     break;
2346   }
2347 
2348   // If we know the value of all of the demanded bits, return this as a
2349   // constant.
2350   if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2351     // Avoid folding to a constant if any OpaqueConstant is involved.
2352     const SDNode *N = Op.getNode();
2353     for (SDNode *Op :
2354          llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2355       if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2356         if (C->isOpaque())
2357           return false;
2358     }
2359     if (VT.isInteger())
2360       return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2361     if (VT.isFloatingPoint())
2362       return TLO.CombineTo(
2363           Op,
2364           TLO.DAG.getConstantFP(
2365               APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2366   }
2367 
2368   return false;
2369 }
2370 
2371 bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2372                                                 const APInt &DemandedElts,
2373                                                 APInt &KnownUndef,
2374                                                 APInt &KnownZero,
2375                                                 DAGCombinerInfo &DCI) const {
2376   SelectionDAG &DAG = DCI.DAG;
2377   TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2378                         !DCI.isBeforeLegalizeOps());
2379 
2380   bool Simplified =
2381       SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2382   if (Simplified) {
2383     DCI.AddToWorklist(Op.getNode());
2384     DCI.CommitTargetLoweringOpt(TLO);
2385   }
2386 
2387   return Simplified;
2388 }
2389 
2390 /// Given a vector binary operation and known undefined elements for each input
2391 /// operand, compute whether each element of the output is undefined.
2392 static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2393                                          const APInt &UndefOp0,
2394                                          const APInt &UndefOp1) {
2395   EVT VT = BO.getValueType();
2396   assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
2397          "Vector binop only");
2398 
2399   EVT EltVT = VT.getVectorElementType();
2400   unsigned NumElts = VT.getVectorNumElements();
2401   assert(UndefOp0.getBitWidth() == NumElts &&
2402          UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
2403 
2404   auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2405                                    const APInt &UndefVals) {
2406     if (UndefVals[Index])
2407       return DAG.getUNDEF(EltVT);
2408 
2409     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2410       // Try hard to make sure that the getNode() call is not creating temporary
2411       // nodes. Ignore opaque integers because they do not constant fold.
2412       SDValue Elt = BV->getOperand(Index);
2413       auto *C = dyn_cast<ConstantSDNode>(Elt);
2414       if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2415         return Elt;
2416     }
2417 
2418     return SDValue();
2419   };
2420 
2421   APInt KnownUndef = APInt::getZero(NumElts);
2422   for (unsigned i = 0; i != NumElts; ++i) {
2423     // If both inputs for this element are either constant or undef and match
2424     // the element type, compute the constant/undef result for this element of
2425     // the vector.
2426     // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2427     // not handle FP constants. The code within getNode() should be refactored
2428     // to avoid the danger of creating a bogus temporary node here.
2429     SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2430     SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2431     if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2432       if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2433         KnownUndef.setBit(i);
2434   }
2435   return KnownUndef;
2436 }
2437 
2438 bool TargetLowering::SimplifyDemandedVectorElts(
2439     SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2440     APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2441     bool AssumeSingleUse) const {
2442   EVT VT = Op.getValueType();
2443   unsigned Opcode = Op.getOpcode();
2444   APInt DemandedElts = OriginalDemandedElts;
2445   unsigned NumElts = DemandedElts.getBitWidth();
2446   assert(VT.isVector() && "Expected vector op");
2447 
2448   KnownUndef = KnownZero = APInt::getZero(NumElts);
2449 
2450   // TODO: For now we assume we know nothing about scalable vectors.
2451   if (VT.isScalableVector())
2452     return false;
2453 
2454   assert(VT.getVectorNumElements() == NumElts &&
2455          "Mask size mismatches value type element count!");
2456 
2457   // Undef operand.
2458   if (Op.isUndef()) {
2459     KnownUndef.setAllBits();
2460     return false;
2461   }
2462 
2463   // If Op has other users, assume that all elements are needed.
2464   if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2465     DemandedElts.setAllBits();
2466 
2467   // Not demanding any elements from Op.
2468   if (DemandedElts == 0) {
2469     KnownUndef.setAllBits();
2470     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2471   }
2472 
2473   // Limit search depth.
2474   if (Depth >= SelectionDAG::MaxRecursionDepth)
2475     return false;
2476 
2477   SDLoc DL(Op);
2478   unsigned EltSizeInBits = VT.getScalarSizeInBits();
2479   bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
2480 
2481   // Helper for demanding the specified elements and all the bits of both binary
2482   // operands.
2483   auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2484     SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2485                                                            TLO.DAG, Depth + 1);
2486     SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2487                                                            TLO.DAG, Depth + 1);
2488     if (NewOp0 || NewOp1) {
2489       SDValue NewOp = TLO.DAG.getNode(
2490           Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2491       return TLO.CombineTo(Op, NewOp);
2492     }
2493     return false;
2494   };
2495 
2496   switch (Opcode) {
2497   case ISD::SCALAR_TO_VECTOR: {
2498     if (!DemandedElts[0]) {
2499       KnownUndef.setAllBits();
2500       return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2501     }
2502     SDValue ScalarSrc = Op.getOperand(0);
2503     if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2504       SDValue Src = ScalarSrc.getOperand(0);
2505       SDValue Idx = ScalarSrc.getOperand(1);
2506       EVT SrcVT = Src.getValueType();
2507 
2508       ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2509 
2510       if (SrcEltCnt.isScalable())
2511         return false;
2512 
2513       unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2514       if (isNullConstant(Idx)) {
2515         APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2516         APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2517         APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2518         if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2519                                        TLO, Depth + 1))
2520           return true;
2521       }
2522     }
2523     KnownUndef.setHighBits(NumElts - 1);
2524     break;
2525   }
2526   case ISD::BITCAST: {
2527     SDValue Src = Op.getOperand(0);
2528     EVT SrcVT = Src.getValueType();
2529 
2530     // We only handle vectors here.
2531     // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2532     if (!SrcVT.isVector())
2533       break;
2534 
2535     // Fast handling of 'identity' bitcasts.
2536     unsigned NumSrcElts = SrcVT.getVectorNumElements();
2537     if (NumSrcElts == NumElts)
2538       return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2539                                         KnownZero, TLO, Depth + 1);
2540 
2541     APInt SrcDemandedElts, SrcZero, SrcUndef;
2542 
2543     // Bitcast from 'large element' src vector to 'small element' vector, we
2544     // must demand a source element if any DemandedElt maps to it.
2545     if ((NumElts % NumSrcElts) == 0) {
2546       unsigned Scale = NumElts / NumSrcElts;
2547       SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2548       if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2549                                      TLO, Depth + 1))
2550         return true;
2551 
2552       // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2553       // of the large element.
2554       // TODO - bigendian once we have test coverage.
2555       if (IsLE) {
2556         unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2557         APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
2558         for (unsigned i = 0; i != NumElts; ++i)
2559           if (DemandedElts[i]) {
2560             unsigned Ofs = (i % Scale) * EltSizeInBits;
2561             SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2562           }
2563 
2564         KnownBits Known;
2565         if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2566                                  TLO, Depth + 1))
2567           return true;
2568       }
2569 
2570       // If the src element is zero/undef then all the output elements will be -
2571       // only demanded elements are guaranteed to be correct.
2572       for (unsigned i = 0; i != NumSrcElts; ++i) {
2573         if (SrcDemandedElts[i]) {
2574           if (SrcZero[i])
2575             KnownZero.setBits(i * Scale, (i + 1) * Scale);
2576           if (SrcUndef[i])
2577             KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2578         }
2579       }
2580     }
2581 
2582     // Bitcast from 'small element' src vector to 'large element' vector, we
2583     // demand all smaller source elements covered by the larger demanded element
2584     // of this vector.
2585     if ((NumSrcElts % NumElts) == 0) {
2586       unsigned Scale = NumSrcElts / NumElts;
2587       SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2588       if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2589                                      TLO, Depth + 1))
2590         return true;
2591 
2592       // If all the src elements covering an output element are zero/undef, then
2593       // the output element will be as well, assuming it was demanded.
2594       for (unsigned i = 0; i != NumElts; ++i) {
2595         if (DemandedElts[i]) {
2596           if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
2597             KnownZero.setBit(i);
2598           if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
2599             KnownUndef.setBit(i);
2600         }
2601       }
2602     }
2603     break;
2604   }
2605   case ISD::BUILD_VECTOR: {
2606     // Check all elements and simplify any unused elements with UNDEF.
2607     if (!DemandedElts.isAllOnes()) {
2608       // Don't simplify BROADCASTS.
2609       if (llvm::any_of(Op->op_values(),
2610                        [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2611         SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2612         bool Updated = false;
2613         for (unsigned i = 0; i != NumElts; ++i) {
2614           if (!DemandedElts[i] && !Ops[i].isUndef()) {
2615             Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2616             KnownUndef.setBit(i);
2617             Updated = true;
2618           }
2619         }
2620         if (Updated)
2621           return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2622       }
2623     }
2624     for (unsigned i = 0; i != NumElts; ++i) {
2625       SDValue SrcOp = Op.getOperand(i);
2626       if (SrcOp.isUndef()) {
2627         KnownUndef.setBit(i);
2628       } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2629                  (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2630         KnownZero.setBit(i);
2631       }
2632     }
2633     break;
2634   }
2635   case ISD::CONCAT_VECTORS: {
2636     EVT SubVT = Op.getOperand(0).getValueType();
2637     unsigned NumSubVecs = Op.getNumOperands();
2638     unsigned NumSubElts = SubVT.getVectorNumElements();
2639     for (unsigned i = 0; i != NumSubVecs; ++i) {
2640       SDValue SubOp = Op.getOperand(i);
2641       APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2642       APInt SubUndef, SubZero;
2643       if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2644                                      Depth + 1))
2645         return true;
2646       KnownUndef.insertBits(SubUndef, i * NumSubElts);
2647       KnownZero.insertBits(SubZero, i * NumSubElts);
2648     }
2649     break;
2650   }
2651   case ISD::INSERT_SUBVECTOR: {
2652     // Demand any elements from the subvector and the remainder from the src its
2653     // inserted into.
2654     SDValue Src = Op.getOperand(0);
2655     SDValue Sub = Op.getOperand(1);
2656     uint64_t Idx = Op.getConstantOperandVal(2);
2657     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2658     APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2659     APInt DemandedSrcElts = DemandedElts;
2660     DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
2661 
2662     APInt SubUndef, SubZero;
2663     if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2664                                    Depth + 1))
2665       return true;
2666 
2667     // If none of the src operand elements are demanded, replace it with undef.
2668     if (!DemandedSrcElts && !Src.isUndef())
2669       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2670                                                TLO.DAG.getUNDEF(VT), Sub,
2671                                                Op.getOperand(2)));
2672 
2673     if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2674                                    TLO, Depth + 1))
2675       return true;
2676     KnownUndef.insertBits(SubUndef, Idx);
2677     KnownZero.insertBits(SubZero, Idx);
2678 
2679     // Attempt to avoid multi-use ops if we don't need anything from them.
2680     if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
2681       SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2682           Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2683       SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2684           Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2685       if (NewSrc || NewSub) {
2686         NewSrc = NewSrc ? NewSrc : Src;
2687         NewSub = NewSub ? NewSub : Sub;
2688         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2689                                         NewSub, Op.getOperand(2));
2690         return TLO.CombineTo(Op, NewOp);
2691       }
2692     }
2693     break;
2694   }
2695   case ISD::EXTRACT_SUBVECTOR: {
2696     // Offset the demanded elts by the subvector index.
2697     SDValue Src = Op.getOperand(0);
2698     if (Src.getValueType().isScalableVector())
2699       break;
2700     uint64_t Idx = Op.getConstantOperandVal(1);
2701     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2702     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2703 
2704     APInt SrcUndef, SrcZero;
2705     if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2706                                    Depth + 1))
2707       return true;
2708     KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2709     KnownZero = SrcZero.extractBits(NumElts, Idx);
2710 
2711     // Attempt to avoid multi-use ops if we don't need anything from them.
2712     if (!DemandedElts.isAllOnes()) {
2713       SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2714           Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2715       if (NewSrc) {
2716         SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2717                                         Op.getOperand(1));
2718         return TLO.CombineTo(Op, NewOp);
2719       }
2720     }
2721     break;
2722   }
2723   case ISD::INSERT_VECTOR_ELT: {
2724     SDValue Vec = Op.getOperand(0);
2725     SDValue Scl = Op.getOperand(1);
2726     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2727 
2728     // For a legal, constant insertion index, if we don't need this insertion
2729     // then strip it, else remove it from the demanded elts.
2730     if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2731       unsigned Idx = CIdx->getZExtValue();
2732       if (!DemandedElts[Idx])
2733         return TLO.CombineTo(Op, Vec);
2734 
2735       APInt DemandedVecElts(DemandedElts);
2736       DemandedVecElts.clearBit(Idx);
2737       if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2738                                      KnownZero, TLO, Depth + 1))
2739         return true;
2740 
2741       KnownUndef.setBitVal(Idx, Scl.isUndef());
2742 
2743       KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2744       break;
2745     }
2746 
2747     APInt VecUndef, VecZero;
2748     if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2749                                    Depth + 1))
2750       return true;
2751     // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2752     break;
2753   }
2754   case ISD::VSELECT: {
2755     // Try to transform the select condition based on the current demanded
2756     // elements.
2757     // TODO: If a condition element is undef, we can choose from one arm of the
2758     //       select (and if one arm is undef, then we can propagate that to the
2759     //       result).
2760     // TODO - add support for constant vselect masks (see IR version of this).
2761     APInt UnusedUndef, UnusedZero;
2762     if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2763                                    UnusedZero, TLO, Depth + 1))
2764       return true;
2765 
2766     // See if we can simplify either vselect operand.
2767     APInt DemandedLHS(DemandedElts);
2768     APInt DemandedRHS(DemandedElts);
2769     APInt UndefLHS, ZeroLHS;
2770     APInt UndefRHS, ZeroRHS;
2771     if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2772                                    ZeroLHS, TLO, Depth + 1))
2773       return true;
2774     if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2775                                    ZeroRHS, TLO, Depth + 1))
2776       return true;
2777 
2778     KnownUndef = UndefLHS & UndefRHS;
2779     KnownZero = ZeroLHS & ZeroRHS;
2780     break;
2781   }
2782   case ISD::VECTOR_SHUFFLE: {
2783     ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2784 
2785     // Collect demanded elements from shuffle operands..
2786     APInt DemandedLHS(NumElts, 0);
2787     APInt DemandedRHS(NumElts, 0);
2788     for (unsigned i = 0; i != NumElts; ++i) {
2789       int M = ShuffleMask[i];
2790       if (M < 0 || !DemandedElts[i])
2791         continue;
2792       assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
2793       if (M < (int)NumElts)
2794         DemandedLHS.setBit(M);
2795       else
2796         DemandedRHS.setBit(M - NumElts);
2797     }
2798 
2799     // See if we can simplify either shuffle operand.
2800     APInt UndefLHS, ZeroLHS;
2801     APInt UndefRHS, ZeroRHS;
2802     if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2803                                    ZeroLHS, TLO, Depth + 1))
2804       return true;
2805     if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2806                                    ZeroRHS, TLO, Depth + 1))
2807       return true;
2808 
2809     // Simplify mask using undef elements from LHS/RHS.
2810     bool Updated = false;
2811     bool IdentityLHS = true, IdentityRHS = true;
2812     SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2813     for (unsigned i = 0; i != NumElts; ++i) {
2814       int &M = NewMask[i];
2815       if (M < 0)
2816         continue;
2817       if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2818           (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2819         Updated = true;
2820         M = -1;
2821       }
2822       IdentityLHS &= (M < 0) || (M == (int)i);
2823       IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2824     }
2825 
2826     // Update legal shuffle masks based on demanded elements if it won't reduce
2827     // to Identity which can cause premature removal of the shuffle mask.
2828     if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2829       SDValue LegalShuffle =
2830           buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2831                                   NewMask, TLO.DAG);
2832       if (LegalShuffle)
2833         return TLO.CombineTo(Op, LegalShuffle);
2834     }
2835 
2836     // Propagate undef/zero elements from LHS/RHS.
2837     for (unsigned i = 0; i != NumElts; ++i) {
2838       int M = ShuffleMask[i];
2839       if (M < 0) {
2840         KnownUndef.setBit(i);
2841       } else if (M < (int)NumElts) {
2842         if (UndefLHS[M])
2843           KnownUndef.setBit(i);
2844         if (ZeroLHS[M])
2845           KnownZero.setBit(i);
2846       } else {
2847         if (UndefRHS[M - NumElts])
2848           KnownUndef.setBit(i);
2849         if (ZeroRHS[M - NumElts])
2850           KnownZero.setBit(i);
2851       }
2852     }
2853     break;
2854   }
2855   case ISD::ANY_EXTEND_VECTOR_INREG:
2856   case ISD::SIGN_EXTEND_VECTOR_INREG:
2857   case ISD::ZERO_EXTEND_VECTOR_INREG: {
2858     APInt SrcUndef, SrcZero;
2859     SDValue Src = Op.getOperand(0);
2860     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2861     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2862     if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2863                                    Depth + 1))
2864       return true;
2865     KnownZero = SrcZero.zextOrTrunc(NumElts);
2866     KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2867 
2868     if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2869         Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2870         DemandedSrcElts == 1) {
2871       // aext - if we just need the bottom element then we can bitcast.
2872       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2873     }
2874 
2875     if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2876       // zext(undef) upper bits are guaranteed to be zero.
2877       if (DemandedElts.isSubsetOf(KnownUndef))
2878         return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2879       KnownUndef.clearAllBits();
2880 
2881       // zext - if we just need the bottom element then we can mask:
2882       // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
2883       if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
2884           Op->isOnlyUserOf(Src.getNode()) &&
2885           Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
2886         SDLoc DL(Op);
2887         EVT SrcVT = Src.getValueType();
2888         EVT SrcSVT = SrcVT.getScalarType();
2889         SmallVector<SDValue> MaskElts;
2890         MaskElts.push_back(TLO.DAG.getAllOnesConstant(DL, SrcSVT));
2891         MaskElts.append(NumSrcElts - 1, TLO.DAG.getConstant(0, DL, SrcSVT));
2892         SDValue Mask = TLO.DAG.getBuildVector(SrcVT, DL, MaskElts);
2893         if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
2894                 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) {
2895           Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold);
2896           return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold));
2897         }
2898       }
2899     }
2900     break;
2901   }
2902 
2903   // TODO: There are more binop opcodes that could be handled here - MIN,
2904   // MAX, saturated math, etc.
2905   case ISD::ADD: {
2906     SDValue Op0 = Op.getOperand(0);
2907     SDValue Op1 = Op.getOperand(1);
2908     if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) {
2909       APInt UndefLHS, ZeroLHS;
2910       if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2911                                      Depth + 1, /*AssumeSingleUse*/ true))
2912         return true;
2913     }
2914     LLVM_FALLTHROUGH;
2915   }
2916   case ISD::OR:
2917   case ISD::XOR:
2918   case ISD::SUB:
2919   case ISD::FADD:
2920   case ISD::FSUB:
2921   case ISD::FMUL:
2922   case ISD::FDIV:
2923   case ISD::FREM: {
2924     SDValue Op0 = Op.getOperand(0);
2925     SDValue Op1 = Op.getOperand(1);
2926 
2927     APInt UndefRHS, ZeroRHS;
2928     if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2929                                    Depth + 1))
2930       return true;
2931     APInt UndefLHS, ZeroLHS;
2932     if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2933                                    Depth + 1))
2934       return true;
2935 
2936     KnownZero = ZeroLHS & ZeroRHS;
2937     KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2938 
2939     // Attempt to avoid multi-use ops if we don't need anything from them.
2940     // TODO - use KnownUndef to relax the demandedelts?
2941     if (!DemandedElts.isAllOnes())
2942       if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2943         return true;
2944     break;
2945   }
2946   case ISD::SHL:
2947   case ISD::SRL:
2948   case ISD::SRA:
2949   case ISD::ROTL:
2950   case ISD::ROTR: {
2951     SDValue Op0 = Op.getOperand(0);
2952     SDValue Op1 = Op.getOperand(1);
2953 
2954     APInt UndefRHS, ZeroRHS;
2955     if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2956                                    Depth + 1))
2957       return true;
2958     APInt UndefLHS, ZeroLHS;
2959     if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2960                                    Depth + 1))
2961       return true;
2962 
2963     KnownZero = ZeroLHS;
2964     KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2965 
2966     // Attempt to avoid multi-use ops if we don't need anything from them.
2967     // TODO - use KnownUndef to relax the demandedelts?
2968     if (!DemandedElts.isAllOnes())
2969       if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2970         return true;
2971     break;
2972   }
2973   case ISD::MUL:
2974   case ISD::AND: {
2975     SDValue Op0 = Op.getOperand(0);
2976     SDValue Op1 = Op.getOperand(1);
2977 
2978     APInt SrcUndef, SrcZero;
2979     if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2980                                    Depth + 1))
2981       return true;
2982     if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2983                                    TLO, Depth + 1))
2984       return true;
2985 
2986     // If either side has a zero element, then the result element is zero, even
2987     // if the other is an UNDEF.
2988     // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2989     // and then handle 'and' nodes with the rest of the binop opcodes.
2990     KnownZero |= SrcZero;
2991     KnownUndef &= SrcUndef;
2992     KnownUndef &= ~KnownZero;
2993 
2994     // Attempt to avoid multi-use ops if we don't need anything from them.
2995     // TODO - use KnownUndef to relax the demandedelts?
2996     if (!DemandedElts.isAllOnes())
2997       if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2998         return true;
2999     break;
3000   }
3001   case ISD::TRUNCATE:
3002   case ISD::SIGN_EXTEND:
3003   case ISD::ZERO_EXTEND:
3004     if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
3005                                    KnownZero, TLO, Depth + 1))
3006       return true;
3007 
3008     if (Op.getOpcode() == ISD::ZERO_EXTEND) {
3009       // zext(undef) upper bits are guaranteed to be zero.
3010       if (DemandedElts.isSubsetOf(KnownUndef))
3011         return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
3012       KnownUndef.clearAllBits();
3013     }
3014     break;
3015   default: {
3016     if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
3017       if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3018                                                   KnownZero, TLO, Depth))
3019         return true;
3020     } else {
3021       KnownBits Known;
3022       APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
3023       if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
3024                                TLO, Depth, AssumeSingleUse))
3025         return true;
3026     }
3027     break;
3028   }
3029   }
3030   assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
3031 
3032   // Constant fold all undef cases.
3033   // TODO: Handle zero cases as well.
3034   if (DemandedElts.isSubsetOf(KnownUndef))
3035     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3036 
3037   return false;
3038 }
3039 
3040 /// Determine which of the bits specified in Mask are known to be either zero or
3041 /// one and return them in the Known.
3042 void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3043                                                    KnownBits &Known,
3044                                                    const APInt &DemandedElts,
3045                                                    const SelectionDAG &DAG,
3046                                                    unsigned Depth) const {
3047   assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3048           Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3049           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3050           Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3051          "Should use MaskedValueIsZero if you don't know whether Op"
3052          " is a target node!");
3053   Known.resetAll();
3054 }
3055 
3056 void TargetLowering::computeKnownBitsForTargetInstr(
3057     GISelKnownBits &Analysis, Register R, KnownBits &Known,
3058     const APInt &DemandedElts, const MachineRegisterInfo &MRI,
3059     unsigned Depth) const {
3060   Known.resetAll();
3061 }
3062 
3063 void TargetLowering::computeKnownBitsForFrameIndex(
3064   const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
3065   // The low bits are known zero if the pointer is aligned.
3066   Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
3067 }
3068 
3069 Align TargetLowering::computeKnownAlignForTargetInstr(
3070   GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
3071   unsigned Depth) const {
3072   return Align(1);
3073 }
3074 
3075 /// This method can be implemented by targets that want to expose additional
3076 /// information about sign bits to the DAG Combiner.
3077 unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
3078                                                          const APInt &,
3079                                                          const SelectionDAG &,
3080                                                          unsigned Depth) const {
3081   assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3082           Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3083           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3084           Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3085          "Should use ComputeNumSignBits if you don't know whether Op"
3086          " is a target node!");
3087   return 1;
3088 }
3089 
3090 unsigned TargetLowering::computeNumSignBitsForTargetInstr(
3091   GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
3092   const MachineRegisterInfo &MRI, unsigned Depth) const {
3093   return 1;
3094 }
3095 
3096 bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
3097     SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3098     TargetLoweringOpt &TLO, unsigned Depth) const {
3099   assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3100           Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3101           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3102           Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3103          "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3104          " is a target node!");
3105   return false;
3106 }
3107 
3108 bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3109     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3110     KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3111   assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3112           Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3113           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3114           Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3115          "Should use SimplifyDemandedBits if you don't know whether Op"
3116          " is a target node!");
3117   computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3118   return false;
3119 }
3120 
3121 SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3122     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3123     SelectionDAG &DAG, unsigned Depth) const {
3124   assert(
3125       (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3126        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3127        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3128        Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3129       "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3130       " is a target node!");
3131   return SDValue();
3132 }
3133 
3134 SDValue
3135 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3136                                         SDValue N1, MutableArrayRef<int> Mask,
3137                                         SelectionDAG &DAG) const {
3138   bool LegalMask = isShuffleMaskLegal(Mask, VT);
3139   if (!LegalMask) {
3140     std::swap(N0, N1);
3141     ShuffleVectorSDNode::commuteMask(Mask);
3142     LegalMask = isShuffleMaskLegal(Mask, VT);
3143   }
3144 
3145   if (!LegalMask)
3146     return SDValue();
3147 
3148   return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3149 }
3150 
3151 const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3152   return nullptr;
3153 }
3154 
3155 bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
3156     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3157     bool PoisonOnly, unsigned Depth) const {
3158   assert(
3159       (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3160        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3161        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3162        Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3163       "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3164       " is a target node!");
3165   return false;
3166 }
3167 
3168 bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3169                                                   const SelectionDAG &DAG,
3170                                                   bool SNaN,
3171                                                   unsigned Depth) const {
3172   assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3173           Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3174           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3175           Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3176          "Should use isKnownNeverNaN if you don't know whether Op"
3177          " is a target node!");
3178   return false;
3179 }
3180 
3181 bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
3182                                                const APInt &DemandedElts,
3183                                                APInt &UndefElts,
3184                                                unsigned Depth) const {
3185   assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3186           Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3187           Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3188           Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3189          "Should use isSplatValue if you don't know whether Op"
3190          " is a target node!");
3191   return false;
3192 }
3193 
3194 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3195 // work with truncating build vectors and vectors with elements of less than
3196 // 8 bits.
3197 bool TargetLowering::isConstTrueVal(SDValue N) const {
3198   if (!N)
3199     return false;
3200 
3201   unsigned EltWidth;
3202   APInt CVal;
3203   if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
3204                                                /*AllowTruncation=*/true)) {
3205     CVal = CN->getAPIntValue();
3206     EltWidth = N.getValueType().getScalarSizeInBits();
3207   } else
3208     return false;
3209 
3210   // If this is a truncating splat, truncate the splat value.
3211   // Otherwise, we may fail to match the expected values below.
3212   if (EltWidth < CVal.getBitWidth())
3213     CVal = CVal.trunc(EltWidth);
3214 
3215   switch (getBooleanContents(N.getValueType())) {
3216   case UndefinedBooleanContent:
3217     return CVal[0];
3218   case ZeroOrOneBooleanContent:
3219     return CVal.isOne();
3220   case ZeroOrNegativeOneBooleanContent:
3221     return CVal.isAllOnes();
3222   }
3223 
3224   llvm_unreachable("Invalid boolean contents");
3225 }
3226 
3227 bool TargetLowering::isConstFalseVal(SDValue N) const {
3228   if (!N)
3229     return false;
3230 
3231   const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3232   if (!CN) {
3233     const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3234     if (!BV)
3235       return false;
3236 
3237     // Only interested in constant splats, we don't care about undef
3238     // elements in identifying boolean constants and getConstantSplatNode
3239     // returns NULL if all ops are undef;
3240     CN = BV->getConstantSplatNode();
3241     if (!CN)
3242       return false;
3243   }
3244 
3245   if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3246     return !CN->getAPIntValue()[0];
3247 
3248   return CN->isZero();
3249 }
3250 
3251 bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3252                                        bool SExt) const {
3253   if (VT == MVT::i1)
3254     return N->isOne();
3255 
3256   TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3257   switch (Cnt) {
3258   case TargetLowering::ZeroOrOneBooleanContent:
3259     // An extended value of 1 is always true, unless its original type is i1,
3260     // in which case it will be sign extended to -1.
3261     return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3262   case TargetLowering::UndefinedBooleanContent:
3263   case TargetLowering::ZeroOrNegativeOneBooleanContent:
3264     return N->isAllOnes() && SExt;
3265   }
3266   llvm_unreachable("Unexpected enumeration.");
3267 }
3268 
3269 /// This helper function of SimplifySetCC tries to optimize the comparison when
3270 /// either operand of the SetCC node is a bitwise-and instruction.
3271 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3272                                          ISD::CondCode Cond, const SDLoc &DL,
3273                                          DAGCombinerInfo &DCI) const {
3274   if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3275     std::swap(N0, N1);
3276 
3277   SelectionDAG &DAG = DCI.DAG;
3278   EVT OpVT = N0.getValueType();
3279   if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3280       (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3281     return SDValue();
3282 
3283   // (X & Y) != 0 --> zextOrTrunc(X & Y)
3284   // iff everything but LSB is known zero:
3285   if (Cond == ISD::SETNE && isNullConstant(N1) &&
3286       (getBooleanContents(OpVT) == TargetLowering::UndefinedBooleanContent ||
3287        getBooleanContents(OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
3288     unsigned NumEltBits = OpVT.getScalarSizeInBits();
3289     APInt UpperBits = APInt::getHighBitsSet(NumEltBits, NumEltBits - 1);
3290     if (DAG.MaskedValueIsZero(N0, UpperBits))
3291       return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT);
3292   }
3293 
3294   // Match these patterns in any of their permutations:
3295   // (X & Y) == Y
3296   // (X & Y) != Y
3297   SDValue X, Y;
3298   if (N0.getOperand(0) == N1) {
3299     X = N0.getOperand(1);
3300     Y = N0.getOperand(0);
3301   } else if (N0.getOperand(1) == N1) {
3302     X = N0.getOperand(0);
3303     Y = N0.getOperand(1);
3304   } else {
3305     return SDValue();
3306   }
3307 
3308   SDValue Zero = DAG.getConstant(0, DL, OpVT);
3309   if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3310     // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3311     // Note that where Y is variable and is known to have at most one bit set
3312     // (for example, if it is Z & 1) we cannot do this; the expressions are not
3313     // equivalent when Y == 0.
3314     assert(OpVT.isInteger());
3315     Cond = ISD::getSetCCInverse(Cond, OpVT);
3316     if (DCI.isBeforeLegalizeOps() ||
3317         isCondCodeLegal(Cond, N0.getSimpleValueType()))
3318       return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3319   } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3320     // If the target supports an 'and-not' or 'and-complement' logic operation,
3321     // try to use that to make a comparison operation more efficient.
3322     // But don't do this transform if the mask is a single bit because there are
3323     // more efficient ways to deal with that case (for example, 'bt' on x86 or
3324     // 'rlwinm' on PPC).
3325 
3326     // Bail out if the compare operand that we want to turn into a zero is
3327     // already a zero (otherwise, infinite loop).
3328     auto *YConst = dyn_cast<ConstantSDNode>(Y);
3329     if (YConst && YConst->isZero())
3330       return SDValue();
3331 
3332     // Transform this into: ~X & Y == 0.
3333     SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3334     SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3335     return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3336   }
3337 
3338   return SDValue();
3339 }
3340 
3341 /// There are multiple IR patterns that could be checking whether certain
3342 /// truncation of a signed number would be lossy or not. The pattern which is
3343 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
3344 /// We are looking for the following pattern: (KeptBits is a constant)
3345 ///   (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3346 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3347 /// KeptBits also can't be 1, that would have been folded to  %x dstcond 0
3348 /// We will unfold it into the natural trunc+sext pattern:
3349 ///   ((%x << C) a>> C) dstcond %x
3350 /// Where  C = bitwidth(x) - KeptBits  and  C u< bitwidth(x)
3351 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3352     EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3353     const SDLoc &DL) const {
3354   // We must be comparing with a constant.
3355   ConstantSDNode *C1;
3356   if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3357     return SDValue();
3358 
3359   // N0 should be:  add %x, (1 << (KeptBits-1))
3360   if (N0->getOpcode() != ISD::ADD)
3361     return SDValue();
3362 
3363   // And we must be 'add'ing a constant.
3364   ConstantSDNode *C01;
3365   if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3366     return SDValue();
3367 
3368   SDValue X = N0->getOperand(0);
3369   EVT XVT = X.getValueType();
3370 
3371   // Validate constants ...
3372 
3373   APInt I1 = C1->getAPIntValue();
3374 
3375   ISD::CondCode NewCond;
3376   if (Cond == ISD::CondCode::SETULT) {
3377     NewCond = ISD::CondCode::SETEQ;
3378   } else if (Cond == ISD::CondCode::SETULE) {
3379     NewCond = ISD::CondCode::SETEQ;
3380     // But need to 'canonicalize' the constant.
3381     I1 += 1;
3382   } else if (Cond == ISD::CondCode::SETUGT) {
3383     NewCond = ISD::CondCode::SETNE;
3384     // But need to 'canonicalize' the constant.
3385     I1 += 1;
3386   } else if (Cond == ISD::CondCode::SETUGE) {
3387     NewCond = ISD::CondCode::SETNE;
3388   } else
3389     return SDValue();
3390 
3391   APInt I01 = C01->getAPIntValue();
3392 
3393   auto checkConstants = [&I1, &I01]() -> bool {
3394     // Both of them must be power-of-two, and the constant from setcc is bigger.
3395     return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3396   };
3397 
3398   if (checkConstants()) {
3399     // Great, e.g. got  icmp ult i16 (add i16 %x, 128), 256
3400   } else {
3401     // What if we invert constants? (and the target predicate)
3402     I1.negate();
3403     I01.negate();
3404     assert(XVT.isInteger());
3405     NewCond = getSetCCInverse(NewCond, XVT);
3406     if (!checkConstants())
3407       return SDValue();
3408     // Great, e.g. got  icmp uge i16 (add i16 %x, -128), -256
3409   }
3410 
3411   // They are power-of-two, so which bit is set?
3412   const unsigned KeptBits = I1.logBase2();
3413   const unsigned KeptBitsMinusOne = I01.logBase2();
3414 
3415   // Magic!
3416   if (KeptBits != (KeptBitsMinusOne + 1))
3417     return SDValue();
3418   assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
3419 
3420   // We don't want to do this in every single case.
3421   SelectionDAG &DAG = DCI.DAG;
3422   if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3423           XVT, KeptBits))
3424     return SDValue();
3425 
3426   const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3427   assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
3428 
3429   // Unfold into:  ((%x << C) a>> C) cond %x
3430   // Where 'cond' will be either 'eq' or 'ne'.
3431   SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3432   SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3433   SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3434   SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3435 
3436   return T2;
3437 }
3438 
3439 // (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
3440 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3441     EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3442     DAGCombinerInfo &DCI, const SDLoc &DL) const {
3443   assert(isConstOrConstSplat(N1C) &&
3444          isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
3445          "Should be a comparison with 0.");
3446   assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3447          "Valid only for [in]equality comparisons.");
3448 
3449   unsigned NewShiftOpcode;
3450   SDValue X, C, Y;
3451 
3452   SelectionDAG &DAG = DCI.DAG;
3453   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3454 
3455   // Look for '(C l>>/<< Y)'.
3456   auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3457     // The shift should be one-use.
3458     if (!V.hasOneUse())
3459       return false;
3460     unsigned OldShiftOpcode = V.getOpcode();
3461     switch (OldShiftOpcode) {
3462     case ISD::SHL:
3463       NewShiftOpcode = ISD::SRL;
3464       break;
3465     case ISD::SRL:
3466       NewShiftOpcode = ISD::SHL;
3467       break;
3468     default:
3469       return false; // must be a logical shift.
3470     }
3471     // We should be shifting a constant.
3472     // FIXME: best to use isConstantOrConstantVector().
3473     C = V.getOperand(0);
3474     ConstantSDNode *CC =
3475         isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3476     if (!CC)
3477       return false;
3478     Y = V.getOperand(1);
3479 
3480     ConstantSDNode *XC =
3481         isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3482     return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3483         X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3484   };
3485 
3486   // LHS of comparison should be an one-use 'and'.
3487   if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3488     return SDValue();
3489 
3490   X = N0.getOperand(0);
3491   SDValue Mask = N0.getOperand(1);
3492 
3493   // 'and' is commutative!
3494   if (!Match(Mask)) {
3495     std::swap(X, Mask);
3496     if (!Match(Mask))
3497       return SDValue();
3498   }
3499 
3500   EVT VT = X.getValueType();
3501 
3502   // Produce:
3503   // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3504   SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3505   SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3506   SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3507   return T2;
3508 }
3509 
3510 /// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3511 /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3512 /// handle the commuted versions of these patterns.
3513 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3514                                            ISD::CondCode Cond, const SDLoc &DL,
3515                                            DAGCombinerInfo &DCI) const {
3516   unsigned BOpcode = N0.getOpcode();
3517   assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
3518          "Unexpected binop");
3519   assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
3520 
3521   // (X + Y) == X --> Y == 0
3522   // (X - Y) == X --> Y == 0
3523   // (X ^ Y) == X --> Y == 0
3524   SelectionDAG &DAG = DCI.DAG;
3525   EVT OpVT = N0.getValueType();
3526   SDValue X = N0.getOperand(0);
3527   SDValue Y = N0.getOperand(1);
3528   if (X == N1)
3529     return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3530 
3531   if (Y != N1)
3532     return SDValue();
3533 
3534   // (X + Y) == Y --> X == 0
3535   // (X ^ Y) == Y --> X == 0
3536   if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3537     return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3538 
3539   // The shift would not be valid if the operands are boolean (i1).
3540   if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3541     return SDValue();
3542 
3543   // (X - Y) == Y --> X == Y << 1
3544   EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3545                                  !DCI.isBeforeLegalize());
3546   SDValue One = DAG.getConstant(1, DL, ShiftVT);
3547   SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3548   if (!DCI.isCalledByLegalizer())
3549     DCI.AddToWorklist(YShl1.getNode());
3550   return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3551 }
3552 
3553 static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3554                                       SDValue N0, const APInt &C1,
3555                                       ISD::CondCode Cond, const SDLoc &dl,
3556                                       SelectionDAG &DAG) {
3557   // Look through truncs that don't change the value of a ctpop.
3558   // FIXME: Add vector support? Need to be careful with setcc result type below.
3559   SDValue CTPOP = N0;
3560   if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3561       N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3562     CTPOP = N0.getOperand(0);
3563 
3564   if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3565     return SDValue();
3566 
3567   EVT CTVT = CTPOP.getValueType();
3568   SDValue CTOp = CTPOP.getOperand(0);
3569 
3570   // If this is a vector CTPOP, keep the CTPOP if it is legal.
3571   // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3572   if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3573     return SDValue();
3574 
3575   // (ctpop x) u< 2 -> (x & x-1) == 0
3576   // (ctpop x) u> 1 -> (x & x-1) != 0
3577   if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3578     unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3579     if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3580       return SDValue();
3581     if (C1 == 0 && (Cond == ISD::SETULT))
3582       return SDValue(); // This is handled elsewhere.
3583 
3584     unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3585 
3586     SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3587     SDValue Result = CTOp;
3588     for (unsigned i = 0; i < Passes; i++) {
3589       SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3590       Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3591     }
3592     ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3593     return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3594   }
3595 
3596   // If ctpop is not supported, expand a power-of-2 comparison based on it.
3597   if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3598     // For scalars, keep CTPOP if it is legal or custom.
3599     if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3600       return SDValue();
3601     // This is based on X86's custom lowering for CTPOP which produces more
3602     // instructions than the expansion here.
3603 
3604     // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3605     // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3606     SDValue Zero = DAG.getConstant(0, dl, CTVT);
3607     SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3608     assert(CTVT.isInteger());
3609     ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3610     SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3611     SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3612     SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3613     SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3614     unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3615     return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3616   }
3617 
3618   return SDValue();
3619 }
3620 
3621 /// Try to simplify a setcc built with the specified operands and cc. If it is
3622 /// unable to simplify it, return a null SDValue.
3623 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3624                                       ISD::CondCode Cond, bool foldBooleans,
3625                                       DAGCombinerInfo &DCI,
3626                                       const SDLoc &dl) const {
3627   SelectionDAG &DAG = DCI.DAG;
3628   const DataLayout &Layout = DAG.getDataLayout();
3629   EVT OpVT = N0.getValueType();
3630 
3631   // Constant fold or commute setcc.
3632   if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3633     return Fold;
3634 
3635   // Ensure that the constant occurs on the RHS and fold constant comparisons.
3636   // TODO: Handle non-splat vector constants. All undef causes trouble.
3637   // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3638   // infinite loop here when we encounter one.
3639   ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3640   if (isConstOrConstSplat(N0) &&
3641       (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3642       (DCI.isBeforeLegalizeOps() ||
3643        isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3644     return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3645 
3646   // If we have a subtract with the same 2 non-constant operands as this setcc
3647   // -- but in reverse order -- then try to commute the operands of this setcc
3648   // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3649   // instruction on some targets.
3650   if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3651       (DCI.isBeforeLegalizeOps() ||
3652        isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3653       DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3654       !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3655     return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3656 
3657   if (auto *N1C = isConstOrConstSplat(N1)) {
3658     const APInt &C1 = N1C->getAPIntValue();
3659 
3660     // Optimize some CTPOP cases.
3661     if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3662       return V;
3663 
3664     // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3665     // equality comparison, then we're just comparing whether X itself is
3666     // zero.
3667     if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
3668         N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3669         isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3670       if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3671         if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3672             ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3673           if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3674             // (srl (ctlz x), 5) == 0  -> X != 0
3675             // (srl (ctlz x), 5) != 1  -> X != 0
3676             Cond = ISD::SETNE;
3677           } else {
3678             // (srl (ctlz x), 5) != 0  -> X == 0
3679             // (srl (ctlz x), 5) == 1  -> X == 0
3680             Cond = ISD::SETEQ;
3681           }
3682           SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3683           return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3684                               Cond);
3685         }
3686       }
3687     }
3688   }
3689 
3690   // FIXME: Support vectors.
3691   if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3692     const APInt &C1 = N1C->getAPIntValue();
3693 
3694     // (zext x) == C --> x == (trunc C)
3695     // (sext x) == C --> x == (trunc C)
3696     if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3697         DCI.isBeforeLegalize() && N0->hasOneUse()) {
3698       unsigned MinBits = N0.getValueSizeInBits();
3699       SDValue PreExt;
3700       bool Signed = false;
3701       if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3702         // ZExt
3703         MinBits = N0->getOperand(0).getValueSizeInBits();
3704         PreExt = N0->getOperand(0);
3705       } else if (N0->getOpcode() == ISD::AND) {
3706         // DAGCombine turns costly ZExts into ANDs
3707         if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3708           if ((C->getAPIntValue()+1).isPowerOf2()) {
3709             MinBits = C->getAPIntValue().countTrailingOnes();
3710             PreExt = N0->getOperand(0);
3711           }
3712       } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3713         // SExt
3714         MinBits = N0->getOperand(0).getValueSizeInBits();
3715         PreExt = N0->getOperand(0);
3716         Signed = true;
3717       } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3718         // ZEXTLOAD / SEXTLOAD
3719         if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3720           MinBits = LN0->getMemoryVT().getSizeInBits();
3721           PreExt = N0;
3722         } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3723           Signed = true;
3724           MinBits = LN0->getMemoryVT().getSizeInBits();
3725           PreExt = N0;
3726         }
3727       }
3728 
3729       // Figure out how many bits we need to preserve this constant.
3730       unsigned ReqdBits = Signed ? C1.getMinSignedBits() : C1.getActiveBits();
3731 
3732       // Make sure we're not losing bits from the constant.
3733       if (MinBits > 0 &&
3734           MinBits < C1.getBitWidth() &&
3735           MinBits >= ReqdBits) {
3736         EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3737         if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3738           // Will get folded away.
3739           SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3740           if (MinBits == 1 && C1 == 1)
3741             // Invert the condition.
3742             return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3743                                 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3744           SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3745           return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3746         }
3747 
3748         // If truncating the setcc operands is not desirable, we can still
3749         // simplify the expression in some cases:
3750         // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3751         // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3752         // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3753         // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3754         // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3755         // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3756         SDValue TopSetCC = N0->getOperand(0);
3757         unsigned N0Opc = N0->getOpcode();
3758         bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3759         if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3760             TopSetCC.getOpcode() == ISD::SETCC &&
3761             (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3762             (isConstFalseVal(N1) ||
3763              isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3764 
3765           bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
3766                          (!N1C->isZero() && Cond == ISD::SETNE);
3767 
3768           if (!Inverse)
3769             return TopSetCC;
3770 
3771           ISD::CondCode InvCond = ISD::getSetCCInverse(
3772               cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3773               TopSetCC.getOperand(0).getValueType());
3774           return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3775                                       TopSetCC.getOperand(1),
3776                                       InvCond);
3777         }
3778       }
3779     }
3780 
3781     // If the LHS is '(and load, const)', the RHS is 0, the test is for
3782     // equality or unsigned, and all 1 bits of the const are in the same
3783     // partial word, see if we can shorten the load.
3784     if (DCI.isBeforeLegalize() &&
3785         !ISD::isSignedIntSetCC(Cond) &&
3786         N0.getOpcode() == ISD::AND && C1 == 0 &&
3787         N0.getNode()->hasOneUse() &&
3788         isa<LoadSDNode>(N0.getOperand(0)) &&
3789         N0.getOperand(0).getNode()->hasOneUse() &&
3790         isa<ConstantSDNode>(N0.getOperand(1))) {
3791       LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3792       APInt bestMask;
3793       unsigned bestWidth = 0, bestOffset = 0;
3794       if (Lod->isSimple() && Lod->isUnindexed()) {
3795         unsigned origWidth = N0.getValueSizeInBits();
3796         unsigned maskWidth = origWidth;
3797         // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3798         // 8 bits, but have to be careful...
3799         if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3800           origWidth = Lod->getMemoryVT().getSizeInBits();
3801         const APInt &Mask = N0.getConstantOperandAPInt(1);
3802         for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3803           APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3804           for (unsigned offset=0; offset<origWidth/width; offset++) {
3805             if (Mask.isSubsetOf(newMask)) {
3806               if (Layout.isLittleEndian())
3807                 bestOffset = (uint64_t)offset * (width/8);
3808               else
3809                 bestOffset = (origWidth/width - offset - 1) * (width/8);
3810               bestMask = Mask.lshr(offset * (width/8) * 8);
3811               bestWidth = width;
3812               break;
3813             }
3814             newMask <<= width;
3815           }
3816         }
3817       }
3818       if (bestWidth) {
3819         EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3820         if (newVT.isRound() &&
3821             shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3822           SDValue Ptr = Lod->getBasePtr();
3823           if (bestOffset != 0)
3824             Ptr =
3825                 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3826           SDValue NewLoad =
3827               DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3828                           Lod->getPointerInfo().getWithOffset(bestOffset),
3829                           Lod->getOriginalAlign());
3830           return DAG.getSetCC(dl, VT,
3831                               DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3832                                       DAG.getConstant(bestMask.trunc(bestWidth),
3833                                                       dl, newVT)),
3834                               DAG.getConstant(0LL, dl, newVT), Cond);
3835         }
3836       }
3837     }
3838 
3839     // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3840     if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3841       unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3842 
3843       // If the comparison constant has bits in the upper part, the
3844       // zero-extended value could never match.
3845       if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3846                                               C1.getBitWidth() - InSize))) {
3847         switch (Cond) {
3848         case ISD::SETUGT:
3849         case ISD::SETUGE:
3850         case ISD::SETEQ:
3851           return DAG.getConstant(0, dl, VT);
3852         case ISD::SETULT:
3853         case ISD::SETULE:
3854         case ISD::SETNE:
3855           return DAG.getConstant(1, dl, VT);
3856         case ISD::SETGT:
3857         case ISD::SETGE:
3858           // True if the sign bit of C1 is set.
3859           return DAG.getConstant(C1.isNegative(), dl, VT);
3860         case ISD::SETLT:
3861         case ISD::SETLE:
3862           // True if the sign bit of C1 isn't set.
3863           return DAG.getConstant(C1.isNonNegative(), dl, VT);
3864         default:
3865           break;
3866         }
3867       }
3868 
3869       // Otherwise, we can perform the comparison with the low bits.
3870       switch (Cond) {
3871       case ISD::SETEQ:
3872       case ISD::SETNE:
3873       case ISD::SETUGT:
3874       case ISD::SETUGE:
3875       case ISD::SETULT:
3876       case ISD::SETULE: {
3877         EVT newVT = N0.getOperand(0).getValueType();
3878         if (DCI.isBeforeLegalizeOps() ||
3879             (isOperationLegal(ISD::SETCC, newVT) &&
3880              isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3881           EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3882           SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3883 
3884           SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3885                                           NewConst, Cond);
3886           return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3887         }
3888         break;
3889       }
3890       default:
3891         break; // todo, be more careful with signed comparisons
3892       }
3893     } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3894                (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3895                !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3896                                       OpVT)) {
3897       EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3898       unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3899       EVT ExtDstTy = N0.getValueType();
3900       unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3901 
3902       // If the constant doesn't fit into the number of bits for the source of
3903       // the sign extension, it is impossible for both sides to be equal.
3904       if (C1.getMinSignedBits() > ExtSrcTyBits)
3905         return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3906 
3907       assert(ExtDstTy == N0.getOperand(0).getValueType() &&
3908              ExtDstTy != ExtSrcTy && "Unexpected types!");
3909       APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3910       SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3911                                    DAG.getConstant(Imm, dl, ExtDstTy));
3912       if (!DCI.isCalledByLegalizer())
3913         DCI.AddToWorklist(ZextOp.getNode());
3914       // Otherwise, make this a use of a zext.
3915       return DAG.getSetCC(dl, VT, ZextOp,
3916                           DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3917     } else if ((N1C->isZero() || N1C->isOne()) &&
3918                (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3919       // SETCC (SETCC), [0|1], [EQ|NE]  -> SETCC
3920       if (N0.getOpcode() == ISD::SETCC &&
3921           isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3922           (N0.getValueType() == MVT::i1 ||
3923            getBooleanContents(N0.getOperand(0).getValueType()) ==
3924                        ZeroOrOneBooleanContent)) {
3925         bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3926         if (TrueWhenTrue)
3927           return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3928         // Invert the condition.
3929         ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3930         CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3931         if (DCI.isBeforeLegalizeOps() ||
3932             isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3933           return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3934       }
3935 
3936       if ((N0.getOpcode() == ISD::XOR ||
3937            (N0.getOpcode() == ISD::AND &&
3938             N0.getOperand(0).getOpcode() == ISD::XOR &&
3939             N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3940           isOneConstant(N0.getOperand(1))) {
3941         // If this is (X^1) == 0/1, swap the RHS and eliminate the xor.  We
3942         // can only do this if the top bits are known zero.
3943         unsigned BitWidth = N0.getValueSizeInBits();
3944         if (DAG.MaskedValueIsZero(N0,
3945                                   APInt::getHighBitsSet(BitWidth,
3946                                                         BitWidth-1))) {
3947           // Okay, get the un-inverted input value.
3948           SDValue Val;
3949           if (N0.getOpcode() == ISD::XOR) {
3950             Val = N0.getOperand(0);
3951           } else {
3952             assert(N0.getOpcode() == ISD::AND &&
3953                     N0.getOperand(0).getOpcode() == ISD::XOR);
3954             // ((X^1)&1)^1 -> X & 1
3955             Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3956                               N0.getOperand(0).getOperand(0),
3957                               N0.getOperand(1));
3958           }
3959 
3960           return DAG.getSetCC(dl, VT, Val, N1,
3961                               Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3962         }
3963       } else if (N1C->isOne()) {
3964         SDValue Op0 = N0;
3965         if (Op0.getOpcode() == ISD::TRUNCATE)
3966           Op0 = Op0.getOperand(0);
3967 
3968         if ((Op0.getOpcode() == ISD::XOR) &&
3969             Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3970             Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3971           SDValue XorLHS = Op0.getOperand(0);
3972           SDValue XorRHS = Op0.getOperand(1);
3973           // Ensure that the input setccs return an i1 type or 0/1 value.
3974           if (Op0.getValueType() == MVT::i1 ||
3975               (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3976                       ZeroOrOneBooleanContent &&
3977                getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3978                         ZeroOrOneBooleanContent)) {
3979             // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3980             Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3981             return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3982           }
3983         }
3984         if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3985           // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3986           if (Op0.getValueType().bitsGT(VT))
3987             Op0 = DAG.getNode(ISD::AND, dl, VT,
3988                           DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3989                           DAG.getConstant(1, dl, VT));
3990           else if (Op0.getValueType().bitsLT(VT))
3991             Op0 = DAG.getNode(ISD::AND, dl, VT,
3992                         DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3993                         DAG.getConstant(1, dl, VT));
3994 
3995           return DAG.getSetCC(dl, VT, Op0,
3996                               DAG.getConstant(0, dl, Op0.getValueType()),
3997                               Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3998         }
3999         if (Op0.getOpcode() == ISD::AssertZext &&
4000             cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
4001           return DAG.getSetCC(dl, VT, Op0,
4002                               DAG.getConstant(0, dl, Op0.getValueType()),
4003                               Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
4004       }
4005     }
4006 
4007     // Given:
4008     //   icmp eq/ne (urem %x, %y), 0
4009     // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
4010     //   icmp eq/ne %x, 0
4011     if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
4012         (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4013       KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
4014       KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
4015       if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
4016         return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
4017     }
4018 
4019     // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
4020     //  and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
4021     if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4022         N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
4023         N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
4024         N1C && N1C->isAllOnes()) {
4025       return DAG.getSetCC(dl, VT, N0.getOperand(0),
4026                           DAG.getConstant(0, dl, OpVT),
4027                           Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
4028     }
4029 
4030     if (SDValue V =
4031             optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
4032       return V;
4033   }
4034 
4035   // These simplifications apply to splat vectors as well.
4036   // TODO: Handle more splat vector cases.
4037   if (auto *N1C = isConstOrConstSplat(N1)) {
4038     const APInt &C1 = N1C->getAPIntValue();
4039 
4040     APInt MinVal, MaxVal;
4041     unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
4042     if (ISD::isSignedIntSetCC(Cond)) {
4043       MinVal = APInt::getSignedMinValue(OperandBitSize);
4044       MaxVal = APInt::getSignedMaxValue(OperandBitSize);
4045     } else {
4046       MinVal = APInt::getMinValue(OperandBitSize);
4047       MaxVal = APInt::getMaxValue(OperandBitSize);
4048     }
4049 
4050     // Canonicalize GE/LE comparisons to use GT/LT comparisons.
4051     if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
4052       // X >= MIN --> true
4053       if (C1 == MinVal)
4054         return DAG.getBoolConstant(true, dl, VT, OpVT);
4055 
4056       if (!VT.isVector()) { // TODO: Support this for vectors.
4057         // X >= C0 --> X > (C0 - 1)
4058         APInt C = C1 - 1;
4059         ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
4060         if ((DCI.isBeforeLegalizeOps() ||
4061              isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4062             (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4063                                   isLegalICmpImmediate(C.getSExtValue())))) {
4064           return DAG.getSetCC(dl, VT, N0,
4065                               DAG.getConstant(C, dl, N1.getValueType()),
4066                               NewCC);
4067         }
4068       }
4069     }
4070 
4071     if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
4072       // X <= MAX --> true
4073       if (C1 == MaxVal)
4074         return DAG.getBoolConstant(true, dl, VT, OpVT);
4075 
4076       // X <= C0 --> X < (C0 + 1)
4077       if (!VT.isVector()) { // TODO: Support this for vectors.
4078         APInt C = C1 + 1;
4079         ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
4080         if ((DCI.isBeforeLegalizeOps() ||
4081              isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4082             (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4083                                   isLegalICmpImmediate(C.getSExtValue())))) {
4084           return DAG.getSetCC(dl, VT, N0,
4085                               DAG.getConstant(C, dl, N1.getValueType()),
4086                               NewCC);
4087         }
4088       }
4089     }
4090 
4091     if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
4092       if (C1 == MinVal)
4093         return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
4094 
4095       // TODO: Support this for vectors after legalize ops.
4096       if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4097         // Canonicalize setlt X, Max --> setne X, Max
4098         if (C1 == MaxVal)
4099           return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4100 
4101         // If we have setult X, 1, turn it into seteq X, 0
4102         if (C1 == MinVal+1)
4103           return DAG.getSetCC(dl, VT, N0,
4104                               DAG.getConstant(MinVal, dl, N0.getValueType()),
4105                               ISD::SETEQ);
4106       }
4107     }
4108 
4109     if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
4110       if (C1 == MaxVal)
4111         return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
4112 
4113       // TODO: Support this for vectors after legalize ops.
4114       if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4115         // Canonicalize setgt X, Min --> setne X, Min
4116         if (C1 == MinVal)
4117           return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4118 
4119         // If we have setugt X, Max-1, turn it into seteq X, Max
4120         if (C1 == MaxVal-1)
4121           return DAG.getSetCC(dl, VT, N0,
4122                               DAG.getConstant(MaxVal, dl, N0.getValueType()),
4123                               ISD::SETEQ);
4124       }
4125     }
4126 
4127     if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4128       // (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
4129       if (C1.isZero())
4130         if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4131                 VT, N0, N1, Cond, DCI, dl))
4132           return CC;
4133 
4134       // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4135       // For example, when high 32-bits of i64 X are known clear:
4136       // all bits clear: (X | (Y<<32)) ==  0 --> (X | Y) ==  0
4137       // all bits set:   (X | (Y<<32)) == -1 --> (X & Y) == -1
4138       bool CmpZero = N1C->getAPIntValue().isZero();
4139       bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
4140       if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4141         // Match or(lo,shl(hi,bw/2)) pattern.
4142         auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4143           unsigned EltBits = V.getScalarValueSizeInBits();
4144           if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4145             return false;
4146           SDValue LHS = V.getOperand(0);
4147           SDValue RHS = V.getOperand(1);
4148           APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4149           // Unshifted element must have zero upperbits.
4150           if (RHS.getOpcode() == ISD::SHL &&
4151               isa<ConstantSDNode>(RHS.getOperand(1)) &&
4152               RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4153               DAG.MaskedValueIsZero(LHS, HiBits)) {
4154             Lo = LHS;
4155             Hi = RHS.getOperand(0);
4156             return true;
4157           }
4158           if (LHS.getOpcode() == ISD::SHL &&
4159               isa<ConstantSDNode>(LHS.getOperand(1)) &&
4160               LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4161               DAG.MaskedValueIsZero(RHS, HiBits)) {
4162             Lo = RHS;
4163             Hi = LHS.getOperand(0);
4164             return true;
4165           }
4166           return false;
4167         };
4168 
4169         auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4170           unsigned EltBits = N0.getScalarValueSizeInBits();
4171           unsigned HalfBits = EltBits / 2;
4172           APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4173           SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4174           SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4175           SDValue NewN0 =
4176               DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4177           SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4178           return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4179         };
4180 
4181         SDValue Lo, Hi;
4182         if (IsConcat(N0, Lo, Hi))
4183           return MergeConcat(Lo, Hi);
4184 
4185         if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4186           SDValue Lo0, Lo1, Hi0, Hi1;
4187           if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4188               IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4189             return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4190                                DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4191           }
4192         }
4193       }
4194     }
4195 
4196     // If we have "setcc X, C0", check to see if we can shrink the immediate
4197     // by changing cc.
4198     // TODO: Support this for vectors after legalize ops.
4199     if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4200       // SETUGT X, SINTMAX  -> SETLT X, 0
4201       // SETUGE X, SINTMIN -> SETLT X, 0
4202       if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4203           (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4204         return DAG.getSetCC(dl, VT, N0,
4205                             DAG.getConstant(0, dl, N1.getValueType()),
4206                             ISD::SETLT);
4207 
4208       // SETULT X, SINTMIN  -> SETGT X, -1
4209       // SETULE X, SINTMAX  -> SETGT X, -1
4210       if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4211           (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4212         return DAG.getSetCC(dl, VT, N0,
4213                             DAG.getAllOnesConstant(dl, N1.getValueType()),
4214                             ISD::SETGT);
4215     }
4216   }
4217 
4218   // Back to non-vector simplifications.
4219   // TODO: Can we do these for vector splats?
4220   if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4221     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4222     const APInt &C1 = N1C->getAPIntValue();
4223     EVT ShValTy = N0.getValueType();
4224 
4225     // Fold bit comparisons when we can. This will result in an
4226     // incorrect value when boolean false is negative one, unless
4227     // the bitsize is 1 in which case the false value is the same
4228     // in practice regardless of the representation.
4229     if ((VT.getSizeInBits() == 1 ||
4230          getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4231         (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4232         (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4233         N0.getOpcode() == ISD::AND) {
4234       if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4235         EVT ShiftTy =
4236             getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4237         if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0  -->  (X & 8) >> 3
4238           // Perform the xform if the AND RHS is a single bit.
4239           unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4240           if (AndRHS->getAPIntValue().isPowerOf2() &&
4241               !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4242             return DAG.getNode(ISD::TRUNCATE, dl, VT,
4243                                DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4244                                            DAG.getConstant(ShCt, dl, ShiftTy)));
4245           }
4246         } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4247           // (X & 8) == 8  -->  (X & 8) >> 3
4248           // Perform the xform if C1 is a single bit.
4249           unsigned ShCt = C1.logBase2();
4250           if (C1.isPowerOf2() &&
4251               !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4252             return DAG.getNode(ISD::TRUNCATE, dl, VT,
4253                                DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4254                                            DAG.getConstant(ShCt, dl, ShiftTy)));
4255           }
4256         }
4257       }
4258     }
4259 
4260     if (C1.getMinSignedBits() <= 64 &&
4261         !isLegalICmpImmediate(C1.getSExtValue())) {
4262       EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4263       // (X & -256) == 256 -> (X >> 8) == 1
4264       if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4265           N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4266         if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4267           const APInt &AndRHSC = AndRHS->getAPIntValue();
4268           if (AndRHSC.isNegatedPowerOf2() && (AndRHSC & C1) == C1) {
4269             unsigned ShiftBits = AndRHSC.countTrailingZeros();
4270             if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4271               SDValue Shift =
4272                 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4273                             DAG.getConstant(ShiftBits, dl, ShiftTy));
4274               SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4275               return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4276             }
4277           }
4278         }
4279       } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4280                  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4281         bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4282         // X <  0x100000000 -> (X >> 32) <  1
4283         // X >= 0x100000000 -> (X >> 32) >= 1
4284         // X <= 0x0ffffffff -> (X >> 32) <  1
4285         // X >  0x0ffffffff -> (X >> 32) >= 1
4286         unsigned ShiftBits;
4287         APInt NewC = C1;
4288         ISD::CondCode NewCond = Cond;
4289         if (AdjOne) {
4290           ShiftBits = C1.countTrailingOnes();
4291           NewC = NewC + 1;
4292           NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4293         } else {
4294           ShiftBits = C1.countTrailingZeros();
4295         }
4296         NewC.lshrInPlace(ShiftBits);
4297         if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4298             isLegalICmpImmediate(NewC.getSExtValue()) &&
4299             !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4300           SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4301                                       DAG.getConstant(ShiftBits, dl, ShiftTy));
4302           SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4303           return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4304         }
4305       }
4306     }
4307   }
4308 
4309   if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4310     auto *CFP = cast<ConstantFPSDNode>(N1);
4311     assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
4312 
4313     // Otherwise, we know the RHS is not a NaN.  Simplify the node to drop the
4314     // constant if knowing that the operand is non-nan is enough.  We prefer to
4315     // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4316     // materialize 0.0.
4317     if (Cond == ISD::SETO || Cond == ISD::SETUO)
4318       return DAG.getSetCC(dl, VT, N0, N0, Cond);
4319 
4320     // setcc (fneg x), C -> setcc swap(pred) x, -C
4321     if (N0.getOpcode() == ISD::FNEG) {
4322       ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4323       if (DCI.isBeforeLegalizeOps() ||
4324           isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4325         SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4326         return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4327       }
4328     }
4329 
4330     // If the condition is not legal, see if we can find an equivalent one
4331     // which is legal.
4332     if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4333       // If the comparison was an awkward floating-point == or != and one of
4334       // the comparison operands is infinity or negative infinity, convert the
4335       // condition to a less-awkward <= or >=.
4336       if (CFP->getValueAPF().isInfinity()) {
4337         bool IsNegInf = CFP->getValueAPF().isNegative();
4338         ISD::CondCode NewCond = ISD::SETCC_INVALID;
4339         switch (Cond) {
4340         case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4341         case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4342         case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4343         case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4344         default: break;
4345         }
4346         if (NewCond != ISD::SETCC_INVALID &&
4347             isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4348           return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4349       }
4350     }
4351   }
4352 
4353   if (N0 == N1) {
4354     // The sext(setcc()) => setcc() optimization relies on the appropriate
4355     // constant being emitted.
4356     assert(!N0.getValueType().isInteger() &&
4357            "Integer types should be handled by FoldSetCC");
4358 
4359     bool EqTrue = ISD::isTrueWhenEqual(Cond);
4360     unsigned UOF = ISD::getUnorderedFlavor(Cond);
4361     if (UOF == 2) // FP operators that are undefined on NaNs.
4362       return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4363     if (UOF == unsigned(EqTrue))
4364       return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4365     // Otherwise, we can't fold it.  However, we can simplify it to SETUO/SETO
4366     // if it is not already.
4367     ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4368     if (NewCond != Cond &&
4369         (DCI.isBeforeLegalizeOps() ||
4370                             isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4371       return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4372   }
4373 
4374   if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4375       N0.getValueType().isInteger()) {
4376     if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4377         N0.getOpcode() == ISD::XOR) {
4378       // Simplify (X+Y) == (X+Z) -->  Y == Z
4379       if (N0.getOpcode() == N1.getOpcode()) {
4380         if (N0.getOperand(0) == N1.getOperand(0))
4381           return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4382         if (N0.getOperand(1) == N1.getOperand(1))
4383           return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4384         if (isCommutativeBinOp(N0.getOpcode())) {
4385           // If X op Y == Y op X, try other combinations.
4386           if (N0.getOperand(0) == N1.getOperand(1))
4387             return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4388                                 Cond);
4389           if (N0.getOperand(1) == N1.getOperand(0))
4390             return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4391                                 Cond);
4392         }
4393       }
4394 
4395       // If RHS is a legal immediate value for a compare instruction, we need
4396       // to be careful about increasing register pressure needlessly.
4397       bool LegalRHSImm = false;
4398 
4399       if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4400         if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4401           // Turn (X+C1) == C2 --> X == C2-C1
4402           if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4403             return DAG.getSetCC(dl, VT, N0.getOperand(0),
4404                                 DAG.getConstant(RHSC->getAPIntValue()-
4405                                                 LHSR->getAPIntValue(),
4406                                 dl, N0.getValueType()), Cond);
4407           }
4408 
4409           // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4410           if (N0.getOpcode() == ISD::XOR)
4411             // If we know that all of the inverted bits are zero, don't bother
4412             // performing the inversion.
4413             if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4414               return
4415                 DAG.getSetCC(dl, VT, N0.getOperand(0),
4416                              DAG.getConstant(LHSR->getAPIntValue() ^
4417                                                RHSC->getAPIntValue(),
4418                                              dl, N0.getValueType()),
4419                              Cond);
4420         }
4421 
4422         // Turn (C1-X) == C2 --> X == C1-C2
4423         if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4424           if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4425             return
4426               DAG.getSetCC(dl, VT, N0.getOperand(1),
4427                            DAG.getConstant(SUBC->getAPIntValue() -
4428                                              RHSC->getAPIntValue(),
4429                                            dl, N0.getValueType()),
4430                            Cond);
4431           }
4432         }
4433 
4434         // Could RHSC fold directly into a compare?
4435         if (RHSC->getValueType(0).getSizeInBits() <= 64)
4436           LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4437       }
4438 
4439       // (X+Y) == X --> Y == 0 and similar folds.
4440       // Don't do this if X is an immediate that can fold into a cmp
4441       // instruction and X+Y has other uses. It could be an induction variable
4442       // chain, and the transform would increase register pressure.
4443       if (!LegalRHSImm || N0.hasOneUse())
4444         if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4445           return V;
4446     }
4447 
4448     if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4449         N1.getOpcode() == ISD::XOR)
4450       if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4451         return V;
4452 
4453     if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4454       return V;
4455   }
4456 
4457   // Fold remainder of division by a constant.
4458   if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4459       N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4460     AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4461 
4462     // When division is cheap or optimizing for minimum size,
4463     // fall through to DIVREM creation by skipping this fold.
4464     if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
4465       if (N0.getOpcode() == ISD::UREM) {
4466         if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4467           return Folded;
4468       } else if (N0.getOpcode() == ISD::SREM) {
4469         if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4470           return Folded;
4471       }
4472     }
4473   }
4474 
4475   // Fold away ALL boolean setcc's.
4476   if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4477     SDValue Temp;
4478     switch (Cond) {
4479     default: llvm_unreachable("Unknown integer setcc!");
4480     case ISD::SETEQ:  // X == Y  -> ~(X^Y)
4481       Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4482       N0 = DAG.getNOT(dl, Temp, OpVT);
4483       if (!DCI.isCalledByLegalizer())
4484         DCI.AddToWorklist(Temp.getNode());
4485       break;
4486     case ISD::SETNE:  // X != Y   -->  (X^Y)
4487       N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4488       break;
4489     case ISD::SETGT:  // X >s Y   -->  X == 0 & Y == 1  -->  ~X & Y
4490     case ISD::SETULT: // X <u Y   -->  X == 0 & Y == 1  -->  ~X & Y
4491       Temp = DAG.getNOT(dl, N0, OpVT);
4492       N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4493       if (!DCI.isCalledByLegalizer())
4494         DCI.AddToWorklist(Temp.getNode());
4495       break;
4496     case ISD::SETLT:  // X <s Y   --> X == 1 & Y == 0  -->  ~Y & X
4497     case ISD::SETUGT: // X >u Y   --> X == 1 & Y == 0  -->  ~Y & X
4498       Temp = DAG.getNOT(dl, N1, OpVT);
4499       N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4500       if (!DCI.isCalledByLegalizer())
4501         DCI.AddToWorklist(Temp.getNode());
4502       break;
4503     case ISD::SETULE: // X <=u Y  --> X == 0 | Y == 1  -->  ~X | Y
4504     case ISD::SETGE:  // X >=s Y  --> X == 0 | Y == 1  -->  ~X | Y
4505       Temp = DAG.getNOT(dl, N0, OpVT);
4506       N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4507       if (!DCI.isCalledByLegalizer())
4508         DCI.AddToWorklist(Temp.getNode());
4509       break;
4510     case ISD::SETUGE: // X >=u Y  --> X == 1 | Y == 0  -->  ~Y | X
4511     case ISD::SETLE:  // X <=s Y  --> X == 1 | Y == 0  -->  ~Y | X
4512       Temp = DAG.getNOT(dl, N1, OpVT);
4513       N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4514       break;
4515     }
4516     if (VT.getScalarType() != MVT::i1) {
4517       if (!DCI.isCalledByLegalizer())
4518         DCI.AddToWorklist(N0.getNode());
4519       // FIXME: If running after legalize, we probably can't do this.
4520       ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4521       N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4522     }
4523     return N0;
4524   }
4525 
4526   // Could not fold it.
4527   return SDValue();
4528 }
4529 
4530 /// Returns true (and the GlobalValue and the offset) if the node is a
4531 /// GlobalAddress + offset.
4532 bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4533                                     int64_t &Offset) const {
4534 
4535   SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4536 
4537   if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4538     GA = GASD->getGlobal();
4539     Offset += GASD->getOffset();
4540     return true;
4541   }
4542 
4543   if (N->getOpcode() == ISD::ADD) {
4544     SDValue N1 = N->getOperand(0);
4545     SDValue N2 = N->getOperand(1);
4546     if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4547       if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4548         Offset += V->getSExtValue();
4549         return true;
4550       }
4551     } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4552       if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4553         Offset += V->getSExtValue();
4554         return true;
4555       }
4556     }
4557   }
4558 
4559   return false;
4560 }
4561 
4562 SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4563                                           DAGCombinerInfo &DCI) const {
4564   // Default implementation: no optimization.
4565   return SDValue();
4566 }
4567 
4568 //===----------------------------------------------------------------------===//
4569 //  Inline Assembler Implementation Methods
4570 //===----------------------------------------------------------------------===//
4571 
4572 TargetLowering::ConstraintType
4573 TargetLowering::getConstraintType(StringRef Constraint) const {
4574   unsigned S = Constraint.size();
4575 
4576   if (S == 1) {
4577     switch (Constraint[0]) {
4578     default: break;
4579     case 'r':
4580       return C_RegisterClass;
4581     case 'm': // memory
4582     case 'o': // offsetable
4583     case 'V': // not offsetable
4584       return C_Memory;
4585     case 'n': // Simple Integer
4586     case 'E': // Floating Point Constant
4587     case 'F': // Floating Point Constant
4588       return C_Immediate;
4589     case 'i': // Simple Integer or Relocatable Constant
4590     case 's': // Relocatable Constant
4591     case 'p': // Address.
4592     case 'X': // Allow ANY value.
4593     case 'I': // Target registers.
4594     case 'J':
4595     case 'K':
4596     case 'L':
4597     case 'M':
4598     case 'N':
4599     case 'O':
4600     case 'P':
4601     case '<':
4602     case '>':
4603       return C_Other;
4604     }
4605   }
4606 
4607   if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4608     if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4609       return C_Memory;
4610     return C_Register;
4611   }
4612   return C_Unknown;
4613 }
4614 
4615 /// Try to replace an X constraint, which matches anything, with another that
4616 /// has more specific requirements based on the type of the corresponding
4617 /// operand.
4618 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4619   if (ConstraintVT.isInteger())
4620     return "r";
4621   if (ConstraintVT.isFloatingPoint())
4622     return "f"; // works for many targets
4623   return nullptr;
4624 }
4625 
4626 SDValue TargetLowering::LowerAsmOutputForConstraint(
4627     SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4628     const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4629   return SDValue();
4630 }
4631 
4632 /// Lower the specified operand into the Ops vector.
4633 /// If it is invalid, don't add anything to Ops.
4634 void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4635                                                   std::string &Constraint,
4636                                                   std::vector<SDValue> &Ops,
4637                                                   SelectionDAG &DAG) const {
4638 
4639   if (Constraint.length() > 1) return;
4640 
4641   char ConstraintLetter = Constraint[0];
4642   switch (ConstraintLetter) {
4643   default: break;
4644   case 'X':    // Allows any operand
4645   case 'i':    // Simple Integer or Relocatable Constant
4646   case 'n':    // Simple Integer
4647   case 's': {  // Relocatable Constant
4648 
4649     ConstantSDNode *C;
4650     uint64_t Offset = 0;
4651 
4652     // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4653     // etc., since getelementpointer is variadic. We can't use
4654     // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4655     // while in this case the GA may be furthest from the root node which is
4656     // likely an ISD::ADD.
4657     while (true) {
4658       if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
4659         // gcc prints these as sign extended.  Sign extend value to 64 bits
4660         // now; without this it would get ZExt'd later in
4661         // ScheduleDAGSDNodes::EmitNode, which is very generic.
4662         bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4663         BooleanContent BCont = getBooleanContents(MVT::i64);
4664         ISD::NodeType ExtOpc =
4665             IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4666         int64_t ExtVal =
4667             ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4668         Ops.push_back(
4669             DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4670         return;
4671       }
4672       if (ConstraintLetter != 'n') {
4673         if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
4674           Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4675                                                    GA->getValueType(0),
4676                                                    Offset + GA->getOffset()));
4677           return;
4678         }
4679         if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
4680           Ops.push_back(DAG.getTargetBlockAddress(
4681               BA->getBlockAddress(), BA->getValueType(0),
4682               Offset + BA->getOffset(), BA->getTargetFlags()));
4683           return;
4684         }
4685         if (isa<BasicBlockSDNode>(Op)) {
4686           Ops.push_back(Op);
4687           return;
4688         }
4689       }
4690       const unsigned OpCode = Op.getOpcode();
4691       if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4692         if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
4693           Op = Op.getOperand(1);
4694         // Subtraction is not commutative.
4695         else if (OpCode == ISD::ADD &&
4696                  (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4697           Op = Op.getOperand(0);
4698         else
4699           return;
4700         Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4701         continue;
4702       }
4703       return;
4704     }
4705     break;
4706   }
4707   }
4708 }
4709 
4710 std::pair<unsigned, const TargetRegisterClass *>
4711 TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4712                                              StringRef Constraint,
4713                                              MVT VT) const {
4714   if (Constraint.empty() || Constraint[0] != '{')
4715     return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4716   assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
4717 
4718   // Remove the braces from around the name.
4719   StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4720 
4721   std::pair<unsigned, const TargetRegisterClass *> R =
4722       std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4723 
4724   // Figure out which register class contains this reg.
4725   for (const TargetRegisterClass *RC : RI->regclasses()) {
4726     // If none of the value types for this register class are valid, we
4727     // can't use it.  For example, 64-bit reg classes on 32-bit targets.
4728     if (!isLegalRC(*RI, *RC))
4729       continue;
4730 
4731     for (const MCPhysReg &PR : *RC) {
4732       if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
4733         std::pair<unsigned, const TargetRegisterClass *> S =
4734             std::make_pair(PR, RC);
4735 
4736         // If this register class has the requested value type, return it,
4737         // otherwise keep searching and return the first class found
4738         // if no other is found which explicitly has the requested type.
4739         if (RI->isTypeLegalForClass(*RC, VT))
4740           return S;
4741         if (!R.second)
4742           R = S;
4743       }
4744     }
4745   }
4746 
4747   return R;
4748 }
4749 
4750 //===----------------------------------------------------------------------===//
4751 // Constraint Selection.
4752 
4753 /// Return true of this is an input operand that is a matching constraint like
4754 /// "4".
4755 bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4756   assert(!ConstraintCode.empty() && "No known constraint!");
4757   return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4758 }
4759 
4760 /// If this is an input matching constraint, this method returns the output
4761 /// operand it matches.
4762 unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4763   assert(!ConstraintCode.empty() && "No known constraint!");
4764   return atoi(ConstraintCode.c_str());
4765 }
4766 
4767 /// Split up the constraint string from the inline assembly value into the
4768 /// specific constraints and their prefixes, and also tie in the associated
4769 /// operand values.
4770 /// If this returns an empty vector, and if the constraint string itself
4771 /// isn't empty, there was an error parsing.
4772 TargetLowering::AsmOperandInfoVector
4773 TargetLowering::ParseConstraints(const DataLayout &DL,
4774                                  const TargetRegisterInfo *TRI,
4775                                  const CallBase &Call) const {
4776   /// Information about all of the constraints.
4777   AsmOperandInfoVector ConstraintOperands;
4778   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4779   unsigned maCount = 0; // Largest number of multiple alternative constraints.
4780 
4781   // Do a prepass over the constraints, canonicalizing them, and building up the
4782   // ConstraintOperands list.
4783   unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4784   unsigned ResNo = 0; // ResNo - The result number of the next output.
4785 
4786   for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4787     ConstraintOperands.emplace_back(std::move(CI));
4788     AsmOperandInfo &OpInfo = ConstraintOperands.back();
4789 
4790     // Update multiple alternative constraint count.
4791     if (OpInfo.multipleAlternatives.size() > maCount)
4792       maCount = OpInfo.multipleAlternatives.size();
4793 
4794     OpInfo.ConstraintVT = MVT::Other;
4795 
4796     // Compute the value type for each operand.
4797     switch (OpInfo.Type) {
4798     case InlineAsm::isOutput:
4799       // Indirect outputs just consume an argument.
4800       if (OpInfo.isIndirect) {
4801         OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
4802         break;
4803       }
4804 
4805       // The return value of the call is this value.  As such, there is no
4806       // corresponding argument.
4807       assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
4808       if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4809         OpInfo.ConstraintVT =
4810             getSimpleValueType(DL, STy->getElementType(ResNo));
4811       } else {
4812         assert(ResNo == 0 && "Asm only has one result!");
4813         OpInfo.ConstraintVT =
4814             getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
4815       }
4816       ++ResNo;
4817       break;
4818     case InlineAsm::isInput:
4819       OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
4820       break;
4821     case InlineAsm::isClobber:
4822       // Nothing to do.
4823       break;
4824     }
4825 
4826     if (OpInfo.CallOperandVal) {
4827       llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4828       if (OpInfo.isIndirect) {
4829         OpTy = Call.getAttributes().getParamElementType(ArgNo);
4830         assert(OpTy && "Indirect opernad must have elementtype attribute");
4831       }
4832 
4833       // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4834       if (StructType *STy = dyn_cast<StructType>(OpTy))
4835         if (STy->getNumElements() == 1)
4836           OpTy = STy->getElementType(0);
4837 
4838       // If OpTy is not a single value, it may be a struct/union that we
4839       // can tile with integers.
4840       if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4841         unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4842         switch (BitSize) {
4843         default: break;
4844         case 1:
4845         case 8:
4846         case 16:
4847         case 32:
4848         case 64:
4849         case 128:
4850           OpInfo.ConstraintVT =
4851               MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4852           break;
4853         }
4854       } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4855         unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4856         OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4857       } else {
4858         OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4859       }
4860 
4861       ArgNo++;
4862     }
4863   }
4864 
4865   // If we have multiple alternative constraints, select the best alternative.
4866   if (!ConstraintOperands.empty()) {
4867     if (maCount) {
4868       unsigned bestMAIndex = 0;
4869       int bestWeight = -1;
4870       // weight:  -1 = invalid match, and 0 = so-so match to 5 = good match.
4871       int weight = -1;
4872       unsigned maIndex;
4873       // Compute the sums of the weights for each alternative, keeping track
4874       // of the best (highest weight) one so far.
4875       for (maIndex = 0; maIndex < maCount; ++maIndex) {
4876         int weightSum = 0;
4877         for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4878              cIndex != eIndex; ++cIndex) {
4879           AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4880           if (OpInfo.Type == InlineAsm::isClobber)
4881             continue;
4882 
4883           // If this is an output operand with a matching input operand,
4884           // look up the matching input. If their types mismatch, e.g. one
4885           // is an integer, the other is floating point, or their sizes are
4886           // different, flag it as an maCantMatch.
4887           if (OpInfo.hasMatchingInput()) {
4888             AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4889             if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4890               if ((OpInfo.ConstraintVT.isInteger() !=
4891                    Input.ConstraintVT.isInteger()) ||
4892                   (OpInfo.ConstraintVT.getSizeInBits() !=
4893                    Input.ConstraintVT.getSizeInBits())) {
4894                 weightSum = -1; // Can't match.
4895                 break;
4896               }
4897             }
4898           }
4899           weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4900           if (weight == -1) {
4901             weightSum = -1;
4902             break;
4903           }
4904           weightSum += weight;
4905         }
4906         // Update best.
4907         if (weightSum > bestWeight) {
4908           bestWeight = weightSum;
4909           bestMAIndex = maIndex;
4910         }
4911       }
4912 
4913       // Now select chosen alternative in each constraint.
4914       for (AsmOperandInfo &cInfo : ConstraintOperands)
4915         if (cInfo.Type != InlineAsm::isClobber)
4916           cInfo.selectAlternative(bestMAIndex);
4917     }
4918   }
4919 
4920   // Check and hook up tied operands, choose constraint code to use.
4921   for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4922        cIndex != eIndex; ++cIndex) {
4923     AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4924 
4925     // If this is an output operand with a matching input operand, look up the
4926     // matching input. If their types mismatch, e.g. one is an integer, the
4927     // other is floating point, or their sizes are different, flag it as an
4928     // error.
4929     if (OpInfo.hasMatchingInput()) {
4930       AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4931 
4932       if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4933         std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4934             getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4935                                          OpInfo.ConstraintVT);
4936         std::pair<unsigned, const TargetRegisterClass *> InputRC =
4937             getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4938                                          Input.ConstraintVT);
4939         if ((OpInfo.ConstraintVT.isInteger() !=
4940              Input.ConstraintVT.isInteger()) ||
4941             (MatchRC.second != InputRC.second)) {
4942           report_fatal_error("Unsupported asm: input constraint"
4943                              " with a matching output constraint of"
4944                              " incompatible type!");
4945         }
4946       }
4947     }
4948   }
4949 
4950   return ConstraintOperands;
4951 }
4952 
4953 /// Return an integer indicating how general CT is.
4954 static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4955   switch (CT) {
4956   case TargetLowering::C_Immediate:
4957   case TargetLowering::C_Other:
4958   case TargetLowering::C_Unknown:
4959     return 0;
4960   case TargetLowering::C_Register:
4961     return 1;
4962   case TargetLowering::C_RegisterClass:
4963     return 2;
4964   case TargetLowering::C_Memory:
4965     return 3;
4966   }
4967   llvm_unreachable("Invalid constraint type");
4968 }
4969 
4970 /// Examine constraint type and operand type and determine a weight value.
4971 /// This object must already have been set up with the operand type
4972 /// and the current alternative constraint selected.
4973 TargetLowering::ConstraintWeight
4974   TargetLowering::getMultipleConstraintMatchWeight(
4975     AsmOperandInfo &info, int maIndex) const {
4976   InlineAsm::ConstraintCodeVector *rCodes;
4977   if (maIndex >= (int)info.multipleAlternatives.size())
4978     rCodes = &info.Codes;
4979   else
4980     rCodes = &info.multipleAlternatives[maIndex].Codes;
4981   ConstraintWeight BestWeight = CW_Invalid;
4982 
4983   // Loop over the options, keeping track of the most general one.
4984   for (const std::string &rCode : *rCodes) {
4985     ConstraintWeight weight =
4986         getSingleConstraintMatchWeight(info, rCode.c_str());
4987     if (weight > BestWeight)
4988       BestWeight = weight;
4989   }
4990 
4991   return BestWeight;
4992 }
4993 
4994 /// Examine constraint type and operand type and determine a weight value.
4995 /// This object must already have been set up with the operand type
4996 /// and the current alternative constraint selected.
4997 TargetLowering::ConstraintWeight
4998   TargetLowering::getSingleConstraintMatchWeight(
4999     AsmOperandInfo &info, const char *constraint) const {
5000   ConstraintWeight weight = CW_Invalid;
5001   Value *CallOperandVal = info.CallOperandVal;
5002     // If we don't have a value, we can't do a match,
5003     // but allow it at the lowest weight.
5004   if (!CallOperandVal)
5005     return CW_Default;
5006   // Look at the constraint type.
5007   switch (*constraint) {
5008     case 'i': // immediate integer.
5009     case 'n': // immediate integer with a known value.
5010       if (isa<ConstantInt>(CallOperandVal))
5011         weight = CW_Constant;
5012       break;
5013     case 's': // non-explicit intregal immediate.
5014       if (isa<GlobalValue>(CallOperandVal))
5015         weight = CW_Constant;
5016       break;
5017     case 'E': // immediate float if host format.
5018     case 'F': // immediate float.
5019       if (isa<ConstantFP>(CallOperandVal))
5020         weight = CW_Constant;
5021       break;
5022     case '<': // memory operand with autodecrement.
5023     case '>': // memory operand with autoincrement.
5024     case 'm': // memory operand.
5025     case 'o': // offsettable memory operand
5026     case 'V': // non-offsettable memory operand
5027       weight = CW_Memory;
5028       break;
5029     case 'r': // general register.
5030     case 'g': // general register, memory operand or immediate integer.
5031               // note: Clang converts "g" to "imr".
5032       if (CallOperandVal->getType()->isIntegerTy())
5033         weight = CW_Register;
5034       break;
5035     case 'X': // any operand.
5036   default:
5037     weight = CW_Default;
5038     break;
5039   }
5040   return weight;
5041 }
5042 
5043 /// If there are multiple different constraints that we could pick for this
5044 /// operand (e.g. "imr") try to pick the 'best' one.
5045 /// This is somewhat tricky: constraints fall into four classes:
5046 ///    Other         -> immediates and magic values
5047 ///    Register      -> one specific register
5048 ///    RegisterClass -> a group of regs
5049 ///    Memory        -> memory
5050 /// Ideally, we would pick the most specific constraint possible: if we have
5051 /// something that fits into a register, we would pick it.  The problem here
5052 /// is that if we have something that could either be in a register or in
5053 /// memory that use of the register could cause selection of *other*
5054 /// operands to fail: they might only succeed if we pick memory.  Because of
5055 /// this the heuristic we use is:
5056 ///
5057 ///  1) If there is an 'other' constraint, and if the operand is valid for
5058 ///     that constraint, use it.  This makes us take advantage of 'i'
5059 ///     constraints when available.
5060 ///  2) Otherwise, pick the most general constraint present.  This prefers
5061 ///     'm' over 'r', for example.
5062 ///
5063 static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
5064                              const TargetLowering &TLI,
5065                              SDValue Op, SelectionDAG *DAG) {
5066   assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
5067   unsigned BestIdx = 0;
5068   TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
5069   int BestGenerality = -1;
5070 
5071   // Loop over the options, keeping track of the most general one.
5072   for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
5073     TargetLowering::ConstraintType CType =
5074       TLI.getConstraintType(OpInfo.Codes[i]);
5075 
5076     // Indirect 'other' or 'immediate' constraints are not allowed.
5077     if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
5078                                CType == TargetLowering::C_Register ||
5079                                CType == TargetLowering::C_RegisterClass))
5080       continue;
5081 
5082     // If this is an 'other' or 'immediate' constraint, see if the operand is
5083     // valid for it. For example, on X86 we might have an 'rI' constraint. If
5084     // the operand is an integer in the range [0..31] we want to use I (saving a
5085     // load of a register), otherwise we must use 'r'.
5086     if ((CType == TargetLowering::C_Other ||
5087          CType == TargetLowering::C_Immediate) && Op.getNode()) {
5088       assert(OpInfo.Codes[i].size() == 1 &&
5089              "Unhandled multi-letter 'other' constraint");
5090       std::vector<SDValue> ResultOps;
5091       TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
5092                                        ResultOps, *DAG);
5093       if (!ResultOps.empty()) {
5094         BestType = CType;
5095         BestIdx = i;
5096         break;
5097       }
5098     }
5099 
5100     // Things with matching constraints can only be registers, per gcc
5101     // documentation.  This mainly affects "g" constraints.
5102     if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
5103       continue;
5104 
5105     // This constraint letter is more general than the previous one, use it.
5106     int Generality = getConstraintGenerality(CType);
5107     if (Generality > BestGenerality) {
5108       BestType = CType;
5109       BestIdx = i;
5110       BestGenerality = Generality;
5111     }
5112   }
5113 
5114   OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
5115   OpInfo.ConstraintType = BestType;
5116 }
5117 
5118 /// Determines the constraint code and constraint type to use for the specific
5119 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5120 void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5121                                             SDValue Op,
5122                                             SelectionDAG *DAG) const {
5123   assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
5124 
5125   // Single-letter constraints ('r') are very common.
5126   if (OpInfo.Codes.size() == 1) {
5127     OpInfo.ConstraintCode = OpInfo.Codes[0];
5128     OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5129   } else {
5130     ChooseConstraint(OpInfo, *this, Op, DAG);
5131   }
5132 
5133   // 'X' matches anything.
5134   if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
5135     // Constants are handled elsewhere.  For Functions, the type here is the
5136     // type of the result, which is not what we want to look at; leave them
5137     // alone.
5138     Value *v = OpInfo.CallOperandVal;
5139     if (isa<ConstantInt>(v) || isa<Function>(v)) {
5140       return;
5141     }
5142 
5143     if (isa<BasicBlock>(v) || isa<BlockAddress>(v)) {
5144       OpInfo.ConstraintCode = "i";
5145       return;
5146     }
5147 
5148     // Otherwise, try to resolve it to something we know about by looking at
5149     // the actual operand type.
5150     if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5151       OpInfo.ConstraintCode = Repl;
5152       OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5153     }
5154   }
5155 }
5156 
5157 /// Given an exact SDIV by a constant, create a multiplication
5158 /// with the multiplicative inverse of the constant.
5159 static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5160                               const SDLoc &dl, SelectionDAG &DAG,
5161                               SmallVectorImpl<SDNode *> &Created) {
5162   SDValue Op0 = N->getOperand(0);
5163   SDValue Op1 = N->getOperand(1);
5164   EVT VT = N->getValueType(0);
5165   EVT SVT = VT.getScalarType();
5166   EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5167   EVT ShSVT = ShVT.getScalarType();
5168 
5169   bool UseSRA = false;
5170   SmallVector<SDValue, 16> Shifts, Factors;
5171 
5172   auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5173     if (C->isZero())
5174       return false;
5175     APInt Divisor = C->getAPIntValue();
5176     unsigned Shift = Divisor.countTrailingZeros();
5177     if (Shift) {
5178       Divisor.ashrInPlace(Shift);
5179       UseSRA = true;
5180     }
5181     // Calculate the multiplicative inverse, using Newton's method.
5182     APInt t;
5183     APInt Factor = Divisor;
5184     while ((t = Divisor * Factor) != 1)
5185       Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5186     Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5187     Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5188     return true;
5189   };
5190 
5191   // Collect all magic values from the build vector.
5192   if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5193     return SDValue();
5194 
5195   SDValue Shift, Factor;
5196   if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5197     Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5198     Factor = DAG.getBuildVector(VT, dl, Factors);
5199   } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5200     assert(Shifts.size() == 1 && Factors.size() == 1 &&
5201            "Expected matchUnaryPredicate to return one element for scalable "
5202            "vectors");
5203     Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5204     Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5205   } else {
5206     assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
5207     Shift = Shifts[0];
5208     Factor = Factors[0];
5209   }
5210 
5211   SDValue Res = Op0;
5212 
5213   // Shift the value upfront if it is even, so the LSB is one.
5214   if (UseSRA) {
5215     // TODO: For UDIV use SRL instead of SRA.
5216     SDNodeFlags Flags;
5217     Flags.setExact(true);
5218     Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5219     Created.push_back(Res.getNode());
5220   }
5221 
5222   return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5223 }
5224 
5225 SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5226                               SelectionDAG &DAG,
5227                               SmallVectorImpl<SDNode *> &Created) const {
5228   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5229   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5230   if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5231     return SDValue(N, 0); // Lower SDIV as SDIV
5232   return SDValue();
5233 }
5234 
5235 /// Given an ISD::SDIV node expressing a divide by constant,
5236 /// return a DAG expression to select that will generate the same value by
5237 /// multiplying by a magic number.
5238 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5239 SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5240                                   bool IsAfterLegalization,
5241                                   SmallVectorImpl<SDNode *> &Created) const {
5242   SDLoc dl(N);
5243   EVT VT = N->getValueType(0);
5244   EVT SVT = VT.getScalarType();
5245   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5246   EVT ShSVT = ShVT.getScalarType();
5247   unsigned EltBits = VT.getScalarSizeInBits();
5248   EVT MulVT;
5249 
5250   // Check to see if we can do this.
5251   // FIXME: We should be more aggressive here.
5252   if (!isTypeLegal(VT)) {
5253     // Limit this to simple scalars for now.
5254     if (VT.isVector() || !VT.isSimple())
5255       return SDValue();
5256 
5257     // If this type will be promoted to a large enough type with a legal
5258     // multiply operation, we can go ahead and do this transform.
5259     if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5260       return SDValue();
5261 
5262     MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5263     if (MulVT.getSizeInBits() < (2 * EltBits) ||
5264         !isOperationLegal(ISD::MUL, MulVT))
5265       return SDValue();
5266   }
5267 
5268   // If the sdiv has an 'exact' bit we can use a simpler lowering.
5269   if (N->getFlags().hasExact())
5270     return BuildExactSDIV(*this, N, dl, DAG, Created);
5271 
5272   SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5273 
5274   auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5275     if (C->isZero())
5276       return false;
5277 
5278     const APInt &Divisor = C->getAPIntValue();
5279     SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);
5280     int NumeratorFactor = 0;
5281     int ShiftMask = -1;
5282 
5283     if (Divisor.isOne() || Divisor.isAllOnes()) {
5284       // If d is +1/-1, we just multiply the numerator by +1/-1.
5285       NumeratorFactor = Divisor.getSExtValue();
5286       magics.Magic = 0;
5287       magics.ShiftAmount = 0;
5288       ShiftMask = 0;
5289     } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
5290       // If d > 0 and m < 0, add the numerator.
5291       NumeratorFactor = 1;
5292     } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
5293       // If d < 0 and m > 0, subtract the numerator.
5294       NumeratorFactor = -1;
5295     }
5296 
5297     MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
5298     Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5299     Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
5300     ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5301     return true;
5302   };
5303 
5304   SDValue N0 = N->getOperand(0);
5305   SDValue N1 = N->getOperand(1);
5306 
5307   // Collect the shifts / magic values from each element.
5308   if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5309     return SDValue();
5310 
5311   SDValue MagicFactor, Factor, Shift, ShiftMask;
5312   if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5313     MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5314     Factor = DAG.getBuildVector(VT, dl, Factors);
5315     Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5316     ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5317   } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5318     assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
5319            Shifts.size() == 1 && ShiftMasks.size() == 1 &&
5320            "Expected matchUnaryPredicate to return one element for scalable "
5321            "vectors");
5322     MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5323     Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5324     Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5325     ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5326   } else {
5327     assert(isa<ConstantSDNode>(N1) && "Expected a constant");
5328     MagicFactor = MagicFactors[0];
5329     Factor = Factors[0];
5330     Shift = Shifts[0];
5331     ShiftMask = ShiftMasks[0];
5332   }
5333 
5334   // Multiply the numerator (operand 0) by the magic value.
5335   // FIXME: We should support doing a MUL in a wider type.
5336   auto GetMULHS = [&](SDValue X, SDValue Y) {
5337     // If the type isn't legal, use a wider mul of the the type calculated
5338     // earlier.
5339     if (!isTypeLegal(VT)) {
5340       X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5341       Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5342       Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5343       Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5344                       DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5345       return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5346     }
5347 
5348     if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5349       return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5350     if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5351       SDValue LoHi =
5352           DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5353       return SDValue(LoHi.getNode(), 1);
5354     }
5355     return SDValue();
5356   };
5357 
5358   SDValue Q = GetMULHS(N0, MagicFactor);
5359   if (!Q)
5360     return SDValue();
5361 
5362   Created.push_back(Q.getNode());
5363 
5364   // (Optionally) Add/subtract the numerator using Factor.
5365   Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5366   Created.push_back(Factor.getNode());
5367   Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5368   Created.push_back(Q.getNode());
5369 
5370   // Shift right algebraic by shift value.
5371   Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5372   Created.push_back(Q.getNode());
5373 
5374   // Extract the sign bit, mask it and add it to the quotient.
5375   SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5376   SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5377   Created.push_back(T.getNode());
5378   T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5379   Created.push_back(T.getNode());
5380   return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5381 }
5382 
5383 /// Given an ISD::UDIV node expressing a divide by constant,
5384 /// return a DAG expression to select that will generate the same value by
5385 /// multiplying by a magic number.
5386 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5387 SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5388                                   bool IsAfterLegalization,
5389                                   SmallVectorImpl<SDNode *> &Created) const {
5390   SDLoc dl(N);
5391   EVT VT = N->getValueType(0);
5392   EVT SVT = VT.getScalarType();
5393   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5394   EVT ShSVT = ShVT.getScalarType();
5395   unsigned EltBits = VT.getScalarSizeInBits();
5396   EVT MulVT;
5397 
5398   // Check to see if we can do this.
5399   // FIXME: We should be more aggressive here.
5400   if (!isTypeLegal(VT)) {
5401     // Limit this to simple scalars for now.
5402     if (VT.isVector() || !VT.isSimple())
5403       return SDValue();
5404 
5405     // If this type will be promoted to a large enough type with a legal
5406     // multiply operation, we can go ahead and do this transform.
5407     if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5408       return SDValue();
5409 
5410     MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5411     if (MulVT.getSizeInBits() < (2 * EltBits) ||
5412         !isOperationLegal(ISD::MUL, MulVT))
5413       return SDValue();
5414   }
5415 
5416   bool UseNPQ = false;
5417   SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5418 
5419   auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5420     if (C->isZero())
5421       return false;
5422     // FIXME: We should use a narrower constant when the upper
5423     // bits are known to be zero.
5424     const APInt& Divisor = C->getAPIntValue();
5425     UnsignedDivisonByConstantInfo magics = UnsignedDivisonByConstantInfo::get(Divisor);
5426     unsigned PreShift = 0, PostShift = 0;
5427 
5428     // If the divisor is even, we can avoid using the expensive fixup by
5429     // shifting the divided value upfront.
5430     if (magics.IsAdd != 0 && !Divisor[0]) {
5431       PreShift = Divisor.countTrailingZeros();
5432       // Get magic number for the shifted divisor.
5433       magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
5434       assert(magics.IsAdd == 0 && "Should use cheap fixup now");
5435     }
5436 
5437     APInt Magic = magics.Magic;
5438 
5439     unsigned SelNPQ;
5440     if (magics.IsAdd == 0 || Divisor.isOne()) {
5441       assert(magics.ShiftAmount < Divisor.getBitWidth() &&
5442              "We shouldn't generate an undefined shift!");
5443       PostShift = magics.ShiftAmount;
5444       SelNPQ = false;
5445     } else {
5446       PostShift = magics.ShiftAmount - 1;
5447       SelNPQ = true;
5448     }
5449 
5450     PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5451     MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5452     NPQFactors.push_back(
5453         DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5454                                : APInt::getZero(EltBits),
5455                         dl, SVT));
5456     PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5457     UseNPQ |= SelNPQ;
5458     return true;
5459   };
5460 
5461   SDValue N0 = N->getOperand(0);
5462   SDValue N1 = N->getOperand(1);
5463 
5464   // Collect the shifts/magic values from each element.
5465   if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5466     return SDValue();
5467 
5468   SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5469   if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5470     PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5471     MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5472     NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5473     PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5474   } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5475     assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
5476            NPQFactors.size() == 1 && PostShifts.size() == 1 &&
5477            "Expected matchUnaryPredicate to return one for scalable vectors");
5478     PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5479     MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5480     NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5481     PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5482   } else {
5483     assert(isa<ConstantSDNode>(N1) && "Expected a constant");
5484     PreShift = PreShifts[0];
5485     MagicFactor = MagicFactors[0];
5486     PostShift = PostShifts[0];
5487   }
5488 
5489   SDValue Q = N0;
5490   Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5491   Created.push_back(Q.getNode());
5492 
5493   // FIXME: We should support doing a MUL in a wider type.
5494   auto GetMULHU = [&](SDValue X, SDValue Y) {
5495     // If the type isn't legal, use a wider mul of the the type calculated
5496     // earlier.
5497     if (!isTypeLegal(VT)) {
5498       X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5499       Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5500       Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5501       Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5502                       DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5503       return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5504     }
5505 
5506     if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5507       return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5508     if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5509       SDValue LoHi =
5510           DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5511       return SDValue(LoHi.getNode(), 1);
5512     }
5513     return SDValue(); // No mulhu or equivalent
5514   };
5515 
5516   // Multiply the numerator (operand 0) by the magic value.
5517   Q = GetMULHU(Q, MagicFactor);
5518   if (!Q)
5519     return SDValue();
5520 
5521   Created.push_back(Q.getNode());
5522 
5523   if (UseNPQ) {
5524     SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5525     Created.push_back(NPQ.getNode());
5526 
5527     // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5528     // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5529     if (VT.isVector())
5530       NPQ = GetMULHU(NPQ, NPQFactor);
5531     else
5532       NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5533 
5534     Created.push_back(NPQ.getNode());
5535 
5536     Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5537     Created.push_back(Q.getNode());
5538   }
5539 
5540   Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5541   Created.push_back(Q.getNode());
5542 
5543   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5544 
5545   SDValue One = DAG.getConstant(1, dl, VT);
5546   SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5547   return DAG.getSelect(dl, VT, IsOne, N0, Q);
5548 }
5549 
5550 /// If all values in Values that *don't* match the predicate are same 'splat'
5551 /// value, then replace all values with that splat value.
5552 /// Else, if AlternativeReplacement was provided, then replace all values that
5553 /// do match predicate with AlternativeReplacement value.
5554 static void
5555 turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5556                           std::function<bool(SDValue)> Predicate,
5557                           SDValue AlternativeReplacement = SDValue()) {
5558   SDValue Replacement;
5559   // Is there a value for which the Predicate does *NOT* match? What is it?
5560   auto SplatValue = llvm::find_if_not(Values, Predicate);
5561   if (SplatValue != Values.end()) {
5562     // Does Values consist only of SplatValue's and values matching Predicate?
5563     if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5564           return Value == *SplatValue || Predicate(Value);
5565         })) // Then we shall replace values matching predicate with SplatValue.
5566       Replacement = *SplatValue;
5567   }
5568   if (!Replacement) {
5569     // Oops, we did not find the "baseline" splat value.
5570     if (!AlternativeReplacement)
5571       return; // Nothing to do.
5572     // Let's replace with provided value then.
5573     Replacement = AlternativeReplacement;
5574   }
5575   std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5576 }
5577 
5578 /// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5579 /// where the divisor is constant and the comparison target is zero,
5580 /// return a DAG expression that will generate the same comparison result
5581 /// using only multiplications, additions and shifts/rotations.
5582 /// Ref: "Hacker's Delight" 10-17.
5583 SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5584                                         SDValue CompTargetNode,
5585                                         ISD::CondCode Cond,
5586                                         DAGCombinerInfo &DCI,
5587                                         const SDLoc &DL) const {
5588   SmallVector<SDNode *, 5> Built;
5589   if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5590                                          DCI, DL, Built)) {
5591     for (SDNode *N : Built)
5592       DCI.AddToWorklist(N);
5593     return Folded;
5594   }
5595 
5596   return SDValue();
5597 }
5598 
5599 SDValue
5600 TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5601                                   SDValue CompTargetNode, ISD::CondCode Cond,
5602                                   DAGCombinerInfo &DCI, const SDLoc &DL,
5603                                   SmallVectorImpl<SDNode *> &Created) const {
5604   // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5605   // - D must be constant, with D = D0 * 2^K where D0 is odd
5606   // - P is the multiplicative inverse of D0 modulo 2^W
5607   // - Q = floor(((2^W) - 1) / D)
5608   // where W is the width of the common type of N and D.
5609   assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5610          "Only applicable for (in)equality comparisons.");
5611 
5612   SelectionDAG &DAG = DCI.DAG;
5613 
5614   EVT VT = REMNode.getValueType();
5615   EVT SVT = VT.getScalarType();
5616   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5617   EVT ShSVT = ShVT.getScalarType();
5618 
5619   // If MUL is unavailable, we cannot proceed in any case.
5620   if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5621     return SDValue();
5622 
5623   bool ComparingWithAllZeros = true;
5624   bool AllComparisonsWithNonZerosAreTautological = true;
5625   bool HadTautologicalLanes = false;
5626   bool AllLanesAreTautological = true;
5627   bool HadEvenDivisor = false;
5628   bool AllDivisorsArePowerOfTwo = true;
5629   bool HadTautologicalInvertedLanes = false;
5630   SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5631 
5632   auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5633     // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5634     if (CDiv->isZero())
5635       return false;
5636 
5637     const APInt &D = CDiv->getAPIntValue();
5638     const APInt &Cmp = CCmp->getAPIntValue();
5639 
5640     ComparingWithAllZeros &= Cmp.isZero();
5641 
5642     // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5643     // if C2 is not less than C1, the comparison is always false.
5644     // But we will only be able to produce the comparison that will give the
5645     // opposive tautological answer. So this lane would need to be fixed up.
5646     bool TautologicalInvertedLane = D.ule(Cmp);
5647     HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5648 
5649     // If all lanes are tautological (either all divisors are ones, or divisor
5650     // is not greater than the constant we are comparing with),
5651     // we will prefer to avoid the fold.
5652     bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
5653     HadTautologicalLanes |= TautologicalLane;
5654     AllLanesAreTautological &= TautologicalLane;
5655 
5656     // If we are comparing with non-zero, we need'll need  to subtract said
5657     // comparison value from the LHS. But there is no point in doing that if
5658     // every lane where we are comparing with non-zero is tautological..
5659     if (!Cmp.isZero())
5660       AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5661 
5662     // Decompose D into D0 * 2^K
5663     unsigned K = D.countTrailingZeros();
5664     assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
5665     APInt D0 = D.lshr(K);
5666 
5667     // D is even if it has trailing zeros.
5668     HadEvenDivisor |= (K != 0);
5669     // D is a power-of-two if D0 is one.
5670     // If all divisors are power-of-two, we will prefer to avoid the fold.
5671     AllDivisorsArePowerOfTwo &= D0.isOne();
5672 
5673     // P = inv(D0, 2^W)
5674     // 2^W requires W + 1 bits, so we have to extend and then truncate.
5675     unsigned W = D.getBitWidth();
5676     APInt P = D0.zext(W + 1)
5677                   .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5678                   .trunc(W);
5679     assert(!P.isZero() && "No multiplicative inverse!"); // unreachable
5680     assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
5681 
5682     // Q = floor((2^W - 1) u/ D)
5683     // R = ((2^W - 1) u% D)
5684     APInt Q, R;
5685     APInt::udivrem(APInt::getAllOnes(W), D, Q, R);
5686 
5687     // If we are comparing with zero, then that comparison constant is okay,
5688     // else it may need to be one less than that.
5689     if (Cmp.ugt(R))
5690       Q -= 1;
5691 
5692     assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
5693            "We are expecting that K is always less than all-ones for ShSVT");
5694 
5695     // If the lane is tautological the result can be constant-folded.
5696     if (TautologicalLane) {
5697       // Set P and K amount to a bogus values so we can try to splat them.
5698       P = 0;
5699       K = -1;
5700       // And ensure that comparison constant is tautological,
5701       // it will always compare true/false.
5702       Q = -1;
5703     }
5704 
5705     PAmts.push_back(DAG.getConstant(P, DL, SVT));
5706     KAmts.push_back(
5707         DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5708     QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5709     return true;
5710   };
5711 
5712   SDValue N = REMNode.getOperand(0);
5713   SDValue D = REMNode.getOperand(1);
5714 
5715   // Collect the values from each element.
5716   if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5717     return SDValue();
5718 
5719   // If all lanes are tautological, the result can be constant-folded.
5720   if (AllLanesAreTautological)
5721     return SDValue();
5722 
5723   // If this is a urem by a powers-of-two, avoid the fold since it can be
5724   // best implemented as a bit test.
5725   if (AllDivisorsArePowerOfTwo)
5726     return SDValue();
5727 
5728   SDValue PVal, KVal, QVal;
5729   if (D.getOpcode() == ISD::BUILD_VECTOR) {
5730     if (HadTautologicalLanes) {
5731       // Try to turn PAmts into a splat, since we don't care about the values
5732       // that are currently '0'. If we can't, just keep '0'`s.
5733       turnVectorIntoSplatVector(PAmts, isNullConstant);
5734       // Try to turn KAmts into a splat, since we don't care about the values
5735       // that are currently '-1'. If we can't, change them to '0'`s.
5736       turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5737                                 DAG.getConstant(0, DL, ShSVT));
5738     }
5739 
5740     PVal = DAG.getBuildVector(VT, DL, PAmts);
5741     KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5742     QVal = DAG.getBuildVector(VT, DL, QAmts);
5743   } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
5744     assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
5745            "Expected matchBinaryPredicate to return one element for "
5746            "SPLAT_VECTORs");
5747     PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
5748     KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
5749     QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
5750   } else {
5751     PVal = PAmts[0];
5752     KVal = KAmts[0];
5753     QVal = QAmts[0];
5754   }
5755 
5756   if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5757     if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
5758       return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5759     assert(CompTargetNode.getValueType() == N.getValueType() &&
5760            "Expecting that the types on LHS and RHS of comparisons match.");
5761     N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5762   }
5763 
5764   // (mul N, P)
5765   SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5766   Created.push_back(Op0.getNode());
5767 
5768   // Rotate right only if any divisor was even. We avoid rotates for all-odd
5769   // divisors as a performance improvement, since rotating by 0 is a no-op.
5770   if (HadEvenDivisor) {
5771     // We need ROTR to do this.
5772     if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5773       return SDValue();
5774     // UREM: (rotr (mul N, P), K)
5775     Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
5776     Created.push_back(Op0.getNode());
5777   }
5778 
5779   // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5780   SDValue NewCC =
5781       DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5782                    ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5783   if (!HadTautologicalInvertedLanes)
5784     return NewCC;
5785 
5786   // If any lanes previously compared always-false, the NewCC will give
5787   // always-true result for them, so we need to fixup those lanes.
5788   // Or the other way around for inequality predicate.
5789   assert(VT.isVector() && "Can/should only get here for vectors.");
5790   Created.push_back(NewCC.getNode());
5791 
5792   // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5793   // if C2 is not less than C1, the comparison is always false.
5794   // But we have produced the comparison that will give the
5795   // opposive tautological answer. So these lanes would need to be fixed up.
5796   SDValue TautologicalInvertedChannels =
5797       DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5798   Created.push_back(TautologicalInvertedChannels.getNode());
5799 
5800   // NOTE: we avoid letting illegal types through even if we're before legalize
5801   // ops – legalization has a hard time producing good code for this.
5802   if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5803     // If we have a vector select, let's replace the comparison results in the
5804     // affected lanes with the correct tautological result.
5805     SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5806                                               DL, SETCCVT, SETCCVT);
5807     return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5808                        Replacement, NewCC);
5809   }
5810 
5811   // Else, we can just invert the comparison result in the appropriate lanes.
5812   //
5813   // NOTE: see the note above VSELECT above.
5814   if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5815     return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5816                        TautologicalInvertedChannels);
5817 
5818   return SDValue(); // Don't know how to lower.
5819 }
5820 
5821 /// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5822 /// where the divisor is constant and the comparison target is zero,
5823 /// return a DAG expression that will generate the same comparison result
5824 /// using only multiplications, additions and shifts/rotations.
5825 /// Ref: "Hacker's Delight" 10-17.
5826 SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5827                                         SDValue CompTargetNode,
5828                                         ISD::CondCode Cond,
5829                                         DAGCombinerInfo &DCI,
5830                                         const SDLoc &DL) const {
5831   SmallVector<SDNode *, 7> Built;
5832   if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5833                                          DCI, DL, Built)) {
5834     assert(Built.size() <= 7 && "Max size prediction failed.");
5835     for (SDNode *N : Built)
5836       DCI.AddToWorklist(N);
5837     return Folded;
5838   }
5839 
5840   return SDValue();
5841 }
5842 
5843 SDValue
5844 TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5845                                   SDValue CompTargetNode, ISD::CondCode Cond,
5846                                   DAGCombinerInfo &DCI, const SDLoc &DL,
5847                                   SmallVectorImpl<SDNode *> &Created) const {
5848   // Fold:
5849   //   (seteq/ne (srem N, D), 0)
5850   // To:
5851   //   (setule/ugt (rotr (add (mul N, P), A), K), Q)
5852   //
5853   // - D must be constant, with D = D0 * 2^K where D0 is odd
5854   // - P is the multiplicative inverse of D0 modulo 2^W
5855   // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5856   // - Q = floor((2 * A) / (2^K))
5857   // where W is the width of the common type of N and D.
5858   assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5859          "Only applicable for (in)equality comparisons.");
5860 
5861   SelectionDAG &DAG = DCI.DAG;
5862 
5863   EVT VT = REMNode.getValueType();
5864   EVT SVT = VT.getScalarType();
5865   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5866   EVT ShSVT = ShVT.getScalarType();
5867 
5868   // If we are after ops legalization, and MUL is unavailable, we can not
5869   // proceed.
5870   if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5871     return SDValue();
5872 
5873   // TODO: Could support comparing with non-zero too.
5874   ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5875   if (!CompTarget || !CompTarget->isZero())
5876     return SDValue();
5877 
5878   bool HadIntMinDivisor = false;
5879   bool HadOneDivisor = false;
5880   bool AllDivisorsAreOnes = true;
5881   bool HadEvenDivisor = false;
5882   bool NeedToApplyOffset = false;
5883   bool AllDivisorsArePowerOfTwo = true;
5884   SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5885 
5886   auto BuildSREMPattern = [&](ConstantSDNode *C) {
5887     // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5888     if (C->isZero())
5889       return false;
5890 
5891     // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
5892 
5893     // WARNING: this fold is only valid for positive divisors!
5894     APInt D = C->getAPIntValue();
5895     if (D.isNegative())
5896       D.negate(); //  `rem %X, -C` is equivalent to `rem %X, C`
5897 
5898     HadIntMinDivisor |= D.isMinSignedValue();
5899 
5900     // If all divisors are ones, we will prefer to avoid the fold.
5901     HadOneDivisor |= D.isOne();
5902     AllDivisorsAreOnes &= D.isOne();
5903 
5904     // Decompose D into D0 * 2^K
5905     unsigned K = D.countTrailingZeros();
5906     assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
5907     APInt D0 = D.lshr(K);
5908 
5909     if (!D.isMinSignedValue()) {
5910       // D is even if it has trailing zeros; unless it's INT_MIN, in which case
5911       // we don't care about this lane in this fold, we'll special-handle it.
5912       HadEvenDivisor |= (K != 0);
5913     }
5914 
5915     // D is a power-of-two if D0 is one. This includes INT_MIN.
5916     // If all divisors are power-of-two, we will prefer to avoid the fold.
5917     AllDivisorsArePowerOfTwo &= D0.isOne();
5918 
5919     // P = inv(D0, 2^W)
5920     // 2^W requires W + 1 bits, so we have to extend and then truncate.
5921     unsigned W = D.getBitWidth();
5922     APInt P = D0.zext(W + 1)
5923                   .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5924                   .trunc(W);
5925     assert(!P.isZero() && "No multiplicative inverse!"); // unreachable
5926     assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
5927 
5928     // A = floor((2^(W - 1) - 1) / D0) & -2^K
5929     APInt A = APInt::getSignedMaxValue(W).udiv(D0);
5930     A.clearLowBits(K);
5931 
5932     if (!D.isMinSignedValue()) {
5933       // If divisor INT_MIN, then we don't care about this lane in this fold,
5934       // we'll special-handle it.
5935       NeedToApplyOffset |= A != 0;
5936     }
5937 
5938     // Q = floor((2 * A) / (2^K))
5939     APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
5940 
5941     assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
5942            "We are expecting that A is always less than all-ones for SVT");
5943     assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
5944            "We are expecting that K is always less than all-ones for ShSVT");
5945 
5946     // If the divisor is 1 the result can be constant-folded. Likewise, we
5947     // don't care about INT_MIN lanes, those can be set to undef if appropriate.
5948     if (D.isOne()) {
5949       // Set P, A and K to a bogus values so we can try to splat them.
5950       P = 0;
5951       A = -1;
5952       K = -1;
5953 
5954       // x ?% 1 == 0  <-->  true  <-->  x u<= -1
5955       Q = -1;
5956     }
5957 
5958     PAmts.push_back(DAG.getConstant(P, DL, SVT));
5959     AAmts.push_back(DAG.getConstant(A, DL, SVT));
5960     KAmts.push_back(
5961         DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5962     QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5963     return true;
5964   };
5965 
5966   SDValue N = REMNode.getOperand(0);
5967   SDValue D = REMNode.getOperand(1);
5968 
5969   // Collect the values from each element.
5970   if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5971     return SDValue();
5972 
5973   // If this is a srem by a one, avoid the fold since it can be constant-folded.
5974   if (AllDivisorsAreOnes)
5975     return SDValue();
5976 
5977   // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
5978   // since it can be best implemented as a bit test.
5979   if (AllDivisorsArePowerOfTwo)
5980     return SDValue();
5981 
5982   SDValue PVal, AVal, KVal, QVal;
5983   if (D.getOpcode() == ISD::BUILD_VECTOR) {
5984     if (HadOneDivisor) {
5985       // Try to turn PAmts into a splat, since we don't care about the values
5986       // that are currently '0'. If we can't, just keep '0'`s.
5987       turnVectorIntoSplatVector(PAmts, isNullConstant);
5988       // Try to turn AAmts into a splat, since we don't care about the
5989       // values that are currently '-1'. If we can't, change them to '0'`s.
5990       turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
5991                                 DAG.getConstant(0, DL, SVT));
5992       // Try to turn KAmts into a splat, since we don't care about the values
5993       // that are currently '-1'. If we can't, change them to '0'`s.
5994       turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5995                                 DAG.getConstant(0, DL, ShSVT));
5996     }
5997 
5998     PVal = DAG.getBuildVector(VT, DL, PAmts);
5999     AVal = DAG.getBuildVector(VT, DL, AAmts);
6000     KVal = DAG.getBuildVector(ShVT, DL, KAmts);
6001     QVal = DAG.getBuildVector(VT, DL, QAmts);
6002   } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
6003     assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
6004            QAmts.size() == 1 &&
6005            "Expected matchUnaryPredicate to return one element for scalable "
6006            "vectors");
6007     PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
6008     AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
6009     KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
6010     QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
6011   } else {
6012     assert(isa<ConstantSDNode>(D) && "Expected a constant");
6013     PVal = PAmts[0];
6014     AVal = AAmts[0];
6015     KVal = KAmts[0];
6016     QVal = QAmts[0];
6017   }
6018 
6019   // (mul N, P)
6020   SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
6021   Created.push_back(Op0.getNode());
6022 
6023   if (NeedToApplyOffset) {
6024     // We need ADD to do this.
6025     if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
6026       return SDValue();
6027 
6028     // (add (mul N, P), A)
6029     Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
6030     Created.push_back(Op0.getNode());
6031   }
6032 
6033   // Rotate right only if any divisor was even. We avoid rotates for all-odd
6034   // divisors as a performance improvement, since rotating by 0 is a no-op.
6035   if (HadEvenDivisor) {
6036     // We need ROTR to do this.
6037     if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
6038       return SDValue();
6039     // SREM: (rotr (add (mul N, P), A), K)
6040     Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
6041     Created.push_back(Op0.getNode());
6042   }
6043 
6044   // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
6045   SDValue Fold =
6046       DAG.getSetCC(DL, SETCCVT, Op0, QVal,
6047                    ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
6048 
6049   // If we didn't have lanes with INT_MIN divisor, then we're done.
6050   if (!HadIntMinDivisor)
6051     return Fold;
6052 
6053   // That fold is only valid for positive divisors. Which effectively means,
6054   // it is invalid for INT_MIN divisors. So if we have such a lane,
6055   // we must fix-up results for said lanes.
6056   assert(VT.isVector() && "Can/should only get here for vectors.");
6057 
6058   // NOTE: we avoid letting illegal types through even if we're before legalize
6059   // ops – legalization has a hard time producing good code for the code that
6060   // follows.
6061   if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
6062       !isOperationLegalOrCustom(ISD::AND, VT) ||
6063       !isOperationLegalOrCustom(Cond, VT) ||
6064       !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))
6065     return SDValue();
6066 
6067   Created.push_back(Fold.getNode());
6068 
6069   SDValue IntMin = DAG.getConstant(
6070       APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
6071   SDValue IntMax = DAG.getConstant(
6072       APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
6073   SDValue Zero =
6074       DAG.getConstant(APInt::getZero(SVT.getScalarSizeInBits()), DL, VT);
6075 
6076   // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
6077   SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
6078   Created.push_back(DivisorIsIntMin.getNode());
6079 
6080   // (N s% INT_MIN) ==/!= 0  <-->  (N & INT_MAX) ==/!= 0
6081   SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
6082   Created.push_back(Masked.getNode());
6083   SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
6084   Created.push_back(MaskedIsZero.getNode());
6085 
6086   // To produce final result we need to blend 2 vectors: 'SetCC' and
6087   // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
6088   // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
6089   // constant-folded, select can get lowered to a shuffle with constant mask.
6090   SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,
6091                                 MaskedIsZero, Fold);
6092 
6093   return Blended;
6094 }
6095 
6096 bool TargetLowering::
6097 verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
6098   if (!isa<ConstantSDNode>(Op.getOperand(0))) {
6099     DAG.getContext()->emitError("argument to '__builtin_return_address' must "
6100                                 "be a constant integer");
6101     return true;
6102   }
6103 
6104   return false;
6105 }
6106 
6107 SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
6108                                          const DenormalMode &Mode) const {
6109   SDLoc DL(Op);
6110   EVT VT = Op.getValueType();
6111   EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
6112   SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
6113   // Testing it with denormal inputs to avoid wrong estimate.
6114   if (Mode.Input == DenormalMode::IEEE) {
6115     // This is specifically a check for the handling of denormal inputs,
6116     // not the result.
6117 
6118     // Test = fabs(X) < SmallestNormal
6119     const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
6120     APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
6121     SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
6122     SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
6123     return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
6124   }
6125   // Test = X == 0.0
6126   return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
6127 }
6128 
6129 SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
6130                                              bool LegalOps, bool OptForSize,
6131                                              NegatibleCost &Cost,
6132                                              unsigned Depth) const {
6133   // fneg is removable even if it has multiple uses.
6134   if (Op.getOpcode() == ISD::FNEG) {
6135     Cost = NegatibleCost::Cheaper;
6136     return Op.getOperand(0);
6137   }
6138 
6139   // Don't recurse exponentially.
6140   if (Depth > SelectionDAG::MaxRecursionDepth)
6141     return SDValue();
6142 
6143   // Pre-increment recursion depth for use in recursive calls.
6144   ++Depth;
6145   const SDNodeFlags Flags = Op->getFlags();
6146   const TargetOptions &Options = DAG.getTarget().Options;
6147   EVT VT = Op.getValueType();
6148   unsigned Opcode = Op.getOpcode();
6149 
6150   // Don't allow anything with multiple uses unless we know it is free.
6151   if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
6152     bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
6153                         isFPExtFree(VT, Op.getOperand(0).getValueType());
6154     if (!IsFreeExtend)
6155       return SDValue();
6156   }
6157 
6158   auto RemoveDeadNode = [&](SDValue N) {
6159     if (N && N.getNode()->use_empty())
6160       DAG.RemoveDeadNode(N.getNode());
6161   };
6162 
6163   SDLoc DL(Op);
6164 
6165   // Because getNegatedExpression can delete nodes we need a handle to keep
6166   // temporary nodes alive in case the recursion manages to create an identical
6167   // node.
6168   std::list<HandleSDNode> Handles;
6169 
6170   switch (Opcode) {
6171   case ISD::ConstantFP: {
6172     // Don't invert constant FP values after legalization unless the target says
6173     // the negated constant is legal.
6174     bool IsOpLegal =
6175         isOperationLegal(ISD::ConstantFP, VT) ||
6176         isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
6177                      OptForSize);
6178 
6179     if (LegalOps && !IsOpLegal)
6180       break;
6181 
6182     APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
6183     V.changeSign();
6184     SDValue CFP = DAG.getConstantFP(V, DL, VT);
6185 
6186     // If we already have the use of the negated floating constant, it is free
6187     // to negate it even it has multiple uses.
6188     if (!Op.hasOneUse() && CFP.use_empty())
6189       break;
6190     Cost = NegatibleCost::Neutral;
6191     return CFP;
6192   }
6193   case ISD::BUILD_VECTOR: {
6194     // Only permit BUILD_VECTOR of constants.
6195     if (llvm::any_of(Op->op_values(), [&](SDValue N) {
6196           return !N.isUndef() && !isa<ConstantFPSDNode>(N);
6197         }))
6198       break;
6199 
6200     bool IsOpLegal =
6201         (isOperationLegal(ISD::ConstantFP, VT) &&
6202          isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
6203         llvm::all_of(Op->op_values(), [&](SDValue N) {
6204           return N.isUndef() ||
6205                  isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
6206                               OptForSize);
6207         });
6208 
6209     if (LegalOps && !IsOpLegal)
6210       break;
6211 
6212     SmallVector<SDValue, 4> Ops;
6213     for (SDValue C : Op->op_values()) {
6214       if (C.isUndef()) {
6215         Ops.push_back(C);
6216         continue;
6217       }
6218       APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
6219       V.changeSign();
6220       Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
6221     }
6222     Cost = NegatibleCost::Neutral;
6223     return DAG.getBuildVector(VT, DL, Ops);
6224   }
6225   case ISD::FADD: {
6226     if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6227       break;
6228 
6229     // After operation legalization, it might not be legal to create new FSUBs.
6230     if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
6231       break;
6232     SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6233 
6234     // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
6235     NegatibleCost CostX = NegatibleCost::Expensive;
6236     SDValue NegX =
6237         getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6238     // Prevent this node from being deleted by the next call.
6239     if (NegX)
6240       Handles.emplace_back(NegX);
6241 
6242     // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
6243     NegatibleCost CostY = NegatibleCost::Expensive;
6244     SDValue NegY =
6245         getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6246 
6247     // We're done with the handles.
6248     Handles.clear();
6249 
6250     // Negate the X if its cost is less or equal than Y.
6251     if (NegX && (CostX <= CostY)) {
6252       Cost = CostX;
6253       SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
6254       if (NegY != N)
6255         RemoveDeadNode(NegY);
6256       return N;
6257     }
6258 
6259     // Negate the Y if it is not expensive.
6260     if (NegY) {
6261       Cost = CostY;
6262       SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
6263       if (NegX != N)
6264         RemoveDeadNode(NegX);
6265       return N;
6266     }
6267     break;
6268   }
6269   case ISD::FSUB: {
6270     // We can't turn -(A-B) into B-A when we honor signed zeros.
6271     if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6272       break;
6273 
6274     SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6275     // fold (fneg (fsub 0, Y)) -> Y
6276     if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
6277       if (C->isZero()) {
6278         Cost = NegatibleCost::Cheaper;
6279         return Y;
6280       }
6281 
6282     // fold (fneg (fsub X, Y)) -> (fsub Y, X)
6283     Cost = NegatibleCost::Neutral;
6284     return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
6285   }
6286   case ISD::FMUL:
6287   case ISD::FDIV: {
6288     SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6289 
6290     // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
6291     NegatibleCost CostX = NegatibleCost::Expensive;
6292     SDValue NegX =
6293         getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6294     // Prevent this node from being deleted by the next call.
6295     if (NegX)
6296       Handles.emplace_back(NegX);
6297 
6298     // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
6299     NegatibleCost CostY = NegatibleCost::Expensive;
6300     SDValue NegY =
6301         getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6302 
6303     // We're done with the handles.
6304     Handles.clear();
6305 
6306     // Negate the X if its cost is less or equal than Y.
6307     if (NegX && (CostX <= CostY)) {
6308       Cost = CostX;
6309       SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
6310       if (NegY != N)
6311         RemoveDeadNode(NegY);
6312       return N;
6313     }
6314 
6315     // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
6316     if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
6317       if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
6318         break;
6319 
6320     // Negate the Y if it is not expensive.
6321     if (NegY) {
6322       Cost = CostY;
6323       SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
6324       if (NegX != N)
6325         RemoveDeadNode(NegX);
6326       return N;
6327     }
6328     break;
6329   }
6330   case ISD::FMA:
6331   case ISD::FMAD: {
6332     if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6333       break;
6334 
6335     SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
6336     NegatibleCost CostZ = NegatibleCost::Expensive;
6337     SDValue NegZ =
6338         getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
6339     // Give up if fail to negate the Z.
6340     if (!NegZ)
6341       break;
6342 
6343     // Prevent this node from being deleted by the next two calls.
6344     Handles.emplace_back(NegZ);
6345 
6346     // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
6347     NegatibleCost CostX = NegatibleCost::Expensive;
6348     SDValue NegX =
6349         getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6350     // Prevent this node from being deleted by the next call.
6351     if (NegX)
6352       Handles.emplace_back(NegX);
6353 
6354     // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
6355     NegatibleCost CostY = NegatibleCost::Expensive;
6356     SDValue NegY =
6357         getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6358 
6359     // We're done with the handles.
6360     Handles.clear();
6361 
6362     // Negate the X if its cost is less or equal than Y.
6363     if (NegX && (CostX <= CostY)) {
6364       Cost = std::min(CostX, CostZ);
6365       SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
6366       if (NegY != N)
6367         RemoveDeadNode(NegY);
6368       return N;
6369     }
6370 
6371     // Negate the Y if it is not expensive.
6372     if (NegY) {
6373       Cost = std::min(CostY, CostZ);
6374       SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
6375       if (NegX != N)
6376         RemoveDeadNode(NegX);
6377       return N;
6378     }
6379     break;
6380   }
6381 
6382   case ISD::FP_EXTEND:
6383   case ISD::FSIN:
6384     if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
6385                                             OptForSize, Cost, Depth))
6386       return DAG.getNode(Opcode, DL, VT, NegV);
6387     break;
6388   case ISD::FP_ROUND:
6389     if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
6390                                             OptForSize, Cost, Depth))
6391       return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
6392     break;
6393   }
6394 
6395   return SDValue();
6396 }
6397 
6398 //===----------------------------------------------------------------------===//
6399 // Legalization Utilities
6400 //===----------------------------------------------------------------------===//
6401 
6402 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
6403                                     SDValue LHS, SDValue RHS,
6404                                     SmallVectorImpl<SDValue> &Result,
6405                                     EVT HiLoVT, SelectionDAG &DAG,
6406                                     MulExpansionKind Kind, SDValue LL,
6407                                     SDValue LH, SDValue RL, SDValue RH) const {
6408   assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
6409          Opcode == ISD::SMUL_LOHI);
6410 
6411   bool HasMULHS = (Kind == MulExpansionKind::Always) ||
6412                   isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
6413   bool HasMULHU = (Kind == MulExpansionKind::Always) ||
6414                   isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
6415   bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
6416                       isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
6417   bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
6418                       isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);
6419 
6420   if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
6421     return false;
6422 
6423   unsigned OuterBitSize = VT.getScalarSizeInBits();
6424   unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
6425 
6426   // LL, LH, RL, and RH must be either all NULL or all set to a value.
6427   assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
6428          (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
6429 
6430   SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
6431   auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
6432                           bool Signed) -> bool {
6433     if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
6434       Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
6435       Hi = SDValue(Lo.getNode(), 1);
6436       return true;
6437     }
6438     if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
6439       Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
6440       Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
6441       return true;
6442     }
6443     return false;
6444   };
6445 
6446   SDValue Lo, Hi;
6447 
6448   if (!LL.getNode() && !RL.getNode() &&
6449       isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6450     LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
6451     RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
6452   }
6453 
6454   if (!LL.getNode())
6455     return false;
6456 
6457   APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
6458   if (DAG.MaskedValueIsZero(LHS, HighMask) &&
6459       DAG.MaskedValueIsZero(RHS, HighMask)) {
6460     // The inputs are both zero-extended.
6461     if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
6462       Result.push_back(Lo);
6463       Result.push_back(Hi);
6464       if (Opcode != ISD::MUL) {
6465         SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6466         Result.push_back(Zero);
6467         Result.push_back(Zero);
6468       }
6469       return true;
6470     }
6471   }
6472 
6473   if (!VT.isVector() && Opcode == ISD::MUL &&
6474       DAG.ComputeNumSignBits(LHS) > InnerBitSize &&
6475       DAG.ComputeNumSignBits(RHS) > InnerBitSize) {
6476     // The input values are both sign-extended.
6477     // TODO non-MUL case?
6478     if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
6479       Result.push_back(Lo);
6480       Result.push_back(Hi);
6481       return true;
6482     }
6483   }
6484 
6485   unsigned ShiftAmount = OuterBitSize - InnerBitSize;
6486   EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
6487   SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
6488 
6489   if (!LH.getNode() && !RH.getNode() &&
6490       isOperationLegalOrCustom(ISD::SRL, VT) &&
6491       isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6492     LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
6493     LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
6494     RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
6495     RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
6496   }
6497 
6498   if (!LH.getNode())
6499     return false;
6500 
6501   if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
6502     return false;
6503 
6504   Result.push_back(Lo);
6505 
6506   if (Opcode == ISD::MUL) {
6507     RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
6508     LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
6509     Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
6510     Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
6511     Result.push_back(Hi);
6512     return true;
6513   }
6514 
6515   // Compute the full width result.
6516   auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
6517     Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
6518     Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6519     Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
6520     return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
6521   };
6522 
6523   SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6524   if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
6525     return false;
6526 
6527   // This is effectively the add part of a multiply-add of half-sized operands,
6528   // so it cannot overflow.
6529   Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6530 
6531   if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
6532     return false;
6533 
6534   SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6535   EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
6536 
6537   bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
6538                   isOperationLegalOrCustom(ISD::ADDE, VT));
6539   if (UseGlue)
6540     Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
6541                        Merge(Lo, Hi));
6542   else
6543     Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
6544                        Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
6545 
6546   SDValue Carry = Next.getValue(1);
6547   Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6548   Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6549 
6550   if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
6551     return false;
6552 
6553   if (UseGlue)
6554     Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
6555                      Carry);
6556   else
6557     Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
6558                      Zero, Carry);
6559 
6560   Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6561 
6562   if (Opcode == ISD::SMUL_LOHI) {
6563     SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6564                                   DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
6565     Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
6566 
6567     NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6568                           DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
6569     Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
6570   }
6571 
6572   Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6573   Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6574   Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6575   return true;
6576 }
6577 
6578 bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
6579                                SelectionDAG &DAG, MulExpansionKind Kind,
6580                                SDValue LL, SDValue LH, SDValue RL,
6581                                SDValue RH) const {
6582   SmallVector<SDValue, 2> Result;
6583   bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
6584                            N->getOperand(0), N->getOperand(1), Result, HiLoVT,
6585                            DAG, Kind, LL, LH, RL, RH);
6586   if (Ok) {
6587     assert(Result.size() == 2);
6588     Lo = Result[0];
6589     Hi = Result[1];
6590   }
6591   return Ok;
6592 }
6593 
6594 // Check that (every element of) Z is undef or not an exact multiple of BW.
6595 static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
6596   return ISD::matchUnaryPredicate(
6597       Z,
6598       [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
6599       true);
6600 }
6601 
6602 SDValue TargetLowering::expandFunnelShift(SDNode *Node,
6603                                           SelectionDAG &DAG) const {
6604   EVT VT = Node->getValueType(0);
6605 
6606   if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6607                         !isOperationLegalOrCustom(ISD::SRL, VT) ||
6608                         !isOperationLegalOrCustom(ISD::SUB, VT) ||
6609                         !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
6610     return SDValue();
6611 
6612   SDValue X = Node->getOperand(0);
6613   SDValue Y = Node->getOperand(1);
6614   SDValue Z = Node->getOperand(2);
6615 
6616   unsigned BW = VT.getScalarSizeInBits();
6617   bool IsFSHL = Node->getOpcode() == ISD::FSHL;
6618   SDLoc DL(SDValue(Node, 0));
6619 
6620   EVT ShVT = Z.getValueType();
6621 
6622   // If a funnel shift in the other direction is more supported, use it.
6623   unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
6624   if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
6625       isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
6626     if (isNonZeroModBitWidthOrUndef(Z, BW)) {
6627       // fshl X, Y, Z -> fshr X, Y, -Z
6628       // fshr X, Y, Z -> fshl X, Y, -Z
6629       SDValue Zero = DAG.getConstant(0, DL, ShVT);
6630       Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);
6631     } else {
6632       // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
6633       // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
6634       SDValue One = DAG.getConstant(1, DL, ShVT);
6635       if (IsFSHL) {
6636         Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
6637         X = DAG.getNode(ISD::SRL, DL, VT, X, One);
6638       } else {
6639         X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
6640         Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
6641       }
6642       Z = DAG.getNOT(DL, Z, ShVT);
6643     }
6644     return DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
6645   }
6646 
6647   SDValue ShX, ShY;
6648   SDValue ShAmt, InvShAmt;
6649   if (isNonZeroModBitWidthOrUndef(Z, BW)) {
6650     // fshl: X << C | Y >> (BW - C)
6651     // fshr: X << (BW - C) | Y >> C
6652     // where C = Z % BW is not zero
6653     SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6654     ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6655     InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
6656     ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
6657     ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
6658   } else {
6659     // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
6660     // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
6661     SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
6662     if (isPowerOf2_32(BW)) {
6663       // Z % BW -> Z & (BW - 1)
6664       ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
6665       // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
6666       InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
6667     } else {
6668       SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6669       ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6670       InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
6671     }
6672 
6673     SDValue One = DAG.getConstant(1, DL, ShVT);
6674     if (IsFSHL) {
6675       ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
6676       SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
6677       ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
6678     } else {
6679       SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
6680       ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
6681       ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
6682     }
6683   }
6684   return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
6685 }
6686 
6687 // TODO: Merge with expandFunnelShift.
6688 SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
6689                                   SelectionDAG &DAG) const {
6690   EVT VT = Node->getValueType(0);
6691   unsigned EltSizeInBits = VT.getScalarSizeInBits();
6692   bool IsLeft = Node->getOpcode() == ISD::ROTL;
6693   SDValue Op0 = Node->getOperand(0);
6694   SDValue Op1 = Node->getOperand(1);
6695   SDLoc DL(SDValue(Node, 0));
6696 
6697   EVT ShVT = Op1.getValueType();
6698   SDValue Zero = DAG.getConstant(0, DL, ShVT);
6699 
6700   // If a rotate in the other direction is more supported, use it.
6701   unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
6702   if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
6703       isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
6704     SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6705     return DAG.getNode(RevRot, DL, VT, Op0, Sub);
6706   }
6707 
6708   if (!AllowVectorOps && VT.isVector() &&
6709       (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6710        !isOperationLegalOrCustom(ISD::SRL, VT) ||
6711        !isOperationLegalOrCustom(ISD::SUB, VT) ||
6712        !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
6713        !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6714     return SDValue();
6715 
6716   unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
6717   unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
6718   SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
6719   SDValue ShVal;
6720   SDValue HsVal;
6721   if (isPowerOf2_32(EltSizeInBits)) {
6722     // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
6723     // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
6724     SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6725     SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
6726     ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
6727     SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
6728     HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
6729   } else {
6730     // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
6731     // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
6732     SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
6733     SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
6734     ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
6735     SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
6736     SDValue One = DAG.getConstant(1, DL, ShVT);
6737     HsVal =
6738         DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
6739   }
6740   return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
6741 }
6742 
6743 void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
6744                                       SelectionDAG &DAG) const {
6745   assert(Node->getNumOperands() == 3 && "Not a double-shift!");
6746   EVT VT = Node->getValueType(0);
6747   unsigned VTBits = VT.getScalarSizeInBits();
6748   assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
6749 
6750   bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
6751   bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
6752   SDValue ShOpLo = Node->getOperand(0);
6753   SDValue ShOpHi = Node->getOperand(1);
6754   SDValue ShAmt = Node->getOperand(2);
6755   EVT ShAmtVT = ShAmt.getValueType();
6756   EVT ShAmtCCVT =
6757       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
6758   SDLoc dl(Node);
6759 
6760   // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
6761   // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
6762   // away during isel.
6763   SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
6764                                   DAG.getConstant(VTBits - 1, dl, ShAmtVT));
6765   SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
6766                                      DAG.getConstant(VTBits - 1, dl, ShAmtVT))
6767                        : DAG.getConstant(0, dl, VT);
6768 
6769   SDValue Tmp2, Tmp3;
6770   if (IsSHL) {
6771     Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
6772     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
6773   } else {
6774     Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
6775     Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
6776   }
6777 
6778   // If the shift amount is larger or equal than the width of a part we don't
6779   // use the result from the FSHL/FSHR. Insert a test and select the appropriate
6780   // values for large shift amounts.
6781   SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
6782                                 DAG.getConstant(VTBits, dl, ShAmtVT));
6783   SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
6784                               DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);
6785 
6786   if (IsSHL) {
6787     Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
6788     Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
6789   } else {
6790     Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
6791     Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
6792   }
6793 }
6794 
6795 bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
6796                                       SelectionDAG &DAG) const {
6797   unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
6798   SDValue Src = Node->getOperand(OpNo);
6799   EVT SrcVT = Src.getValueType();
6800   EVT DstVT = Node->getValueType(0);
6801   SDLoc dl(SDValue(Node, 0));
6802 
6803   // FIXME: Only f32 to i64 conversions are supported.
6804   if (SrcVT != MVT::f32 || DstVT != MVT::i64)
6805     return false;
6806 
6807   if (Node->isStrictFPOpcode())
6808     // When a NaN is converted to an integer a trap is allowed. We can't
6809     // use this expansion here because it would eliminate that trap. Other
6810     // traps are also allowed and cannot be eliminated. See
6811     // IEEE 754-2008 sec 5.8.
6812     return false;
6813 
6814   // Expand f32 -> i64 conversion
6815   // This algorithm comes from compiler-rt's implementation of fixsfdi:
6816   // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
6817   unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6818   EVT IntVT = SrcVT.changeTypeToInteger();
6819   EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
6820 
6821   SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
6822   SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
6823   SDValue Bias = DAG.getConstant(127, dl, IntVT);
6824   SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
6825   SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
6826   SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
6827 
6828   SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
6829 
6830   SDValue ExponentBits = DAG.getNode(
6831       ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
6832       DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
6833   SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
6834 
6835   SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
6836                              DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
6837                              DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
6838   Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
6839 
6840   SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
6841                           DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
6842                           DAG.getConstant(0x00800000, dl, IntVT));
6843 
6844   R = DAG.getZExtOrTrunc(R, dl, DstVT);
6845 
6846   R = DAG.getSelectCC(
6847       dl, Exponent, ExponentLoBit,
6848       DAG.getNode(ISD::SHL, dl, DstVT, R,
6849                   DAG.getZExtOrTrunc(
6850                       DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
6851                       dl, IntShVT)),
6852       DAG.getNode(ISD::SRL, dl, DstVT, R,
6853                   DAG.getZExtOrTrunc(
6854                       DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
6855                       dl, IntShVT)),
6856       ISD::SETGT);
6857 
6858   SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
6859                             DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
6860 
6861   Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
6862                            DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
6863   return true;
6864 }
6865 
6866 bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
6867                                       SDValue &Chain,
6868                                       SelectionDAG &DAG) const {
6869   SDLoc dl(SDValue(Node, 0));
6870   unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
6871   SDValue Src = Node->getOperand(OpNo);
6872 
6873   EVT SrcVT = Src.getValueType();
6874   EVT DstVT = Node->getValueType(0);
6875   EVT SetCCVT =
6876       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
6877   EVT DstSetCCVT =
6878       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
6879 
6880   // Only expand vector types if we have the appropriate vector bit operations.
6881   unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
6882                                                    ISD::FP_TO_SINT;
6883   if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
6884                            !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))
6885     return false;
6886 
6887   // If the maximum float value is smaller then the signed integer range,
6888   // the destination signmask can't be represented by the float, so we can
6889   // just use FP_TO_SINT directly.
6890   const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
6891   APFloat APF(APFSem, APInt::getZero(SrcVT.getScalarSizeInBits()));
6892   APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
6893   if (APFloat::opOverflow &
6894       APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
6895     if (Node->isStrictFPOpcode()) {
6896       Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6897                            { Node->getOperand(0), Src });
6898       Chain = Result.getValue(1);
6899     } else
6900       Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6901     return true;
6902   }
6903 
6904   // Don't expand it if there isn't cheap fsub instruction.
6905   if (!isOperationLegalOrCustom(
6906           Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
6907     return false;
6908 
6909   SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
6910   SDValue Sel;
6911 
6912   if (Node->isStrictFPOpcode()) {
6913     Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
6914                        Node->getOperand(0), /*IsSignaling*/ true);
6915     Chain = Sel.getValue(1);
6916   } else {
6917     Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
6918   }
6919 
6920   bool Strict = Node->isStrictFPOpcode() ||
6921                 shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
6922 
6923   if (Strict) {
6924     // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
6925     // signmask then offset (the result of which should be fully representable).
6926     // Sel = Src < 0x8000000000000000
6927     // FltOfs = select Sel, 0, 0x8000000000000000
6928     // IntOfs = select Sel, 0, 0x8000000000000000
6929     // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
6930 
6931     // TODO: Should any fast-math-flags be set for the FSUB?
6932     SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
6933                                    DAG.getConstantFP(0.0, dl, SrcVT), Cst);
6934     Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
6935     SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
6936                                    DAG.getConstant(0, dl, DstVT),
6937                                    DAG.getConstant(SignMask, dl, DstVT));
6938     SDValue SInt;
6939     if (Node->isStrictFPOpcode()) {
6940       SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
6941                                 { Chain, Src, FltOfs });
6942       SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6943                          { Val.getValue(1), Val });
6944       Chain = SInt.getValue(1);
6945     } else {
6946       SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
6947       SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
6948     }
6949     Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
6950   } else {
6951     // Expand based on maximum range of FP_TO_SINT:
6952     // True = fp_to_sint(Src)
6953     // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
6954     // Result = select (Src < 0x8000000000000000), True, False
6955 
6956     SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6957     // TODO: Should any fast-math-flags be set for the FSUB?
6958     SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
6959                                 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
6960     False = DAG.getNode(ISD::XOR, dl, DstVT, False,
6961                         DAG.getConstant(SignMask, dl, DstVT));
6962     Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
6963     Result = DAG.getSelect(dl, DstVT, Sel, True, False);
6964   }
6965   return true;
6966 }
6967 
6968 bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
6969                                       SDValue &Chain,
6970                                       SelectionDAG &DAG) const {
6971   // This transform is not correct for converting 0 when rounding mode is set
6972   // to round toward negative infinity which will produce -0.0. So disable under
6973   // strictfp.
6974   if (Node->isStrictFPOpcode())
6975     return false;
6976 
6977   SDValue Src = Node->getOperand(0);
6978   EVT SrcVT = Src.getValueType();
6979   EVT DstVT = Node->getValueType(0);
6980 
6981   if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
6982     return false;
6983 
6984   // Only expand vector types if we have the appropriate vector bit operations.
6985   if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
6986                            !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
6987                            !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
6988                            !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||
6989                            !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))
6990     return false;
6991 
6992   SDLoc dl(SDValue(Node, 0));
6993   EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());
6994 
6995   // Implementation of unsigned i64 to f64 following the algorithm in
6996   // __floatundidf in compiler_rt.  This implementation performs rounding
6997   // correctly in all rounding modes with the exception of converting 0
6998   // when rounding toward negative infinity. In that case the fsub will produce
6999   // -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.
7000   SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
7001   SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
7002       BitsToDouble(UINT64_C(0x4530000000100000)), dl, DstVT);
7003   SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
7004   SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
7005   SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
7006 
7007   SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
7008   SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
7009   SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
7010   SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
7011   SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
7012   SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
7013   SDValue HiSub =
7014       DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
7015   Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
7016   return true;
7017 }
7018 
7019 SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
7020                                               SelectionDAG &DAG) const {
7021   SDLoc dl(Node);
7022   unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
7023     ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
7024   EVT VT = Node->getValueType(0);
7025 
7026   if (VT.isScalableVector())
7027     report_fatal_error(
7028         "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
7029 
7030   if (isOperationLegalOrCustom(NewOp, VT)) {
7031     SDValue Quiet0 = Node->getOperand(0);
7032     SDValue Quiet1 = Node->getOperand(1);
7033 
7034     if (!Node->getFlags().hasNoNaNs()) {
7035       // Insert canonicalizes if it's possible we need to quiet to get correct
7036       // sNaN behavior.
7037       if (!DAG.isKnownNeverSNaN(Quiet0)) {
7038         Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
7039                              Node->getFlags());
7040       }
7041       if (!DAG.isKnownNeverSNaN(Quiet1)) {
7042         Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
7043                              Node->getFlags());
7044       }
7045     }
7046 
7047     return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
7048   }
7049 
7050   // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
7051   // instead if there are no NaNs.
7052   if (Node->getFlags().hasNoNaNs()) {
7053     unsigned IEEE2018Op =
7054         Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
7055     if (isOperationLegalOrCustom(IEEE2018Op, VT)) {
7056       return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
7057                          Node->getOperand(1), Node->getFlags());
7058     }
7059   }
7060 
7061   // If none of the above worked, but there are no NaNs, then expand to
7062   // a compare/select sequence.  This is required for correctness since
7063   // InstCombine might have canonicalized a fcmp+select sequence to a
7064   // FMINNUM/FMAXNUM node.  If we were to fall through to the default
7065   // expansion to libcall, we might introduce a link-time dependency
7066   // on libm into a file that originally did not have one.
7067   if (Node->getFlags().hasNoNaNs()) {
7068     ISD::CondCode Pred =
7069         Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
7070     SDValue Op1 = Node->getOperand(0);
7071     SDValue Op2 = Node->getOperand(1);
7072     SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);
7073     // Copy FMF flags, but always set the no-signed-zeros flag
7074     // as this is implied by the FMINNUM/FMAXNUM semantics.
7075     SDNodeFlags Flags = Node->getFlags();
7076     Flags.setNoSignedZeros(true);
7077     SelCC->setFlags(Flags);
7078     return SelCC;
7079   }
7080 
7081   return SDValue();
7082 }
7083 
7084 // Only expand vector types if we have the appropriate vector bit operations.
7085 static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
7086   assert(VT.isVector() && "Expected vector type");
7087   unsigned Len = VT.getScalarSizeInBits();
7088   return TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
7089          TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
7090          TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
7091          (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) &&
7092          TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT);
7093 }
7094 
7095 SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
7096   SDLoc dl(Node);
7097   EVT VT = Node->getValueType(0);
7098   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7099   SDValue Op = Node->getOperand(0);
7100   unsigned Len = VT.getScalarSizeInBits();
7101   assert(VT.isInteger() && "CTPOP not implemented for this type.");
7102 
7103   // TODO: Add support for irregular type lengths.
7104   if (!(Len <= 128 && Len % 8 == 0))
7105     return SDValue();
7106 
7107   // Only expand vector types if we have the appropriate vector bit operations.
7108   if (VT.isVector() && !canExpandVectorCTPOP(*this, VT))
7109     return SDValue();
7110 
7111   // This is the "best" algorithm from
7112   // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
7113   SDValue Mask55 =
7114       DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
7115   SDValue Mask33 =
7116       DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
7117   SDValue Mask0F =
7118       DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
7119   SDValue Mask01 =
7120       DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
7121 
7122   // v = v - ((v >> 1) & 0x55555555...)
7123   Op = DAG.getNode(ISD::SUB, dl, VT, Op,
7124                    DAG.getNode(ISD::AND, dl, VT,
7125                                DAG.getNode(ISD::SRL, dl, VT, Op,
7126                                            DAG.getConstant(1, dl, ShVT)),
7127                                Mask55));
7128   // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
7129   Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
7130                    DAG.getNode(ISD::AND, dl, VT,
7131                                DAG.getNode(ISD::SRL, dl, VT, Op,
7132                                            DAG.getConstant(2, dl, ShVT)),
7133                                Mask33));
7134   // v = (v + (v >> 4)) & 0x0F0F0F0F...
7135   Op = DAG.getNode(ISD::AND, dl, VT,
7136                    DAG.getNode(ISD::ADD, dl, VT, Op,
7137                                DAG.getNode(ISD::SRL, dl, VT, Op,
7138                                            DAG.getConstant(4, dl, ShVT))),
7139                    Mask0F);
7140   // v = (v * 0x01010101...) >> (Len - 8)
7141   if (Len > 8)
7142     Op =
7143         DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
7144                     DAG.getConstant(Len - 8, dl, ShVT));
7145 
7146   return Op;
7147 }
7148 
7149 SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
7150   SDLoc dl(Node);
7151   EVT VT = Node->getValueType(0);
7152   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7153   SDValue Op = Node->getOperand(0);
7154   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7155 
7156   // If the non-ZERO_UNDEF version is supported we can use that instead.
7157   if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
7158       isOperationLegalOrCustom(ISD::CTLZ, VT))
7159     return DAG.getNode(ISD::CTLZ, dl, VT, Op);
7160 
7161   // If the ZERO_UNDEF version is supported use that and handle the zero case.
7162   if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
7163     EVT SetCCVT =
7164         getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7165     SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
7166     SDValue Zero = DAG.getConstant(0, dl, VT);
7167     SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
7168     return DAG.getSelect(dl, VT, SrcIsZero,
7169                          DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
7170   }
7171 
7172   // Only expand vector types if we have the appropriate vector bit operations.
7173   // This includes the operations needed to expand CTPOP if it isn't supported.
7174   if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
7175                         (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
7176                          !canExpandVectorCTPOP(*this, VT)) ||
7177                         !isOperationLegalOrCustom(ISD::SRL, VT) ||
7178                         !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
7179     return SDValue();
7180 
7181   // for now, we do this:
7182   // x = x | (x >> 1);
7183   // x = x | (x >> 2);
7184   // ...
7185   // x = x | (x >>16);
7186   // x = x | (x >>32); // for 64-bit input
7187   // return popcount(~x);
7188   //
7189   // Ref: "Hacker's Delight" by Henry Warren
7190   for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {
7191     SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
7192     Op = DAG.getNode(ISD::OR, dl, VT, Op,
7193                      DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
7194   }
7195   Op = DAG.getNOT(dl, Op, VT);
7196   return DAG.getNode(ISD::CTPOP, dl, VT, Op);
7197 }
7198 
7199 SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
7200   SDLoc dl(Node);
7201   EVT VT = Node->getValueType(0);
7202   SDValue Op = Node->getOperand(0);
7203   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7204 
7205   // If the non-ZERO_UNDEF version is supported we can use that instead.
7206   if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
7207       isOperationLegalOrCustom(ISD::CTTZ, VT))
7208     return DAG.getNode(ISD::CTTZ, dl, VT, Op);
7209 
7210   // If the ZERO_UNDEF version is supported use that and handle the zero case.
7211   if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
7212     EVT SetCCVT =
7213         getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7214     SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
7215     SDValue Zero = DAG.getConstant(0, dl, VT);
7216     SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
7217     return DAG.getSelect(dl, VT, SrcIsZero,
7218                          DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
7219   }
7220 
7221   // Only expand vector types if we have the appropriate vector bit operations.
7222   // This includes the operations needed to expand CTPOP if it isn't supported.
7223   if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
7224                         (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
7225                          !isOperationLegalOrCustom(ISD::CTLZ, VT) &&
7226                          !canExpandVectorCTPOP(*this, VT)) ||
7227                         !isOperationLegalOrCustom(ISD::SUB, VT) ||
7228                         !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
7229                         !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
7230     return SDValue();
7231 
7232   // for now, we use: { return popcount(~x & (x - 1)); }
7233   // unless the target has ctlz but not ctpop, in which case we use:
7234   // { return 32 - nlz(~x & (x-1)); }
7235   // Ref: "Hacker's Delight" by Henry Warren
7236   SDValue Tmp = DAG.getNode(
7237       ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
7238       DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
7239 
7240   // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
7241   if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
7242     return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
7243                        DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
7244   }
7245 
7246   return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
7247 }
7248 
7249 SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
7250                                   bool IsNegative) const {
7251   SDLoc dl(N);
7252   EVT VT = N->getValueType(0);
7253   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7254   SDValue Op = N->getOperand(0);
7255 
7256   // abs(x) -> smax(x,sub(0,x))
7257   if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
7258       isOperationLegal(ISD::SMAX, VT)) {
7259     SDValue Zero = DAG.getConstant(0, dl, VT);
7260     return DAG.getNode(ISD::SMAX, dl, VT, Op,
7261                        DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7262   }
7263 
7264   // abs(x) -> umin(x,sub(0,x))
7265   if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
7266       isOperationLegal(ISD::UMIN, VT)) {
7267     SDValue Zero = DAG.getConstant(0, dl, VT);
7268     return DAG.getNode(ISD::UMIN, dl, VT, Op,
7269                        DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7270   }
7271 
7272   // 0 - abs(x) -> smin(x, sub(0,x))
7273   if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
7274       isOperationLegal(ISD::SMIN, VT)) {
7275     SDValue Zero = DAG.getConstant(0, dl, VT);
7276     return DAG.getNode(ISD::SMIN, dl, VT, Op,
7277                        DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7278   }
7279 
7280   // Only expand vector types if we have the appropriate vector operations.
7281   if (VT.isVector() &&
7282       (!isOperationLegalOrCustom(ISD::SRA, VT) ||
7283        (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
7284        (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
7285        !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
7286     return SDValue();
7287 
7288   SDValue Shift =
7289       DAG.getNode(ISD::SRA, dl, VT, Op,
7290                   DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));
7291   if (!IsNegative) {
7292     SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
7293     return DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
7294   }
7295 
7296   // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
7297   SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
7298   return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
7299 }
7300 
7301 SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
7302   SDLoc dl(N);
7303   EVT VT = N->getValueType(0);
7304   SDValue Op = N->getOperand(0);
7305 
7306   if (!VT.isSimple())
7307     return SDValue();
7308 
7309   EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
7310   SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
7311   switch (VT.getSimpleVT().getScalarType().SimpleTy) {
7312   default:
7313     return SDValue();
7314   case MVT::i16:
7315     // Use a rotate by 8. This can be further expanded if necessary.
7316     return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7317   case MVT::i32:
7318     Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7319     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7320     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7321     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7322     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
7323                        DAG.getConstant(0xFF0000, dl, VT));
7324     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
7325     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
7326     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
7327     return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
7328   case MVT::i64:
7329     Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
7330     Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
7331     Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7332     Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7333     Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7334     Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7335     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
7336     Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
7337     Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
7338                        DAG.getConstant(255ULL<<48, dl, VT));
7339     Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
7340                        DAG.getConstant(255ULL<<40, dl, VT));
7341     Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
7342                        DAG.getConstant(255ULL<<32, dl, VT));
7343     Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
7344                        DAG.getConstant(255ULL<<24, dl, VT));
7345     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
7346                        DAG.getConstant(255ULL<<16, dl, VT));
7347     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
7348                        DAG.getConstant(255ULL<<8 , dl, VT));
7349     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
7350     Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
7351     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
7352     Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
7353     Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
7354     Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
7355     return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
7356   }
7357 }
7358 
7359 SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
7360   SDLoc dl(N);
7361   EVT VT = N->getValueType(0);
7362   SDValue Op = N->getOperand(0);
7363   EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
7364   unsigned Sz = VT.getScalarSizeInBits();
7365 
7366   SDValue Tmp, Tmp2, Tmp3;
7367 
7368   // If we can, perform BSWAP first and then the mask+swap the i4, then i2
7369   // and finally the i1 pairs.
7370   // TODO: We can easily support i4/i2 legal types if any target ever does.
7371   if (Sz >= 8 && isPowerOf2_32(Sz)) {
7372     // Create the masks - repeating the pattern every byte.
7373     APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
7374     APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
7375     APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));
7376 
7377     // BSWAP if the type is wider than a single byte.
7378     Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
7379 
7380     // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
7381     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));
7382     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));
7383     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));
7384     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
7385     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7386 
7387     // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
7388     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));
7389     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));
7390     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));
7391     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
7392     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7393 
7394     // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
7395     Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));
7396     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));
7397     Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));
7398     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
7399     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7400     return Tmp;
7401   }
7402 
7403   Tmp = DAG.getConstant(0, dl, VT);
7404   for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
7405     if (I < J)
7406       Tmp2 =
7407           DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
7408     else
7409       Tmp2 =
7410           DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
7411 
7412     APInt Shift(Sz, 1);
7413     Shift <<= J;
7414     Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
7415     Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
7416   }
7417 
7418   return Tmp;
7419 }
7420 
7421 std::pair<SDValue, SDValue>
7422 TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
7423                                     SelectionDAG &DAG) const {
7424   SDLoc SL(LD);
7425   SDValue Chain = LD->getChain();
7426   SDValue BasePTR = LD->getBasePtr();
7427   EVT SrcVT = LD->getMemoryVT();
7428   EVT DstVT = LD->getValueType(0);
7429   ISD::LoadExtType ExtType = LD->getExtensionType();
7430 
7431   if (SrcVT.isScalableVector())
7432     report_fatal_error("Cannot scalarize scalable vector loads");
7433 
7434   unsigned NumElem = SrcVT.getVectorNumElements();
7435 
7436   EVT SrcEltVT = SrcVT.getScalarType();
7437   EVT DstEltVT = DstVT.getScalarType();
7438 
7439   // A vector must always be stored in memory as-is, i.e. without any padding
7440   // between the elements, since various code depend on it, e.g. in the
7441   // handling of a bitcast of a vector type to int, which may be done with a
7442   // vector store followed by an integer load. A vector that does not have
7443   // elements that are byte-sized must therefore be stored as an integer
7444   // built out of the extracted vector elements.
7445   if (!SrcEltVT.isByteSized()) {
7446     unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
7447     EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);
7448 
7449     unsigned NumSrcBits = SrcVT.getSizeInBits();
7450     EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);
7451 
7452     unsigned SrcEltBits = SrcEltVT.getSizeInBits();
7453     SDValue SrcEltBitMask = DAG.getConstant(
7454         APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);
7455 
7456     // Load the whole vector and avoid masking off the top bits as it makes
7457     // the codegen worse.
7458     SDValue Load =
7459         DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
7460                        LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),
7461                        LD->getMemOperand()->getFlags(), LD->getAAInfo());
7462 
7463     SmallVector<SDValue, 8> Vals;
7464     for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7465       unsigned ShiftIntoIdx =
7466           (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
7467       SDValue ShiftAmount =
7468           DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),
7469                                      LoadVT, SL, /*LegalTypes=*/false);
7470       SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
7471       SDValue Elt =
7472           DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
7473       SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
7474 
7475       if (ExtType != ISD::NON_EXTLOAD) {
7476         unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
7477         Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
7478       }
7479 
7480       Vals.push_back(Scalar);
7481     }
7482 
7483     SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
7484     return std::make_pair(Value, Load.getValue(1));
7485   }
7486 
7487   unsigned Stride = SrcEltVT.getSizeInBits() / 8;
7488   assert(SrcEltVT.isByteSized());
7489 
7490   SmallVector<SDValue, 8> Vals;
7491   SmallVector<SDValue, 8> LoadChains;
7492 
7493   for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7494     SDValue ScalarLoad =
7495         DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
7496                        LD->getPointerInfo().getWithOffset(Idx * Stride),
7497                        SrcEltVT, LD->getOriginalAlign(),
7498                        LD->getMemOperand()->getFlags(), LD->getAAInfo());
7499 
7500     BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::Fixed(Stride));
7501 
7502     Vals.push_back(ScalarLoad.getValue(0));
7503     LoadChains.push_back(ScalarLoad.getValue(1));
7504   }
7505 
7506   SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
7507   SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
7508 
7509   return std::make_pair(Value, NewChain);
7510 }
7511 
7512 SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
7513                                              SelectionDAG &DAG) const {
7514   SDLoc SL(ST);
7515 
7516   SDValue Chain = ST->getChain();
7517   SDValue BasePtr = ST->getBasePtr();
7518   SDValue Value = ST->getValue();
7519   EVT StVT = ST->getMemoryVT();
7520 
7521   if (StVT.isScalableVector())
7522     report_fatal_error("Cannot scalarize scalable vector stores");
7523 
7524   // The type of the data we want to save
7525   EVT RegVT = Value.getValueType();
7526   EVT RegSclVT = RegVT.getScalarType();
7527 
7528   // The type of data as saved in memory.
7529   EVT MemSclVT = StVT.getScalarType();
7530 
7531   unsigned NumElem = StVT.getVectorNumElements();
7532 
7533   // A vector must always be stored in memory as-is, i.e. without any padding
7534   // between the elements, since various code depend on it, e.g. in the
7535   // handling of a bitcast of a vector type to int, which may be done with a
7536   // vector store followed by an integer load. A vector that does not have
7537   // elements that are byte-sized must therefore be stored as an integer
7538   // built out of the extracted vector elements.
7539   if (!MemSclVT.isByteSized()) {
7540     unsigned NumBits = StVT.getSizeInBits();
7541     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
7542 
7543     SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
7544 
7545     for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7546       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
7547                                 DAG.getVectorIdxConstant(Idx, SL));
7548       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
7549       SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
7550       unsigned ShiftIntoIdx =
7551           (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
7552       SDValue ShiftAmount =
7553           DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
7554       SDValue ShiftedElt =
7555           DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
7556       CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
7557     }
7558 
7559     return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
7560                         ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
7561                         ST->getAAInfo());
7562   }
7563 
7564   // Store Stride in bytes
7565   unsigned Stride = MemSclVT.getSizeInBits() / 8;
7566   assert(Stride && "Zero stride!");
7567   // Extract each of the elements from the original vector and save them into
7568   // memory individually.
7569   SmallVector<SDValue, 8> Stores;
7570   for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7571     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
7572                               DAG.getVectorIdxConstant(Idx, SL));
7573 
7574     SDValue Ptr =
7575         DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Idx * Stride));
7576 
7577     // This scalar TruncStore may be illegal, but we legalize it later.
7578     SDValue Store = DAG.getTruncStore(
7579         Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
7580         MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
7581         ST->getAAInfo());
7582 
7583     Stores.push_back(Store);
7584   }
7585 
7586   return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
7587 }
7588 
7589 std::pair<SDValue, SDValue>
7590 TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
7591   assert(LD->getAddressingMode() == ISD::UNINDEXED &&
7592          "unaligned indexed loads not implemented!");
7593   SDValue Chain = LD->getChain();
7594   SDValue Ptr = LD->getBasePtr();
7595   EVT VT = LD->getValueType(0);
7596   EVT LoadedVT = LD->getMemoryVT();
7597   SDLoc dl(LD);
7598   auto &MF = DAG.getMachineFunction();
7599 
7600   if (VT.isFloatingPoint() || VT.isVector()) {
7601     EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
7602     if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
7603       if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
7604           LoadedVT.isVector()) {
7605         // Scalarize the load and let the individual components be handled.
7606         return scalarizeVectorLoad(LD, DAG);
7607       }
7608 
7609       // Expand to a (misaligned) integer load of the same size,
7610       // then bitconvert to floating point or vector.
7611       SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
7612                                     LD->getMemOperand());
7613       SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
7614       if (LoadedVT != VT)
7615         Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
7616                              ISD::ANY_EXTEND, dl, VT, Result);
7617 
7618       return std::make_pair(Result, newLoad.getValue(1));
7619     }
7620 
7621     // Copy the value to a (aligned) stack slot using (unaligned) integer
7622     // loads and stores, then do a (aligned) load from the stack slot.
7623     MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
7624     unsigned LoadedBytes = LoadedVT.getStoreSize();
7625     unsigned RegBytes = RegVT.getSizeInBits() / 8;
7626     unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
7627 
7628     // Make sure the stack slot is also aligned for the register type.
7629     SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
7630     auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
7631     SmallVector<SDValue, 8> Stores;
7632     SDValue StackPtr = StackBase;
7633     unsigned Offset = 0;
7634 
7635     EVT PtrVT = Ptr.getValueType();
7636     EVT StackPtrVT = StackPtr.getValueType();
7637 
7638     SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7639     SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7640 
7641     // Do all but one copies using the full register width.
7642     for (unsigned i = 1; i < NumRegs; i++) {
7643       // Load one integer register's worth from the original location.
7644       SDValue Load = DAG.getLoad(
7645           RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
7646           LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
7647           LD->getAAInfo());
7648       // Follow the load with a store to the stack slot.  Remember the store.
7649       Stores.push_back(DAG.getStore(
7650           Load.getValue(1), dl, Load, StackPtr,
7651           MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
7652       // Increment the pointers.
7653       Offset += RegBytes;
7654 
7655       Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
7656       StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
7657     }
7658 
7659     // The last copy may be partial.  Do an extending load.
7660     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
7661                                   8 * (LoadedBytes - Offset));
7662     SDValue Load =
7663         DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
7664                        LD->getPointerInfo().getWithOffset(Offset), MemVT,
7665                        LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
7666                        LD->getAAInfo());
7667     // Follow the load with a store to the stack slot.  Remember the store.
7668     // On big-endian machines this requires a truncating store to ensure
7669     // that the bits end up in the right place.
7670     Stores.push_back(DAG.getTruncStore(
7671         Load.getValue(1), dl, Load, StackPtr,
7672         MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
7673 
7674     // The order of the stores doesn't matter - say it with a TokenFactor.
7675     SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7676 
7677     // Finally, perform the original load only redirected to the stack slot.
7678     Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
7679                           MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
7680                           LoadedVT);
7681 
7682     // Callers expect a MERGE_VALUES node.
7683     return std::make_pair(Load, TF);
7684   }
7685 
7686   assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
7687          "Unaligned load of unsupported type.");
7688 
7689   // Compute the new VT that is half the size of the old one.  This is an
7690   // integer MVT.
7691   unsigned NumBits = LoadedVT.getSizeInBits();
7692   EVT NewLoadedVT;
7693   NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
7694   NumBits >>= 1;
7695 
7696   Align Alignment = LD->getOriginalAlign();
7697   unsigned IncrementSize = NumBits / 8;
7698   ISD::LoadExtType HiExtType = LD->getExtensionType();
7699 
7700   // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
7701   if (HiExtType == ISD::NON_EXTLOAD)
7702     HiExtType = ISD::ZEXTLOAD;
7703 
7704   // Load the value in two parts
7705   SDValue Lo, Hi;
7706   if (DAG.getDataLayout().isLittleEndian()) {
7707     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7708                         NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7709                         LD->getAAInfo());
7710 
7711     Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7712     Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
7713                         LD->getPointerInfo().getWithOffset(IncrementSize),
7714                         NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7715                         LD->getAAInfo());
7716   } else {
7717     Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7718                         NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7719                         LD->getAAInfo());
7720 
7721     Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7722     Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
7723                         LD->getPointerInfo().getWithOffset(IncrementSize),
7724                         NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7725                         LD->getAAInfo());
7726   }
7727 
7728   // aggregate the two parts
7729   SDValue ShiftAmount =
7730       DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
7731                                                     DAG.getDataLayout()));
7732   SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
7733   Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
7734 
7735   SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
7736                              Hi.getValue(1));
7737 
7738   return std::make_pair(Result, TF);
7739 }
7740 
7741 SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
7742                                              SelectionDAG &DAG) const {
7743   assert(ST->getAddressingMode() == ISD::UNINDEXED &&
7744          "unaligned indexed stores not implemented!");
7745   SDValue Chain = ST->getChain();
7746   SDValue Ptr = ST->getBasePtr();
7747   SDValue Val = ST->getValue();
7748   EVT VT = Val.getValueType();
7749   Align Alignment = ST->getOriginalAlign();
7750   auto &MF = DAG.getMachineFunction();
7751   EVT StoreMemVT = ST->getMemoryVT();
7752 
7753   SDLoc dl(ST);
7754   if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
7755     EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
7756     if (isTypeLegal(intVT)) {
7757       if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
7758           StoreMemVT.isVector()) {
7759         // Scalarize the store and let the individual components be handled.
7760         SDValue Result = scalarizeVectorStore(ST, DAG);
7761         return Result;
7762       }
7763       // Expand to a bitconvert of the value to the integer type of the
7764       // same size, then a (misaligned) int store.
7765       // FIXME: Does not handle truncating floating point stores!
7766       SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
7767       Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
7768                             Alignment, ST->getMemOperand()->getFlags());
7769       return Result;
7770     }
7771     // Do a (aligned) store to a stack slot, then copy from the stack slot
7772     // to the final destination using (unaligned) integer loads and stores.
7773     MVT RegVT = getRegisterType(
7774         *DAG.getContext(),
7775         EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
7776     EVT PtrVT = Ptr.getValueType();
7777     unsigned StoredBytes = StoreMemVT.getStoreSize();
7778     unsigned RegBytes = RegVT.getSizeInBits() / 8;
7779     unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
7780 
7781     // Make sure the stack slot is also aligned for the register type.
7782     SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
7783     auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
7784 
7785     // Perform the original store, only redirected to the stack slot.
7786     SDValue Store = DAG.getTruncStore(
7787         Chain, dl, Val, StackPtr,
7788         MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);
7789 
7790     EVT StackPtrVT = StackPtr.getValueType();
7791 
7792     SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7793     SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7794     SmallVector<SDValue, 8> Stores;
7795     unsigned Offset = 0;
7796 
7797     // Do all but one copies using the full register width.
7798     for (unsigned i = 1; i < NumRegs; i++) {
7799       // Load one integer register's worth from the stack slot.
7800       SDValue Load = DAG.getLoad(
7801           RegVT, dl, Store, StackPtr,
7802           MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
7803       // Store it to the final location.  Remember the store.
7804       Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
7805                                     ST->getPointerInfo().getWithOffset(Offset),
7806                                     ST->getOriginalAlign(),
7807                                     ST->getMemOperand()->getFlags()));
7808       // Increment the pointers.
7809       Offset += RegBytes;
7810       StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
7811       Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
7812     }
7813 
7814     // The last store may be partial.  Do a truncating store.  On big-endian
7815     // machines this requires an extending load from the stack slot to ensure
7816     // that the bits are in the right place.
7817     EVT LoadMemVT =
7818         EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));
7819 
7820     // Load from the stack slot.
7821     SDValue Load = DAG.getExtLoad(
7822         ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
7823         MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);
7824 
7825     Stores.push_back(
7826         DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
7827                           ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
7828                           ST->getOriginalAlign(),
7829                           ST->getMemOperand()->getFlags(), ST->getAAInfo()));
7830     // The order of the stores doesn't matter - say it with a TokenFactor.
7831     SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7832     return Result;
7833   }
7834 
7835   assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
7836          "Unaligned store of unknown type.");
7837   // Get the half-size VT
7838   EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
7839   unsigned NumBits = NewStoredVT.getFixedSizeInBits();
7840   unsigned IncrementSize = NumBits / 8;
7841 
7842   // Divide the stored value in two parts.
7843   SDValue ShiftAmount = DAG.getConstant(
7844       NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout()));
7845   SDValue Lo = Val;
7846   SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
7847 
7848   // Store the two parts
7849   SDValue Store1, Store2;
7850   Store1 = DAG.getTruncStore(Chain, dl,
7851                              DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
7852                              Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
7853                              ST->getMemOperand()->getFlags());
7854 
7855   Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7856   Store2 = DAG.getTruncStore(
7857       Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
7858       ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
7859       ST->getMemOperand()->getFlags(), ST->getAAInfo());
7860 
7861   SDValue Result =
7862       DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
7863   return Result;
7864 }
7865 
7866 SDValue
7867 TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
7868                                        const SDLoc &DL, EVT DataVT,
7869                                        SelectionDAG &DAG,
7870                                        bool IsCompressedMemory) const {
7871   SDValue Increment;
7872   EVT AddrVT = Addr.getValueType();
7873   EVT MaskVT = Mask.getValueType();
7874   assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
7875          "Incompatible types of Data and Mask");
7876   if (IsCompressedMemory) {
7877     if (DataVT.isScalableVector())
7878       report_fatal_error(
7879           "Cannot currently handle compressed memory with scalable vectors");
7880     // Incrementing the pointer according to number of '1's in the mask.
7881     EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
7882     SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
7883     if (MaskIntVT.getSizeInBits() < 32) {
7884       MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
7885       MaskIntVT = MVT::i32;
7886     }
7887 
7888     // Count '1's with POPCNT.
7889     Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
7890     Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
7891     // Scale is an element size in bytes.
7892     SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
7893                                     AddrVT);
7894     Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
7895   } else if (DataVT.isScalableVector()) {
7896     Increment = DAG.getVScale(DL, AddrVT,
7897                               APInt(AddrVT.getFixedSizeInBits(),
7898                                     DataVT.getStoreSize().getKnownMinSize()));
7899   } else
7900     Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);
7901 
7902   return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
7903 }
7904 
7905 static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
7906                                        EVT VecVT, const SDLoc &dl,
7907                                        ElementCount SubEC) {
7908   assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
7909          "Cannot index a scalable vector within a fixed-width vector");
7910 
7911   unsigned NElts = VecVT.getVectorMinNumElements();
7912   unsigned NumSubElts = SubEC.getKnownMinValue();
7913   EVT IdxVT = Idx.getValueType();
7914 
7915   if (VecVT.isScalableVector() && !SubEC.isScalable()) {
7916     // If this is a constant index and we know the value plus the number of the
7917     // elements in the subvector minus one is less than the minimum number of
7918     // elements then it's safe to return Idx.
7919     if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))
7920       if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
7921         return Idx;
7922     SDValue VS =
7923         DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));
7924     unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
7925     SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,
7926                               DAG.getConstant(NumSubElts, dl, IdxVT));
7927     return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);
7928   }
7929   if (isPowerOf2_32(NElts) && NumSubElts == 1) {
7930     APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));
7931     return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
7932                        DAG.getConstant(Imm, dl, IdxVT));
7933   }
7934   unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
7935   return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
7936                      DAG.getConstant(MaxIndex, dl, IdxVT));
7937 }
7938 
7939 SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
7940                                                 SDValue VecPtr, EVT VecVT,
7941                                                 SDValue Index) const {
7942   return getVectorSubVecPointer(
7943       DAG, VecPtr, VecVT,
7944       EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1),
7945       Index);
7946 }
7947 
7948 SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG,
7949                                                SDValue VecPtr, EVT VecVT,
7950                                                EVT SubVecVT,
7951                                                SDValue Index) const {
7952   SDLoc dl(Index);
7953   // Make sure the index type is big enough to compute in.
7954   Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
7955 
7956   EVT EltVT = VecVT.getVectorElementType();
7957 
7958   // Calculate the element offset and add it to the pointer.
7959   unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
7960   assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
7961          "Converting bits to bytes lost precision");
7962   assert(SubVecVT.getVectorElementType() == EltVT &&
7963          "Sub-vector must be a vector with matching element type");
7964   Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
7965                                   SubVecVT.getVectorElementCount());
7966 
7967   EVT IdxVT = Index.getValueType();
7968   if (SubVecVT.isScalableVector())
7969     Index =
7970         DAG.getNode(ISD::MUL, dl, IdxVT, Index,
7971                     DAG.getVScale(dl, IdxVT, APInt(IdxVT.getSizeInBits(), 1)));
7972 
7973   Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
7974                       DAG.getConstant(EltSize, dl, IdxVT));
7975   return DAG.getMemBasePlusOffset(VecPtr, Index, dl);
7976 }
7977 
7978 //===----------------------------------------------------------------------===//
7979 // Implementation of Emulated TLS Model
7980 //===----------------------------------------------------------------------===//
7981 
7982 SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
7983                                                 SelectionDAG &DAG) const {
7984   // Access to address of TLS varialbe xyz is lowered to a function call:
7985   //   __emutls_get_address( address of global variable named "__emutls_v.xyz" )
7986   EVT PtrVT = getPointerTy(DAG.getDataLayout());
7987   PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
7988   SDLoc dl(GA);
7989 
7990   ArgListTy Args;
7991   ArgListEntry Entry;
7992   std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
7993   Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
7994   StringRef EmuTlsVarName(NameString);
7995   GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
7996   assert(EmuTlsVar && "Cannot find EmuTlsVar ");
7997   Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
7998   Entry.Ty = VoidPtrType;
7999   Args.push_back(Entry);
8000 
8001   SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
8002 
8003   TargetLowering::CallLoweringInfo CLI(DAG);
8004   CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
8005   CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
8006   std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
8007 
8008   // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
8009   // At last for X86 targets, maybe good for other targets too?
8010   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
8011   MFI.setAdjustsStack(true); // Is this only for X86 target?
8012   MFI.setHasCalls(true);
8013 
8014   assert((GA->getOffset() == 0) &&
8015          "Emulated TLS must have zero offset in GlobalAddressSDNode");
8016   return CallResult.first;
8017 }
8018 
8019 SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
8020                                                 SelectionDAG &DAG) const {
8021   assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
8022   if (!isCtlzFast())
8023     return SDValue();
8024   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
8025   SDLoc dl(Op);
8026   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
8027     if (C->isZero() && CC == ISD::SETEQ) {
8028       EVT VT = Op.getOperand(0).getValueType();
8029       SDValue Zext = Op.getOperand(0);
8030       if (VT.bitsLT(MVT::i32)) {
8031         VT = MVT::i32;
8032         Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
8033       }
8034       unsigned Log2b = Log2_32(VT.getSizeInBits());
8035       SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
8036       SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
8037                                 DAG.getConstant(Log2b, dl, MVT::i32));
8038       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
8039     }
8040   }
8041   return SDValue();
8042 }
8043 
8044 // Convert redundant addressing modes (e.g. scaling is redundant
8045 // when accessing bytes).
8046 ISD::MemIndexType
8047 TargetLowering::getCanonicalIndexType(ISD::MemIndexType IndexType, EVT MemVT,
8048                                       SDValue Offsets) const {
8049   bool IsScaledIndex =
8050       (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::UNSIGNED_SCALED);
8051   bool IsSignedIndex =
8052       (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::SIGNED_UNSCALED);
8053 
8054   // Scaling is unimportant for bytes, canonicalize to unscaled.
8055   if (IsScaledIndex && MemVT.getScalarType() == MVT::i8)
8056     return IsSignedIndex ? ISD::SIGNED_UNSCALED : ISD::UNSIGNED_UNSCALED;
8057 
8058   return IndexType;
8059 }
8060 
8061 SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
8062   SDValue Op0 = Node->getOperand(0);
8063   SDValue Op1 = Node->getOperand(1);
8064   EVT VT = Op0.getValueType();
8065   unsigned Opcode = Node->getOpcode();
8066   SDLoc DL(Node);
8067 
8068   // umin(x,y) -> sub(x,usubsat(x,y))
8069   if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
8070       isOperationLegal(ISD::USUBSAT, VT)) {
8071     return DAG.getNode(ISD::SUB, DL, VT, Op0,
8072                        DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
8073   }
8074 
8075   // umax(x,y) -> add(x,usubsat(y,x))
8076   if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
8077       isOperationLegal(ISD::USUBSAT, VT)) {
8078     return DAG.getNode(ISD::ADD, DL, VT, Op0,
8079                        DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
8080   }
8081 
8082   // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
8083   ISD::CondCode CC;
8084   switch (Opcode) {
8085   default: llvm_unreachable("How did we get here?");
8086   case ISD::SMAX: CC = ISD::SETGT; break;
8087   case ISD::SMIN: CC = ISD::SETLT; break;
8088   case ISD::UMAX: CC = ISD::SETUGT; break;
8089   case ISD::UMIN: CC = ISD::SETULT; break;
8090   }
8091 
8092   // FIXME: Should really try to split the vector in case it's legal on a
8093   // subvector.
8094   if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
8095     return DAG.UnrollVectorOp(Node);
8096 
8097   EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8098   SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
8099   return DAG.getSelect(DL, VT, Cond, Op0, Op1);
8100 }
8101 
8102 SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
8103   unsigned Opcode = Node->getOpcode();
8104   SDValue LHS = Node->getOperand(0);
8105   SDValue RHS = Node->getOperand(1);
8106   EVT VT = LHS.getValueType();
8107   SDLoc dl(Node);
8108 
8109   assert(VT == RHS.getValueType() && "Expected operands to be the same type");
8110   assert(VT.isInteger() && "Expected operands to be integers");
8111 
8112   // usub.sat(a, b) -> umax(a, b) - b
8113   if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
8114     SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
8115     return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
8116   }
8117 
8118   // uadd.sat(a, b) -> umin(a, ~b) + b
8119   if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
8120     SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
8121     SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
8122     return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
8123   }
8124 
8125   unsigned OverflowOp;
8126   switch (Opcode) {
8127   case ISD::SADDSAT:
8128     OverflowOp = ISD::SADDO;
8129     break;
8130   case ISD::UADDSAT:
8131     OverflowOp = ISD::UADDO;
8132     break;
8133   case ISD::SSUBSAT:
8134     OverflowOp = ISD::SSUBO;
8135     break;
8136   case ISD::USUBSAT:
8137     OverflowOp = ISD::USUBO;
8138     break;
8139   default:
8140     llvm_unreachable("Expected method to receive signed or unsigned saturation "
8141                      "addition or subtraction node.");
8142   }
8143 
8144   // FIXME: Should really try to split the vector in case it's legal on a
8145   // subvector.
8146   if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
8147     return DAG.UnrollVectorOp(Node);
8148 
8149   unsigned BitWidth = LHS.getScalarValueSizeInBits();
8150   EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8151   SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8152   SDValue SumDiff = Result.getValue(0);
8153   SDValue Overflow = Result.getValue(1);
8154   SDValue Zero = DAG.getConstant(0, dl, VT);
8155   SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
8156 
8157   if (Opcode == ISD::UADDSAT) {
8158     if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
8159       // (LHS + RHS) | OverflowMask
8160       SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
8161       return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
8162     }
8163     // Overflow ? 0xffff.... : (LHS + RHS)
8164     return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
8165   }
8166 
8167   if (Opcode == ISD::USUBSAT) {
8168     if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
8169       // (LHS - RHS) & ~OverflowMask
8170       SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
8171       SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
8172       return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
8173     }
8174     // Overflow ? 0 : (LHS - RHS)
8175     return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
8176   }
8177 
8178   // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
8179   APInt MinVal = APInt::getSignedMinValue(BitWidth);
8180   SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
8181   SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,
8182                               DAG.getConstant(BitWidth - 1, dl, VT));
8183   Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);
8184   return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
8185 }
8186 
8187 SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
8188   unsigned Opcode = Node->getOpcode();
8189   bool IsSigned = Opcode == ISD::SSHLSAT;
8190   SDValue LHS = Node->getOperand(0);
8191   SDValue RHS = Node->getOperand(1);
8192   EVT VT = LHS.getValueType();
8193   SDLoc dl(Node);
8194 
8195   assert((Node->getOpcode() == ISD::SSHLSAT ||
8196           Node->getOpcode() == ISD::USHLSAT) &&
8197           "Expected a SHLSAT opcode");
8198   assert(VT == RHS.getValueType() && "Expected operands to be the same type");
8199   assert(VT.isInteger() && "Expected operands to be integers");
8200 
8201   // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
8202 
8203   unsigned BW = VT.getScalarSizeInBits();
8204   SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
8205   SDValue Orig =
8206       DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);
8207 
8208   SDValue SatVal;
8209   if (IsSigned) {
8210     SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
8211     SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
8212     SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT),
8213                              SatMin, SatMax, ISD::SETLT);
8214   } else {
8215     SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
8216   }
8217   Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE);
8218 
8219   return Result;
8220 }
8221 
8222 SDValue
8223 TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
8224   assert((Node->getOpcode() == ISD::SMULFIX ||
8225           Node->getOpcode() == ISD::UMULFIX ||
8226           Node->getOpcode() == ISD::SMULFIXSAT ||
8227           Node->getOpcode() == ISD::UMULFIXSAT) &&
8228          "Expected a fixed point multiplication opcode");
8229 
8230   SDLoc dl(Node);
8231   SDValue LHS = Node->getOperand(0);
8232   SDValue RHS = Node->getOperand(1);
8233   EVT VT = LHS.getValueType();
8234   unsigned Scale = Node->getConstantOperandVal(2);
8235   bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
8236                      Node->getOpcode() == ISD::UMULFIXSAT);
8237   bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
8238                  Node->getOpcode() == ISD::SMULFIXSAT);
8239   EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8240   unsigned VTSize = VT.getScalarSizeInBits();
8241 
8242   if (!Scale) {
8243     // [us]mul.fix(a, b, 0) -> mul(a, b)
8244     if (!Saturating) {
8245       if (isOperationLegalOrCustom(ISD::MUL, VT))
8246         return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8247     } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
8248       SDValue Result =
8249           DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8250       SDValue Product = Result.getValue(0);
8251       SDValue Overflow = Result.getValue(1);
8252       SDValue Zero = DAG.getConstant(0, dl, VT);
8253 
8254       APInt MinVal = APInt::getSignedMinValue(VTSize);
8255       APInt MaxVal = APInt::getSignedMaxValue(VTSize);
8256       SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
8257       SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8258       // Xor the inputs, if resulting sign bit is 0 the product will be
8259       // positive, else negative.
8260       SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
8261       SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
8262       Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
8263       return DAG.getSelect(dl, VT, Overflow, Result, Product);
8264     } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
8265       SDValue Result =
8266           DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8267       SDValue Product = Result.getValue(0);
8268       SDValue Overflow = Result.getValue(1);
8269 
8270       APInt MaxVal = APInt::getMaxValue(VTSize);
8271       SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8272       return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
8273     }
8274   }
8275 
8276   assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
8277          "Expected scale to be less than the number of bits if signed or at "
8278          "most the number of bits if unsigned.");
8279   assert(LHS.getValueType() == RHS.getValueType() &&
8280          "Expected both operands to be the same type");
8281 
8282   // Get the upper and lower bits of the result.
8283   SDValue Lo, Hi;
8284   unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
8285   unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
8286   if (isOperationLegalOrCustom(LoHiOp, VT)) {
8287     SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
8288     Lo = Result.getValue(0);
8289     Hi = Result.getValue(1);
8290   } else if (isOperationLegalOrCustom(HiOp, VT)) {
8291     Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8292     Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
8293   } else if (VT.isVector()) {
8294     return SDValue();
8295   } else {
8296     report_fatal_error("Unable to expand fixed point multiplication.");
8297   }
8298 
8299   if (Scale == VTSize)
8300     // Result is just the top half since we'd be shifting by the width of the
8301     // operand. Overflow impossible so this works for both UMULFIX and
8302     // UMULFIXSAT.
8303     return Hi;
8304 
8305   // The result will need to be shifted right by the scale since both operands
8306   // are scaled. The result is given to us in 2 halves, so we only want part of
8307   // both in the result.
8308   EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
8309   SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
8310                                DAG.getConstant(Scale, dl, ShiftTy));
8311   if (!Saturating)
8312     return Result;
8313 
8314   if (!Signed) {
8315     // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
8316     // widened multiplication) aren't all zeroes.
8317 
8318     // Saturate to max if ((Hi >> Scale) != 0),
8319     // which is the same as if (Hi > ((1 << Scale) - 1))
8320     APInt MaxVal = APInt::getMaxValue(VTSize);
8321     SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
8322                                       dl, VT);
8323     Result = DAG.getSelectCC(dl, Hi, LowMask,
8324                              DAG.getConstant(MaxVal, dl, VT), Result,
8325                              ISD::SETUGT);
8326 
8327     return Result;
8328   }
8329 
8330   // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
8331   // widened multiplication) aren't all ones or all zeroes.
8332 
8333   SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
8334   SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
8335 
8336   if (Scale == 0) {
8337     SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
8338                                DAG.getConstant(VTSize - 1, dl, ShiftTy));
8339     SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
8340     // Saturated to SatMin if wide product is negative, and SatMax if wide
8341     // product is positive ...
8342     SDValue Zero = DAG.getConstant(0, dl, VT);
8343     SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
8344                                                ISD::SETLT);
8345     // ... but only if we overflowed.
8346     return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
8347   }
8348 
8349   //  We handled Scale==0 above so all the bits to examine is in Hi.
8350 
8351   // Saturate to max if ((Hi >> (Scale - 1)) > 0),
8352   // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
8353   SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
8354                                     dl, VT);
8355   Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
8356   // Saturate to min if (Hi >> (Scale - 1)) < -1),
8357   // which is the same as if (HI < (-1 << (Scale - 1))
8358   SDValue HighMask =
8359       DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
8360                       dl, VT);
8361   Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
8362   return Result;
8363 }
8364 
8365 SDValue
8366 TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
8367                                     SDValue LHS, SDValue RHS,
8368                                     unsigned Scale, SelectionDAG &DAG) const {
8369   assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
8370           Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
8371          "Expected a fixed point division opcode");
8372 
8373   EVT VT = LHS.getValueType();
8374   bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
8375   bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
8376   EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8377 
8378   // If there is enough room in the type to upscale the LHS or downscale the
8379   // RHS before the division, we can perform it in this type without having to
8380   // resize. For signed operations, the LHS headroom is the number of
8381   // redundant sign bits, and for unsigned ones it is the number of zeroes.
8382   // The headroom for the RHS is the number of trailing zeroes.
8383   unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
8384                             : DAG.computeKnownBits(LHS).countMinLeadingZeros();
8385   unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();
8386 
8387   // For signed saturating operations, we need to be able to detect true integer
8388   // division overflow; that is, when you have MIN / -EPS. However, this
8389   // is undefined behavior and if we emit divisions that could take such
8390   // values it may cause undesired behavior (arithmetic exceptions on x86, for
8391   // example).
8392   // Avoid this by requiring an extra bit so that we never get this case.
8393   // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
8394   // signed saturating division, we need to emit a whopping 32-bit division.
8395   if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
8396     return SDValue();
8397 
8398   unsigned LHSShift = std::min(LHSLead, Scale);
8399   unsigned RHSShift = Scale - LHSShift;
8400 
8401   // At this point, we know that if we shift the LHS up by LHSShift and the
8402   // RHS down by RHSShift, we can emit a regular division with a final scaling
8403   // factor of Scale.
8404 
8405   EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
8406   if (LHSShift)
8407     LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
8408                       DAG.getConstant(LHSShift, dl, ShiftTy));
8409   if (RHSShift)
8410     RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
8411                       DAG.getConstant(RHSShift, dl, ShiftTy));
8412 
8413   SDValue Quot;
8414   if (Signed) {
8415     // For signed operations, if the resulting quotient is negative and the
8416     // remainder is nonzero, subtract 1 from the quotient to round towards
8417     // negative infinity.
8418     SDValue Rem;
8419     // FIXME: Ideally we would always produce an SDIVREM here, but if the
8420     // type isn't legal, SDIVREM cannot be expanded. There is no reason why
8421     // we couldn't just form a libcall, but the type legalizer doesn't do it.
8422     if (isTypeLegal(VT) &&
8423         isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
8424       Quot = DAG.getNode(ISD::SDIVREM, dl,
8425                          DAG.getVTList(VT, VT),
8426                          LHS, RHS);
8427       Rem = Quot.getValue(1);
8428       Quot = Quot.getValue(0);
8429     } else {
8430       Quot = DAG.getNode(ISD::SDIV, dl, VT,
8431                          LHS, RHS);
8432       Rem = DAG.getNode(ISD::SREM, dl, VT,
8433                         LHS, RHS);
8434     }
8435     SDValue Zero = DAG.getConstant(0, dl, VT);
8436     SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
8437     SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
8438     SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
8439     SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
8440     SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
8441                                DAG.getConstant(1, dl, VT));
8442     Quot = DAG.getSelect(dl, VT,
8443                          DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
8444                          Sub1, Quot);
8445   } else
8446     Quot = DAG.getNode(ISD::UDIV, dl, VT,
8447                        LHS, RHS);
8448 
8449   return Quot;
8450 }
8451 
8452 void TargetLowering::expandUADDSUBO(
8453     SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
8454   SDLoc dl(Node);
8455   SDValue LHS = Node->getOperand(0);
8456   SDValue RHS = Node->getOperand(1);
8457   bool IsAdd = Node->getOpcode() == ISD::UADDO;
8458 
8459   // If ADD/SUBCARRY is legal, use that instead.
8460   unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
8461   if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
8462     SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
8463     SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
8464                                     { LHS, RHS, CarryIn });
8465     Result = SDValue(NodeCarry.getNode(), 0);
8466     Overflow = SDValue(NodeCarry.getNode(), 1);
8467     return;
8468   }
8469 
8470   Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
8471                             LHS.getValueType(), LHS, RHS);
8472 
8473   EVT ResultType = Node->getValueType(1);
8474   EVT SetCCType = getSetCCResultType(
8475       DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
8476   ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
8477   SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
8478   Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8479 }
8480 
8481 void TargetLowering::expandSADDSUBO(
8482     SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
8483   SDLoc dl(Node);
8484   SDValue LHS = Node->getOperand(0);
8485   SDValue RHS = Node->getOperand(1);
8486   bool IsAdd = Node->getOpcode() == ISD::SADDO;
8487 
8488   Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
8489                             LHS.getValueType(), LHS, RHS);
8490 
8491   EVT ResultType = Node->getValueType(1);
8492   EVT OType = getSetCCResultType(
8493       DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
8494 
8495   // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
8496   unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
8497   if (isOperationLegal(OpcSat, LHS.getValueType())) {
8498     SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
8499     SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
8500     Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8501     return;
8502   }
8503 
8504   SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
8505 
8506   // For an addition, the result should be less than one of the operands (LHS)
8507   // if and only if the other operand (RHS) is negative, otherwise there will
8508   // be overflow.
8509   // For a subtraction, the result should be less than one of the operands
8510   // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
8511   // otherwise there will be overflow.
8512   SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
8513   SDValue ConditionRHS =
8514       DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);
8515 
8516   Overflow = DAG.getBoolExtOrTrunc(
8517       DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
8518       ResultType, ResultType);
8519 }
8520 
8521 bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
8522                                 SDValue &Overflow, SelectionDAG &DAG) const {
8523   SDLoc dl(Node);
8524   EVT VT = Node->getValueType(0);
8525   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8526   SDValue LHS = Node->getOperand(0);
8527   SDValue RHS = Node->getOperand(1);
8528   bool isSigned = Node->getOpcode() == ISD::SMULO;
8529 
8530   // For power-of-two multiplications we can use a simpler shift expansion.
8531   if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
8532     const APInt &C = RHSC->getAPIntValue();
8533     // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
8534     if (C.isPowerOf2()) {
8535       // smulo(x, signed_min) is same as umulo(x, signed_min).
8536       bool UseArithShift = isSigned && !C.isMinSignedValue();
8537       EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout());
8538       SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);
8539       Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
8540       Overflow = DAG.getSetCC(dl, SetCCVT,
8541           DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
8542                       dl, VT, Result, ShiftAmt),
8543           LHS, ISD::SETNE);
8544       return true;
8545     }
8546   }
8547 
8548   EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
8549   if (VT.isVector())
8550     WideVT =
8551         EVT::getVectorVT(*DAG.getContext(), WideVT, VT.getVectorElementCount());
8552 
8553   SDValue BottomHalf;
8554   SDValue TopHalf;
8555   static const unsigned Ops[2][3] =
8556       { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
8557         { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
8558   if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
8559     BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8560     TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
8561   } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
8562     BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
8563                              RHS);
8564     TopHalf = BottomHalf.getValue(1);
8565   } else if (isTypeLegal(WideVT)) {
8566     LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
8567     RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
8568     SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
8569     BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
8570     SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,
8571         getShiftAmountTy(WideVT, DAG.getDataLayout()));
8572     TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
8573                           DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
8574   } else {
8575     if (VT.isVector())
8576       return false;
8577 
8578     // We can fall back to a libcall with an illegal type for the MUL if we
8579     // have a libcall big enough.
8580     // Also, we can fall back to a division in some cases, but that's a big
8581     // performance hit in the general case.
8582     RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
8583     if (WideVT == MVT::i16)
8584       LC = RTLIB::MUL_I16;
8585     else if (WideVT == MVT::i32)
8586       LC = RTLIB::MUL_I32;
8587     else if (WideVT == MVT::i64)
8588       LC = RTLIB::MUL_I64;
8589     else if (WideVT == MVT::i128)
8590       LC = RTLIB::MUL_I128;
8591     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
8592 
8593     SDValue HiLHS;
8594     SDValue HiRHS;
8595     if (isSigned) {
8596       // The high part is obtained by SRA'ing all but one of the bits of low
8597       // part.
8598       unsigned LoSize = VT.getFixedSizeInBits();
8599       HiLHS =
8600           DAG.getNode(ISD::SRA, dl, VT, LHS,
8601                       DAG.getConstant(LoSize - 1, dl,
8602                                       getPointerTy(DAG.getDataLayout())));
8603       HiRHS =
8604           DAG.getNode(ISD::SRA, dl, VT, RHS,
8605                       DAG.getConstant(LoSize - 1, dl,
8606                                       getPointerTy(DAG.getDataLayout())));
8607     } else {
8608         HiLHS = DAG.getConstant(0, dl, VT);
8609         HiRHS = DAG.getConstant(0, dl, VT);
8610     }
8611 
8612     // Here we're passing the 2 arguments explicitly as 4 arguments that are
8613     // pre-lowered to the correct types. This all depends upon WideVT not
8614     // being a legal type for the architecture and thus has to be split to
8615     // two arguments.
8616     SDValue Ret;
8617     TargetLowering::MakeLibCallOptions CallOptions;
8618     CallOptions.setSExt(isSigned);
8619     CallOptions.setIsPostTypeLegalization(true);
8620     if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
8621       // Halves of WideVT are packed into registers in different order
8622       // depending on platform endianness. This is usually handled by
8623       // the C calling convention, but we can't defer to it in
8624       // the legalizer.
8625       SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
8626       Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
8627     } else {
8628       SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
8629       Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
8630     }
8631     assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
8632            "Ret value is a collection of constituent nodes holding result.");
8633     if (DAG.getDataLayout().isLittleEndian()) {
8634       // Same as above.
8635       BottomHalf = Ret.getOperand(0);
8636       TopHalf = Ret.getOperand(1);
8637     } else {
8638       BottomHalf = Ret.getOperand(1);
8639       TopHalf = Ret.getOperand(0);
8640     }
8641   }
8642 
8643   Result = BottomHalf;
8644   if (isSigned) {
8645     SDValue ShiftAmt = DAG.getConstant(
8646         VT.getScalarSizeInBits() - 1, dl,
8647         getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
8648     SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
8649     Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
8650   } else {
8651     Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
8652                             DAG.getConstant(0, dl, VT), ISD::SETNE);
8653   }
8654 
8655   // Truncate the result if SetCC returns a larger type than needed.
8656   EVT RType = Node->getValueType(1);
8657   if (RType.bitsLT(Overflow.getValueType()))
8658     Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);
8659 
8660   assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
8661          "Unexpected result type for S/UMULO legalization");
8662   return true;
8663 }
8664 
8665 SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
8666   SDLoc dl(Node);
8667   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
8668   SDValue Op = Node->getOperand(0);
8669   EVT VT = Op.getValueType();
8670 
8671   if (VT.isScalableVector())
8672     report_fatal_error(
8673         "Expanding reductions for scalable vectors is undefined.");
8674 
8675   // Try to use a shuffle reduction for power of two vectors.
8676   if (VT.isPow2VectorType()) {
8677     while (VT.getVectorNumElements() > 1) {
8678       EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
8679       if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
8680         break;
8681 
8682       SDValue Lo, Hi;
8683       std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
8684       Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi);
8685       VT = HalfVT;
8686     }
8687   }
8688 
8689   EVT EltVT = VT.getVectorElementType();
8690   unsigned NumElts = VT.getVectorNumElements();
8691 
8692   SmallVector<SDValue, 8> Ops;
8693   DAG.ExtractVectorElements(Op, Ops, 0, NumElts);
8694 
8695   SDValue Res = Ops[0];
8696   for (unsigned i = 1; i < NumElts; i++)
8697     Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());
8698 
8699   // Result type may be wider than element type.
8700   if (EltVT != Node->getValueType(0))
8701     Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
8702   return Res;
8703 }
8704 
8705 SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
8706   SDLoc dl(Node);
8707   SDValue AccOp = Node->getOperand(0);
8708   SDValue VecOp = Node->getOperand(1);
8709   SDNodeFlags Flags = Node->getFlags();
8710 
8711   EVT VT = VecOp.getValueType();
8712   EVT EltVT = VT.getVectorElementType();
8713 
8714   if (VT.isScalableVector())
8715     report_fatal_error(
8716         "Expanding reductions for scalable vectors is undefined.");
8717 
8718   unsigned NumElts = VT.getVectorNumElements();
8719 
8720   SmallVector<SDValue, 8> Ops;
8721   DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);
8722 
8723   unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
8724 
8725   SDValue Res = AccOp;
8726   for (unsigned i = 0; i < NumElts; i++)
8727     Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
8728 
8729   return Res;
8730 }
8731 
8732 bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
8733                                SelectionDAG &DAG) const {
8734   EVT VT = Node->getValueType(0);
8735   SDLoc dl(Node);
8736   bool isSigned = Node->getOpcode() == ISD::SREM;
8737   unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
8738   unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
8739   SDValue Dividend = Node->getOperand(0);
8740   SDValue Divisor = Node->getOperand(1);
8741   if (isOperationLegalOrCustom(DivRemOpc, VT)) {
8742     SDVTList VTs = DAG.getVTList(VT, VT);
8743     Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
8744     return true;
8745   }
8746   if (isOperationLegalOrCustom(DivOpc, VT)) {
8747     // X % Y -> X-X/Y*Y
8748     SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
8749     SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
8750     Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
8751     return true;
8752   }
8753   return false;
8754 }
8755 
8756 SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
8757                                             SelectionDAG &DAG) const {
8758   bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
8759   SDLoc dl(SDValue(Node, 0));
8760   SDValue Src = Node->getOperand(0);
8761 
8762   // DstVT is the result type, while SatVT is the size to which we saturate
8763   EVT SrcVT = Src.getValueType();
8764   EVT DstVT = Node->getValueType(0);
8765 
8766   EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
8767   unsigned SatWidth = SatVT.getScalarSizeInBits();
8768   unsigned DstWidth = DstVT.getScalarSizeInBits();
8769   assert(SatWidth <= DstWidth &&
8770          "Expected saturation width smaller than result width");
8771 
8772   // Determine minimum and maximum integer values and their corresponding
8773   // floating-point values.
8774   APInt MinInt, MaxInt;
8775   if (IsSigned) {
8776     MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth);
8777     MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth);
8778   } else {
8779     MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth);
8780     MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth);
8781   }
8782 
8783   // We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as
8784   // libcall emission cannot handle this. Large result types will fail.
8785   if (SrcVT == MVT::f16) {
8786     Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
8787     SrcVT = Src.getValueType();
8788   }
8789 
8790   APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
8791   APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));
8792 
8793   APFloat::opStatus MinStatus =
8794       MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
8795   APFloat::opStatus MaxStatus =
8796       MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
8797   bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8798                              !(MaxStatus & APFloat::opStatus::opInexact);
8799 
8800   SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
8801   SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
8802 
8803   // If the integer bounds are exactly representable as floats and min/max are
8804   // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
8805   // of comparisons and selects.
8806   bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&
8807                      isOperationLegal(ISD::FMAXNUM, SrcVT);
8808   if (AreExactFloatBounds && MinMaxLegal) {
8809     SDValue Clamped = Src;
8810 
8811     // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8812     Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);
8813     // Clamp by MaxFloat from above. NaN cannot occur.
8814     Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);
8815     // Convert clamped value to integer.
8816     SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
8817                                   dl, DstVT, Clamped);
8818 
8819     // In the unsigned case we're done, because we mapped NaN to MinFloat,
8820     // which will cast to zero.
8821     if (!IsSigned)
8822       return FpToInt;
8823 
8824     // Otherwise, select 0 if Src is NaN.
8825     SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
8826     return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
8827                            ISD::CondCode::SETUO);
8828   }
8829 
8830   SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
8831   SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
8832 
8833   // Result of direct conversion. The assumption here is that the operation is
8834   // non-trapping and it's fine to apply it to an out-of-range value if we
8835   // select it away later.
8836   SDValue FpToInt =
8837       DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);
8838 
8839   SDValue Select = FpToInt;
8840 
8841   // If Src ULT MinFloat, select MinInt. In particular, this also selects
8842   // MinInt if Src is NaN.
8843   Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,
8844                            ISD::CondCode::SETULT);
8845   // If Src OGT MaxFloat, select MaxInt.
8846   Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
8847                            ISD::CondCode::SETOGT);
8848 
8849   // In the unsigned case we are done, because we mapped NaN to MinInt, which
8850   // is already zero.
8851   if (!IsSigned)
8852     return Select;
8853 
8854   // Otherwise, select 0 if Src is NaN.
8855   SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
8856   return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
8857 }
8858 
8859 SDValue TargetLowering::expandVectorSplice(SDNode *Node,
8860                                            SelectionDAG &DAG) const {
8861   assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!");
8862   assert(Node->getValueType(0).isScalableVector() &&
8863          "Fixed length vector types expected to use SHUFFLE_VECTOR!");
8864 
8865   EVT VT = Node->getValueType(0);
8866   SDValue V1 = Node->getOperand(0);
8867   SDValue V2 = Node->getOperand(1);
8868   int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();
8869   SDLoc DL(Node);
8870 
8871   // Expand through memory thusly:
8872   //  Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
8873   //  Store V1, Ptr
8874   //  Store V2, Ptr + sizeof(V1)
8875   //  If (Imm < 0)
8876   //    TrailingElts = -Imm
8877   //    Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))
8878   //  else
8879   //    Ptr = Ptr + (Imm * sizeof(VT.Elt))
8880   //  Res = Load Ptr
8881 
8882   Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8883 
8884   EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
8885                                VT.getVectorElementCount() * 2);
8886   SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
8887   EVT PtrVT = StackPtr.getValueType();
8888   auto &MF = DAG.getMachineFunction();
8889   auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
8890   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
8891 
8892   // Store the lo part of CONCAT_VECTORS(V1, V2)
8893   SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);
8894   // Store the hi part of CONCAT_VECTORS(V1, V2)
8895   SDValue OffsetToV2 = DAG.getVScale(
8896       DL, PtrVT,
8897       APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
8898   SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);
8899   SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);
8900 
8901   if (Imm >= 0) {
8902     // Load back the required element. getVectorElementPointer takes care of
8903     // clamping the index if it's out-of-bounds.
8904     StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));
8905     // Load the spliced result
8906     return DAG.getLoad(VT, DL, StoreV2, StackPtr,
8907                        MachinePointerInfo::getUnknownStack(MF));
8908   }
8909 
8910   uint64_t TrailingElts = -Imm;
8911 
8912   // NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.
8913   TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();
8914   SDValue TrailingBytes =
8915       DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);
8916 
8917   if (TrailingElts > VT.getVectorMinNumElements()) {
8918     SDValue VLBytes = DAG.getVScale(
8919         DL, PtrVT,
8920         APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
8921     TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);
8922   }
8923 
8924   // Calculate the start address of the spliced result.
8925   StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);
8926 
8927   // Load the spliced result
8928   return DAG.getLoad(VT, DL, StoreV2, StackPtr2,
8929                      MachinePointerInfo::getUnknownStack(MF));
8930 }
8931 
8932 bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
8933                                            SDValue &LHS, SDValue &RHS,
8934                                            SDValue &CC, bool &NeedInvert,
8935                                            const SDLoc &dl, SDValue &Chain,
8936                                            bool IsSignaling) const {
8937   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8938   MVT OpVT = LHS.getSimpleValueType();
8939   ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
8940   NeedInvert = false;
8941   switch (TLI.getCondCodeAction(CCCode, OpVT)) {
8942   default:
8943     llvm_unreachable("Unknown condition code action!");
8944   case TargetLowering::Legal:
8945     // Nothing to do.
8946     break;
8947   case TargetLowering::Expand: {
8948     ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
8949     if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8950       std::swap(LHS, RHS);
8951       CC = DAG.getCondCode(InvCC);
8952       return true;
8953     }
8954     // Swapping operands didn't work. Try inverting the condition.
8955     bool NeedSwap = false;
8956     InvCC = getSetCCInverse(CCCode, OpVT);
8957     if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8958       // If inverting the condition is not enough, try swapping operands
8959       // on top of it.
8960       InvCC = ISD::getSetCCSwappedOperands(InvCC);
8961       NeedSwap = true;
8962     }
8963     if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8964       CC = DAG.getCondCode(InvCC);
8965       NeedInvert = true;
8966       if (NeedSwap)
8967         std::swap(LHS, RHS);
8968       return true;
8969     }
8970 
8971     ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
8972     unsigned Opc = 0;
8973     switch (CCCode) {
8974     default:
8975       llvm_unreachable("Don't know how to expand this condition!");
8976     case ISD::SETUO:
8977       if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
8978         CC1 = ISD::SETUNE;
8979         CC2 = ISD::SETUNE;
8980         Opc = ISD::OR;
8981         break;
8982       }
8983       assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
8984              "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
8985       NeedInvert = true;
8986       LLVM_FALLTHROUGH;
8987     case ISD::SETO:
8988       assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
8989              "If SETO is expanded, SETOEQ must be legal!");
8990       CC1 = ISD::SETOEQ;
8991       CC2 = ISD::SETOEQ;
8992       Opc = ISD::AND;
8993       break;
8994     case ISD::SETONE:
8995     case ISD::SETUEQ:
8996       // If the SETUO or SETO CC isn't legal, we might be able to use
8997       // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
8998       // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
8999       // the operands.
9000       CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
9001       if (!TLI.isCondCodeLegal(CC2, OpVT) &&
9002           (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
9003            TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
9004         CC1 = ISD::SETOGT;
9005         CC2 = ISD::SETOLT;
9006         Opc = ISD::OR;
9007         NeedInvert = ((unsigned)CCCode & 0x8U);
9008         break;
9009       }
9010       LLVM_FALLTHROUGH;
9011     case ISD::SETOEQ:
9012     case ISD::SETOGT:
9013     case ISD::SETOGE:
9014     case ISD::SETOLT:
9015     case ISD::SETOLE:
9016     case ISD::SETUNE:
9017     case ISD::SETUGT:
9018     case ISD::SETUGE:
9019     case ISD::SETULT:
9020     case ISD::SETULE:
9021       // If we are floating point, assign and break, otherwise fall through.
9022       if (!OpVT.isInteger()) {
9023         // We can use the 4th bit to tell if we are the unordered
9024         // or ordered version of the opcode.
9025         CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
9026         Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
9027         CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
9028         break;
9029       }
9030       // Fallthrough if we are unsigned integer.
9031       LLVM_FALLTHROUGH;
9032     case ISD::SETLE:
9033     case ISD::SETGT:
9034     case ISD::SETGE:
9035     case ISD::SETLT:
9036     case ISD::SETNE:
9037     case ISD::SETEQ:
9038       // If all combinations of inverting the condition and swapping operands
9039       // didn't work then we have no means to expand the condition.
9040       llvm_unreachable("Don't know how to expand this condition!");
9041     }
9042 
9043     SDValue SetCC1, SetCC2;
9044     if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
9045       // If we aren't the ordered or unorder operation,
9046       // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
9047       SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
9048       SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
9049     } else {
9050       // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
9051       SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
9052       SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
9053     }
9054     if (Chain)
9055       Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
9056                           SetCC2.getValue(1));
9057     LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
9058     RHS = SDValue();
9059     CC = SDValue();
9060     return true;
9061   }
9062   }
9063   return false;
9064 }
9065