xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (revision 95eb4b873b6a8b527c5bd78d7191975dfca38998)
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/FloatingPointMode.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/Analysis/ConstantFolding.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/CodeGen/ISDOpcodes.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineJumpTableInfo.h"
26 #include "llvm/CodeGen/MachineMemOperand.h"
27 #include "llvm/CodeGen/MachineValueType.h"
28 #include "llvm/CodeGen/RuntimeLibcalls.h"
29 #include "llvm/CodeGen/SelectionDAG.h"
30 #include "llvm/CodeGen/SelectionDAGNodes.h"
31 #include "llvm/CodeGen/TargetFrameLowering.h"
32 #include "llvm/CodeGen/TargetLowering.h"
33 #include "llvm/CodeGen/TargetSubtargetInfo.h"
34 #include "llvm/CodeGen/ValueTypes.h"
35 #include "llvm/IR/CallingConv.h"
36 #include "llvm/IR/Constants.h"
37 #include "llvm/IR/DataLayout.h"
38 #include "llvm/IR/DerivedTypes.h"
39 #include "llvm/IR/Function.h"
40 #include "llvm/IR/Metadata.h"
41 #include "llvm/IR/Type.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/Compiler.h"
44 #include "llvm/Support/Debug.h"
45 #include "llvm/Support/ErrorHandling.h"
46 #include "llvm/Support/MathExtras.h"
47 #include "llvm/Support/raw_ostream.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Target/TargetOptions.h"
50 #include <cassert>
51 #include <cstdint>
52 #include <tuple>
53 #include <utility>
54 
55 using namespace llvm;
56 
57 #define DEBUG_TYPE "legalizedag"
58 
59 namespace {
60 
61 /// Keeps track of state when getting the sign of a floating-point value as an
62 /// integer.
63 struct FloatSignAsInt {
64   EVT FloatVT;
65   SDValue Chain;
66   SDValue FloatPtr;
67   SDValue IntPtr;
68   MachinePointerInfo IntPointerInfo;
69   MachinePointerInfo FloatPointerInfo;
70   SDValue IntValue;
71   APInt SignMask;
72   uint8_t SignBit;
73 };
74 
75 //===----------------------------------------------------------------------===//
76 /// This takes an arbitrary SelectionDAG as input and
77 /// hacks on it until the target machine can handle it.  This involves
78 /// eliminating value sizes the machine cannot handle (promoting small sizes to
79 /// large sizes or splitting up large values into small values) as well as
80 /// eliminating operations the machine cannot handle.
81 ///
82 /// This code also does a small amount of optimization and recognition of idioms
83 /// as part of its processing.  For example, if a target does not support a
84 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
85 /// will attempt merge setcc and brc instructions into brcc's.
86 class SelectionDAGLegalize {
87   const TargetMachine &TM;
88   const TargetLowering &TLI;
89   SelectionDAG &DAG;
90 
91   /// The set of nodes which have already been legalized. We hold a
92   /// reference to it in order to update as necessary on node deletion.
93   SmallPtrSetImpl<SDNode *> &LegalizedNodes;
94 
95   /// A set of all the nodes updated during legalization.
96   SmallSetVector<SDNode *, 16> *UpdatedNodes;
97 
98   EVT getSetCCResultType(EVT VT) const {
99     return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
100   }
101 
102   // Libcall insertion helpers.
103 
104 public:
105   SelectionDAGLegalize(SelectionDAG &DAG,
106                        SmallPtrSetImpl<SDNode *> &LegalizedNodes,
107                        SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
108       : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
109         LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
110 
111   /// Legalizes the given operation.
112   void LegalizeOp(SDNode *Node);
113 
114 private:
115   SDValue OptimizeFloatStore(StoreSDNode *ST);
116 
117   void LegalizeLoadOps(SDNode *Node);
118   void LegalizeStoreOps(SDNode *Node);
119 
120   /// Some targets cannot handle a variable
121   /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
122   /// is necessary to spill the vector being inserted into to memory, perform
123   /// the insert there, and then read the result back.
124   SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
125                                          const SDLoc &dl);
126   SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
127                                   const SDLoc &dl);
128 
129   /// Return a vector shuffle operation which
130   /// performs the same shuffe in terms of order or result bytes, but on a type
131   /// whose vector element type is narrower than the original shuffle type.
132   /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
133   SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
134                                      SDValue N1, SDValue N2,
135                                      ArrayRef<int> Mask) const;
136 
137   std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
138                         TargetLowering::ArgListTy &&Args, bool isSigned);
139   std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 
141   void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142   void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
143                        SmallVectorImpl<SDValue> &Results);
144   void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
145                        RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
146                        RTLIB::Libcall Call_F128,
147                        RTLIB::Libcall Call_PPCF128,
148                        SmallVectorImpl<SDValue> &Results);
149   SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150                            RTLIB::Libcall Call_I8,
151                            RTLIB::Libcall Call_I16,
152                            RTLIB::Libcall Call_I32,
153                            RTLIB::Libcall Call_I64,
154                            RTLIB::Libcall Call_I128);
155   void ExpandArgFPLibCall(SDNode *Node,
156                           RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
157                           RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
158                           RTLIB::Libcall Call_PPCF128,
159                           SmallVectorImpl<SDValue> &Results);
160   void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161   void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 
163   SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
164                            const SDLoc &dl);
165   SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
166                            const SDLoc &dl, SDValue ChainIn);
167   SDValue ExpandBUILD_VECTOR(SDNode *Node);
168   SDValue ExpandSPLAT_VECTOR(SDNode *Node);
169   SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
170   void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
171                                 SmallVectorImpl<SDValue> &Results);
172   void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
173                          SDValue Value) const;
174   SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
175                           SDValue NewIntValue) const;
176   SDValue ExpandFCOPYSIGN(SDNode *Node) const;
177   SDValue ExpandFABS(SDNode *Node) const;
178   SDValue ExpandFNEG(SDNode *Node) const;
179   SDValue expandLdexp(SDNode *Node) const;
180   SDValue expandFrexp(SDNode *Node) const;
181 
182   SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
183   void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
184                              SmallVectorImpl<SDValue> &Results);
185   void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
186                              SmallVectorImpl<SDValue> &Results);
187   SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
188 
189   SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
190 
191   SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
192   SDValue ExpandInsertToVectorThroughStack(SDValue Op);
193   SDValue ExpandVectorBuildThroughStack(SDNode* Node);
194 
195   SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
196   SDValue ExpandConstant(ConstantSDNode *CP);
197 
198   // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
199   bool ExpandNode(SDNode *Node);
200   void ConvertNodeToLibcall(SDNode *Node);
201   void PromoteNode(SDNode *Node);
202 
203 public:
204   // Node replacement helpers
205 
206   void ReplacedNode(SDNode *N) {
207     LegalizedNodes.erase(N);
208     if (UpdatedNodes)
209       UpdatedNodes->insert(N);
210   }
211 
212   void ReplaceNode(SDNode *Old, SDNode *New) {
213     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
214                dbgs() << "     with:      "; New->dump(&DAG));
215 
216     assert(Old->getNumValues() == New->getNumValues() &&
217            "Replacing one node with another that produces a different number "
218            "of values!");
219     DAG.ReplaceAllUsesWith(Old, New);
220     if (UpdatedNodes)
221       UpdatedNodes->insert(New);
222     ReplacedNode(Old);
223   }
224 
225   void ReplaceNode(SDValue Old, SDValue New) {
226     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
227                dbgs() << "     with:      "; New->dump(&DAG));
228 
229     DAG.ReplaceAllUsesWith(Old, New);
230     if (UpdatedNodes)
231       UpdatedNodes->insert(New.getNode());
232     ReplacedNode(Old.getNode());
233   }
234 
235   void ReplaceNode(SDNode *Old, const SDValue *New) {
236     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
237 
238     DAG.ReplaceAllUsesWith(Old, New);
239     for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
240       LLVM_DEBUG(dbgs() << (i == 0 ? "     with:      " : "      and:      ");
241                  New[i]->dump(&DAG));
242       if (UpdatedNodes)
243         UpdatedNodes->insert(New[i].getNode());
244     }
245     ReplacedNode(Old);
246   }
247 
248   void ReplaceNodeWithValue(SDValue Old, SDValue New) {
249     LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
250                dbgs() << "     with:      "; New->dump(&DAG));
251 
252     DAG.ReplaceAllUsesOfValueWith(Old, New);
253     if (UpdatedNodes)
254       UpdatedNodes->insert(New.getNode());
255     ReplacedNode(Old.getNode());
256   }
257 };
258 
259 } // end anonymous namespace
260 
261 /// Return a vector shuffle operation which
262 /// performs the same shuffle in terms of order or result bytes, but on a type
263 /// whose vector element type is narrower than the original shuffle type.
264 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
265 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
266     EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
267     ArrayRef<int> Mask) const {
268   unsigned NumMaskElts = VT.getVectorNumElements();
269   unsigned NumDestElts = NVT.getVectorNumElements();
270   unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
271 
272   assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
273 
274   if (NumEltsGrowth == 1)
275     return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
276 
277   SmallVector<int, 8> NewMask;
278   for (unsigned i = 0; i != NumMaskElts; ++i) {
279     int Idx = Mask[i];
280     for (unsigned j = 0; j != NumEltsGrowth; ++j) {
281       if (Idx < 0)
282         NewMask.push_back(-1);
283       else
284         NewMask.push_back(Idx * NumEltsGrowth + j);
285     }
286   }
287   assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
288   assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
289   return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
290 }
291 
292 /// Expands the ConstantFP node to an integer constant or
293 /// a load from the constant pool.
294 SDValue
295 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
296   bool Extend = false;
297   SDLoc dl(CFP);
298 
299   // If a FP immediate is precise when represented as a float and if the
300   // target can do an extending load from float to double, we put it into
301   // the constant pool as a float, even if it's is statically typed as a
302   // double.  This shrinks FP constants and canonicalizes them for targets where
303   // an FP extending load is the same cost as a normal load (such as on the x87
304   // fp stack or PPC FP unit).
305   EVT VT = CFP->getValueType(0);
306   ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
307   if (!UseCP) {
308     assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
309     return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
310                            (VT == MVT::f64) ? MVT::i64 : MVT::i32);
311   }
312 
313   APFloat APF = CFP->getValueAPF();
314   EVT OrigVT = VT;
315   EVT SVT = VT;
316 
317   // We don't want to shrink SNaNs. Converting the SNaN back to its real type
318   // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
319   if (!APF.isSignaling()) {
320     while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
321       SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
322       if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
323           // Only do this if the target has a native EXTLOAD instruction from
324           // smaller type.
325           TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
326           TLI.ShouldShrinkFPConstant(OrigVT)) {
327         Type *SType = SVT.getTypeForEVT(*DAG.getContext());
328         LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
329             Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
330         VT = SVT;
331         Extend = true;
332       }
333     }
334   }
335 
336   SDValue CPIdx =
337       DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
338   Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
339   if (Extend) {
340     SDValue Result = DAG.getExtLoad(
341         ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
342         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
343         Alignment);
344     return Result;
345   }
346   SDValue Result = DAG.getLoad(
347       OrigVT, dl, DAG.getEntryNode(), CPIdx,
348       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
349   return Result;
350 }
351 
352 /// Expands the Constant node to a load from the constant pool.
353 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
354   SDLoc dl(CP);
355   EVT VT = CP->getValueType(0);
356   SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
357                                       TLI.getPointerTy(DAG.getDataLayout()));
358   Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
359   SDValue Result = DAG.getLoad(
360       VT, dl, DAG.getEntryNode(), CPIdx,
361       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
362   return Result;
363 }
364 
365 /// Some target cannot handle a variable insertion index for the
366 /// INSERT_VECTOR_ELT instruction.  In this case, it
367 /// is necessary to spill the vector being inserted into to memory, perform
368 /// the insert there, and then read the result back.
369 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
370                                                              SDValue Val,
371                                                              SDValue Idx,
372                                                              const SDLoc &dl) {
373   SDValue Tmp1 = Vec;
374   SDValue Tmp2 = Val;
375   SDValue Tmp3 = Idx;
376 
377   // If the target doesn't support this, we have to spill the input vector
378   // to a temporary stack slot, update the element, then reload it.  This is
379   // badness.  We could also load the value into a vector register (either
380   // with a "move to register" or "extload into register" instruction, then
381   // permute it into place, if the idx is a constant and if the idx is
382   // supported by the target.
383   EVT VT    = Tmp1.getValueType();
384   EVT EltVT = VT.getVectorElementType();
385   SDValue StackPtr = DAG.CreateStackTemporary(VT);
386 
387   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
388 
389   // Store the vector.
390   SDValue Ch = DAG.getStore(
391       DAG.getEntryNode(), dl, Tmp1, StackPtr,
392       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
393 
394   SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
395 
396   // Store the scalar value.
397   Ch = DAG.getTruncStore(
398       Ch, dl, Tmp2, StackPtr2,
399       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
400   // Load the updated vector.
401   return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
402                                                DAG.getMachineFunction(), SPFI));
403 }
404 
405 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
406                                                       SDValue Idx,
407                                                       const SDLoc &dl) {
408   if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
409     // SCALAR_TO_VECTOR requires that the type of the value being inserted
410     // match the element type of the vector being created, except for
411     // integers in which case the inserted value can be over width.
412     EVT EltVT = Vec.getValueType().getVectorElementType();
413     if (Val.getValueType() == EltVT ||
414         (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
415       SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
416                                   Vec.getValueType(), Val);
417 
418       unsigned NumElts = Vec.getValueType().getVectorNumElements();
419       // We generate a shuffle of InVec and ScVec, so the shuffle mask
420       // should be 0,1,2,3,4,5... with the appropriate element replaced with
421       // elt 0 of the RHS.
422       SmallVector<int, 8> ShufOps;
423       for (unsigned i = 0; i != NumElts; ++i)
424         ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
425 
426       return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
427     }
428   }
429   return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
430 }
431 
432 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
433   if (!ISD::isNormalStore(ST))
434     return SDValue();
435 
436   LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
437   // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
438   // FIXME: move this to the DAG Combiner!  Note that we can't regress due
439   // to phase ordering between legalized code and the dag combiner.  This
440   // probably means that we need to integrate dag combiner and legalizer
441   // together.
442   // We generally can't do this one for long doubles.
443   SDValue Chain = ST->getChain();
444   SDValue Ptr = ST->getBasePtr();
445   SDValue Value = ST->getValue();
446   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
447   AAMDNodes AAInfo = ST->getAAInfo();
448   SDLoc dl(ST);
449 
450   // Don't optimise TargetConstantFP
451   if (Value.getOpcode() == ISD::TargetConstantFP)
452     return SDValue();
453 
454   if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
455     if (CFP->getValueType(0) == MVT::f32 &&
456         TLI.isTypeLegal(MVT::i32)) {
457       SDValue Con = DAG.getConstant(CFP->getValueAPF().
458                                       bitcastToAPInt().zextOrTrunc(32),
459                                     SDLoc(CFP), MVT::i32);
460       return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
461                           ST->getOriginalAlign(), MMOFlags, AAInfo);
462     }
463 
464     if (CFP->getValueType(0) == MVT::f64 &&
465         !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
466       // If this target supports 64-bit registers, do a single 64-bit store.
467       if (TLI.isTypeLegal(MVT::i64)) {
468         SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
469                                       zextOrTrunc(64), SDLoc(CFP), MVT::i64);
470         return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
471                             ST->getOriginalAlign(), MMOFlags, AAInfo);
472       }
473 
474       if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
475         // Otherwise, if the target supports 32-bit registers, use 2 32-bit
476         // stores.  If the target supports neither 32- nor 64-bits, this
477         // xform is certainly not worth it.
478         const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
479         SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
480         SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
481         if (DAG.getDataLayout().isBigEndian())
482           std::swap(Lo, Hi);
483 
484         Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
485                           ST->getOriginalAlign(), MMOFlags, AAInfo);
486         Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
487         Hi = DAG.getStore(Chain, dl, Hi, Ptr,
488                           ST->getPointerInfo().getWithOffset(4),
489                           ST->getOriginalAlign(), MMOFlags, AAInfo);
490 
491         return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
492       }
493     }
494   }
495   return SDValue();
496 }
497 
498 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
499   StoreSDNode *ST = cast<StoreSDNode>(Node);
500   SDValue Chain = ST->getChain();
501   SDValue Ptr = ST->getBasePtr();
502   SDLoc dl(Node);
503 
504   MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
505   AAMDNodes AAInfo = ST->getAAInfo();
506 
507   if (!ST->isTruncatingStore()) {
508     LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
509     if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
510       ReplaceNode(ST, OptStore);
511       return;
512     }
513 
514     SDValue Value = ST->getValue();
515     MVT VT = Value.getSimpleValueType();
516     switch (TLI.getOperationAction(ISD::STORE, VT)) {
517     default: llvm_unreachable("This action is not supported yet!");
518     case TargetLowering::Legal: {
519       // If this is an unaligned store and the target doesn't support it,
520       // expand it.
521       EVT MemVT = ST->getMemoryVT();
522       const DataLayout &DL = DAG.getDataLayout();
523       if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
524                                               *ST->getMemOperand())) {
525         LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
526         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
527         ReplaceNode(SDValue(ST, 0), Result);
528       } else
529         LLVM_DEBUG(dbgs() << "Legal store\n");
530       break;
531     }
532     case TargetLowering::Custom: {
533       LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
534       SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
535       if (Res && Res != SDValue(Node, 0))
536         ReplaceNode(SDValue(Node, 0), Res);
537       return;
538     }
539     case TargetLowering::Promote: {
540       MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
541       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
542              "Can only promote stores to same size type");
543       Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
544       SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
545                                     ST->getOriginalAlign(), MMOFlags, AAInfo);
546       ReplaceNode(SDValue(Node, 0), Result);
547       break;
548     }
549     }
550     return;
551   }
552 
553   LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
554   SDValue Value = ST->getValue();
555   EVT StVT = ST->getMemoryVT();
556   TypeSize StWidth = StVT.getSizeInBits();
557   TypeSize StSize = StVT.getStoreSizeInBits();
558   auto &DL = DAG.getDataLayout();
559 
560   if (StWidth != StSize) {
561     // Promote to a byte-sized store with upper bits zero if not
562     // storing an integral number of bytes.  For example, promote
563     // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
564     EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
565     Value = DAG.getZeroExtendInReg(Value, dl, StVT);
566     SDValue Result =
567         DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
568                           ST->getOriginalAlign(), MMOFlags, AAInfo);
569     ReplaceNode(SDValue(Node, 0), Result);
570   } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
571     // If not storing a power-of-2 number of bits, expand as two stores.
572     assert(!StVT.isVector() && "Unsupported truncstore!");
573     unsigned StWidthBits = StWidth.getFixedValue();
574     unsigned LogStWidth = Log2_32(StWidthBits);
575     assert(LogStWidth < 32);
576     unsigned RoundWidth = 1 << LogStWidth;
577     assert(RoundWidth < StWidthBits);
578     unsigned ExtraWidth = StWidthBits - RoundWidth;
579     assert(ExtraWidth < RoundWidth);
580     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
581            "Store size not an integral number of bytes!");
582     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
583     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
584     SDValue Lo, Hi;
585     unsigned IncrementSize;
586 
587     if (DL.isLittleEndian()) {
588       // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
589       // Store the bottom RoundWidth bits.
590       Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
591                              RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
592 
593       // Store the remaining ExtraWidth bits.
594       IncrementSize = RoundWidth / 8;
595       Ptr =
596           DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
597       Hi = DAG.getNode(
598           ISD::SRL, dl, Value.getValueType(), Value,
599           DAG.getConstant(RoundWidth, dl,
600                           TLI.getShiftAmountTy(Value.getValueType(), DL)));
601       Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
602                              ST->getPointerInfo().getWithOffset(IncrementSize),
603                              ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
604     } else {
605       // Big endian - avoid unaligned stores.
606       // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
607       // Store the top RoundWidth bits.
608       Hi = DAG.getNode(
609           ISD::SRL, dl, Value.getValueType(), Value,
610           DAG.getConstant(ExtraWidth, dl,
611                           TLI.getShiftAmountTy(Value.getValueType(), DL)));
612       Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
613                              ST->getOriginalAlign(), MMOFlags, AAInfo);
614 
615       // Store the remaining ExtraWidth bits.
616       IncrementSize = RoundWidth / 8;
617       Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
618                         DAG.getConstant(IncrementSize, dl,
619                                         Ptr.getValueType()));
620       Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
621                              ST->getPointerInfo().getWithOffset(IncrementSize),
622                              ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
623     }
624 
625     // The order of the stores doesn't matter.
626     SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
627     ReplaceNode(SDValue(Node, 0), Result);
628   } else {
629     switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
630     default: llvm_unreachable("This action is not supported yet!");
631     case TargetLowering::Legal: {
632       EVT MemVT = ST->getMemoryVT();
633       // If this is an unaligned store and the target doesn't support it,
634       // expand it.
635       if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
636                                               *ST->getMemOperand())) {
637         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
638         ReplaceNode(SDValue(ST, 0), Result);
639       }
640       break;
641     }
642     case TargetLowering::Custom: {
643       SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
644       if (Res && Res != SDValue(Node, 0))
645         ReplaceNode(SDValue(Node, 0), Res);
646       return;
647     }
648     case TargetLowering::Expand:
649       assert(!StVT.isVector() &&
650              "Vector Stores are handled in LegalizeVectorOps");
651 
652       SDValue Result;
653 
654       // TRUNCSTORE:i16 i32 -> STORE i16
655       if (TLI.isTypeLegal(StVT)) {
656         Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
657         Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
658                               ST->getOriginalAlign(), MMOFlags, AAInfo);
659       } else {
660         // The in-memory type isn't legal. Truncate to the type it would promote
661         // to, and then do a truncstore.
662         Value = DAG.getNode(ISD::TRUNCATE, dl,
663                             TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
664                             Value);
665         Result =
666             DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
667                               ST->getOriginalAlign(), MMOFlags, AAInfo);
668       }
669 
670       ReplaceNode(SDValue(Node, 0), Result);
671       break;
672     }
673   }
674 }
675 
676 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
677   LoadSDNode *LD = cast<LoadSDNode>(Node);
678   SDValue Chain = LD->getChain();  // The chain.
679   SDValue Ptr = LD->getBasePtr();  // The base pointer.
680   SDValue Value;                   // The value returned by the load op.
681   SDLoc dl(Node);
682 
683   ISD::LoadExtType ExtType = LD->getExtensionType();
684   if (ExtType == ISD::NON_EXTLOAD) {
685     LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
686     MVT VT = Node->getSimpleValueType(0);
687     SDValue RVal = SDValue(Node, 0);
688     SDValue RChain = SDValue(Node, 1);
689 
690     switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
691     default: llvm_unreachable("This action is not supported yet!");
692     case TargetLowering::Legal: {
693       EVT MemVT = LD->getMemoryVT();
694       const DataLayout &DL = DAG.getDataLayout();
695       // If this is an unaligned load and the target doesn't support it,
696       // expand it.
697       if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
698                                               *LD->getMemOperand())) {
699         std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
700       }
701       break;
702     }
703     case TargetLowering::Custom:
704       if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
705         RVal = Res;
706         RChain = Res.getValue(1);
707       }
708       break;
709 
710     case TargetLowering::Promote: {
711       MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
712       assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
713              "Can only promote loads to same size type");
714 
715       SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
716       RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
717       RChain = Res.getValue(1);
718       break;
719     }
720     }
721     if (RChain.getNode() != Node) {
722       assert(RVal.getNode() != Node && "Load must be completely replaced");
723       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
724       DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
725       if (UpdatedNodes) {
726         UpdatedNodes->insert(RVal.getNode());
727         UpdatedNodes->insert(RChain.getNode());
728       }
729       ReplacedNode(Node);
730     }
731     return;
732   }
733 
734   LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
735   EVT SrcVT = LD->getMemoryVT();
736   TypeSize SrcWidth = SrcVT.getSizeInBits();
737   MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
738   AAMDNodes AAInfo = LD->getAAInfo();
739 
740   if (SrcWidth != SrcVT.getStoreSizeInBits() &&
741       // Some targets pretend to have an i1 loading operation, and actually
742       // load an i8.  This trick is correct for ZEXTLOAD because the top 7
743       // bits are guaranteed to be zero; it helps the optimizers understand
744       // that these bits are zero.  It is also useful for EXTLOAD, since it
745       // tells the optimizers that those bits are undefined.  It would be
746       // nice to have an effective generic way of getting these benefits...
747       // Until such a way is found, don't insist on promoting i1 here.
748       (SrcVT != MVT::i1 ||
749        TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
750          TargetLowering::Promote)) {
751     // Promote to a byte-sized load if not loading an integral number of
752     // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
753     unsigned NewWidth = SrcVT.getStoreSizeInBits();
754     EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
755     SDValue Ch;
756 
757     // The extra bits are guaranteed to be zero, since we stored them that
758     // way.  A zext load from NVT thus automatically gives zext from SrcVT.
759 
760     ISD::LoadExtType NewExtType =
761       ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
762 
763     SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
764                                     Chain, Ptr, LD->getPointerInfo(), NVT,
765                                     LD->getOriginalAlign(), MMOFlags, AAInfo);
766 
767     Ch = Result.getValue(1); // The chain.
768 
769     if (ExtType == ISD::SEXTLOAD)
770       // Having the top bits zero doesn't help when sign extending.
771       Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
772                            Result.getValueType(),
773                            Result, DAG.getValueType(SrcVT));
774     else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
775       // All the top bits are guaranteed to be zero - inform the optimizers.
776       Result = DAG.getNode(ISD::AssertZext, dl,
777                            Result.getValueType(), Result,
778                            DAG.getValueType(SrcVT));
779 
780     Value = Result;
781     Chain = Ch;
782   } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
783     // If not loading a power-of-2 number of bits, expand as two loads.
784     assert(!SrcVT.isVector() && "Unsupported extload!");
785     unsigned SrcWidthBits = SrcWidth.getFixedValue();
786     unsigned LogSrcWidth = Log2_32(SrcWidthBits);
787     assert(LogSrcWidth < 32);
788     unsigned RoundWidth = 1 << LogSrcWidth;
789     assert(RoundWidth < SrcWidthBits);
790     unsigned ExtraWidth = SrcWidthBits - RoundWidth;
791     assert(ExtraWidth < RoundWidth);
792     assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
793            "Load size not an integral number of bytes!");
794     EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
795     EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
796     SDValue Lo, Hi, Ch;
797     unsigned IncrementSize;
798     auto &DL = DAG.getDataLayout();
799 
800     if (DL.isLittleEndian()) {
801       // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
802       // Load the bottom RoundWidth bits.
803       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
804                           LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
805                           MMOFlags, AAInfo);
806 
807       // Load the remaining ExtraWidth bits.
808       IncrementSize = RoundWidth / 8;
809       Ptr =
810           DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
811       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
812                           LD->getPointerInfo().getWithOffset(IncrementSize),
813                           ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
814 
815       // Build a factor node to remember that this load is independent of
816       // the other one.
817       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
818                        Hi.getValue(1));
819 
820       // Move the top bits to the right place.
821       Hi = DAG.getNode(
822           ISD::SHL, dl, Hi.getValueType(), Hi,
823           DAG.getConstant(RoundWidth, dl,
824                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
825 
826       // Join the hi and lo parts.
827       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
828     } else {
829       // Big endian - avoid unaligned loads.
830       // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
831       // Load the top RoundWidth bits.
832       Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
833                           LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
834                           MMOFlags, AAInfo);
835 
836       // Load the remaining ExtraWidth bits.
837       IncrementSize = RoundWidth / 8;
838       Ptr =
839           DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
840       Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
841                           LD->getPointerInfo().getWithOffset(IncrementSize),
842                           ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
843 
844       // Build a factor node to remember that this load is independent of
845       // the other one.
846       Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
847                        Hi.getValue(1));
848 
849       // Move the top bits to the right place.
850       Hi = DAG.getNode(
851           ISD::SHL, dl, Hi.getValueType(), Hi,
852           DAG.getConstant(ExtraWidth, dl,
853                           TLI.getShiftAmountTy(Hi.getValueType(), DL)));
854 
855       // Join the hi and lo parts.
856       Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
857     }
858 
859     Chain = Ch;
860   } else {
861     bool isCustom = false;
862     switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
863                                  SrcVT.getSimpleVT())) {
864     default: llvm_unreachable("This action is not supported yet!");
865     case TargetLowering::Custom:
866       isCustom = true;
867       [[fallthrough]];
868     case TargetLowering::Legal:
869       Value = SDValue(Node, 0);
870       Chain = SDValue(Node, 1);
871 
872       if (isCustom) {
873         if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
874           Value = Res;
875           Chain = Res.getValue(1);
876         }
877       } else {
878         // If this is an unaligned load and the target doesn't support it,
879         // expand it.
880         EVT MemVT = LD->getMemoryVT();
881         const DataLayout &DL = DAG.getDataLayout();
882         if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
883                                     *LD->getMemOperand())) {
884           std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
885         }
886       }
887       break;
888 
889     case TargetLowering::Expand: {
890       EVT DestVT = Node->getValueType(0);
891       if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
892         // If the source type is not legal, see if there is a legal extload to
893         // an intermediate type that we can then extend further.
894         EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
895         if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
896             (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
897              TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
898           // If we are loading a legal type, this is a non-extload followed by a
899           // full extend.
900           ISD::LoadExtType MidExtType =
901               (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
902 
903           SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
904                                         SrcVT, LD->getMemOperand());
905           unsigned ExtendOp =
906               ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
907           Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
908           Chain = Load.getValue(1);
909           break;
910         }
911 
912         // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
913         // normal undefined upper bits behavior to allow using an in-reg extend
914         // with the illegal FP type, so load as an integer and do the
915         // from-integer conversion.
916         EVT SVT = SrcVT.getScalarType();
917         if (SVT == MVT::f16 || SVT == MVT::bf16) {
918           EVT ISrcVT = SrcVT.changeTypeToInteger();
919           EVT IDestVT = DestVT.changeTypeToInteger();
920           EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
921 
922           SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
923                                           Ptr, ISrcVT, LD->getMemOperand());
924           Value =
925               DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
926                           dl, DestVT, Result);
927           Chain = Result.getValue(1);
928           break;
929         }
930       }
931 
932       assert(!SrcVT.isVector() &&
933              "Vector Loads are handled in LegalizeVectorOps");
934 
935       // FIXME: This does not work for vectors on most targets.  Sign-
936       // and zero-extend operations are currently folded into extending
937       // loads, whether they are legal or not, and then we end up here
938       // without any support for legalizing them.
939       assert(ExtType != ISD::EXTLOAD &&
940              "EXTLOAD should always be supported!");
941       // Turn the unsupported load into an EXTLOAD followed by an
942       // explicit zero/sign extend inreg.
943       SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
944                                       Node->getValueType(0),
945                                       Chain, Ptr, SrcVT,
946                                       LD->getMemOperand());
947       SDValue ValRes;
948       if (ExtType == ISD::SEXTLOAD)
949         ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
950                              Result.getValueType(),
951                              Result, DAG.getValueType(SrcVT));
952       else
953         ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
954       Value = ValRes;
955       Chain = Result.getValue(1);
956       break;
957     }
958     }
959   }
960 
961   // Since loads produce two values, make sure to remember that we legalized
962   // both of them.
963   if (Chain.getNode() != Node) {
964     assert(Value.getNode() != Node && "Load must be completely replaced");
965     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
966     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
967     if (UpdatedNodes) {
968       UpdatedNodes->insert(Value.getNode());
969       UpdatedNodes->insert(Chain.getNode());
970     }
971     ReplacedNode(Node);
972   }
973 }
974 
975 /// Return a legal replacement for the given operation, with all legal operands.
976 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
977   LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
978 
979   // Allow illegal target nodes and illegal registers.
980   if (Node->getOpcode() == ISD::TargetConstant ||
981       Node->getOpcode() == ISD::Register)
982     return;
983 
984 #ifndef NDEBUG
985   for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
986     assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
987              TargetLowering::TypeLegal &&
988            "Unexpected illegal type!");
989 
990   for (const SDValue &Op : Node->op_values())
991     assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
992               TargetLowering::TypeLegal ||
993             Op.getOpcode() == ISD::TargetConstant ||
994             Op.getOpcode() == ISD::Register) &&
995             "Unexpected illegal type!");
996 #endif
997 
998   // Figure out the correct action; the way to query this varies by opcode
999   TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1000   bool SimpleFinishLegalizing = true;
1001   switch (Node->getOpcode()) {
1002   case ISD::INTRINSIC_W_CHAIN:
1003   case ISD::INTRINSIC_WO_CHAIN:
1004   case ISD::INTRINSIC_VOID:
1005   case ISD::STACKSAVE:
1006     Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1007     break;
1008   case ISD::GET_DYNAMIC_AREA_OFFSET:
1009     Action = TLI.getOperationAction(Node->getOpcode(),
1010                                     Node->getValueType(0));
1011     break;
1012   case ISD::VAARG:
1013     Action = TLI.getOperationAction(Node->getOpcode(),
1014                                     Node->getValueType(0));
1015     if (Action != TargetLowering::Promote)
1016       Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1017     break;
1018   case ISD::SET_FPENV:
1019   case ISD::SET_FPMODE:
1020     Action = TLI.getOperationAction(Node->getOpcode(),
1021                                     Node->getOperand(1).getValueType());
1022     break;
1023   case ISD::FP_TO_FP16:
1024   case ISD::FP_TO_BF16:
1025   case ISD::SINT_TO_FP:
1026   case ISD::UINT_TO_FP:
1027   case ISD::EXTRACT_VECTOR_ELT:
1028   case ISD::LROUND:
1029   case ISD::LLROUND:
1030   case ISD::LRINT:
1031   case ISD::LLRINT:
1032     Action = TLI.getOperationAction(Node->getOpcode(),
1033                                     Node->getOperand(0).getValueType());
1034     break;
1035   case ISD::STRICT_FP_TO_FP16:
1036   case ISD::STRICT_SINT_TO_FP:
1037   case ISD::STRICT_UINT_TO_FP:
1038   case ISD::STRICT_LRINT:
1039   case ISD::STRICT_LLRINT:
1040   case ISD::STRICT_LROUND:
1041   case ISD::STRICT_LLROUND:
1042     // These pseudo-ops are the same as the other STRICT_ ops except
1043     // they are registered with setOperationAction() using the input type
1044     // instead of the output type.
1045     Action = TLI.getOperationAction(Node->getOpcode(),
1046                                     Node->getOperand(1).getValueType());
1047     break;
1048   case ISD::SIGN_EXTEND_INREG: {
1049     EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1050     Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1051     break;
1052   }
1053   case ISD::ATOMIC_STORE:
1054     Action = TLI.getOperationAction(Node->getOpcode(),
1055                                     Node->getOperand(1).getValueType());
1056     break;
1057   case ISD::SELECT_CC:
1058   case ISD::STRICT_FSETCC:
1059   case ISD::STRICT_FSETCCS:
1060   case ISD::SETCC:
1061   case ISD::SETCCCARRY:
1062   case ISD::VP_SETCC:
1063   case ISD::BR_CC: {
1064     unsigned Opc = Node->getOpcode();
1065     unsigned CCOperand = Opc == ISD::SELECT_CC                         ? 4
1066                          : Opc == ISD::STRICT_FSETCC                   ? 3
1067                          : Opc == ISD::STRICT_FSETCCS                  ? 3
1068                          : Opc == ISD::SETCCCARRY                      ? 3
1069                          : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1070                                                                        : 1;
1071     unsigned CompareOperand = Opc == ISD::BR_CC            ? 2
1072                               : Opc == ISD::STRICT_FSETCC  ? 1
1073                               : Opc == ISD::STRICT_FSETCCS ? 1
1074                                                            : 0;
1075     MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1076     ISD::CondCode CCCode =
1077         cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1078     Action = TLI.getCondCodeAction(CCCode, OpVT);
1079     if (Action == TargetLowering::Legal) {
1080       if (Node->getOpcode() == ISD::SELECT_CC)
1081         Action = TLI.getOperationAction(Node->getOpcode(),
1082                                         Node->getValueType(0));
1083       else
1084         Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1085     }
1086     break;
1087   }
1088   case ISD::LOAD:
1089   case ISD::STORE:
1090     // FIXME: Model these properly.  LOAD and STORE are complicated, and
1091     // STORE expects the unlegalized operand in some cases.
1092     SimpleFinishLegalizing = false;
1093     break;
1094   case ISD::CALLSEQ_START:
1095   case ISD::CALLSEQ_END:
1096     // FIXME: This shouldn't be necessary.  These nodes have special properties
1097     // dealing with the recursive nature of legalization.  Removing this
1098     // special case should be done as part of making LegalizeDAG non-recursive.
1099     SimpleFinishLegalizing = false;
1100     break;
1101   case ISD::EXTRACT_ELEMENT:
1102   case ISD::GET_ROUNDING:
1103   case ISD::MERGE_VALUES:
1104   case ISD::EH_RETURN:
1105   case ISD::FRAME_TO_ARGS_OFFSET:
1106   case ISD::EH_DWARF_CFA:
1107   case ISD::EH_SJLJ_SETJMP:
1108   case ISD::EH_SJLJ_LONGJMP:
1109   case ISD::EH_SJLJ_SETUP_DISPATCH:
1110     // These operations lie about being legal: when they claim to be legal,
1111     // they should actually be expanded.
1112     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1113     if (Action == TargetLowering::Legal)
1114       Action = TargetLowering::Expand;
1115     break;
1116   case ISD::INIT_TRAMPOLINE:
1117   case ISD::ADJUST_TRAMPOLINE:
1118   case ISD::FRAMEADDR:
1119   case ISD::RETURNADDR:
1120   case ISD::ADDROFRETURNADDR:
1121   case ISD::SPONENTRY:
1122     // These operations lie about being legal: when they claim to be legal,
1123     // they should actually be custom-lowered.
1124     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1125     if (Action == TargetLowering::Legal)
1126       Action = TargetLowering::Custom;
1127     break;
1128   case ISD::READCYCLECOUNTER:
1129     // READCYCLECOUNTER returns an i64, even if type legalization might have
1130     // expanded that to several smaller types.
1131     Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1132     break;
1133   case ISD::READ_REGISTER:
1134   case ISD::WRITE_REGISTER:
1135     // Named register is legal in the DAG, but blocked by register name
1136     // selection if not implemented by target (to chose the correct register)
1137     // They'll be converted to Copy(To/From)Reg.
1138     Action = TargetLowering::Legal;
1139     break;
1140   case ISD::UBSANTRAP:
1141     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1142     if (Action == TargetLowering::Expand) {
1143       // replace ISD::UBSANTRAP with ISD::TRAP
1144       SDValue NewVal;
1145       NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1146                            Node->getOperand(0));
1147       ReplaceNode(Node, NewVal.getNode());
1148       LegalizeOp(NewVal.getNode());
1149       return;
1150     }
1151     break;
1152   case ISD::DEBUGTRAP:
1153     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1154     if (Action == TargetLowering::Expand) {
1155       // replace ISD::DEBUGTRAP with ISD::TRAP
1156       SDValue NewVal;
1157       NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1158                            Node->getOperand(0));
1159       ReplaceNode(Node, NewVal.getNode());
1160       LegalizeOp(NewVal.getNode());
1161       return;
1162     }
1163     break;
1164   case ISD::SADDSAT:
1165   case ISD::UADDSAT:
1166   case ISD::SSUBSAT:
1167   case ISD::USUBSAT:
1168   case ISD::SSHLSAT:
1169   case ISD::USHLSAT:
1170   case ISD::FP_TO_SINT_SAT:
1171   case ISD::FP_TO_UINT_SAT:
1172     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1173     break;
1174   case ISD::SMULFIX:
1175   case ISD::SMULFIXSAT:
1176   case ISD::UMULFIX:
1177   case ISD::UMULFIXSAT:
1178   case ISD::SDIVFIX:
1179   case ISD::SDIVFIXSAT:
1180   case ISD::UDIVFIX:
1181   case ISD::UDIVFIXSAT: {
1182     unsigned Scale = Node->getConstantOperandVal(2);
1183     Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1184                                               Node->getValueType(0), Scale);
1185     break;
1186   }
1187   case ISD::MSCATTER:
1188     Action = TLI.getOperationAction(Node->getOpcode(),
1189                     cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1190     break;
1191   case ISD::MSTORE:
1192     Action = TLI.getOperationAction(Node->getOpcode(),
1193                     cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1194     break;
1195   case ISD::VP_SCATTER:
1196     Action = TLI.getOperationAction(
1197         Node->getOpcode(),
1198         cast<VPScatterSDNode>(Node)->getValue().getValueType());
1199     break;
1200   case ISD::VP_STORE:
1201     Action = TLI.getOperationAction(
1202         Node->getOpcode(),
1203         cast<VPStoreSDNode>(Node)->getValue().getValueType());
1204     break;
1205   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1206     Action = TLI.getOperationAction(
1207         Node->getOpcode(),
1208         cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1209     break;
1210   case ISD::VECREDUCE_FADD:
1211   case ISD::VECREDUCE_FMUL:
1212   case ISD::VECREDUCE_ADD:
1213   case ISD::VECREDUCE_MUL:
1214   case ISD::VECREDUCE_AND:
1215   case ISD::VECREDUCE_OR:
1216   case ISD::VECREDUCE_XOR:
1217   case ISD::VECREDUCE_SMAX:
1218   case ISD::VECREDUCE_SMIN:
1219   case ISD::VECREDUCE_UMAX:
1220   case ISD::VECREDUCE_UMIN:
1221   case ISD::VECREDUCE_FMAX:
1222   case ISD::VECREDUCE_FMIN:
1223   case ISD::VECREDUCE_FMAXIMUM:
1224   case ISD::VECREDUCE_FMINIMUM:
1225   case ISD::IS_FPCLASS:
1226     Action = TLI.getOperationAction(
1227         Node->getOpcode(), Node->getOperand(0).getValueType());
1228     break;
1229   case ISD::VECREDUCE_SEQ_FADD:
1230   case ISD::VECREDUCE_SEQ_FMUL:
1231   case ISD::VP_REDUCE_FADD:
1232   case ISD::VP_REDUCE_FMUL:
1233   case ISD::VP_REDUCE_ADD:
1234   case ISD::VP_REDUCE_MUL:
1235   case ISD::VP_REDUCE_AND:
1236   case ISD::VP_REDUCE_OR:
1237   case ISD::VP_REDUCE_XOR:
1238   case ISD::VP_REDUCE_SMAX:
1239   case ISD::VP_REDUCE_SMIN:
1240   case ISD::VP_REDUCE_UMAX:
1241   case ISD::VP_REDUCE_UMIN:
1242   case ISD::VP_REDUCE_FMAX:
1243   case ISD::VP_REDUCE_FMIN:
1244   case ISD::VP_REDUCE_SEQ_FADD:
1245   case ISD::VP_REDUCE_SEQ_FMUL:
1246     Action = TLI.getOperationAction(
1247         Node->getOpcode(), Node->getOperand(1).getValueType());
1248     break;
1249   default:
1250     if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1251       Action = TLI.getCustomOperationAction(*Node);
1252     } else {
1253       Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1254     }
1255     break;
1256   }
1257 
1258   if (SimpleFinishLegalizing) {
1259     SDNode *NewNode = Node;
1260     switch (Node->getOpcode()) {
1261     default: break;
1262     case ISD::SHL:
1263     case ISD::SRL:
1264     case ISD::SRA:
1265     case ISD::ROTL:
1266     case ISD::ROTR: {
1267       // Legalizing shifts/rotates requires adjusting the shift amount
1268       // to the appropriate width.
1269       SDValue Op0 = Node->getOperand(0);
1270       SDValue Op1 = Node->getOperand(1);
1271       if (!Op1.getValueType().isVector()) {
1272         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1273         // The getShiftAmountOperand() may create a new operand node or
1274         // return the existing one. If new operand is created we need
1275         // to update the parent node.
1276         // Do not try to legalize SAO here! It will be automatically legalized
1277         // in the next round.
1278         if (SAO != Op1)
1279           NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1280       }
1281     }
1282     break;
1283     case ISD::FSHL:
1284     case ISD::FSHR:
1285     case ISD::SRL_PARTS:
1286     case ISD::SRA_PARTS:
1287     case ISD::SHL_PARTS: {
1288       // Legalizing shifts/rotates requires adjusting the shift amount
1289       // to the appropriate width.
1290       SDValue Op0 = Node->getOperand(0);
1291       SDValue Op1 = Node->getOperand(1);
1292       SDValue Op2 = Node->getOperand(2);
1293       if (!Op2.getValueType().isVector()) {
1294         SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1295         // The getShiftAmountOperand() may create a new operand node or
1296         // return the existing one. If new operand is created we need
1297         // to update the parent node.
1298         if (SAO != Op2)
1299           NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1300       }
1301       break;
1302     }
1303     }
1304 
1305     if (NewNode != Node) {
1306       ReplaceNode(Node, NewNode);
1307       Node = NewNode;
1308     }
1309     switch (Action) {
1310     case TargetLowering::Legal:
1311       LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1312       return;
1313     case TargetLowering::Custom:
1314       LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1315       // FIXME: The handling for custom lowering with multiple results is
1316       // a complete mess.
1317       if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1318         if (!(Res.getNode() != Node || Res.getResNo() != 0))
1319           return;
1320 
1321         if (Node->getNumValues() == 1) {
1322           // Verify the new types match the original. Glue is waived because
1323           // ISD::ADDC can be legalized by replacing Glue with an integer type.
1324           assert((Res.getValueType() == Node->getValueType(0) ||
1325                   Node->getValueType(0) == MVT::Glue) &&
1326                  "Type mismatch for custom legalized operation");
1327           LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1328           // We can just directly replace this node with the lowered value.
1329           ReplaceNode(SDValue(Node, 0), Res);
1330           return;
1331         }
1332 
1333         SmallVector<SDValue, 8> ResultVals;
1334         for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1335           // Verify the new types match the original. Glue is waived because
1336           // ISD::ADDC can be legalized by replacing Glue with an integer type.
1337           assert((Res->getValueType(i) == Node->getValueType(i) ||
1338                   Node->getValueType(i) == MVT::Glue) &&
1339                  "Type mismatch for custom legalized operation");
1340           ResultVals.push_back(Res.getValue(i));
1341         }
1342         LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1343         ReplaceNode(Node, ResultVals.data());
1344         return;
1345       }
1346       LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1347       [[fallthrough]];
1348     case TargetLowering::Expand:
1349       if (ExpandNode(Node))
1350         return;
1351       [[fallthrough]];
1352     case TargetLowering::LibCall:
1353       ConvertNodeToLibcall(Node);
1354       return;
1355     case TargetLowering::Promote:
1356       PromoteNode(Node);
1357       return;
1358     }
1359   }
1360 
1361   switch (Node->getOpcode()) {
1362   default:
1363 #ifndef NDEBUG
1364     dbgs() << "NODE: ";
1365     Node->dump( &DAG);
1366     dbgs() << "\n";
1367 #endif
1368     llvm_unreachable("Do not know how to legalize this operator!");
1369 
1370   case ISD::CALLSEQ_START:
1371   case ISD::CALLSEQ_END:
1372     break;
1373   case ISD::LOAD:
1374     return LegalizeLoadOps(Node);
1375   case ISD::STORE:
1376     return LegalizeStoreOps(Node);
1377   }
1378 }
1379 
1380 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1381   SDValue Vec = Op.getOperand(0);
1382   SDValue Idx = Op.getOperand(1);
1383   SDLoc dl(Op);
1384 
1385   // Before we generate a new store to a temporary stack slot, see if there is
1386   // already one that we can use. There often is because when we scalarize
1387   // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1388   // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1389   // the vector. If all are expanded here, we don't want one store per vector
1390   // element.
1391 
1392   // Caches for hasPredecessorHelper
1393   SmallPtrSet<const SDNode *, 32> Visited;
1394   SmallVector<const SDNode *, 16> Worklist;
1395   Visited.insert(Op.getNode());
1396   Worklist.push_back(Idx.getNode());
1397   SDValue StackPtr, Ch;
1398   for (SDNode *User : Vec.getNode()->uses()) {
1399     if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1400       if (ST->isIndexed() || ST->isTruncatingStore() ||
1401           ST->getValue() != Vec)
1402         continue;
1403 
1404       // Make sure that nothing else could have stored into the destination of
1405       // this store.
1406       if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1407         continue;
1408 
1409       // If the index is dependent on the store we will introduce a cycle when
1410       // creating the load (the load uses the index, and by replacing the chain
1411       // we will make the index dependent on the load). Also, the store might be
1412       // dependent on the extractelement and introduce a cycle when creating
1413       // the load.
1414       if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1415           ST->hasPredecessor(Op.getNode()))
1416         continue;
1417 
1418       StackPtr = ST->getBasePtr();
1419       Ch = SDValue(ST, 0);
1420       break;
1421     }
1422   }
1423 
1424   EVT VecVT = Vec.getValueType();
1425 
1426   if (!Ch.getNode()) {
1427     // Store the value to a temporary stack slot, then LOAD the returned part.
1428     StackPtr = DAG.CreateStackTemporary(VecVT);
1429     Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1430                       MachinePointerInfo());
1431   }
1432 
1433   SDValue NewLoad;
1434   Align ElementAlignment =
1435       std::min(cast<StoreSDNode>(Ch)->getAlign(),
1436                DAG.getDataLayout().getPrefTypeAlign(
1437                    Op.getValueType().getTypeForEVT(*DAG.getContext())));
1438 
1439   if (Op.getValueType().isVector()) {
1440     StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1441                                           Op.getValueType(), Idx);
1442     NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1443                           MachinePointerInfo(), ElementAlignment);
1444   } else {
1445     StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1446     NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1447                              MachinePointerInfo(), VecVT.getVectorElementType(),
1448                              ElementAlignment);
1449   }
1450 
1451   // Replace the chain going out of the store, by the one out of the load.
1452   DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1453 
1454   // We introduced a cycle though, so update the loads operands, making sure
1455   // to use the original store's chain as an incoming chain.
1456   SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1457                                           NewLoad->op_end());
1458   NewLoadOperands[0] = Ch;
1459   NewLoad =
1460       SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1461   return NewLoad;
1462 }
1463 
1464 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1465   assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1466 
1467   SDValue Vec  = Op.getOperand(0);
1468   SDValue Part = Op.getOperand(1);
1469   SDValue Idx  = Op.getOperand(2);
1470   SDLoc dl(Op);
1471 
1472   // Store the value to a temporary stack slot, then LOAD the returned part.
1473   EVT VecVT = Vec.getValueType();
1474   EVT SubVecVT = Part.getValueType();
1475   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1476   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1477   MachinePointerInfo PtrInfo =
1478       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1479 
1480   // First store the whole vector.
1481   SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1482 
1483   // Then store the inserted part.
1484   SDValue SubStackPtr =
1485       TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1486 
1487   // Store the subvector.
1488   Ch = DAG.getStore(
1489       Ch, dl, Part, SubStackPtr,
1490       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1491 
1492   // Finally, load the updated vector.
1493   return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1494 }
1495 
1496 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1497   assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1498           Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1499          "Unexpected opcode!");
1500 
1501   // We can't handle this case efficiently.  Allocate a sufficiently
1502   // aligned object on the stack, store each operand into it, then load
1503   // the result as a vector.
1504   // Create the stack frame object.
1505   EVT VT = Node->getValueType(0);
1506   EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1507                                            : Node->getOperand(0).getValueType();
1508   SDLoc dl(Node);
1509   SDValue FIPtr = DAG.CreateStackTemporary(VT);
1510   int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1511   MachinePointerInfo PtrInfo =
1512       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1513 
1514   // Emit a store of each element to the stack slot.
1515   SmallVector<SDValue, 8> Stores;
1516   unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1517   assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1518 
1519   // If the destination vector element type of a BUILD_VECTOR is narrower than
1520   // the source element type, only store the bits necessary.
1521   bool Truncate = isa<BuildVectorSDNode>(Node) &&
1522                   MemVT.bitsLT(Node->getOperand(0).getValueType());
1523 
1524   // Store (in the right endianness) the elements to memory.
1525   for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1526     // Ignore undef elements.
1527     if (Node->getOperand(i).isUndef()) continue;
1528 
1529     unsigned Offset = TypeByteSize*i;
1530 
1531     SDValue Idx =
1532         DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1533 
1534     if (Truncate)
1535       Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1536                                          Node->getOperand(i), Idx,
1537                                          PtrInfo.getWithOffset(Offset), MemVT));
1538     else
1539       Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1540                                     Idx, PtrInfo.getWithOffset(Offset)));
1541   }
1542 
1543   SDValue StoreChain;
1544   if (!Stores.empty())    // Not all undef elements?
1545     StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1546   else
1547     StoreChain = DAG.getEntryNode();
1548 
1549   // Result is a load from the stack slot.
1550   return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1551 }
1552 
1553 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1554 /// containing the sign bit if the target has no integer value capable of
1555 /// holding all bits of the floating-point value.
1556 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1557                                              const SDLoc &DL,
1558                                              SDValue Value) const {
1559   EVT FloatVT = Value.getValueType();
1560   unsigned NumBits = FloatVT.getScalarSizeInBits();
1561   State.FloatVT = FloatVT;
1562   EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1563   // Convert to an integer of the same size.
1564   if (TLI.isTypeLegal(IVT)) {
1565     State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1566     State.SignMask = APInt::getSignMask(NumBits);
1567     State.SignBit = NumBits - 1;
1568     return;
1569   }
1570 
1571   auto &DataLayout = DAG.getDataLayout();
1572   // Store the float to memory, then load the sign part out as an integer.
1573   MVT LoadTy = TLI.getRegisterType(MVT::i8);
1574   // First create a temporary that is aligned for both the load and store.
1575   SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1576   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1577   // Then store the float to it.
1578   State.FloatPtr = StackPtr;
1579   MachineFunction &MF = DAG.getMachineFunction();
1580   State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1581   State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1582                              State.FloatPointerInfo);
1583 
1584   SDValue IntPtr;
1585   if (DataLayout.isBigEndian()) {
1586     assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1587     // Load out a legal integer with the same sign bit as the float.
1588     IntPtr = StackPtr;
1589     State.IntPointerInfo = State.FloatPointerInfo;
1590   } else {
1591     // Advance the pointer so that the loaded byte will contain the sign bit.
1592     unsigned ByteOffset = (NumBits / 8) - 1;
1593     IntPtr =
1594         DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1595     State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1596                                                              ByteOffset);
1597   }
1598 
1599   State.IntPtr = IntPtr;
1600   State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1601                                   State.IntPointerInfo, MVT::i8);
1602   State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1603   State.SignBit = 7;
1604 }
1605 
1606 /// Replace the integer value produced by getSignAsIntValue() with a new value
1607 /// and cast the result back to a floating-point type.
1608 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1609                                               const SDLoc &DL,
1610                                               SDValue NewIntValue) const {
1611   if (!State.Chain)
1612     return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1613 
1614   // Override the part containing the sign bit in the value stored on the stack.
1615   SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1616                                     State.IntPointerInfo, MVT::i8);
1617   return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1618                      State.FloatPointerInfo);
1619 }
1620 
1621 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1622   SDLoc DL(Node);
1623   SDValue Mag = Node->getOperand(0);
1624   SDValue Sign = Node->getOperand(1);
1625 
1626   // Get sign bit into an integer value.
1627   FloatSignAsInt SignAsInt;
1628   getSignAsIntValue(SignAsInt, DL, Sign);
1629 
1630   EVT IntVT = SignAsInt.IntValue.getValueType();
1631   SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1632   SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1633                                 SignMask);
1634 
1635   // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1636   EVT FloatVT = Mag.getValueType();
1637   if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1638       TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1639     SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1640     SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1641     SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1642                                 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1643     return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1644   }
1645 
1646   // Transform Mag value to integer, and clear the sign bit.
1647   FloatSignAsInt MagAsInt;
1648   getSignAsIntValue(MagAsInt, DL, Mag);
1649   EVT MagVT = MagAsInt.IntValue.getValueType();
1650   SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1651   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1652                                     ClearSignMask);
1653 
1654   // Get the signbit at the right position for MagAsInt.
1655   int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1656   EVT ShiftVT = IntVT;
1657   if (SignBit.getScalarValueSizeInBits() <
1658       ClearedSign.getScalarValueSizeInBits()) {
1659     SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1660     ShiftVT = MagVT;
1661   }
1662   if (ShiftAmount > 0) {
1663     SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1664     SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1665   } else if (ShiftAmount < 0) {
1666     SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1667     SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1668   }
1669   if (SignBit.getScalarValueSizeInBits() >
1670       ClearedSign.getScalarValueSizeInBits()) {
1671     SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1672   }
1673 
1674   // Store the part with the modified sign and convert back to float.
1675   SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1676   return modifySignAsInt(MagAsInt, DL, CopiedSign);
1677 }
1678 
1679 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1680   // Get the sign bit as an integer.
1681   SDLoc DL(Node);
1682   FloatSignAsInt SignAsInt;
1683   getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1684   EVT IntVT = SignAsInt.IntValue.getValueType();
1685 
1686   // Flip the sign.
1687   SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1688   SDValue SignFlip =
1689       DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1690 
1691   // Convert back to float.
1692   return modifySignAsInt(SignAsInt, DL, SignFlip);
1693 }
1694 
1695 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1696   SDLoc DL(Node);
1697   SDValue Value = Node->getOperand(0);
1698 
1699   // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1700   EVT FloatVT = Value.getValueType();
1701   if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1702     SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1703     return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1704   }
1705 
1706   // Transform value to integer, clear the sign bit and transform back.
1707   FloatSignAsInt ValueAsInt;
1708   getSignAsIntValue(ValueAsInt, DL, Value);
1709   EVT IntVT = ValueAsInt.IntValue.getValueType();
1710   SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1711   SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1712                                     ClearSignMask);
1713   return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1714 }
1715 
1716 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1717                                            SmallVectorImpl<SDValue> &Results) {
1718   Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1719   assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1720           " not tell us which reg is the stack pointer!");
1721   SDLoc dl(Node);
1722   EVT VT = Node->getValueType(0);
1723   SDValue Tmp1 = SDValue(Node, 0);
1724   SDValue Tmp2 = SDValue(Node, 1);
1725   SDValue Tmp3 = Node->getOperand(2);
1726   SDValue Chain = Tmp1.getOperand(0);
1727 
1728   // Chain the dynamic stack allocation so that it doesn't modify the stack
1729   // pointer when other instructions are using the stack.
1730   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1731 
1732   SDValue Size  = Tmp2.getOperand(1);
1733   SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1734   Chain = SP.getValue(1);
1735   Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1736   const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1737   unsigned Opc =
1738     TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1739     ISD::ADD : ISD::SUB;
1740 
1741   Align StackAlign = TFL->getStackAlign();
1742   Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size);       // Value
1743   if (Alignment > StackAlign)
1744     Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1745                        DAG.getConstant(-Alignment.value(), dl, VT));
1746   Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
1747 
1748   Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1749 
1750   Results.push_back(Tmp1);
1751   Results.push_back(Tmp2);
1752 }
1753 
1754 /// Emit a store/load combination to the stack.  This stores
1755 /// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
1756 /// a load from the stack slot to DestVT, extending it if needed.
1757 /// The resultant code need not be legal.
1758 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1759                                                EVT DestVT, const SDLoc &dl) {
1760   return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1761 }
1762 
1763 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1764                                                EVT DestVT, const SDLoc &dl,
1765                                                SDValue Chain) {
1766   EVT SrcVT = SrcOp.getValueType();
1767   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1768   Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1769 
1770   // Don't convert with stack if the load/store is expensive.
1771   if ((SrcVT.bitsGT(SlotVT) &&
1772        !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1773       (SlotVT.bitsLT(DestVT) &&
1774        !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1775     return SDValue();
1776 
1777   // Create the stack frame object.
1778   Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1779       SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1780   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1781 
1782   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1783   int SPFI = StackPtrFI->getIndex();
1784   MachinePointerInfo PtrInfo =
1785       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1786 
1787   // Emit a store to the stack slot.  Use a truncstore if the input value is
1788   // later than DestVT.
1789   SDValue Store;
1790 
1791   if (SrcVT.bitsGT(SlotVT))
1792     Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1793                               SlotVT, SrcAlign);
1794   else {
1795     assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1796     Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1797   }
1798 
1799   // Result is a load from the stack slot.
1800   if (SlotVT.bitsEq(DestVT))
1801     return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1802 
1803   assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1804   return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1805                         DestAlign);
1806 }
1807 
1808 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1809   SDLoc dl(Node);
1810   // Create a vector sized/aligned stack slot, store the value to element #0,
1811   // then load the whole vector back out.
1812   SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1813 
1814   FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1815   int SPFI = StackPtrFI->getIndex();
1816 
1817   SDValue Ch = DAG.getTruncStore(
1818       DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1819       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1820       Node->getValueType(0).getVectorElementType());
1821   return DAG.getLoad(
1822       Node->getValueType(0), dl, Ch, StackPtr,
1823       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1824 }
1825 
1826 static bool
1827 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1828                      const TargetLowering &TLI, SDValue &Res) {
1829   unsigned NumElems = Node->getNumOperands();
1830   SDLoc dl(Node);
1831   EVT VT = Node->getValueType(0);
1832 
1833   // Try to group the scalars into pairs, shuffle the pairs together, then
1834   // shuffle the pairs of pairs together, etc. until the vector has
1835   // been built. This will work only if all of the necessary shuffle masks
1836   // are legal.
1837 
1838   // We do this in two phases; first to check the legality of the shuffles,
1839   // and next, assuming that all shuffles are legal, to create the new nodes.
1840   for (int Phase = 0; Phase < 2; ++Phase) {
1841     SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1842                                                               NewIntermedVals;
1843     for (unsigned i = 0; i < NumElems; ++i) {
1844       SDValue V = Node->getOperand(i);
1845       if (V.isUndef())
1846         continue;
1847 
1848       SDValue Vec;
1849       if (Phase)
1850         Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1851       IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1852     }
1853 
1854     while (IntermedVals.size() > 2) {
1855       NewIntermedVals.clear();
1856       for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1857         // This vector and the next vector are shuffled together (simply to
1858         // append the one to the other).
1859         SmallVector<int, 16> ShuffleVec(NumElems, -1);
1860 
1861         SmallVector<int, 16> FinalIndices;
1862         FinalIndices.reserve(IntermedVals[i].second.size() +
1863                              IntermedVals[i+1].second.size());
1864 
1865         int k = 0;
1866         for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1867              ++j, ++k) {
1868           ShuffleVec[k] = j;
1869           FinalIndices.push_back(IntermedVals[i].second[j]);
1870         }
1871         for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1872              ++j, ++k) {
1873           ShuffleVec[k] = NumElems + j;
1874           FinalIndices.push_back(IntermedVals[i+1].second[j]);
1875         }
1876 
1877         SDValue Shuffle;
1878         if (Phase)
1879           Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1880                                          IntermedVals[i+1].first,
1881                                          ShuffleVec);
1882         else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1883           return false;
1884         NewIntermedVals.push_back(
1885             std::make_pair(Shuffle, std::move(FinalIndices)));
1886       }
1887 
1888       // If we had an odd number of defined values, then append the last
1889       // element to the array of new vectors.
1890       if ((IntermedVals.size() & 1) != 0)
1891         NewIntermedVals.push_back(IntermedVals.back());
1892 
1893       IntermedVals.swap(NewIntermedVals);
1894     }
1895 
1896     assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1897            "Invalid number of intermediate vectors");
1898     SDValue Vec1 = IntermedVals[0].first;
1899     SDValue Vec2;
1900     if (IntermedVals.size() > 1)
1901       Vec2 = IntermedVals[1].first;
1902     else if (Phase)
1903       Vec2 = DAG.getUNDEF(VT);
1904 
1905     SmallVector<int, 16> ShuffleVec(NumElems, -1);
1906     for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1907       ShuffleVec[IntermedVals[0].second[i]] = i;
1908     for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1909       ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1910 
1911     if (Phase)
1912       Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1913     else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1914       return false;
1915   }
1916 
1917   return true;
1918 }
1919 
1920 /// Expand a BUILD_VECTOR node on targets that don't
1921 /// support the operation, but do support the resultant vector type.
1922 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1923   unsigned NumElems = Node->getNumOperands();
1924   SDValue Value1, Value2;
1925   SDLoc dl(Node);
1926   EVT VT = Node->getValueType(0);
1927   EVT OpVT = Node->getOperand(0).getValueType();
1928   EVT EltVT = VT.getVectorElementType();
1929 
1930   // If the only non-undef value is the low element, turn this into a
1931   // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
1932   bool isOnlyLowElement = true;
1933   bool MoreThanTwoValues = false;
1934   bool isConstant = true;
1935   for (unsigned i = 0; i < NumElems; ++i) {
1936     SDValue V = Node->getOperand(i);
1937     if (V.isUndef())
1938       continue;
1939     if (i > 0)
1940       isOnlyLowElement = false;
1941     if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1942       isConstant = false;
1943 
1944     if (!Value1.getNode()) {
1945       Value1 = V;
1946     } else if (!Value2.getNode()) {
1947       if (V != Value1)
1948         Value2 = V;
1949     } else if (V != Value1 && V != Value2) {
1950       MoreThanTwoValues = true;
1951     }
1952   }
1953 
1954   if (!Value1.getNode())
1955     return DAG.getUNDEF(VT);
1956 
1957   if (isOnlyLowElement)
1958     return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1959 
1960   // If all elements are constants, create a load from the constant pool.
1961   if (isConstant) {
1962     SmallVector<Constant*, 16> CV;
1963     for (unsigned i = 0, e = NumElems; i != e; ++i) {
1964       if (ConstantFPSDNode *V =
1965           dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1966         CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1967       } else if (ConstantSDNode *V =
1968                  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1969         if (OpVT==EltVT)
1970           CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1971         else {
1972           // If OpVT and EltVT don't match, EltVT is not legal and the
1973           // element values have been promoted/truncated earlier.  Undo this;
1974           // we don't want a v16i8 to become a v16i32 for example.
1975           const ConstantInt *CI = V->getConstantIntValue();
1976           CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1977                                         CI->getZExtValue()));
1978         }
1979       } else {
1980         assert(Node->getOperand(i).isUndef());
1981         Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1982         CV.push_back(UndefValue::get(OpNTy));
1983       }
1984     }
1985     Constant *CP = ConstantVector::get(CV);
1986     SDValue CPIdx =
1987         DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1988     Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1989     return DAG.getLoad(
1990         VT, dl, DAG.getEntryNode(), CPIdx,
1991         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1992         Alignment);
1993   }
1994 
1995   SmallSet<SDValue, 16> DefinedValues;
1996   for (unsigned i = 0; i < NumElems; ++i) {
1997     if (Node->getOperand(i).isUndef())
1998       continue;
1999     DefinedValues.insert(Node->getOperand(i));
2000   }
2001 
2002   if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
2003     if (!MoreThanTwoValues) {
2004       SmallVector<int, 8> ShuffleVec(NumElems, -1);
2005       for (unsigned i = 0; i < NumElems; ++i) {
2006         SDValue V = Node->getOperand(i);
2007         if (V.isUndef())
2008           continue;
2009         ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2010       }
2011       if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2012         // Get the splatted value into the low element of a vector register.
2013         SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2014         SDValue Vec2;
2015         if (Value2.getNode())
2016           Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2017         else
2018           Vec2 = DAG.getUNDEF(VT);
2019 
2020         // Return shuffle(LowValVec, undef, <0,0,0,0>)
2021         return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2022       }
2023     } else {
2024       SDValue Res;
2025       if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2026         return Res;
2027     }
2028   }
2029 
2030   // Otherwise, we can't handle this case efficiently.
2031   return ExpandVectorBuildThroughStack(Node);
2032 }
2033 
2034 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2035   SDLoc DL(Node);
2036   EVT VT = Node->getValueType(0);
2037   SDValue SplatVal = Node->getOperand(0);
2038 
2039   return DAG.getSplatBuildVector(VT, DL, SplatVal);
2040 }
2041 
2042 // Expand a node into a call to a libcall, returning the value as the first
2043 // result and the chain as the second.  If the result value does not fit into a
2044 // register, return the lo part and set the hi part to the by-reg argument in
2045 // the first.  If it does fit into a single register, return the result and
2046 // leave the Hi part unset.
2047 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2048                                             TargetLowering::ArgListTy &&Args,
2049                                             bool isSigned) {
2050   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2051                                          TLI.getPointerTy(DAG.getDataLayout()));
2052 
2053   EVT RetVT = Node->getValueType(0);
2054   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2055 
2056   // By default, the input chain to this libcall is the entry node of the
2057   // function. If the libcall is going to be emitted as a tail call then
2058   // TLI.isUsedByReturnOnly will change it to the right chain if the return
2059   // node which is being folded has a non-entry input chain.
2060   SDValue InChain = DAG.getEntryNode();
2061 
2062   // isTailCall may be true since the callee does not reference caller stack
2063   // frame. Check if it's in the right position and that the return types match.
2064   SDValue TCChain = InChain;
2065   const Function &F = DAG.getMachineFunction().getFunction();
2066   bool isTailCall =
2067       TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2068       (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2069   if (isTailCall)
2070     InChain = TCChain;
2071 
2072   TargetLowering::CallLoweringInfo CLI(DAG);
2073   bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2074   CLI.setDebugLoc(SDLoc(Node))
2075       .setChain(InChain)
2076       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2077                     std::move(Args))
2078       .setTailCall(isTailCall)
2079       .setSExtResult(signExtend)
2080       .setZExtResult(!signExtend)
2081       .setIsPostTypeLegalization(true);
2082 
2083   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2084 
2085   if (!CallInfo.second.getNode()) {
2086     LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2087     // It's a tailcall, return the chain (which is the DAG root).
2088     return {DAG.getRoot(), DAG.getRoot()};
2089   }
2090 
2091   LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2092   return CallInfo;
2093 }
2094 
2095 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2096                                             bool isSigned) {
2097   TargetLowering::ArgListTy Args;
2098   TargetLowering::ArgListEntry Entry;
2099   for (const SDValue &Op : Node->op_values()) {
2100     EVT ArgVT = Op.getValueType();
2101     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2102     Entry.Node = Op;
2103     Entry.Ty = ArgTy;
2104     Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2105     Entry.IsZExt = !Entry.IsSExt;
2106     Args.push_back(Entry);
2107   }
2108 
2109   return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2110 }
2111 
2112 void SelectionDAGLegalize::ExpandFrexpLibCall(
2113     SDNode *Node, SmallVectorImpl<SDValue> &Results) {
2114   SDLoc dl(Node);
2115   EVT VT = Node->getValueType(0);
2116   EVT ExpVT = Node->getValueType(1);
2117 
2118   SDValue FPOp = Node->getOperand(0);
2119 
2120   EVT ArgVT = FPOp.getValueType();
2121   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2122 
2123   TargetLowering::ArgListEntry FPArgEntry;
2124   FPArgEntry.Node = FPOp;
2125   FPArgEntry.Ty = ArgTy;
2126 
2127   SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2128   TargetLowering::ArgListEntry PtrArgEntry;
2129   PtrArgEntry.Node = StackSlot;
2130   PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2131                                     DAG.getDataLayout().getAllocaAddrSpace());
2132 
2133   TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2134 
2135   RTLIB::Libcall LC = RTLIB::getFREXP(VT);
2136   auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2137 
2138   // FIXME: Get type of int for libcall declaration and cast
2139 
2140   int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2141   auto PtrInfo =
2142       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2143 
2144   SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2145   SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2146                                     LoadExp.getValue(1), DAG.getRoot());
2147   DAG.setRoot(OutputChain);
2148 
2149   Results.push_back(Call);
2150   Results.push_back(LoadExp);
2151 }
2152 
2153 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2154                                            RTLIB::Libcall LC,
2155                                            SmallVectorImpl<SDValue> &Results) {
2156   if (LC == RTLIB::UNKNOWN_LIBCALL)
2157     llvm_unreachable("Can't create an unknown libcall!");
2158 
2159   if (Node->isStrictFPOpcode()) {
2160     EVT RetVT = Node->getValueType(0);
2161     SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2162     TargetLowering::MakeLibCallOptions CallOptions;
2163     // FIXME: This doesn't support tail calls.
2164     std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2165                                                       Ops, CallOptions,
2166                                                       SDLoc(Node),
2167                                                       Node->getOperand(0));
2168     Results.push_back(Tmp.first);
2169     Results.push_back(Tmp.second);
2170   } else {
2171     SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2172     Results.push_back(Tmp);
2173   }
2174 }
2175 
2176 /// Expand the node to a libcall based on the result type.
2177 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2178                                            RTLIB::Libcall Call_F32,
2179                                            RTLIB::Libcall Call_F64,
2180                                            RTLIB::Libcall Call_F80,
2181                                            RTLIB::Libcall Call_F128,
2182                                            RTLIB::Libcall Call_PPCF128,
2183                                            SmallVectorImpl<SDValue> &Results) {
2184   RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2185                                           Call_F32, Call_F64, Call_F80,
2186                                           Call_F128, Call_PPCF128);
2187   ExpandFPLibCall(Node, LC, Results);
2188 }
2189 
2190 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2191                                                RTLIB::Libcall Call_I8,
2192                                                RTLIB::Libcall Call_I16,
2193                                                RTLIB::Libcall Call_I32,
2194                                                RTLIB::Libcall Call_I64,
2195                                                RTLIB::Libcall Call_I128) {
2196   RTLIB::Libcall LC;
2197   switch (Node->getSimpleValueType(0).SimpleTy) {
2198   default: llvm_unreachable("Unexpected request for libcall!");
2199   case MVT::i8:   LC = Call_I8; break;
2200   case MVT::i16:  LC = Call_I16; break;
2201   case MVT::i32:  LC = Call_I32; break;
2202   case MVT::i64:  LC = Call_I64; break;
2203   case MVT::i128: LC = Call_I128; break;
2204   }
2205   return ExpandLibCall(LC, Node, isSigned).first;
2206 }
2207 
2208 /// Expand the node to a libcall based on first argument type (for instance
2209 /// lround and its variant).
2210 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2211                                             RTLIB::Libcall Call_F32,
2212                                             RTLIB::Libcall Call_F64,
2213                                             RTLIB::Libcall Call_F80,
2214                                             RTLIB::Libcall Call_F128,
2215                                             RTLIB::Libcall Call_PPCF128,
2216                                             SmallVectorImpl<SDValue> &Results) {
2217   EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2218   RTLIB::Libcall LC = RTLIB::getFPLibCall(InVT.getSimpleVT(),
2219                                           Call_F32, Call_F64, Call_F80,
2220                                           Call_F128, Call_PPCF128);
2221   ExpandFPLibCall(Node, LC, Results);
2222 }
2223 
2224 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2225 void
2226 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2227                                           SmallVectorImpl<SDValue> &Results) {
2228   unsigned Opcode = Node->getOpcode();
2229   bool isSigned = Opcode == ISD::SDIVREM;
2230 
2231   RTLIB::Libcall LC;
2232   switch (Node->getSimpleValueType(0).SimpleTy) {
2233   default: llvm_unreachable("Unexpected request for libcall!");
2234   case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
2235   case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2236   case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2237   case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2238   case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2239   }
2240 
2241   // The input chain to this libcall is the entry node of the function.
2242   // Legalizing the call will automatically add the previous call to the
2243   // dependence.
2244   SDValue InChain = DAG.getEntryNode();
2245 
2246   EVT RetVT = Node->getValueType(0);
2247   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2248 
2249   TargetLowering::ArgListTy Args;
2250   TargetLowering::ArgListEntry Entry;
2251   for (const SDValue &Op : Node->op_values()) {
2252     EVT ArgVT = Op.getValueType();
2253     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2254     Entry.Node = Op;
2255     Entry.Ty = ArgTy;
2256     Entry.IsSExt = isSigned;
2257     Entry.IsZExt = !isSigned;
2258     Args.push_back(Entry);
2259   }
2260 
2261   // Also pass the return address of the remainder.
2262   SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2263   Entry.Node = FIPtr;
2264   Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2265   Entry.IsSExt = isSigned;
2266   Entry.IsZExt = !isSigned;
2267   Args.push_back(Entry);
2268 
2269   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2270                                          TLI.getPointerTy(DAG.getDataLayout()));
2271 
2272   SDLoc dl(Node);
2273   TargetLowering::CallLoweringInfo CLI(DAG);
2274   CLI.setDebugLoc(dl)
2275       .setChain(InChain)
2276       .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2277                     std::move(Args))
2278       .setSExtResult(isSigned)
2279       .setZExtResult(!isSigned);
2280 
2281   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2282 
2283   // Remainder is loaded back from the stack frame.
2284   SDValue Rem =
2285       DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2286   Results.push_back(CallInfo.first);
2287   Results.push_back(Rem);
2288 }
2289 
2290 /// Return true if sincos libcall is available.
2291 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2292   RTLIB::Libcall LC;
2293   switch (Node->getSimpleValueType(0).SimpleTy) {
2294   default: llvm_unreachable("Unexpected request for libcall!");
2295   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
2296   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
2297   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
2298   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
2299   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2300   }
2301   return TLI.getLibcallName(LC) != nullptr;
2302 }
2303 
2304 /// Only issue sincos libcall if both sin and cos are needed.
2305 static bool useSinCos(SDNode *Node) {
2306   unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2307     ? ISD::FCOS : ISD::FSIN;
2308 
2309   SDValue Op0 = Node->getOperand(0);
2310   for (const SDNode *User : Op0.getNode()->uses()) {
2311     if (User == Node)
2312       continue;
2313     // The other user might have been turned into sincos already.
2314     if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2315       return true;
2316   }
2317   return false;
2318 }
2319 
2320 /// Issue libcalls to sincos to compute sin / cos pairs.
2321 void
2322 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2323                                           SmallVectorImpl<SDValue> &Results) {
2324   RTLIB::Libcall LC;
2325   switch (Node->getSimpleValueType(0).SimpleTy) {
2326   default: llvm_unreachable("Unexpected request for libcall!");
2327   case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
2328   case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
2329   case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
2330   case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
2331   case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2332   }
2333 
2334   // The input chain to this libcall is the entry node of the function.
2335   // Legalizing the call will automatically add the previous call to the
2336   // dependence.
2337   SDValue InChain = DAG.getEntryNode();
2338 
2339   EVT RetVT = Node->getValueType(0);
2340   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2341 
2342   TargetLowering::ArgListTy Args;
2343   TargetLowering::ArgListEntry Entry;
2344 
2345   // Pass the argument.
2346   Entry.Node = Node->getOperand(0);
2347   Entry.Ty = RetTy;
2348   Entry.IsSExt = false;
2349   Entry.IsZExt = false;
2350   Args.push_back(Entry);
2351 
2352   // Pass the return address of sin.
2353   SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2354   Entry.Node = SinPtr;
2355   Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2356   Entry.IsSExt = false;
2357   Entry.IsZExt = false;
2358   Args.push_back(Entry);
2359 
2360   // Also pass the return address of the cos.
2361   SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2362   Entry.Node = CosPtr;
2363   Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2364   Entry.IsSExt = false;
2365   Entry.IsZExt = false;
2366   Args.push_back(Entry);
2367 
2368   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2369                                          TLI.getPointerTy(DAG.getDataLayout()));
2370 
2371   SDLoc dl(Node);
2372   TargetLowering::CallLoweringInfo CLI(DAG);
2373   CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2374       TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2375       std::move(Args));
2376 
2377   std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2378 
2379   Results.push_back(
2380       DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2381   Results.push_back(
2382       DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2383 }
2384 
2385 SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2386   SDLoc dl(Node);
2387   EVT VT = Node->getValueType(0);
2388   SDValue X = Node->getOperand(0);
2389   SDValue N = Node->getOperand(1);
2390   EVT ExpVT = N.getValueType();
2391   EVT AsIntVT = VT.changeTypeToInteger();
2392   if (AsIntVT == EVT()) // TODO: How to handle f80?
2393     return SDValue();
2394 
2395   if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2396     return SDValue();
2397 
2398   SDNodeFlags NSW;
2399   NSW.setNoSignedWrap(true);
2400   SDNodeFlags NUW_NSW;
2401   NUW_NSW.setNoUnsignedWrap(true);
2402   NUW_NSW.setNoSignedWrap(true);
2403 
2404   EVT SetCCVT =
2405       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2406   const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2407 
2408   const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2409   const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2410   const int Precision = APFloat::semanticsPrecision(FltSem);
2411 
2412   const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2413   const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2414 
2415   const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2416 
2417   const APFloat One(FltSem, "1.0");
2418   APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2419 
2420   // Offset by precision to avoid denormal range.
2421   APFloat ScaleDownK =
2422       scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2423 
2424   // TODO: Should really introduce control flow and use a block for the >
2425   // MaxExp, < MinExp cases
2426 
2427   // First, handle exponents Exp > MaxExp and scale down.
2428   SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2429 
2430   SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2431   SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2432   SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2433   SDValue DecN1 =
2434       DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2435 
2436   SDValue ScaleUpTwice =
2437       DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2438 
2439   const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2440   SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2441   SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2442 
2443   SDValue SelectN_Big =
2444       DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2445   SDValue SelectX_Big =
2446       DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2447 
2448   // Now handle exponents Exp < MinExp
2449   SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2450 
2451   SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2452   SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2453 
2454   SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2455 
2456   SDValue ClampMinVal =
2457       DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2458   SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2459   SDValue IncN1 =
2460       DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2461 
2462   const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2463   SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2464   SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2465 
2466   SDValue ScaleDownTwice = DAG.getSetCC(
2467       dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2468       ISD::SETULT);
2469 
2470   SDValue SelectN_Small =
2471       DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2472   SDValue SelectX_Small =
2473       DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2474 
2475   // Now combine the two out of range exponent handling cases with the base
2476   // case.
2477   SDValue NewX = DAG.getNode(
2478       ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2479       DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2480 
2481   SDValue NewN = DAG.getNode(
2482       ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2483       DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2484 
2485   SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2486 
2487   SDValue ExponentShiftAmt =
2488       DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2489   SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2490 
2491   SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2492                               ExponentShiftAmt, NUW_NSW);
2493   SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2494   return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2495 }
2496 
2497 SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2498   SDLoc dl(Node);
2499   SDValue Val = Node->getOperand(0);
2500   EVT VT = Val.getValueType();
2501   EVT ExpVT = Node->getValueType(1);
2502   EVT AsIntVT = VT.changeTypeToInteger();
2503   if (AsIntVT == EVT()) // TODO: How to handle f80?
2504     return SDValue();
2505 
2506   const fltSemantics &FltSem = SelectionDAG::EVTToAPFloatSemantics(VT);
2507   const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2508   const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2509   const unsigned BitSize = VT.getScalarSizeInBits();
2510 
2511   // TODO: Could introduce control flow and skip over the denormal handling.
2512 
2513   // scale_up = fmul value, scalbn(1.0, precision + 1)
2514   // extracted_exp = (bitcast value to uint) >> precision - 1
2515   // biased_exp = extracted_exp + min_exp
2516   // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2517   //
2518   // is_denormal = val < smallest_normalized
2519   // computed_fract = is_denormal ? scale_up : extracted_fract
2520   // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2521   //
2522   // result_0 =  (!isfinite(val) || iszero(val)) ? val : computed_fract
2523   // result_1 =  (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2524 
2525   SDValue NegSmallestNormalizedInt = DAG.getConstant(
2526       APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2527       AsIntVT);
2528 
2529   SDValue SmallestNormalizedInt = DAG.getConstant(
2530       APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2531       AsIntVT);
2532 
2533   // Masks out the exponent bits.
2534   SDValue ExpMask =
2535       DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2536 
2537   // Mask out the exponent part of the value.
2538   //
2539   // e.g, for f32 FractSignMaskVal = 0x807fffff
2540   APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2541   FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2542 
2543   APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2544   SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2545 
2546   SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2547 
2548   const APFloat One(FltSem, "1.0");
2549   // Scale a possible denormal input.
2550   // e.g., for f64, 0x1p+54
2551   APFloat ScaleUpKVal =
2552       scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2553 
2554   SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2555   SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2556 
2557   EVT SetCCVT =
2558       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2559 
2560   SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2561 
2562   SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2563 
2564   SDValue AddNegSmallestNormal =
2565       DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2566   SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2567                                       NegSmallestNormalizedInt, ISD::SETULE);
2568 
2569   SDValue IsDenormal =
2570       DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2571 
2572   SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2573   SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2574 
2575   SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2576   SDValue ScaledSelect =
2577       DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2578 
2579   SDValue ExpMaskScaled =
2580       DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2581 
2582   SDValue ScaledValue =
2583       DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2584 
2585   // Extract the exponent bits.
2586   SDValue ExponentShiftAmt =
2587       DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2588   SDValue ShiftedExp =
2589       DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2590   SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2591 
2592   SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2593   SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2594   SDValue DenormalExpBias =
2595       DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2596 
2597   SDValue MaskedFractAsInt =
2598       DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2599   const APFloat Half(FltSem, "0.5");
2600   SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2601   SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2602   SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2603 
2604   SDValue ComputedExp =
2605       DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2606 
2607   SDValue Result0 =
2608       DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2609 
2610   SDValue Result1 =
2611       DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2612 
2613   return DAG.getMergeValues({Result0, Result1}, dl);
2614 }
2615 
2616 /// This function is responsible for legalizing a
2617 /// INT_TO_FP operation of the specified operand when the target requests that
2618 /// we expand it.  At this point, we know that the result and operand types are
2619 /// legal for the target.
2620 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2621                                                    SDValue &Chain) {
2622   bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2623                    Node->getOpcode() == ISD::SINT_TO_FP);
2624   EVT DestVT = Node->getValueType(0);
2625   SDLoc dl(Node);
2626   unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2627   SDValue Op0 = Node->getOperand(OpNo);
2628   EVT SrcVT = Op0.getValueType();
2629 
2630   // TODO: Should any fast-math-flags be set for the created nodes?
2631   LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2632   if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2633       (DestVT.bitsLE(MVT::f64) ||
2634        TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2635                                                      : ISD::FP_EXTEND,
2636                             DestVT))) {
2637     LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2638                          "expansion\n");
2639 
2640     // Get the stack frame index of a 8 byte buffer.
2641     SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2642 
2643     SDValue Lo = Op0;
2644     // if signed map to unsigned space
2645     if (isSigned) {
2646       // Invert sign bit (signed to unsigned mapping).
2647       Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2648                        DAG.getConstant(0x80000000u, dl, MVT::i32));
2649     }
2650     // Initial hi portion of constructed double.
2651     SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2652 
2653     // If this a big endian target, swap the lo and high data.
2654     if (DAG.getDataLayout().isBigEndian())
2655       std::swap(Lo, Hi);
2656 
2657     SDValue MemChain = DAG.getEntryNode();
2658 
2659     // Store the lo of the constructed double.
2660     SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2661                                   MachinePointerInfo());
2662     // Store the hi of the constructed double.
2663     SDValue HiPtr =
2664         DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2665     SDValue Store2 =
2666         DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2667     MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2668 
2669     // load the constructed double
2670     SDValue Load =
2671         DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2672     // FP constant to bias correct the final result
2673     SDValue Bias = DAG.getConstantFP(
2674         isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2675                  : llvm::bit_cast<double>(0x4330000000000000ULL),
2676         dl, MVT::f64);
2677     // Subtract the bias and get the final result.
2678     SDValue Sub;
2679     SDValue Result;
2680     if (Node->isStrictFPOpcode()) {
2681       Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2682                         {Node->getOperand(0), Load, Bias});
2683       Chain = Sub.getValue(1);
2684       if (DestVT != Sub.getValueType()) {
2685         std::pair<SDValue, SDValue> ResultPair;
2686         ResultPair =
2687             DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2688         Result = ResultPair.first;
2689         Chain = ResultPair.second;
2690       }
2691       else
2692         Result = Sub;
2693     } else {
2694       Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2695       Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2696     }
2697     return Result;
2698   }
2699 
2700   if (isSigned)
2701     return SDValue();
2702 
2703   // TODO: Generalize this for use with other types.
2704   if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2705       (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2706     LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2707     // For unsigned conversions, convert them to signed conversions using the
2708     // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2709     // should be valid for i32->f32 as well.
2710 
2711     // More generally this transform should be valid if there are 3 more bits
2712     // in the integer type than the significand. Rounding uses the first bit
2713     // after the width of the significand and the OR of all bits after that. So
2714     // we need to be able to OR the shifted out bit into one of the bits that
2715     // participate in the OR.
2716 
2717     // TODO: This really should be implemented using a branch rather than a
2718     // select.  We happen to get lucky and machinesink does the right
2719     // thing most of the time.  This would be a good candidate for a
2720     // pseudo-op, or, even better, for whole-function isel.
2721     EVT SetCCVT = getSetCCResultType(SrcVT);
2722 
2723     SDValue SignBitTest = DAG.getSetCC(
2724         dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2725 
2726     EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2727     SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2728     SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2729     SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2730     SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2731     SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2732 
2733     SDValue Slow, Fast;
2734     if (Node->isStrictFPOpcode()) {
2735       // In strict mode, we must avoid spurious exceptions, and therefore
2736       // must make sure to only emit a single STRICT_SINT_TO_FP.
2737       SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2738       Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2739                          { Node->getOperand(0), InCvt });
2740       Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2741                          { Fast.getValue(1), Fast, Fast });
2742       Chain = Slow.getValue(1);
2743       // The STRICT_SINT_TO_FP inherits the exception mode from the
2744       // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2745       // never raise any exception.
2746       SDNodeFlags Flags;
2747       Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2748       Fast->setFlags(Flags);
2749       Flags.setNoFPExcept(true);
2750       Slow->setFlags(Flags);
2751     } else {
2752       SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2753       Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2754       Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2755     }
2756 
2757     return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2758   }
2759 
2760   // Don't expand it if there isn't cheap fadd.
2761   if (!TLI.isOperationLegalOrCustom(
2762           Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2763     return SDValue();
2764 
2765   // The following optimization is valid only if every value in SrcVT (when
2766   // treated as signed) is representable in DestVT.  Check that the mantissa
2767   // size of DestVT is >= than the number of bits in SrcVT -1.
2768   assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2769              SrcVT.getSizeInBits() - 1 &&
2770          "Cannot perform lossless SINT_TO_FP!");
2771 
2772   SDValue Tmp1;
2773   if (Node->isStrictFPOpcode()) {
2774     Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2775                        { Node->getOperand(0), Op0 });
2776   } else
2777     Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2778 
2779   SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2780                                  DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2781   SDValue Zero = DAG.getIntPtrConstant(0, dl),
2782           Four = DAG.getIntPtrConstant(4, dl);
2783   SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2784                                     SignSet, Four, Zero);
2785 
2786   // If the sign bit of the integer is set, the large number will be treated
2787   // as a negative number.  To counteract this, the dynamic code adds an
2788   // offset depending on the data type.
2789   uint64_t FF;
2790   switch (SrcVT.getSimpleVT().SimpleTy) {
2791   default:
2792     return SDValue();
2793   case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
2794   case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
2795   case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
2796   case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
2797   }
2798   if (DAG.getDataLayout().isLittleEndian())
2799     FF <<= 32;
2800   Constant *FudgeFactor = ConstantInt::get(
2801                                        Type::getInt64Ty(*DAG.getContext()), FF);
2802 
2803   SDValue CPIdx =
2804       DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2805   Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2806   CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2807   Alignment = commonAlignment(Alignment, 4);
2808   SDValue FudgeInReg;
2809   if (DestVT == MVT::f32)
2810     FudgeInReg = DAG.getLoad(
2811         MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2812         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2813         Alignment);
2814   else {
2815     SDValue Load = DAG.getExtLoad(
2816         ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2817         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2818         Alignment);
2819     HandleSDNode Handle(Load);
2820     LegalizeOp(Load.getNode());
2821     FudgeInReg = Handle.getValue();
2822   }
2823 
2824   if (Node->isStrictFPOpcode()) {
2825     SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2826                                  { Tmp1.getValue(1), Tmp1, FudgeInReg });
2827     Chain = Result.getValue(1);
2828     return Result;
2829   }
2830 
2831   return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2832 }
2833 
2834 /// This function is responsible for legalizing a
2835 /// *INT_TO_FP operation of the specified operand when the target requests that
2836 /// we promote it.  At this point, we know that the result and operand types are
2837 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2838 /// operation that takes a larger input.
2839 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2840     SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2841   bool IsStrict = N->isStrictFPOpcode();
2842   bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2843                   N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2844   EVT DestVT = N->getValueType(0);
2845   SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2846   unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2847   unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2848 
2849   // First step, figure out the appropriate *INT_TO_FP operation to use.
2850   EVT NewInTy = LegalOp.getValueType();
2851 
2852   unsigned OpToUse = 0;
2853 
2854   // Scan for the appropriate larger type to use.
2855   while (true) {
2856     NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2857     assert(NewInTy.isInteger() && "Ran out of possibilities!");
2858 
2859     // If the target supports SINT_TO_FP of this type, use it.
2860     if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2861       OpToUse = SIntOp;
2862       break;
2863     }
2864     if (IsSigned)
2865       continue;
2866 
2867     // If the target supports UINT_TO_FP of this type, use it.
2868     if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2869       OpToUse = UIntOp;
2870       break;
2871     }
2872 
2873     // Otherwise, try a larger type.
2874   }
2875 
2876   // Okay, we found the operation and type to use.  Zero extend our input to the
2877   // desired type then run the operation on it.
2878   if (IsStrict) {
2879     SDValue Res =
2880         DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2881                     {N->getOperand(0),
2882                      DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2883                                  dl, NewInTy, LegalOp)});
2884     Results.push_back(Res);
2885     Results.push_back(Res.getValue(1));
2886     return;
2887   }
2888 
2889   Results.push_back(
2890       DAG.getNode(OpToUse, dl, DestVT,
2891                   DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2892                               dl, NewInTy, LegalOp)));
2893 }
2894 
2895 /// This function is responsible for legalizing a
2896 /// FP_TO_*INT operation of the specified operand when the target requests that
2897 /// we promote it.  At this point, we know that the result and operand types are
2898 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2899 /// operation that returns a larger result.
2900 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2901                                                  SmallVectorImpl<SDValue> &Results) {
2902   bool IsStrict = N->isStrictFPOpcode();
2903   bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2904                   N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2905   EVT DestVT = N->getValueType(0);
2906   SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2907   // First step, figure out the appropriate FP_TO*INT operation to use.
2908   EVT NewOutTy = DestVT;
2909 
2910   unsigned OpToUse = 0;
2911 
2912   // Scan for the appropriate larger type to use.
2913   while (true) {
2914     NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2915     assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2916 
2917     // A larger signed type can hold all unsigned values of the requested type,
2918     // so using FP_TO_SINT is valid
2919     OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2920     if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2921       break;
2922 
2923     // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2924     OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2925     if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2926       break;
2927 
2928     // Otherwise, try a larger type.
2929   }
2930 
2931   // Okay, we found the operation and type to use.
2932   SDValue Operation;
2933   if (IsStrict) {
2934     SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2935     Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2936   } else
2937     Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2938 
2939   // Truncate the result of the extended FP_TO_*INT operation to the desired
2940   // size.
2941   SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2942   Results.push_back(Trunc);
2943   if (IsStrict)
2944     Results.push_back(Operation.getValue(1));
2945 }
2946 
2947 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2948 /// the result and operand types are legal and there must be a legal
2949 /// FP_TO_*INT_SAT operation for a larger result type.
2950 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2951                                                         const SDLoc &dl) {
2952   unsigned Opcode = Node->getOpcode();
2953 
2954   // Scan for the appropriate larger type to use.
2955   EVT NewOutTy = Node->getValueType(0);
2956   while (true) {
2957     NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2958     assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2959 
2960     if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2961       break;
2962   }
2963 
2964   // Saturation width is determined by second operand, so we don't have to
2965   // perform any fixup and can directly truncate the result.
2966   SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2967                                Node->getOperand(1));
2968   return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2969 }
2970 
2971 /// Open code the operations for PARITY of the specified operation.
2972 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2973   EVT VT = Op.getValueType();
2974   EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2975   unsigned Sz = VT.getScalarSizeInBits();
2976 
2977   // If CTPOP is legal, use it. Otherwise use shifts and xor.
2978   SDValue Result;
2979   if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2980     Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2981   } else {
2982     Result = Op;
2983     for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2984       SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2985                                   DAG.getConstant(1ULL << (--i), dl, ShVT));
2986       Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2987     }
2988   }
2989 
2990   return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2991 }
2992 
2993 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2994   LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2995   SmallVector<SDValue, 8> Results;
2996   SDLoc dl(Node);
2997   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2998   bool NeedInvert;
2999   switch (Node->getOpcode()) {
3000   case ISD::ABS:
3001     if ((Tmp1 = TLI.expandABS(Node, DAG)))
3002       Results.push_back(Tmp1);
3003     break;
3004   case ISD::ABDS:
3005   case ISD::ABDU:
3006     if ((Tmp1 = TLI.expandABD(Node, DAG)))
3007       Results.push_back(Tmp1);
3008     break;
3009   case ISD::CTPOP:
3010     if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3011       Results.push_back(Tmp1);
3012     break;
3013   case ISD::CTLZ:
3014   case ISD::CTLZ_ZERO_UNDEF:
3015     if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3016       Results.push_back(Tmp1);
3017     break;
3018   case ISD::CTTZ:
3019   case ISD::CTTZ_ZERO_UNDEF:
3020     if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3021       Results.push_back(Tmp1);
3022     break;
3023   case ISD::BITREVERSE:
3024     if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3025       Results.push_back(Tmp1);
3026     break;
3027   case ISD::BSWAP:
3028     if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3029       Results.push_back(Tmp1);
3030     break;
3031   case ISD::PARITY:
3032     Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3033     break;
3034   case ISD::FRAMEADDR:
3035   case ISD::RETURNADDR:
3036   case ISD::FRAME_TO_ARGS_OFFSET:
3037     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3038     break;
3039   case ISD::EH_DWARF_CFA: {
3040     SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3041                                         TLI.getPointerTy(DAG.getDataLayout()));
3042     SDValue Offset = DAG.getNode(ISD::ADD, dl,
3043                                  CfaArg.getValueType(),
3044                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3045                                              CfaArg.getValueType()),
3046                                  CfaArg);
3047     SDValue FA = DAG.getNode(
3048         ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3049         DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3050     Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3051                                   FA, Offset));
3052     break;
3053   }
3054   case ISD::GET_ROUNDING:
3055     Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3056     Results.push_back(Node->getOperand(0));
3057     break;
3058   case ISD::EH_RETURN:
3059   case ISD::EH_LABEL:
3060   case ISD::PREFETCH:
3061   case ISD::VAEND:
3062   case ISD::EH_SJLJ_LONGJMP:
3063     // If the target didn't expand these, there's nothing to do, so just
3064     // preserve the chain and be done.
3065     Results.push_back(Node->getOperand(0));
3066     break;
3067   case ISD::READCYCLECOUNTER:
3068     // If the target didn't expand this, just return 'zero' and preserve the
3069     // chain.
3070     Results.append(Node->getNumValues() - 1,
3071                    DAG.getConstant(0, dl, Node->getValueType(0)));
3072     Results.push_back(Node->getOperand(0));
3073     break;
3074   case ISD::EH_SJLJ_SETJMP:
3075     // If the target didn't expand this, just return 'zero' and preserve the
3076     // chain.
3077     Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3078     Results.push_back(Node->getOperand(0));
3079     break;
3080   case ISD::ATOMIC_LOAD: {
3081     // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3082     SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3083     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3084     SDValue Swap = DAG.getAtomicCmpSwap(
3085         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3086         Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3087         cast<AtomicSDNode>(Node)->getMemOperand());
3088     Results.push_back(Swap.getValue(0));
3089     Results.push_back(Swap.getValue(1));
3090     break;
3091   }
3092   case ISD::ATOMIC_STORE: {
3093     // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3094     SDValue Swap = DAG.getAtomic(
3095         ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3096         Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3097         cast<AtomicSDNode>(Node)->getMemOperand());
3098     Results.push_back(Swap.getValue(1));
3099     break;
3100   }
3101   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3102     // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3103     // splits out the success value as a comparison. Expanding the resulting
3104     // ATOMIC_CMP_SWAP will produce a libcall.
3105     SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3106     SDValue Res = DAG.getAtomicCmpSwap(
3107         ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3108         Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3109         Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3110 
3111     SDValue ExtRes = Res;
3112     SDValue LHS = Res;
3113     SDValue RHS = Node->getOperand(1);
3114 
3115     EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3116     EVT OuterType = Node->getValueType(0);
3117     switch (TLI.getExtendForAtomicOps()) {
3118     case ISD::SIGN_EXTEND:
3119       LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3120                         DAG.getValueType(AtomicType));
3121       RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3122                         Node->getOperand(2), DAG.getValueType(AtomicType));
3123       ExtRes = LHS;
3124       break;
3125     case ISD::ZERO_EXTEND:
3126       LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3127                         DAG.getValueType(AtomicType));
3128       RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3129       ExtRes = LHS;
3130       break;
3131     case ISD::ANY_EXTEND:
3132       LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3133       RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3134       break;
3135     default:
3136       llvm_unreachable("Invalid atomic op extension");
3137     }
3138 
3139     SDValue Success =
3140         DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3141 
3142     Results.push_back(ExtRes.getValue(0));
3143     Results.push_back(Success);
3144     Results.push_back(Res.getValue(1));
3145     break;
3146   }
3147   case ISD::ATOMIC_LOAD_SUB: {
3148     SDLoc DL(Node);
3149     EVT VT = Node->getValueType(0);
3150     SDValue RHS = Node->getOperand(2);
3151     AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3152     if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3153         cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3154       RHS = RHS->getOperand(0);
3155     SDValue NewRHS =
3156         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3157     SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3158                                 Node->getOperand(0), Node->getOperand(1),
3159                                 NewRHS, AN->getMemOperand());
3160     Results.push_back(Res);
3161     Results.push_back(Res.getValue(1));
3162     break;
3163   }
3164   case ISD::DYNAMIC_STACKALLOC:
3165     ExpandDYNAMIC_STACKALLOC(Node, Results);
3166     break;
3167   case ISD::MERGE_VALUES:
3168     for (unsigned i = 0; i < Node->getNumValues(); i++)
3169       Results.push_back(Node->getOperand(i));
3170     break;
3171   case ISD::UNDEF: {
3172     EVT VT = Node->getValueType(0);
3173     if (VT.isInteger())
3174       Results.push_back(DAG.getConstant(0, dl, VT));
3175     else {
3176       assert(VT.isFloatingPoint() && "Unknown value type!");
3177       Results.push_back(DAG.getConstantFP(0, dl, VT));
3178     }
3179     break;
3180   }
3181   case ISD::STRICT_FP_ROUND:
3182     // When strict mode is enforced we can't do expansion because it
3183     // does not honor the "strict" properties. Only libcall is allowed.
3184     if (TLI.isStrictFPEnabled())
3185       break;
3186     // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3187     // since this operation is more efficient than stack operation.
3188     if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3189                                        Node->getValueType(0))
3190         == TargetLowering::Legal)
3191       break;
3192     // We fall back to use stack operation when the FP_ROUND operation
3193     // isn't available.
3194     if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3195                                  Node->getValueType(0), dl,
3196                                  Node->getOperand(0)))) {
3197       ReplaceNode(Node, Tmp1.getNode());
3198       LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3199       return true;
3200     }
3201     break;
3202   case ISD::FP_ROUND: {
3203     EVT VT = Node->getValueType(0);
3204     if (VT.getScalarType() == MVT::bf16) {
3205       Results.push_back(
3206           DAG.getNode(ISD::FP_TO_BF16, SDLoc(Node), VT, Node->getOperand(0)));
3207       break;
3208     }
3209 
3210     LLVM_FALLTHROUGH;
3211   }
3212   case ISD::BITCAST:
3213     if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3214                                  Node->getValueType(0), dl)))
3215       Results.push_back(Tmp1);
3216     break;
3217   case ISD::STRICT_FP_EXTEND:
3218     // When strict mode is enforced we can't do expansion because it
3219     // does not honor the "strict" properties. Only libcall is allowed.
3220     if (TLI.isStrictFPEnabled())
3221       break;
3222     // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3223     // since this operation is more efficient than stack operation.
3224     if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3225                                        Node->getValueType(0))
3226         == TargetLowering::Legal)
3227       break;
3228     // We fall back to use stack operation when the FP_EXTEND operation
3229     // isn't available.
3230     if ((Tmp1 = EmitStackConvert(
3231              Node->getOperand(1), Node->getOperand(1).getValueType(),
3232              Node->getValueType(0), dl, Node->getOperand(0)))) {
3233       ReplaceNode(Node, Tmp1.getNode());
3234       LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3235       return true;
3236     }
3237     break;
3238   case ISD::FP_EXTEND: {
3239     SDValue Op = Node->getOperand(0);
3240     EVT SrcVT = Op.getValueType();
3241     EVT DstVT = Node->getValueType(0);
3242     if (SrcVT.getScalarType() == MVT::bf16) {
3243       Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3244       break;
3245     }
3246 
3247     if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3248       Results.push_back(Tmp1);
3249     break;
3250   }
3251   case ISD::BF16_TO_FP: {
3252     // Always expand bf16 to f32 casts, they lower to ext + shift.
3253     //
3254     // Note that the operand of this code can be bf16 or an integer type in case
3255     // bf16 is not supported on the target and was softened.
3256     SDValue Op = Node->getOperand(0);
3257     if (Op.getValueType() == MVT::bf16) {
3258       Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3259                        DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3260     } else {
3261       Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3262     }
3263     Op = DAG.getNode(
3264         ISD::SHL, dl, MVT::i32, Op,
3265         DAG.getConstant(16, dl,
3266                         TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3267     Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3268     // Add fp_extend in case the output is bigger than f32.
3269     if (Node->getValueType(0) != MVT::f32)
3270       Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3271     Results.push_back(Op);
3272     break;
3273   }
3274   case ISD::FP_TO_BF16: {
3275     SDValue Op = Node->getOperand(0);
3276     if (Op.getValueType() != MVT::f32)
3277       Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3278                        DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3279     Op = DAG.getNode(
3280         ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3281         DAG.getConstant(16, dl,
3282                         TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3283     // The result of this node can be bf16 or an integer type in case bf16 is
3284     // not supported on the target and was softened to i16 for storage.
3285     if (Node->getValueType(0) == MVT::bf16) {
3286       Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3287                        DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3288     } else {
3289       Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3290     }
3291     Results.push_back(Op);
3292     break;
3293   }
3294   case ISD::SIGN_EXTEND_INREG: {
3295     EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3296     EVT VT = Node->getValueType(0);
3297 
3298     // An in-register sign-extend of a boolean is a negation:
3299     // 'true' (1) sign-extended is -1.
3300     // 'false' (0) sign-extended is 0.
3301     // However, we must mask the high bits of the source operand because the
3302     // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3303 
3304     // TODO: Do this for vectors too?
3305     if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3306       SDValue One = DAG.getConstant(1, dl, VT);
3307       SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3308       SDValue Zero = DAG.getConstant(0, dl, VT);
3309       SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3310       Results.push_back(Neg);
3311       break;
3312     }
3313 
3314     // NOTE: we could fall back on load/store here too for targets without
3315     // SRA.  However, it is doubtful that any exist.
3316     EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3317     unsigned BitsDiff = VT.getScalarSizeInBits() -
3318                         ExtraVT.getScalarSizeInBits();
3319     SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3320     Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3321                        Node->getOperand(0), ShiftCst);
3322     Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3323     Results.push_back(Tmp1);
3324     break;
3325   }
3326   case ISD::UINT_TO_FP:
3327   case ISD::STRICT_UINT_TO_FP:
3328     if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3329       Results.push_back(Tmp1);
3330       if (Node->isStrictFPOpcode())
3331         Results.push_back(Tmp2);
3332       break;
3333     }
3334     [[fallthrough]];
3335   case ISD::SINT_TO_FP:
3336   case ISD::STRICT_SINT_TO_FP:
3337     if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3338       Results.push_back(Tmp1);
3339       if (Node->isStrictFPOpcode())
3340         Results.push_back(Tmp2);
3341     }
3342     break;
3343   case ISD::FP_TO_SINT:
3344     if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3345       Results.push_back(Tmp1);
3346     break;
3347   case ISD::STRICT_FP_TO_SINT:
3348     if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3349       ReplaceNode(Node, Tmp1.getNode());
3350       LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3351       return true;
3352     }
3353     break;
3354   case ISD::FP_TO_UINT:
3355     if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3356       Results.push_back(Tmp1);
3357     break;
3358   case ISD::STRICT_FP_TO_UINT:
3359     if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3360       // Relink the chain.
3361       DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3362       // Replace the new UINT result.
3363       ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3364       LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3365       return true;
3366     }
3367     break;
3368   case ISD::FP_TO_SINT_SAT:
3369   case ISD::FP_TO_UINT_SAT:
3370     Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3371     break;
3372   case ISD::VAARG:
3373     Results.push_back(DAG.expandVAArg(Node));
3374     Results.push_back(Results[0].getValue(1));
3375     break;
3376   case ISD::VACOPY:
3377     Results.push_back(DAG.expandVACopy(Node));
3378     break;
3379   case ISD::EXTRACT_VECTOR_ELT:
3380     if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3381       // This must be an access of the only element.  Return it.
3382       Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3383                          Node->getOperand(0));
3384     else
3385       Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3386     Results.push_back(Tmp1);
3387     break;
3388   case ISD::EXTRACT_SUBVECTOR:
3389     Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3390     break;
3391   case ISD::INSERT_SUBVECTOR:
3392     Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3393     break;
3394   case ISD::CONCAT_VECTORS:
3395     Results.push_back(ExpandVectorBuildThroughStack(Node));
3396     break;
3397   case ISD::SCALAR_TO_VECTOR:
3398     Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3399     break;
3400   case ISD::INSERT_VECTOR_ELT:
3401     Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3402                                               Node->getOperand(1),
3403                                               Node->getOperand(2), dl));
3404     break;
3405   case ISD::VECTOR_SHUFFLE: {
3406     SmallVector<int, 32> NewMask;
3407     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3408 
3409     EVT VT = Node->getValueType(0);
3410     EVT EltVT = VT.getVectorElementType();
3411     SDValue Op0 = Node->getOperand(0);
3412     SDValue Op1 = Node->getOperand(1);
3413     if (!TLI.isTypeLegal(EltVT)) {
3414       EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3415 
3416       // BUILD_VECTOR operands are allowed to be wider than the element type.
3417       // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3418       // it.
3419       if (NewEltVT.bitsLT(EltVT)) {
3420         // Convert shuffle node.
3421         // If original node was v4i64 and the new EltVT is i32,
3422         // cast operands to v8i32 and re-build the mask.
3423 
3424         // Calculate new VT, the size of the new VT should be equal to original.
3425         EVT NewVT =
3426             EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3427                              VT.getSizeInBits() / NewEltVT.getSizeInBits());
3428         assert(NewVT.bitsEq(VT));
3429 
3430         // cast operands to new VT
3431         Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3432         Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3433 
3434         // Convert the shuffle mask
3435         unsigned int factor =
3436                          NewVT.getVectorNumElements()/VT.getVectorNumElements();
3437 
3438         // EltVT gets smaller
3439         assert(factor > 0);
3440 
3441         for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3442           if (Mask[i] < 0) {
3443             for (unsigned fi = 0; fi < factor; ++fi)
3444               NewMask.push_back(Mask[i]);
3445           }
3446           else {
3447             for (unsigned fi = 0; fi < factor; ++fi)
3448               NewMask.push_back(Mask[i]*factor+fi);
3449           }
3450         }
3451         Mask = NewMask;
3452         VT = NewVT;
3453       }
3454       EltVT = NewEltVT;
3455     }
3456     unsigned NumElems = VT.getVectorNumElements();
3457     SmallVector<SDValue, 16> Ops;
3458     for (unsigned i = 0; i != NumElems; ++i) {
3459       if (Mask[i] < 0) {
3460         Ops.push_back(DAG.getUNDEF(EltVT));
3461         continue;
3462       }
3463       unsigned Idx = Mask[i];
3464       if (Idx < NumElems)
3465         Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3466                                   DAG.getVectorIdxConstant(Idx, dl)));
3467       else
3468         Ops.push_back(
3469             DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3470                         DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3471     }
3472 
3473     Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3474     // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3475     Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3476     Results.push_back(Tmp1);
3477     break;
3478   }
3479   case ISD::VECTOR_SPLICE: {
3480     Results.push_back(TLI.expandVectorSplice(Node, DAG));
3481     break;
3482   }
3483   case ISD::EXTRACT_ELEMENT: {
3484     EVT OpTy = Node->getOperand(0).getValueType();
3485     if (Node->getConstantOperandVal(1)) {
3486       // 1 -> Hi
3487       Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3488                          DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3489                                          TLI.getShiftAmountTy(
3490                                              Node->getOperand(0).getValueType(),
3491                                              DAG.getDataLayout())));
3492       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3493     } else {
3494       // 0 -> Lo
3495       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3496                          Node->getOperand(0));
3497     }
3498     Results.push_back(Tmp1);
3499     break;
3500   }
3501   case ISD::STACKSAVE:
3502     // Expand to CopyFromReg if the target set
3503     // StackPointerRegisterToSaveRestore.
3504     if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3505       Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3506                                            Node->getValueType(0)));
3507       Results.push_back(Results[0].getValue(1));
3508     } else {
3509       Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3510       Results.push_back(Node->getOperand(0));
3511     }
3512     break;
3513   case ISD::STACKRESTORE:
3514     // Expand to CopyToReg if the target set
3515     // StackPointerRegisterToSaveRestore.
3516     if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3517       Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3518                                          Node->getOperand(1)));
3519     } else {
3520       Results.push_back(Node->getOperand(0));
3521     }
3522     break;
3523   case ISD::GET_DYNAMIC_AREA_OFFSET:
3524     Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3525     Results.push_back(Results[0].getValue(0));
3526     break;
3527   case ISD::FCOPYSIGN:
3528     Results.push_back(ExpandFCOPYSIGN(Node));
3529     break;
3530   case ISD::FNEG:
3531     Results.push_back(ExpandFNEG(Node));
3532     break;
3533   case ISD::FABS:
3534     Results.push_back(ExpandFABS(Node));
3535     break;
3536   case ISD::IS_FPCLASS: {
3537     auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));
3538     if (SDValue Expanded =
3539             TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3540                                  Test, Node->getFlags(), SDLoc(Node), DAG))
3541       Results.push_back(Expanded);
3542     break;
3543   }
3544   case ISD::SMIN:
3545   case ISD::SMAX:
3546   case ISD::UMIN:
3547   case ISD::UMAX: {
3548     // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3549     ISD::CondCode Pred;
3550     switch (Node->getOpcode()) {
3551     default: llvm_unreachable("How did we get here?");
3552     case ISD::SMAX: Pred = ISD::SETGT; break;
3553     case ISD::SMIN: Pred = ISD::SETLT; break;
3554     case ISD::UMAX: Pred = ISD::SETUGT; break;
3555     case ISD::UMIN: Pred = ISD::SETULT; break;
3556     }
3557     Tmp1 = Node->getOperand(0);
3558     Tmp2 = Node->getOperand(1);
3559     Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3560     Results.push_back(Tmp1);
3561     break;
3562   }
3563   case ISD::FMINNUM:
3564   case ISD::FMAXNUM: {
3565     if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3566       Results.push_back(Expanded);
3567     break;
3568   }
3569   case ISD::FSIN:
3570   case ISD::FCOS: {
3571     EVT VT = Node->getValueType(0);
3572     // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3573     // fcos which share the same operand and both are used.
3574     if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3575          isSinCosLibcallAvailable(Node, TLI))
3576         && useSinCos(Node)) {
3577       SDVTList VTs = DAG.getVTList(VT, VT);
3578       Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3579       if (Node->getOpcode() == ISD::FCOS)
3580         Tmp1 = Tmp1.getValue(1);
3581       Results.push_back(Tmp1);
3582     }
3583     break;
3584   }
3585   case ISD::FLDEXP:
3586   case ISD::STRICT_FLDEXP: {
3587     EVT VT = Node->getValueType(0);
3588     RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3589     // Use the LibCall instead, it is very likely faster
3590     // FIXME: Use separate LibCall action.
3591     if (TLI.getLibcallName(LC))
3592       break;
3593 
3594     if (SDValue Expanded = expandLdexp(Node)) {
3595       Results.push_back(Expanded);
3596       if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3597         Results.push_back(Expanded.getValue(1));
3598     }
3599 
3600     break;
3601   }
3602   case ISD::FFREXP: {
3603     RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3604     // Use the LibCall instead, it is very likely faster
3605     // FIXME: Use separate LibCall action.
3606     if (TLI.getLibcallName(LC))
3607       break;
3608 
3609     if (SDValue Expanded = expandFrexp(Node)) {
3610       Results.push_back(Expanded);
3611       Results.push_back(Expanded.getValue(1));
3612     }
3613     break;
3614   }
3615   case ISD::FMAD:
3616     llvm_unreachable("Illegal fmad should never be formed");
3617 
3618   case ISD::FP16_TO_FP:
3619     if (Node->getValueType(0) != MVT::f32) {
3620       // We can extend to types bigger than f32 in two steps without changing
3621       // the result. Since "f16 -> f32" is much more commonly available, give
3622       // CodeGen the option of emitting that before resorting to a libcall.
3623       SDValue Res =
3624           DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3625       Results.push_back(
3626           DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3627     }
3628     break;
3629   case ISD::STRICT_FP16_TO_FP:
3630     if (Node->getValueType(0) != MVT::f32) {
3631       // We can extend to types bigger than f32 in two steps without changing
3632       // the result. Since "f16 -> f32" is much more commonly available, give
3633       // CodeGen the option of emitting that before resorting to a libcall.
3634       SDValue Res =
3635           DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
3636                       {Node->getOperand(0), Node->getOperand(1)});
3637       Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3638                         {Node->getValueType(0), MVT::Other},
3639                         {Res.getValue(1), Res});
3640       Results.push_back(Res);
3641       Results.push_back(Res.getValue(1));
3642     }
3643     break;
3644   case ISD::FP_TO_FP16:
3645     LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3646     if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3647       SDValue Op = Node->getOperand(0);
3648       MVT SVT = Op.getSimpleValueType();
3649       if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3650           TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3651         // Under fastmath, we can expand this node into a fround followed by
3652         // a float-half conversion.
3653         SDValue FloatVal =
3654             DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3655                         DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3656         Results.push_back(
3657             DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3658       }
3659     }
3660     break;
3661   case ISD::ConstantFP: {
3662     ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3663     // Check to see if this FP immediate is already legal.
3664     // If this is a legal constant, turn it into a TargetConstantFP node.
3665     if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3666                           DAG.shouldOptForSize()))
3667       Results.push_back(ExpandConstantFP(CFP, true));
3668     break;
3669   }
3670   case ISD::Constant: {
3671     ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3672     Results.push_back(ExpandConstant(CP));
3673     break;
3674   }
3675   case ISD::FSUB: {
3676     EVT VT = Node->getValueType(0);
3677     if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3678         TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3679       const SDNodeFlags Flags = Node->getFlags();
3680       Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3681       Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3682       Results.push_back(Tmp1);
3683     }
3684     break;
3685   }
3686   case ISD::SUB: {
3687     EVT VT = Node->getValueType(0);
3688     assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3689            TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3690            "Don't know how to expand this subtraction!");
3691     Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3692     Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3693     Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3694     break;
3695   }
3696   case ISD::UREM:
3697   case ISD::SREM:
3698     if (TLI.expandREM(Node, Tmp1, DAG))
3699       Results.push_back(Tmp1);
3700     break;
3701   case ISD::UDIV:
3702   case ISD::SDIV: {
3703     bool isSigned = Node->getOpcode() == ISD::SDIV;
3704     unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3705     EVT VT = Node->getValueType(0);
3706     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3707       SDVTList VTs = DAG.getVTList(VT, VT);
3708       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3709                          Node->getOperand(1));
3710       Results.push_back(Tmp1);
3711     }
3712     break;
3713   }
3714   case ISD::MULHU:
3715   case ISD::MULHS: {
3716     unsigned ExpandOpcode =
3717         Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3718     EVT VT = Node->getValueType(0);
3719     SDVTList VTs = DAG.getVTList(VT, VT);
3720 
3721     Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3722                        Node->getOperand(1));
3723     Results.push_back(Tmp1.getValue(1));
3724     break;
3725   }
3726   case ISD::UMUL_LOHI:
3727   case ISD::SMUL_LOHI: {
3728     SDValue LHS = Node->getOperand(0);
3729     SDValue RHS = Node->getOperand(1);
3730     MVT VT = LHS.getSimpleValueType();
3731     unsigned MULHOpcode =
3732         Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3733 
3734     if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3735       Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3736       Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3737       break;
3738     }
3739 
3740     SmallVector<SDValue, 4> Halves;
3741     EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3742     assert(TLI.isTypeLegal(HalfType));
3743     if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3744                            HalfType, DAG,
3745                            TargetLowering::MulExpansionKind::Always)) {
3746       for (unsigned i = 0; i < 2; ++i) {
3747         SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3748         SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3749         SDValue Shift = DAG.getConstant(
3750             HalfType.getScalarSizeInBits(), dl,
3751             TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3752         Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3753         Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3754       }
3755       break;
3756     }
3757     break;
3758   }
3759   case ISD::MUL: {
3760     EVT VT = Node->getValueType(0);
3761     SDVTList VTs = DAG.getVTList(VT, VT);
3762     // See if multiply or divide can be lowered using two-result operations.
3763     // We just need the low half of the multiply; try both the signed
3764     // and unsigned forms. If the target supports both SMUL_LOHI and
3765     // UMUL_LOHI, form a preference by checking which forms of plain
3766     // MULH it supports.
3767     bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3768     bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3769     bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3770     bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3771     unsigned OpToUse = 0;
3772     if (HasSMUL_LOHI && !HasMULHS) {
3773       OpToUse = ISD::SMUL_LOHI;
3774     } else if (HasUMUL_LOHI && !HasMULHU) {
3775       OpToUse = ISD::UMUL_LOHI;
3776     } else if (HasSMUL_LOHI) {
3777       OpToUse = ISD::SMUL_LOHI;
3778     } else if (HasUMUL_LOHI) {
3779       OpToUse = ISD::UMUL_LOHI;
3780     }
3781     if (OpToUse) {
3782       Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3783                                     Node->getOperand(1)));
3784       break;
3785     }
3786 
3787     SDValue Lo, Hi;
3788     EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3789     if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3790         TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3791         TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3792         TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3793         TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3794                       TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3795       Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3796       Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3797       SDValue Shift =
3798           DAG.getConstant(HalfType.getSizeInBits(), dl,
3799                           TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3800       Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3801       Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3802     }
3803     break;
3804   }
3805   case ISD::FSHL:
3806   case ISD::FSHR:
3807     if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3808       Results.push_back(Expanded);
3809     break;
3810   case ISD::ROTL:
3811   case ISD::ROTR:
3812     if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3813       Results.push_back(Expanded);
3814     break;
3815   case ISD::SADDSAT:
3816   case ISD::UADDSAT:
3817   case ISD::SSUBSAT:
3818   case ISD::USUBSAT:
3819     Results.push_back(TLI.expandAddSubSat(Node, DAG));
3820     break;
3821   case ISD::SSHLSAT:
3822   case ISD::USHLSAT:
3823     Results.push_back(TLI.expandShlSat(Node, DAG));
3824     break;
3825   case ISD::SMULFIX:
3826   case ISD::SMULFIXSAT:
3827   case ISD::UMULFIX:
3828   case ISD::UMULFIXSAT:
3829     Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3830     break;
3831   case ISD::SDIVFIX:
3832   case ISD::SDIVFIXSAT:
3833   case ISD::UDIVFIX:
3834   case ISD::UDIVFIXSAT:
3835     if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3836                                             Node->getOperand(0),
3837                                             Node->getOperand(1),
3838                                             Node->getConstantOperandVal(2),
3839                                             DAG)) {
3840       Results.push_back(V);
3841       break;
3842     }
3843     // FIXME: We might want to retry here with a wider type if we fail, if that
3844     // type is legal.
3845     // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3846     // <= 128 (which is the case for all of the default Embedded-C types),
3847     // we will only get here with types and scales that we could always expand
3848     // if we were allowed to generate libcalls to division functions of illegal
3849     // type. But we cannot do that.
3850     llvm_unreachable("Cannot expand DIVFIX!");
3851   case ISD::UADDO_CARRY:
3852   case ISD::USUBO_CARRY: {
3853     SDValue LHS = Node->getOperand(0);
3854     SDValue RHS = Node->getOperand(1);
3855     SDValue Carry = Node->getOperand(2);
3856 
3857     bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3858 
3859     // Initial add of the 2 operands.
3860     unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3861     EVT VT = LHS.getValueType();
3862     SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3863 
3864     // Initial check for overflow.
3865     EVT CarryType = Node->getValueType(1);
3866     EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3867     ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3868     SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3869 
3870     // Add of the sum and the carry.
3871     SDValue One = DAG.getConstant(1, dl, VT);
3872     SDValue CarryExt =
3873         DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3874     SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3875 
3876     // Second check for overflow. If we are adding, we can only overflow if the
3877     // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3878     // If we are subtracting, we can only overflow if the initial sum is 0 and
3879     // the carry is set, resulting in a new sum of all 1s.
3880     SDValue Zero = DAG.getConstant(0, dl, VT);
3881     SDValue Overflow2 =
3882         IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3883               : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3884     Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3885                             DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3886 
3887     SDValue ResultCarry =
3888         DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3889 
3890     Results.push_back(Sum2);
3891     Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3892     break;
3893   }
3894   case ISD::SADDO:
3895   case ISD::SSUBO: {
3896     SDValue Result, Overflow;
3897     TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3898     Results.push_back(Result);
3899     Results.push_back(Overflow);
3900     break;
3901   }
3902   case ISD::UADDO:
3903   case ISD::USUBO: {
3904     SDValue Result, Overflow;
3905     TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3906     Results.push_back(Result);
3907     Results.push_back(Overflow);
3908     break;
3909   }
3910   case ISD::UMULO:
3911   case ISD::SMULO: {
3912     SDValue Result, Overflow;
3913     if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3914       Results.push_back(Result);
3915       Results.push_back(Overflow);
3916     }
3917     break;
3918   }
3919   case ISD::BUILD_PAIR: {
3920     EVT PairTy = Node->getValueType(0);
3921     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3922     Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3923     Tmp2 = DAG.getNode(
3924         ISD::SHL, dl, PairTy, Tmp2,
3925         DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3926                         TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3927     Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3928     break;
3929   }
3930   case ISD::SELECT:
3931     Tmp1 = Node->getOperand(0);
3932     Tmp2 = Node->getOperand(1);
3933     Tmp3 = Node->getOperand(2);
3934     if (Tmp1.getOpcode() == ISD::SETCC) {
3935       Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3936                              Tmp2, Tmp3,
3937                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3938     } else {
3939       Tmp1 = DAG.getSelectCC(dl, Tmp1,
3940                              DAG.getConstant(0, dl, Tmp1.getValueType()),
3941                              Tmp2, Tmp3, ISD::SETNE);
3942     }
3943     Tmp1->setFlags(Node->getFlags());
3944     Results.push_back(Tmp1);
3945     break;
3946   case ISD::BR_JT: {
3947     SDValue Chain = Node->getOperand(0);
3948     SDValue Table = Node->getOperand(1);
3949     SDValue Index = Node->getOperand(2);
3950     int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3951 
3952     const DataLayout &TD = DAG.getDataLayout();
3953     EVT PTy = TLI.getPointerTy(TD);
3954 
3955     unsigned EntrySize =
3956       DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3957 
3958     // For power-of-two jumptable entry sizes convert multiplication to a shift.
3959     // This transformation needs to be done here since otherwise the MIPS
3960     // backend will end up emitting a three instruction multiply sequence
3961     // instead of a single shift and MSP430 will call a runtime function.
3962     if (llvm::isPowerOf2_32(EntrySize))
3963       Index = DAG.getNode(
3964           ISD::SHL, dl, Index.getValueType(), Index,
3965           DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3966     else
3967       Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3968                           DAG.getConstant(EntrySize, dl, Index.getValueType()));
3969     SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3970                                Index, Table);
3971 
3972     EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3973     SDValue LD = DAG.getExtLoad(
3974         ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3975         MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3976     Addr = LD;
3977     if (TLI.isJumpTableRelative()) {
3978       // For PIC, the sequence is:
3979       // BRIND(load(Jumptable + index) + RelocBase)
3980       // RelocBase can be JumpTable, GOT or some sort of global base.
3981       Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3982                           TLI.getPICJumpTableRelocBase(Table, DAG));
3983     }
3984 
3985     Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
3986     Results.push_back(Tmp1);
3987     break;
3988   }
3989   case ISD::BRCOND:
3990     // Expand brcond's setcc into its constituent parts and create a BR_CC
3991     // Node.
3992     Tmp1 = Node->getOperand(0);
3993     Tmp2 = Node->getOperand(1);
3994     if (Tmp2.getOpcode() == ISD::SETCC &&
3995         TLI.isOperationLegalOrCustom(ISD::BR_CC,
3996                                      Tmp2.getOperand(0).getValueType())) {
3997       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3998                          Tmp2.getOperand(0), Tmp2.getOperand(1),
3999                          Node->getOperand(2));
4000     } else {
4001       // We test only the i1 bit.  Skip the AND if UNDEF or another AND.
4002       if (Tmp2.isUndef() ||
4003           (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
4004         Tmp3 = Tmp2;
4005       else
4006         Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
4007                            DAG.getConstant(1, dl, Tmp2.getValueType()));
4008       Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4009                          DAG.getCondCode(ISD::SETNE), Tmp3,
4010                          DAG.getConstant(0, dl, Tmp3.getValueType()),
4011                          Node->getOperand(2));
4012     }
4013     Results.push_back(Tmp1);
4014     break;
4015   case ISD::SETCC:
4016   case ISD::VP_SETCC:
4017   case ISD::STRICT_FSETCC:
4018   case ISD::STRICT_FSETCCS: {
4019     bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4020     bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4021                     Node->getOpcode() == ISD::STRICT_FSETCCS;
4022     bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4023     SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4024     unsigned Offset = IsStrict ? 1 : 0;
4025     Tmp1 = Node->getOperand(0 + Offset);
4026     Tmp2 = Node->getOperand(1 + Offset);
4027     Tmp3 = Node->getOperand(2 + Offset);
4028     SDValue Mask, EVL;
4029     if (IsVP) {
4030       Mask = Node->getOperand(3 + Offset);
4031       EVL = Node->getOperand(4 + Offset);
4032     }
4033     bool Legalized = TLI.LegalizeSetCCCondCode(
4034         DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4035         Chain, IsSignaling);
4036 
4037     if (Legalized) {
4038       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4039       // condition code, create a new SETCC node.
4040       if (Tmp3.getNode()) {
4041         if (IsStrict) {
4042           Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4043                              {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4044           Chain = Tmp1.getValue(1);
4045         } else if (IsVP) {
4046           Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4047                              {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4048         } else {
4049           Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4050                              Tmp2, Tmp3, Node->getFlags());
4051         }
4052       }
4053 
4054       // If we expanded the SETCC by inverting the condition code, then wrap
4055       // the existing SETCC in a NOT to restore the intended condition.
4056       if (NeedInvert) {
4057         if (!IsVP)
4058           Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4059         else
4060           Tmp1 =
4061               DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4062       }
4063 
4064       Results.push_back(Tmp1);
4065       if (IsStrict)
4066         Results.push_back(Chain);
4067 
4068       break;
4069     }
4070 
4071     // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4072     // understand if this code is useful for strict nodes.
4073     assert(!IsStrict && "Don't know how to expand for strict nodes.");
4074 
4075     // Otherwise, SETCC for the given comparison type must be completely
4076     // illegal; expand it into a SELECT_CC.
4077     // FIXME: This drops the mask/evl for VP_SETCC.
4078     EVT VT = Node->getValueType(0);
4079     EVT Tmp1VT = Tmp1.getValueType();
4080     Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4081                        DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4082                        DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4083     Tmp1->setFlags(Node->getFlags());
4084     Results.push_back(Tmp1);
4085     break;
4086   }
4087   case ISD::SELECT_CC: {
4088     // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4089     Tmp1 = Node->getOperand(0);   // LHS
4090     Tmp2 = Node->getOperand(1);   // RHS
4091     Tmp3 = Node->getOperand(2);   // True
4092     Tmp4 = Node->getOperand(3);   // False
4093     EVT VT = Node->getValueType(0);
4094     SDValue Chain;
4095     SDValue CC = Node->getOperand(4);
4096     ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4097 
4098     if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4099       // If the condition code is legal, then we need to expand this
4100       // node using SETCC and SELECT.
4101       EVT CmpVT = Tmp1.getValueType();
4102       assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4103              "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4104              "expanded.");
4105       EVT CCVT = getSetCCResultType(CmpVT);
4106       SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4107       Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4108       break;
4109     }
4110 
4111     // SELECT_CC is legal, so the condition code must not be.
4112     bool Legalized = false;
4113     // Try to legalize by inverting the condition.  This is for targets that
4114     // might support an ordered version of a condition, but not the unordered
4115     // version (or vice versa).
4116     ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4117     if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4118       // Use the new condition code and swap true and false
4119       Legalized = true;
4120       Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4121       Tmp1->setFlags(Node->getFlags());
4122     } else {
4123       // If The inverse is not legal, then try to swap the arguments using
4124       // the inverse condition code.
4125       ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
4126       if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4127         // The swapped inverse condition is legal, so swap true and false,
4128         // lhs and rhs.
4129         Legalized = true;
4130         Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4131         Tmp1->setFlags(Node->getFlags());
4132       }
4133     }
4134 
4135     if (!Legalized) {
4136       Legalized = TLI.LegalizeSetCCCondCode(
4137           DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4138           /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4139 
4140       assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4141 
4142       // If we expanded the SETCC by inverting the condition code, then swap
4143       // the True/False operands to match.
4144       if (NeedInvert)
4145         std::swap(Tmp3, Tmp4);
4146 
4147       // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4148       // condition code, create a new SELECT_CC node.
4149       if (CC.getNode()) {
4150         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4151                            Tmp1, Tmp2, Tmp3, Tmp4, CC);
4152       } else {
4153         Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4154         CC = DAG.getCondCode(ISD::SETNE);
4155         Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4156                            Tmp2, Tmp3, Tmp4, CC);
4157       }
4158       Tmp1->setFlags(Node->getFlags());
4159     }
4160     Results.push_back(Tmp1);
4161     break;
4162   }
4163   case ISD::BR_CC: {
4164     // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4165     SDValue Chain;
4166     Tmp1 = Node->getOperand(0);              // Chain
4167     Tmp2 = Node->getOperand(2);              // LHS
4168     Tmp3 = Node->getOperand(3);              // RHS
4169     Tmp4 = Node->getOperand(1);              // CC
4170 
4171     bool Legalized = TLI.LegalizeSetCCCondCode(
4172         DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4173         /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4174     (void)Legalized;
4175     assert(Legalized && "Can't legalize BR_CC with legal condition!");
4176 
4177     // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4178     // node.
4179     if (Tmp4.getNode()) {
4180       assert(!NeedInvert && "Don't know how to invert BR_CC!");
4181 
4182       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4183                          Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4184     } else {
4185       Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4186       Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4187       Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4188                          Tmp2, Tmp3, Node->getOperand(4));
4189     }
4190     Results.push_back(Tmp1);
4191     break;
4192   }
4193   case ISD::BUILD_VECTOR:
4194     Results.push_back(ExpandBUILD_VECTOR(Node));
4195     break;
4196   case ISD::SPLAT_VECTOR:
4197     Results.push_back(ExpandSPLAT_VECTOR(Node));
4198     break;
4199   case ISD::SRA:
4200   case ISD::SRL:
4201   case ISD::SHL: {
4202     // Scalarize vector SRA/SRL/SHL.
4203     EVT VT = Node->getValueType(0);
4204     assert(VT.isVector() && "Unable to legalize non-vector shift");
4205     assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4206     unsigned NumElem = VT.getVectorNumElements();
4207 
4208     SmallVector<SDValue, 8> Scalars;
4209     for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4210       SDValue Ex =
4211           DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),
4212                       Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4213       SDValue Sh =
4214           DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(),
4215                       Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4216       Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4217                                     VT.getScalarType(), Ex, Sh));
4218     }
4219 
4220     SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4221     Results.push_back(Result);
4222     break;
4223   }
4224   case ISD::VECREDUCE_FADD:
4225   case ISD::VECREDUCE_FMUL:
4226   case ISD::VECREDUCE_ADD:
4227   case ISD::VECREDUCE_MUL:
4228   case ISD::VECREDUCE_AND:
4229   case ISD::VECREDUCE_OR:
4230   case ISD::VECREDUCE_XOR:
4231   case ISD::VECREDUCE_SMAX:
4232   case ISD::VECREDUCE_SMIN:
4233   case ISD::VECREDUCE_UMAX:
4234   case ISD::VECREDUCE_UMIN:
4235   case ISD::VECREDUCE_FMAX:
4236   case ISD::VECREDUCE_FMIN:
4237   case ISD::VECREDUCE_FMAXIMUM:
4238   case ISD::VECREDUCE_FMINIMUM:
4239     Results.push_back(TLI.expandVecReduce(Node, DAG));
4240     break;
4241   case ISD::GLOBAL_OFFSET_TABLE:
4242   case ISD::GlobalAddress:
4243   case ISD::GlobalTLSAddress:
4244   case ISD::ExternalSymbol:
4245   case ISD::ConstantPool:
4246   case ISD::JumpTable:
4247   case ISD::INTRINSIC_W_CHAIN:
4248   case ISD::INTRINSIC_WO_CHAIN:
4249   case ISD::INTRINSIC_VOID:
4250     // FIXME: Custom lowering for these operations shouldn't return null!
4251     // Return true so that we don't call ConvertNodeToLibcall which also won't
4252     // do anything.
4253     return true;
4254   }
4255 
4256   if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4257     // FIXME: We were asked to expand a strict floating-point operation,
4258     // but there is currently no expansion implemented that would preserve
4259     // the "strict" properties.  For now, we just fall back to the non-strict
4260     // version if that is legal on the target.  The actual mutation of the
4261     // operation will happen in SelectionDAGISel::DoInstructionSelection.
4262     switch (Node->getOpcode()) {
4263     default:
4264       if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4265                                          Node->getValueType(0))
4266           == TargetLowering::Legal)
4267         return true;
4268       break;
4269     case ISD::STRICT_FSUB: {
4270       if (TLI.getStrictFPOperationAction(
4271               ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4272         return true;
4273       if (TLI.getStrictFPOperationAction(
4274               ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4275         break;
4276 
4277       EVT VT = Node->getValueType(0);
4278       const SDNodeFlags Flags = Node->getFlags();
4279       SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4280       SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4281                                  {Node->getOperand(0), Node->getOperand(1), Neg},
4282                          Flags);
4283 
4284       Results.push_back(Fadd);
4285       Results.push_back(Fadd.getValue(1));
4286       break;
4287     }
4288     case ISD::STRICT_SINT_TO_FP:
4289     case ISD::STRICT_UINT_TO_FP:
4290     case ISD::STRICT_LRINT:
4291     case ISD::STRICT_LLRINT:
4292     case ISD::STRICT_LROUND:
4293     case ISD::STRICT_LLROUND:
4294       // These are registered by the operand type instead of the value
4295       // type. Reflect that here.
4296       if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4297                                          Node->getOperand(1).getValueType())
4298           == TargetLowering::Legal)
4299         return true;
4300       break;
4301     }
4302   }
4303 
4304   // Replace the original node with the legalized result.
4305   if (Results.empty()) {
4306     LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4307     return false;
4308   }
4309 
4310   LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4311   ReplaceNode(Node, Results.data());
4312   return true;
4313 }
4314 
4315 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4316   LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4317   SmallVector<SDValue, 8> Results;
4318   SDLoc dl(Node);
4319   // FIXME: Check flags on the node to see if we can use a finite call.
4320   unsigned Opc = Node->getOpcode();
4321   switch (Opc) {
4322   case ISD::ATOMIC_FENCE: {
4323     // If the target didn't lower this, lower it to '__sync_synchronize()' call
4324     // FIXME: handle "fence singlethread" more efficiently.
4325     TargetLowering::ArgListTy Args;
4326 
4327     TargetLowering::CallLoweringInfo CLI(DAG);
4328     CLI.setDebugLoc(dl)
4329         .setChain(Node->getOperand(0))
4330         .setLibCallee(
4331             CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4332             DAG.getExternalSymbol("__sync_synchronize",
4333                                   TLI.getPointerTy(DAG.getDataLayout())),
4334             std::move(Args));
4335 
4336     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4337 
4338     Results.push_back(CallResult.second);
4339     break;
4340   }
4341   // By default, atomic intrinsics are marked Legal and lowered. Targets
4342   // which don't support them directly, however, may want libcalls, in which
4343   // case they mark them Expand, and we get here.
4344   case ISD::ATOMIC_SWAP:
4345   case ISD::ATOMIC_LOAD_ADD:
4346   case ISD::ATOMIC_LOAD_SUB:
4347   case ISD::ATOMIC_LOAD_AND:
4348   case ISD::ATOMIC_LOAD_CLR:
4349   case ISD::ATOMIC_LOAD_OR:
4350   case ISD::ATOMIC_LOAD_XOR:
4351   case ISD::ATOMIC_LOAD_NAND:
4352   case ISD::ATOMIC_LOAD_MIN:
4353   case ISD::ATOMIC_LOAD_MAX:
4354   case ISD::ATOMIC_LOAD_UMIN:
4355   case ISD::ATOMIC_LOAD_UMAX:
4356   case ISD::ATOMIC_CMP_SWAP: {
4357     MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4358     AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4359     RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4360     EVT RetVT = Node->getValueType(0);
4361     TargetLowering::MakeLibCallOptions CallOptions;
4362     SmallVector<SDValue, 4> Ops;
4363     if (TLI.getLibcallName(LC)) {
4364       // If outline atomic available, prepare its arguments and expand.
4365       Ops.append(Node->op_begin() + 2, Node->op_end());
4366       Ops.push_back(Node->getOperand(1));
4367 
4368     } else {
4369       LC = RTLIB::getSYNC(Opc, VT);
4370       assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4371              "Unexpected atomic op or value type!");
4372       // Arguments for expansion to sync libcall
4373       Ops.append(Node->op_begin() + 1, Node->op_end());
4374     }
4375     std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4376                                                       Ops, CallOptions,
4377                                                       SDLoc(Node),
4378                                                       Node->getOperand(0));
4379     Results.push_back(Tmp.first);
4380     Results.push_back(Tmp.second);
4381     break;
4382   }
4383   case ISD::TRAP: {
4384     // If this operation is not supported, lower it to 'abort()' call
4385     TargetLowering::ArgListTy Args;
4386     TargetLowering::CallLoweringInfo CLI(DAG);
4387     CLI.setDebugLoc(dl)
4388         .setChain(Node->getOperand(0))
4389         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4390                       DAG.getExternalSymbol(
4391                           "abort", TLI.getPointerTy(DAG.getDataLayout())),
4392                       std::move(Args));
4393     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4394 
4395     Results.push_back(CallResult.second);
4396     break;
4397   }
4398   case ISD::FMINNUM:
4399   case ISD::STRICT_FMINNUM:
4400     ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4401                     RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4402                     RTLIB::FMIN_PPCF128, Results);
4403     break;
4404   // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4405   // libcall legalization for these nodes, but there is no default expasion for
4406   // these nodes either (see PR63267 for example).
4407   case ISD::FMAXNUM:
4408   case ISD::STRICT_FMAXNUM:
4409     ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4410                     RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4411                     RTLIB::FMAX_PPCF128, Results);
4412     break;
4413   case ISD::FSQRT:
4414   case ISD::STRICT_FSQRT:
4415     ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4416                     RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4417                     RTLIB::SQRT_PPCF128, Results);
4418     break;
4419   case ISD::FCBRT:
4420     ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4421                     RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4422                     RTLIB::CBRT_PPCF128, Results);
4423     break;
4424   case ISD::FSIN:
4425   case ISD::STRICT_FSIN:
4426     ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4427                     RTLIB::SIN_F80, RTLIB::SIN_F128,
4428                     RTLIB::SIN_PPCF128, Results);
4429     break;
4430   case ISD::FCOS:
4431   case ISD::STRICT_FCOS:
4432     ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4433                     RTLIB::COS_F80, RTLIB::COS_F128,
4434                     RTLIB::COS_PPCF128, Results);
4435     break;
4436   case ISD::FSINCOS:
4437     // Expand into sincos libcall.
4438     ExpandSinCosLibCall(Node, Results);
4439     break;
4440   case ISD::FLOG:
4441   case ISD::STRICT_FLOG:
4442     ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4443                     RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4444     break;
4445   case ISD::FLOG2:
4446   case ISD::STRICT_FLOG2:
4447     ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4448                     RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4449     break;
4450   case ISD::FLOG10:
4451   case ISD::STRICT_FLOG10:
4452     ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4453                     RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4454     break;
4455   case ISD::FEXP:
4456   case ISD::STRICT_FEXP:
4457     ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4458                     RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4459     break;
4460   case ISD::FEXP2:
4461   case ISD::STRICT_FEXP2:
4462     ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4463                     RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4464     break;
4465   case ISD::FEXP10:
4466     ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4467                     RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4468     break;
4469   case ISD::FTRUNC:
4470   case ISD::STRICT_FTRUNC:
4471     ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4472                     RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4473                     RTLIB::TRUNC_PPCF128, Results);
4474     break;
4475   case ISD::FFLOOR:
4476   case ISD::STRICT_FFLOOR:
4477     ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4478                     RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4479                     RTLIB::FLOOR_PPCF128, Results);
4480     break;
4481   case ISD::FCEIL:
4482   case ISD::STRICT_FCEIL:
4483     ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4484                     RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4485                     RTLIB::CEIL_PPCF128, Results);
4486     break;
4487   case ISD::FRINT:
4488   case ISD::STRICT_FRINT:
4489     ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4490                     RTLIB::RINT_F80, RTLIB::RINT_F128,
4491                     RTLIB::RINT_PPCF128, Results);
4492     break;
4493   case ISD::FNEARBYINT:
4494   case ISD::STRICT_FNEARBYINT:
4495     ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4496                     RTLIB::NEARBYINT_F64,
4497                     RTLIB::NEARBYINT_F80,
4498                     RTLIB::NEARBYINT_F128,
4499                     RTLIB::NEARBYINT_PPCF128, Results);
4500     break;
4501   case ISD::FROUND:
4502   case ISD::STRICT_FROUND:
4503     ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4504                     RTLIB::ROUND_F64,
4505                     RTLIB::ROUND_F80,
4506                     RTLIB::ROUND_F128,
4507                     RTLIB::ROUND_PPCF128, Results);
4508     break;
4509   case ISD::FROUNDEVEN:
4510   case ISD::STRICT_FROUNDEVEN:
4511     ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4512                     RTLIB::ROUNDEVEN_F64,
4513                     RTLIB::ROUNDEVEN_F80,
4514                     RTLIB::ROUNDEVEN_F128,
4515                     RTLIB::ROUNDEVEN_PPCF128, Results);
4516     break;
4517   case ISD::FLDEXP:
4518   case ISD::STRICT_FLDEXP:
4519     ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4520                     RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4521     break;
4522   case ISD::FFREXP: {
4523     ExpandFrexpLibCall(Node, Results);
4524     break;
4525   }
4526   case ISD::FPOWI:
4527   case ISD::STRICT_FPOWI: {
4528     RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4529     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4530     if (!TLI.getLibcallName(LC)) {
4531       // Some targets don't have a powi libcall; use pow instead.
4532       if (Node->isStrictFPOpcode()) {
4533         SDValue Exponent =
4534             DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4535                         {Node->getValueType(0), Node->getValueType(1)},
4536                         {Node->getOperand(0), Node->getOperand(2)});
4537         SDValue FPOW =
4538             DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4539                         {Node->getValueType(0), Node->getValueType(1)},
4540                         {Exponent.getValue(1), Node->getOperand(1), Exponent});
4541         Results.push_back(FPOW);
4542         Results.push_back(FPOW.getValue(1));
4543       } else {
4544         SDValue Exponent =
4545             DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4546                         Node->getOperand(1));
4547         Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4548                                       Node->getValueType(0),
4549                                       Node->getOperand(0), Exponent));
4550       }
4551       break;
4552     }
4553     unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4554     bool ExponentHasSizeOfInt =
4555         DAG.getLibInfo().getIntSize() ==
4556         Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4557     if (!ExponentHasSizeOfInt) {
4558       // If the exponent does not match with sizeof(int) a libcall to
4559       // RTLIB::POWI would use the wrong type for the argument.
4560       DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4561       Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4562       break;
4563     }
4564     ExpandFPLibCall(Node, LC, Results);
4565     break;
4566   }
4567   case ISD::FPOW:
4568   case ISD::STRICT_FPOW:
4569     ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4570                     RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4571     break;
4572   case ISD::LROUND:
4573   case ISD::STRICT_LROUND:
4574     ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4575                        RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4576                        RTLIB::LROUND_F128,
4577                        RTLIB::LROUND_PPCF128, Results);
4578     break;
4579   case ISD::LLROUND:
4580   case ISD::STRICT_LLROUND:
4581     ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4582                        RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4583                        RTLIB::LLROUND_F128,
4584                        RTLIB::LLROUND_PPCF128, Results);
4585     break;
4586   case ISD::LRINT:
4587   case ISD::STRICT_LRINT:
4588     ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4589                        RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4590                        RTLIB::LRINT_F128,
4591                        RTLIB::LRINT_PPCF128, Results);
4592     break;
4593   case ISD::LLRINT:
4594   case ISD::STRICT_LLRINT:
4595     ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4596                        RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4597                        RTLIB::LLRINT_F128,
4598                        RTLIB::LLRINT_PPCF128, Results);
4599     break;
4600   case ISD::FDIV:
4601   case ISD::STRICT_FDIV:
4602     ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4603                     RTLIB::DIV_F80, RTLIB::DIV_F128,
4604                     RTLIB::DIV_PPCF128, Results);
4605     break;
4606   case ISD::FREM:
4607   case ISD::STRICT_FREM:
4608     ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4609                     RTLIB::REM_F80, RTLIB::REM_F128,
4610                     RTLIB::REM_PPCF128, Results);
4611     break;
4612   case ISD::FMA:
4613   case ISD::STRICT_FMA:
4614     ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4615                     RTLIB::FMA_F80, RTLIB::FMA_F128,
4616                     RTLIB::FMA_PPCF128, Results);
4617     break;
4618   case ISD::FADD:
4619   case ISD::STRICT_FADD:
4620     ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4621                     RTLIB::ADD_F80, RTLIB::ADD_F128,
4622                     RTLIB::ADD_PPCF128, Results);
4623     break;
4624   case ISD::FMUL:
4625   case ISD::STRICT_FMUL:
4626     ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4627                     RTLIB::MUL_F80, RTLIB::MUL_F128,
4628                     RTLIB::MUL_PPCF128, Results);
4629     break;
4630   case ISD::FP16_TO_FP:
4631     if (Node->getValueType(0) == MVT::f32) {
4632       Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
4633     }
4634     break;
4635   case ISD::STRICT_FP16_TO_FP: {
4636     if (Node->getValueType(0) == MVT::f32) {
4637       TargetLowering::MakeLibCallOptions CallOptions;
4638       std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4639           DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4640           SDLoc(Node), Node->getOperand(0));
4641       Results.push_back(Tmp.first);
4642       Results.push_back(Tmp.second);
4643     }
4644     break;
4645   }
4646   case ISD::FP_TO_FP16: {
4647     RTLIB::Libcall LC =
4648         RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4649     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4650     Results.push_back(ExpandLibCall(LC, Node, false).first);
4651     break;
4652   }
4653   case ISD::FP_TO_BF16: {
4654     RTLIB::Libcall LC =
4655         RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4656     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4657     Results.push_back(ExpandLibCall(LC, Node, false).first);
4658     break;
4659   }
4660   case ISD::STRICT_SINT_TO_FP:
4661   case ISD::STRICT_UINT_TO_FP:
4662   case ISD::SINT_TO_FP:
4663   case ISD::UINT_TO_FP: {
4664     // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4665     bool IsStrict = Node->isStrictFPOpcode();
4666     bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4667                   Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4668     EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4669     EVT RVT = Node->getValueType(0);
4670     EVT NVT = EVT();
4671     SDLoc dl(Node);
4672 
4673     // Even if the input is legal, no libcall may exactly match, eg. we don't
4674     // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4675     // eg: i13 -> fp. Then, look for an appropriate libcall.
4676     RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4677     for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4678          t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4679          ++t) {
4680       NVT = (MVT::SimpleValueType)t;
4681       // The source needs to big enough to hold the operand.
4682       if (NVT.bitsGE(SVT))
4683         LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4684                     : RTLIB::getUINTTOFP(NVT, RVT);
4685     }
4686     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4687 
4688     SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4689     // Sign/zero extend the argument if the libcall takes a larger type.
4690     SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4691                              NVT, Node->getOperand(IsStrict ? 1 : 0));
4692     TargetLowering::MakeLibCallOptions CallOptions;
4693     CallOptions.setSExt(Signed);
4694     std::pair<SDValue, SDValue> Tmp =
4695         TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4696     Results.push_back(Tmp.first);
4697     if (IsStrict)
4698       Results.push_back(Tmp.second);
4699     break;
4700   }
4701   case ISD::FP_TO_SINT:
4702   case ISD::FP_TO_UINT:
4703   case ISD::STRICT_FP_TO_SINT:
4704   case ISD::STRICT_FP_TO_UINT: {
4705     // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4706     bool IsStrict = Node->isStrictFPOpcode();
4707     bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4708                   Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4709 
4710     SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4711     EVT SVT = Op.getValueType();
4712     EVT RVT = Node->getValueType(0);
4713     EVT NVT = EVT();
4714     SDLoc dl(Node);
4715 
4716     // Even if the result is legal, no libcall may exactly match, eg. we don't
4717     // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4718     // eg: fp -> i32. Then, look for an appropriate libcall.
4719     RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4720     for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4721          IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4722          ++IntVT) {
4723       NVT = (MVT::SimpleValueType)IntVT;
4724       // The type needs to big enough to hold the result.
4725       if (NVT.bitsGE(RVT))
4726         LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4727                     : RTLIB::getFPTOUINT(SVT, NVT);
4728     }
4729     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4730 
4731     SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4732     TargetLowering::MakeLibCallOptions CallOptions;
4733     std::pair<SDValue, SDValue> Tmp =
4734         TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4735 
4736     // Truncate the result if the libcall returns a larger type.
4737     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4738     if (IsStrict)
4739       Results.push_back(Tmp.second);
4740     break;
4741   }
4742 
4743   case ISD::FP_ROUND:
4744   case ISD::STRICT_FP_ROUND: {
4745     // X = FP_ROUND(Y, TRUNC)
4746     // TRUNC is a flag, which is always an integer that is zero or one.
4747     // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4748     // is known to not change the value of Y.
4749     // We can only expand it into libcall if the TRUNC is 0.
4750     bool IsStrict = Node->isStrictFPOpcode();
4751     SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4752     SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4753     EVT VT = Node->getValueType(0);
4754     assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4755            "Unable to expand as libcall if it is not normal rounding");
4756 
4757     RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4758     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4759 
4760     TargetLowering::MakeLibCallOptions CallOptions;
4761     std::pair<SDValue, SDValue> Tmp =
4762         TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4763     Results.push_back(Tmp.first);
4764     if (IsStrict)
4765       Results.push_back(Tmp.second);
4766     break;
4767   }
4768   case ISD::FP_EXTEND: {
4769     Results.push_back(
4770         ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4771                                       Node->getValueType(0)),
4772                       Node, false).first);
4773     break;
4774   }
4775   case ISD::STRICT_FP_EXTEND:
4776   case ISD::STRICT_FP_TO_FP16: {
4777     RTLIB::Libcall LC =
4778         Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4779             ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4780             : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4781                               Node->getValueType(0));
4782     assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4783 
4784     TargetLowering::MakeLibCallOptions CallOptions;
4785     std::pair<SDValue, SDValue> Tmp =
4786         TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4787                         CallOptions, SDLoc(Node), Node->getOperand(0));
4788     Results.push_back(Tmp.first);
4789     Results.push_back(Tmp.second);
4790     break;
4791   }
4792   case ISD::FSUB:
4793   case ISD::STRICT_FSUB:
4794     ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4795                     RTLIB::SUB_F80, RTLIB::SUB_F128,
4796                     RTLIB::SUB_PPCF128, Results);
4797     break;
4798   case ISD::SREM:
4799     Results.push_back(ExpandIntLibCall(Node, true,
4800                                        RTLIB::SREM_I8,
4801                                        RTLIB::SREM_I16, RTLIB::SREM_I32,
4802                                        RTLIB::SREM_I64, RTLIB::SREM_I128));
4803     break;
4804   case ISD::UREM:
4805     Results.push_back(ExpandIntLibCall(Node, false,
4806                                        RTLIB::UREM_I8,
4807                                        RTLIB::UREM_I16, RTLIB::UREM_I32,
4808                                        RTLIB::UREM_I64, RTLIB::UREM_I128));
4809     break;
4810   case ISD::SDIV:
4811     Results.push_back(ExpandIntLibCall(Node, true,
4812                                        RTLIB::SDIV_I8,
4813                                        RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4814                                        RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4815     break;
4816   case ISD::UDIV:
4817     Results.push_back(ExpandIntLibCall(Node, false,
4818                                        RTLIB::UDIV_I8,
4819                                        RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4820                                        RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4821     break;
4822   case ISD::SDIVREM:
4823   case ISD::UDIVREM:
4824     // Expand into divrem libcall
4825     ExpandDivRemLibCall(Node, Results);
4826     break;
4827   case ISD::MUL:
4828     Results.push_back(ExpandIntLibCall(Node, false,
4829                                        RTLIB::MUL_I8,
4830                                        RTLIB::MUL_I16, RTLIB::MUL_I32,
4831                                        RTLIB::MUL_I64, RTLIB::MUL_I128));
4832     break;
4833   case ISD::CTLZ_ZERO_UNDEF:
4834     switch (Node->getSimpleValueType(0).SimpleTy) {
4835     default:
4836       llvm_unreachable("LibCall explicitly requested, but not available");
4837     case MVT::i32:
4838       Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first);
4839       break;
4840     case MVT::i64:
4841       Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first);
4842       break;
4843     case MVT::i128:
4844       Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first);
4845       break;
4846     }
4847     break;
4848   case ISD::RESET_FPENV: {
4849     // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
4850     // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
4851     SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4852     SDValue Chain = Node->getOperand(0);
4853     Results.push_back(
4854         DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));
4855     break;
4856   }
4857   case ISD::GET_FPENV_MEM: {
4858     SDValue Chain = Node->getOperand(0);
4859     SDValue EnvPtr = Node->getOperand(1);
4860     Results.push_back(
4861         DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4862     break;
4863   }
4864   case ISD::SET_FPENV_MEM: {
4865     SDValue Chain = Node->getOperand(0);
4866     SDValue EnvPtr = Node->getOperand(1);
4867     Results.push_back(
4868         DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4869     break;
4870   }
4871   case ISD::GET_FPMODE: {
4872     // Call fegetmode, which saves control modes into a stack slot. Then load
4873     // the value to return from the stack.
4874     EVT ModeVT = Node->getValueType(0);
4875     SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4876     int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4877     SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4878                                               Node->getOperand(0), dl);
4879     SDValue LdInst = DAG.getLoad(
4880         ModeVT, dl, Chain, StackPtr,
4881         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4882     Results.push_back(LdInst);
4883     Results.push_back(LdInst.getValue(1));
4884     break;
4885   }
4886   case ISD::SET_FPMODE: {
4887     // Move control modes to stack slot and then call fesetmode with the pointer
4888     // to the slot as argument.
4889     SDValue Mode = Node->getOperand(1);
4890     EVT ModeVT = Mode.getValueType();
4891     SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4892     int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4893     SDValue StInst = DAG.getStore(
4894         Node->getOperand(0), dl, Mode, StackPtr,
4895         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4896     Results.push_back(
4897         DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4898     break;
4899   }
4900   case ISD::RESET_FPMODE: {
4901     // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
4902     // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
4903     // target must provide custom lowering.
4904     const DataLayout &DL = DAG.getDataLayout();
4905     EVT PtrTy = TLI.getPointerTy(DL);
4906     SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy);
4907     Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4908                                                 Node->getOperand(0), dl));
4909     break;
4910   }
4911   }
4912 
4913   // Replace the original node with the legalized result.
4914   if (!Results.empty()) {
4915     LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4916     ReplaceNode(Node, Results.data());
4917   } else
4918     LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4919 }
4920 
4921 // Determine the vector type to use in place of an original scalar element when
4922 // promoting equally sized vectors.
4923 static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4924                                         MVT EltVT, MVT NewEltVT) {
4925   unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4926   MVT MidVT = OldEltsPerNewElt == 1
4927                   ? NewEltVT
4928                   : MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4929   assert(TLI.isTypeLegal(MidVT) && "unexpected");
4930   return MidVT;
4931 }
4932 
4933 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4934   LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4935   SmallVector<SDValue, 8> Results;
4936   MVT OVT = Node->getSimpleValueType(0);
4937   if (Node->getOpcode() == ISD::UINT_TO_FP ||
4938       Node->getOpcode() == ISD::SINT_TO_FP ||
4939       Node->getOpcode() == ISD::SETCC ||
4940       Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4941       Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4942     OVT = Node->getOperand(0).getSimpleValueType();
4943   }
4944   if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4945       Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4946       Node->getOpcode() == ISD::STRICT_FSETCC ||
4947       Node->getOpcode() == ISD::STRICT_FSETCCS)
4948     OVT = Node->getOperand(1).getSimpleValueType();
4949   if (Node->getOpcode() == ISD::BR_CC ||
4950       Node->getOpcode() == ISD::SELECT_CC)
4951     OVT = Node->getOperand(2).getSimpleValueType();
4952   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4953   SDLoc dl(Node);
4954   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4955   switch (Node->getOpcode()) {
4956   case ISD::CTTZ:
4957   case ISD::CTTZ_ZERO_UNDEF:
4958   case ISD::CTLZ:
4959   case ISD::CTLZ_ZERO_UNDEF:
4960   case ISD::CTPOP:
4961     // Zero extend the argument unless its cttz, then use any_extend.
4962     if (Node->getOpcode() == ISD::CTTZ ||
4963         Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4964       Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4965     else
4966       Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4967 
4968     if (Node->getOpcode() == ISD::CTTZ) {
4969       // The count is the same in the promoted type except if the original
4970       // value was zero.  This can be handled by setting the bit just off
4971       // the top of the original type.
4972       auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4973                                         OVT.getSizeInBits());
4974       Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4975                          DAG.getConstant(TopBit, dl, NVT));
4976     }
4977     // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4978     // already the correct result.
4979     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4980     if (Node->getOpcode() == ISD::CTLZ ||
4981         Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4982       // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4983       Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4984                           DAG.getConstant(NVT.getSizeInBits() -
4985                                           OVT.getSizeInBits(), dl, NVT));
4986     }
4987     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4988     break;
4989   case ISD::BITREVERSE:
4990   case ISD::BSWAP: {
4991     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4992     Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4993     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4994     Tmp1 = DAG.getNode(
4995         ISD::SRL, dl, NVT, Tmp1,
4996         DAG.getConstant(DiffBits, dl,
4997                         TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4998 
4999     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5000     break;
5001   }
5002   case ISD::FP_TO_UINT:
5003   case ISD::STRICT_FP_TO_UINT:
5004   case ISD::FP_TO_SINT:
5005   case ISD::STRICT_FP_TO_SINT:
5006     PromoteLegalFP_TO_INT(Node, dl, Results);
5007     break;
5008   case ISD::FP_TO_UINT_SAT:
5009   case ISD::FP_TO_SINT_SAT:
5010     Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
5011     break;
5012   case ISD::UINT_TO_FP:
5013   case ISD::STRICT_UINT_TO_FP:
5014   case ISD::SINT_TO_FP:
5015   case ISD::STRICT_SINT_TO_FP:
5016     PromoteLegalINT_TO_FP(Node, dl, Results);
5017     break;
5018   case ISD::VAARG: {
5019     SDValue Chain = Node->getOperand(0); // Get the chain.
5020     SDValue Ptr = Node->getOperand(1); // Get the pointer.
5021 
5022     unsigned TruncOp;
5023     if (OVT.isVector()) {
5024       TruncOp = ISD::BITCAST;
5025     } else {
5026       assert(OVT.isInteger()
5027         && "VAARG promotion is supported only for vectors or integer types");
5028       TruncOp = ISD::TRUNCATE;
5029     }
5030 
5031     // Perform the larger operation, then convert back
5032     Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
5033              Node->getConstantOperandVal(3));
5034     Chain = Tmp1.getValue(1);
5035 
5036     Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
5037 
5038     // Modified the chain result - switch anything that used the old chain to
5039     // use the new one.
5040     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
5041     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
5042     if (UpdatedNodes) {
5043       UpdatedNodes->insert(Tmp2.getNode());
5044       UpdatedNodes->insert(Chain.getNode());
5045     }
5046     ReplacedNode(Node);
5047     break;
5048   }
5049   case ISD::MUL:
5050   case ISD::SDIV:
5051   case ISD::SREM:
5052   case ISD::UDIV:
5053   case ISD::UREM:
5054   case ISD::SMIN:
5055   case ISD::SMAX:
5056   case ISD::UMIN:
5057   case ISD::UMAX:
5058   case ISD::AND:
5059   case ISD::OR:
5060   case ISD::XOR: {
5061     unsigned ExtOp, TruncOp;
5062     if (OVT.isVector()) {
5063       ExtOp   = ISD::BITCAST;
5064       TruncOp = ISD::BITCAST;
5065     } else {
5066       assert(OVT.isInteger() && "Cannot promote logic operation");
5067 
5068       switch (Node->getOpcode()) {
5069       default:
5070         ExtOp = ISD::ANY_EXTEND;
5071         break;
5072       case ISD::SDIV:
5073       case ISD::SREM:
5074       case ISD::SMIN:
5075       case ISD::SMAX:
5076         ExtOp = ISD::SIGN_EXTEND;
5077         break;
5078       case ISD::UDIV:
5079       case ISD::UREM:
5080         ExtOp = ISD::ZERO_EXTEND;
5081         break;
5082       case ISD::UMIN:
5083       case ISD::UMAX:
5084         if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5085           ExtOp = ISD::SIGN_EXTEND;
5086         else
5087           ExtOp = ISD::ZERO_EXTEND;
5088         break;
5089       }
5090       TruncOp = ISD::TRUNCATE;
5091     }
5092     // Promote each of the values to the new type.
5093     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5094     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5095     // Perform the larger operation, then convert back
5096     Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5097     Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5098     break;
5099   }
5100   case ISD::UMUL_LOHI:
5101   case ISD::SMUL_LOHI: {
5102     // Promote to a multiply in a wider integer type.
5103     unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5104                                                          : ISD::SIGN_EXTEND;
5105     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5106     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5107     Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
5108 
5109     auto &DL = DAG.getDataLayout();
5110     unsigned OriginalSize = OVT.getScalarSizeInBits();
5111     Tmp2 = DAG.getNode(
5112         ISD::SRL, dl, NVT, Tmp1,
5113         DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
5114     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5115     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5116     break;
5117   }
5118   case ISD::SELECT: {
5119     unsigned ExtOp, TruncOp;
5120     if (Node->getValueType(0).isVector() ||
5121         Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
5122       ExtOp   = ISD::BITCAST;
5123       TruncOp = ISD::BITCAST;
5124     } else if (Node->getValueType(0).isInteger()) {
5125       ExtOp   = ISD::ANY_EXTEND;
5126       TruncOp = ISD::TRUNCATE;
5127     } else {
5128       ExtOp   = ISD::FP_EXTEND;
5129       TruncOp = ISD::FP_ROUND;
5130     }
5131     Tmp1 = Node->getOperand(0);
5132     // Promote each of the values to the new type.
5133     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5134     Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5135     // Perform the larger operation, then round down.
5136     Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5137     Tmp1->setFlags(Node->getFlags());
5138     if (TruncOp != ISD::FP_ROUND)
5139       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
5140     else
5141       Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
5142                          DAG.getIntPtrConstant(0, dl));
5143     Results.push_back(Tmp1);
5144     break;
5145   }
5146   case ISD::VECTOR_SHUFFLE: {
5147     ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
5148 
5149     // Cast the two input vectors.
5150     Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
5151     Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
5152 
5153     // Convert the shuffle mask to the right # elements.
5154     Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5155     Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
5156     Results.push_back(Tmp1);
5157     break;
5158   }
5159   case ISD::VECTOR_SPLICE: {
5160     Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
5161     Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
5162     Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
5163                        Node->getOperand(2));
5164     Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
5165     break;
5166   }
5167   case ISD::SELECT_CC: {
5168     SDValue Cond = Node->getOperand(4);
5169     ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
5170     // Type of the comparison operands.
5171     MVT CVT = Node->getSimpleValueType(0);
5172     assert(CVT == OVT && "not handled");
5173 
5174     unsigned ExtOp = ISD::FP_EXTEND;
5175     if (NVT.isInteger()) {
5176       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5177     }
5178 
5179     // Promote the comparison operands, if needed.
5180     if (TLI.isCondCodeLegal(CCCode, CVT)) {
5181       Tmp1 = Node->getOperand(0);
5182       Tmp2 = Node->getOperand(1);
5183     } else {
5184       Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5185       Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5186     }
5187     // Cast the true/false operands.
5188     Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5189     Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5190 
5191     Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5192                        Node->getFlags());
5193 
5194     // Cast the result back to the original type.
5195     if (ExtOp != ISD::FP_EXTEND)
5196       Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
5197     else
5198       Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
5199                          DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
5200 
5201     Results.push_back(Tmp1);
5202     break;
5203   }
5204   case ISD::SETCC:
5205   case ISD::STRICT_FSETCC:
5206   case ISD::STRICT_FSETCCS: {
5207     unsigned ExtOp = ISD::FP_EXTEND;
5208     if (NVT.isInteger()) {
5209       ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
5210       if (isSignedIntSetCC(CCCode) ||
5211           TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT))
5212         ExtOp = ISD::SIGN_EXTEND;
5213       else
5214         ExtOp = ISD::ZERO_EXTEND;
5215     }
5216     if (Node->isStrictFPOpcode()) {
5217       SDValue InChain = Node->getOperand(0);
5218       std::tie(Tmp1, std::ignore) =
5219           DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
5220       std::tie(Tmp2, std::ignore) =
5221           DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
5222       SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
5223       SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5224       SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5225       Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
5226                                     {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5227                                     Node->getFlags()));
5228       Results.push_back(Results.back().getValue(1));
5229       break;
5230     }
5231     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5232     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5233     Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
5234                                   Tmp2, Node->getOperand(2), Node->getFlags()));
5235     break;
5236   }
5237   case ISD::BR_CC: {
5238     unsigned ExtOp = ISD::FP_EXTEND;
5239     if (NVT.isInteger()) {
5240       ISD::CondCode CCCode =
5241         cast<CondCodeSDNode>(Node->getOperand(1))->get();
5242       ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5243     }
5244     Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5245     Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5246     Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
5247                                   Node->getOperand(0), Node->getOperand(1),
5248                                   Tmp1, Tmp2, Node->getOperand(4)));
5249     break;
5250   }
5251   case ISD::FADD:
5252   case ISD::FSUB:
5253   case ISD::FMUL:
5254   case ISD::FDIV:
5255   case ISD::FREM:
5256   case ISD::FMINNUM:
5257   case ISD::FMAXNUM:
5258   case ISD::FMINIMUM:
5259   case ISD::FMAXIMUM:
5260   case ISD::FPOW:
5261     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5262     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5263     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5264                        Node->getFlags());
5265     Results.push_back(
5266         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5267                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5268     break;
5269   case ISD::STRICT_FADD:
5270   case ISD::STRICT_FSUB:
5271   case ISD::STRICT_FMUL:
5272   case ISD::STRICT_FDIV:
5273   case ISD::STRICT_FMINNUM:
5274   case ISD::STRICT_FMAXNUM:
5275   case ISD::STRICT_FREM:
5276   case ISD::STRICT_FPOW:
5277     Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5278                        {Node->getOperand(0), Node->getOperand(1)});
5279     Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5280                        {Node->getOperand(0), Node->getOperand(2)});
5281     Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5282                        Tmp2.getValue(1));
5283     Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5284                        {Tmp3, Tmp1, Tmp2});
5285     Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5286                        {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5287     Results.push_back(Tmp1);
5288     Results.push_back(Tmp1.getValue(1));
5289     break;
5290   case ISD::FMA:
5291     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5292     Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5293     Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
5294     Results.push_back(
5295         DAG.getNode(ISD::FP_ROUND, dl, OVT,
5296                     DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5297                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5298     break;
5299   case ISD::STRICT_FMA:
5300     Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5301                        {Node->getOperand(0), Node->getOperand(1)});
5302     Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5303                        {Node->getOperand(0), Node->getOperand(2)});
5304     Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5305                        {Node->getOperand(0), Node->getOperand(3)});
5306     Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5307                        Tmp2.getValue(1), Tmp3.getValue(1));
5308     Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5309                        {Tmp4, Tmp1, Tmp2, Tmp3});
5310     Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5311                        {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5312     Results.push_back(Tmp4);
5313     Results.push_back(Tmp4.getValue(1));
5314     break;
5315   case ISD::FCOPYSIGN:
5316   case ISD::FLDEXP:
5317   case ISD::FPOWI: {
5318     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5319     Tmp2 = Node->getOperand(1);
5320     Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5321 
5322     // fcopysign doesn't change anything but the sign bit, so
5323     //   (fp_round (fcopysign (fpext a), b))
5324     // is as precise as
5325     //   (fp_round (fpext a))
5326     // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5327     const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5328     Results.push_back(
5329         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5330                     DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true)));
5331     break;
5332   }
5333   case ISD::STRICT_FPOWI:
5334     Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5335                        {Node->getOperand(0), Node->getOperand(1)});
5336     Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5337                        {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5338     Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5339                        {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5340     Results.push_back(Tmp3);
5341     Results.push_back(Tmp3.getValue(1));
5342     break;
5343   case ISD::FFREXP: {
5344     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5345     Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1);
5346 
5347     Results.push_back(
5348         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5349                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5350 
5351     Results.push_back(Tmp2.getValue(1));
5352     break;
5353   }
5354   case ISD::FFLOOR:
5355   case ISD::FCEIL:
5356   case ISD::FRINT:
5357   case ISD::FNEARBYINT:
5358   case ISD::FROUND:
5359   case ISD::FROUNDEVEN:
5360   case ISD::FTRUNC:
5361   case ISD::FNEG:
5362   case ISD::FSQRT:
5363   case ISD::FSIN:
5364   case ISD::FCOS:
5365   case ISD::FLOG:
5366   case ISD::FLOG2:
5367   case ISD::FLOG10:
5368   case ISD::FABS:
5369   case ISD::FEXP:
5370   case ISD::FEXP2:
5371   case ISD::FEXP10:
5372   case ISD::FCANONICALIZE:
5373     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5374     Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5375     Results.push_back(
5376         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5377                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5378     break;
5379   case ISD::STRICT_FFLOOR:
5380   case ISD::STRICT_FCEIL:
5381   case ISD::STRICT_FRINT:
5382   case ISD::STRICT_FNEARBYINT:
5383   case ISD::STRICT_FROUND:
5384   case ISD::STRICT_FROUNDEVEN:
5385   case ISD::STRICT_FTRUNC:
5386   case ISD::STRICT_FSQRT:
5387   case ISD::STRICT_FSIN:
5388   case ISD::STRICT_FCOS:
5389   case ISD::STRICT_FLOG:
5390   case ISD::STRICT_FLOG2:
5391   case ISD::STRICT_FLOG10:
5392   case ISD::STRICT_FEXP:
5393   case ISD::STRICT_FEXP2:
5394     Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5395                        {Node->getOperand(0), Node->getOperand(1)});
5396     Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5397                        {Tmp1.getValue(1), Tmp1});
5398     Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5399                        {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5400     Results.push_back(Tmp3);
5401     Results.push_back(Tmp3.getValue(1));
5402     break;
5403   case ISD::BUILD_VECTOR: {
5404     MVT EltVT = OVT.getVectorElementType();
5405     MVT NewEltVT = NVT.getVectorElementType();
5406 
5407     // Handle bitcasts to a different vector type with the same total bit size
5408     //
5409     // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5410     //  =>
5411     //  v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5412 
5413     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5414            "Invalid promote type for build_vector");
5415     assert(NewEltVT.bitsLE(EltVT) && "not handled");
5416 
5417     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5418 
5419     SmallVector<SDValue, 8> NewOps;
5420     for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
5421       SDValue Op = Node->getOperand(I);
5422       NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
5423     }
5424 
5425     SDLoc SL(Node);
5426     SDValue Concat =
5427         DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5428                     SL, NVT, NewOps);
5429     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5430     Results.push_back(CvtVec);
5431     break;
5432   }
5433   case ISD::EXTRACT_VECTOR_ELT: {
5434     MVT EltVT = OVT.getVectorElementType();
5435     MVT NewEltVT = NVT.getVectorElementType();
5436 
5437     // Handle bitcasts to a different vector type with the same total bit size.
5438     //
5439     // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5440     //  =>
5441     //  v4i32:castx = bitcast x:v2i64
5442     //
5443     // i64 = bitcast
5444     //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5445     //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
5446     //
5447 
5448     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5449            "Invalid promote type for extract_vector_elt");
5450     assert(NewEltVT.bitsLT(EltVT) && "not handled");
5451 
5452     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5453     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5454 
5455     SDValue Idx = Node->getOperand(1);
5456     EVT IdxVT = Idx.getValueType();
5457     SDLoc SL(Node);
5458     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5459     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5460 
5461     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5462 
5463     SmallVector<SDValue, 8> NewOps;
5464     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5465       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5466       SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5467 
5468       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5469                                 CastVec, TmpIdx);
5470       NewOps.push_back(Elt);
5471     }
5472 
5473     SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5474     Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
5475     break;
5476   }
5477   case ISD::INSERT_VECTOR_ELT: {
5478     MVT EltVT = OVT.getVectorElementType();
5479     MVT NewEltVT = NVT.getVectorElementType();
5480 
5481     // Handle bitcasts to a different vector type with the same total bit size
5482     //
5483     // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5484     //  =>
5485     //  v4i32:castx = bitcast x:v2i64
5486     //  v2i32:casty = bitcast y:i64
5487     //
5488     // v2i64 = bitcast
5489     //   (v4i32 insert_vector_elt
5490     //       (v4i32 insert_vector_elt v4i32:castx,
5491     //                                (extract_vector_elt casty, 0), 2 * z),
5492     //        (extract_vector_elt casty, 1), (2 * z + 1))
5493 
5494     assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5495            "Invalid promote type for insert_vector_elt");
5496     assert(NewEltVT.bitsLT(EltVT) && "not handled");
5497 
5498     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5499     unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5500 
5501     SDValue Val = Node->getOperand(1);
5502     SDValue Idx = Node->getOperand(2);
5503     EVT IdxVT = Idx.getValueType();
5504     SDLoc SL(Node);
5505 
5506     SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
5507     SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5508 
5509     SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5510     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5511 
5512     SDValue NewVec = CastVec;
5513     for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5514       SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5515       SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5516 
5517       SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5518                                 CastVal, IdxOffset);
5519 
5520       NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
5521                            NewVec, Elt, InEltIdx);
5522     }
5523 
5524     Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
5525     break;
5526   }
5527   case ISD::SCALAR_TO_VECTOR: {
5528     MVT EltVT = OVT.getVectorElementType();
5529     MVT NewEltVT = NVT.getVectorElementType();
5530 
5531     // Handle bitcasts to different vector type with the same total bit size.
5532     //
5533     // e.g. v2i64 = scalar_to_vector x:i64
5534     //   =>
5535     //  concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5536     //
5537 
5538     MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5539     SDValue Val = Node->getOperand(0);
5540     SDLoc SL(Node);
5541 
5542     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5543     SDValue Undef = DAG.getUNDEF(MidVT);
5544 
5545     SmallVector<SDValue, 8> NewElts;
5546     NewElts.push_back(CastVal);
5547     for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5548       NewElts.push_back(Undef);
5549 
5550     SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
5551     SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5552     Results.push_back(CvtVec);
5553     break;
5554   }
5555   case ISD::ATOMIC_SWAP: {
5556     AtomicSDNode *AM = cast<AtomicSDNode>(Node);
5557     SDLoc SL(Node);
5558     SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
5559     assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5560            "unexpected promotion type");
5561     assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5562            "unexpected atomic_swap with illegal type");
5563 
5564     SDValue NewAtomic
5565       = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5566                       DAG.getVTList(NVT, MVT::Other),
5567                       { AM->getChain(), AM->getBasePtr(), CastVal },
5568                       AM->getMemOperand());
5569     Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
5570     Results.push_back(NewAtomic.getValue(1));
5571     break;
5572   }
5573   case ISD::SPLAT_VECTOR: {
5574     SDValue Scalar = Node->getOperand(0);
5575     MVT ScalarType = Scalar.getSimpleValueType();
5576     MVT NewScalarType = NVT.getVectorElementType();
5577     if (ScalarType.isInteger()) {
5578       Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar);
5579       Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5580       Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5581       break;
5582     }
5583     Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar);
5584     Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5585     Results.push_back(
5586         DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5587                     DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5588     break;
5589   }
5590   }
5591 
5592   // Replace the original node with the legalized result.
5593   if (!Results.empty()) {
5594     LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5595     ReplaceNode(Node, Results.data());
5596   } else
5597     LLVM_DEBUG(dbgs() << "Could not promote node\n");
5598 }
5599 
5600 /// This is the entry point for the file.
5601 void SelectionDAG::Legalize() {
5602   AssignTopologicalOrder();
5603 
5604   SmallPtrSet<SDNode *, 16> LegalizedNodes;
5605   // Use a delete listener to remove nodes which were deleted during
5606   // legalization from LegalizeNodes. This is needed to handle the situation
5607   // where a new node is allocated by the object pool to the same address of a
5608   // previously deleted node.
5609   DAGNodeDeletedListener DeleteListener(
5610       *this,
5611       [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
5612 
5613   SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5614 
5615   // Visit all the nodes. We start in topological order, so that we see
5616   // nodes with their original operands intact. Legalization can produce
5617   // new nodes which may themselves need to be legalized. Iterate until all
5618   // nodes have been legalized.
5619   while (true) {
5620     bool AnyLegalized = false;
5621     for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5622       --NI;
5623 
5624       SDNode *N = &*NI;
5625       if (N->use_empty() && N != getRoot().getNode()) {
5626         ++NI;
5627         DeleteNode(N);
5628         continue;
5629       }
5630 
5631       if (LegalizedNodes.insert(N).second) {
5632         AnyLegalized = true;
5633         Legalizer.LegalizeOp(N);
5634 
5635         if (N->use_empty() && N != getRoot().getNode()) {
5636           ++NI;
5637           DeleteNode(N);
5638         }
5639       }
5640     }
5641     if (!AnyLegalized)
5642       break;
5643 
5644   }
5645 
5646   // Remove dead nodes now.
5647   RemoveDeadNodes();
5648 }
5649 
5650 bool SelectionDAG::LegalizeOp(SDNode *N,
5651                               SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5652   SmallPtrSet<SDNode *, 16> LegalizedNodes;
5653   SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5654 
5655   // Directly insert the node in question, and legalize it. This will recurse
5656   // as needed through operands.
5657   LegalizedNodes.insert(N);
5658   Legalizer.LegalizeOp(N);
5659 
5660   return LegalizedNodes.count(N);
5661 }
5662