xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp (revision 643ac419fafba89f5adda0e0ea75b538727453fb)
1 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::LegalizeVectors method.
10 //
11 // The vector legalizer looks for vector operations which might need to be
12 // scalarized and legalizes them. This is a separate step from Legalize because
13 // scalarizing can introduce illegal types.  For example, suppose we have an
14 // ISD::SDIV of type v2i64 on x86-32.  The type is legal (for example, addition
15 // on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16 // operation, which introduces nodes with the illegal type i64 which must be
17 // expanded.  Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18 // the operation must be unrolled, which introduces nodes with the illegal
19 // type i8 which must be promoted.
20 //
21 // This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22 // or operations that happen to take a vector which are custom-lowered;
23 // the legalization for such operations never produces nodes
24 // with illegal types, so it's okay to put off legalizing them until
25 // SelectionDAG::Legalize runs.
26 //
27 //===----------------------------------------------------------------------===//
28 
29 #include "llvm/ADT/APInt.h"
30 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/CodeGen/ISDOpcodes.h"
33 #include "llvm/CodeGen/MachineMemOperand.h"
34 #include "llvm/CodeGen/SelectionDAG.h"
35 #include "llvm/CodeGen/SelectionDAGNodes.h"
36 #include "llvm/CodeGen/TargetLowering.h"
37 #include "llvm/CodeGen/ValueTypes.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/Compiler.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/MachineValueType.h"
44 #include "llvm/Support/MathExtras.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <utility>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "legalizevectorops"
53 
54 namespace {
55 
56 class VectorLegalizer {
57   SelectionDAG& DAG;
58   const TargetLowering &TLI;
59   bool Changed = false; // Keep track of whether anything changed
60 
61   /// For nodes that are of legal width, and that have more than one use, this
62   /// map indicates what regularized operand to use.  This allows us to avoid
63   /// legalizing the same thing more than once.
64   SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
65 
66   /// Adds a node to the translation cache.
67   void AddLegalizedOperand(SDValue From, SDValue To) {
68     LegalizedNodes.insert(std::make_pair(From, To));
69     // If someone requests legalization of the new node, return itself.
70     if (From != To)
71       LegalizedNodes.insert(std::make_pair(To, To));
72   }
73 
74   /// Legalizes the given node.
75   SDValue LegalizeOp(SDValue Op);
76 
77   /// Assuming the node is legal, "legalize" the results.
78   SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79 
80   /// Make sure Results are legal and update the translation cache.
81   SDValue RecursivelyLegalizeResults(SDValue Op,
82                                      MutableArrayRef<SDValue> Results);
83 
84   /// Wrapper to interface LowerOperation with a vector of Results.
85   /// Returns false if the target wants to use default expansion. Otherwise
86   /// returns true. If return is true and the Results are empty, then the
87   /// target wants to keep the input node as is.
88   bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89 
90   /// Implements unrolling a VSETCC.
91   SDValue UnrollVSETCC(SDNode *Node);
92 
93   /// Implement expand-based legalization of vector operations.
94   ///
95   /// This is just a high-level routine to dispatch to specific code paths for
96   /// operations to legalize them.
97   void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98 
99   /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100   /// FP_TO_SINT isn't legal.
101   void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102 
103   /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104   /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105   void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106 
107   /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108   SDValue ExpandSEXTINREG(SDNode *Node);
109 
110   /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111   ///
112   /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113   /// type. The contents of the bits in the extended part of each element are
114   /// undef.
115   SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116 
117   /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118   ///
119   /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120   /// type, then shifts left and arithmetic shifts right to introduce a sign
121   /// extension.
122   SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123 
124   /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125   ///
126   /// Shuffles the low lanes of the operand into place and blends zeros into
127   /// the remaining lanes, finally bitcasting to the proper type.
128   SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129 
130   /// Expand bswap of vectors into a shuffle if legal.
131   SDValue ExpandBSWAP(SDNode *Node);
132 
133   /// Implement vselect in terms of XOR, AND, OR when blend is not
134   /// supported by the target.
135   SDValue ExpandVSELECT(SDNode *Node);
136   SDValue ExpandVP_SELECT(SDNode *Node);
137   SDValue ExpandVP_MERGE(SDNode *Node);
138   SDValue ExpandSELECT(SDNode *Node);
139   std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
140   SDValue ExpandStore(SDNode *N);
141   SDValue ExpandFNEG(SDNode *Node);
142   void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143   void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144   void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145   void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146   void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147   void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148   void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149   void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150   void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 
152   void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 
154   /// Implements vector promotion.
155   ///
156   /// This is essentially just bitcasting the operands to a different type and
157   /// bitcasting the result back to the original type.
158   void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
159 
160   /// Implements [SU]INT_TO_FP vector promotion.
161   ///
162   /// This is a [zs]ext of the input operand to a larger integer type.
163   void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
164 
165   /// Implements FP_TO_[SU]INT vector promotion of the result type.
166   ///
167   /// It is promoted to a larger integer type.  The result is then
168   /// truncated back to the original type.
169   void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
170 
171 public:
172   VectorLegalizer(SelectionDAG& dag) :
173       DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
174 
175   /// Begin legalizer the vector operations in the DAG.
176   bool Run();
177 };
178 
179 } // end anonymous namespace
180 
181 bool VectorLegalizer::Run() {
182   // Before we start legalizing vector nodes, check if there are any vectors.
183   bool HasVectors = false;
184   for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
185        E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
186     // Check if the values of the nodes contain vectors. We don't need to check
187     // the operands because we are going to check their values at some point.
188     HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
189 
190     // If we found a vector node we can start the legalization.
191     if (HasVectors)
192       break;
193   }
194 
195   // If this basic block has no vectors then no need to legalize vectors.
196   if (!HasVectors)
197     return false;
198 
199   // The legalize process is inherently a bottom-up recursive process (users
200   // legalize their uses before themselves).  Given infinite stack space, we
201   // could just start legalizing on the root and traverse the whole graph.  In
202   // practice however, this causes us to run out of stack space on large basic
203   // blocks.  To avoid this problem, compute an ordering of the nodes where each
204   // node is only legalized after all of its operands are legalized.
205   DAG.AssignTopologicalOrder();
206   for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
207        E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
208     LegalizeOp(SDValue(&*I, 0));
209 
210   // Finally, it's possible the root changed.  Get the new root.
211   SDValue OldRoot = DAG.getRoot();
212   assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
213   DAG.setRoot(LegalizedNodes[OldRoot]);
214 
215   LegalizedNodes.clear();
216 
217   // Remove dead nodes now.
218   DAG.RemoveDeadNodes();
219 
220   return Changed;
221 }
222 
223 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
224   assert(Op->getNumValues() == Result->getNumValues() &&
225          "Unexpected number of results");
226   // Generic legalization: just pass the operand through.
227   for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
228     AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
229   return SDValue(Result, Op.getResNo());
230 }
231 
232 SDValue
233 VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
234                                             MutableArrayRef<SDValue> Results) {
235   assert(Results.size() == Op->getNumValues() &&
236          "Unexpected number of results");
237   // Make sure that the generated code is itself legal.
238   for (unsigned i = 0, e = Results.size(); i != e; ++i) {
239     Results[i] = LegalizeOp(Results[i]);
240     AddLegalizedOperand(Op.getValue(i), Results[i]);
241   }
242 
243   return Results[Op.getResNo()];
244 }
245 
246 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
247   // Note that LegalizeOp may be reentered even from single-use nodes, which
248   // means that we always must cache transformed nodes.
249   DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
250   if (I != LegalizedNodes.end()) return I->second;
251 
252   // Legalize the operands
253   SmallVector<SDValue, 8> Ops;
254   for (const SDValue &Oper : Op->op_values())
255     Ops.push_back(LegalizeOp(Oper));
256 
257   SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
258 
259   bool HasVectorValueOrOp =
260       llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
261       llvm::any_of(Node->op_values(),
262                    [](SDValue O) { return O.getValueType().isVector(); });
263   if (!HasVectorValueOrOp)
264     return TranslateLegalizeResults(Op, Node);
265 
266   TargetLowering::LegalizeAction Action = TargetLowering::Legal;
267   EVT ValVT;
268   switch (Op.getOpcode()) {
269   default:
270     return TranslateLegalizeResults(Op, Node);
271   case ISD::LOAD: {
272     LoadSDNode *LD = cast<LoadSDNode>(Node);
273     ISD::LoadExtType ExtType = LD->getExtensionType();
274     EVT LoadedVT = LD->getMemoryVT();
275     if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
276       Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
277     break;
278   }
279   case ISD::STORE: {
280     StoreSDNode *ST = cast<StoreSDNode>(Node);
281     EVT StVT = ST->getMemoryVT();
282     MVT ValVT = ST->getValue().getSimpleValueType();
283     if (StVT.isVector() && ST->isTruncatingStore())
284       Action = TLI.getTruncStoreAction(ValVT, StVT);
285     break;
286   }
287   case ISD::MERGE_VALUES:
288     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
289     // This operation lies about being legal: when it claims to be legal,
290     // it should actually be expanded.
291     if (Action == TargetLowering::Legal)
292       Action = TargetLowering::Expand;
293     break;
294 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
295   case ISD::STRICT_##DAGN:
296 #include "llvm/IR/ConstrainedOps.def"
297     ValVT = Node->getValueType(0);
298     if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
299         Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
300       ValVT = Node->getOperand(1).getValueType();
301     Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
302     // If we're asked to expand a strict vector floating-point operation,
303     // by default we're going to simply unroll it.  That is usually the
304     // best approach, except in the case where the resulting strict (scalar)
305     // operations would themselves use the fallback mutation to non-strict.
306     // In that specific case, just do the fallback on the vector op.
307     if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
308         TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
309             TargetLowering::Legal) {
310       EVT EltVT = ValVT.getVectorElementType();
311       if (TLI.getOperationAction(Node->getOpcode(), EltVT)
312           == TargetLowering::Expand &&
313           TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
314           == TargetLowering::Legal)
315         Action = TargetLowering::Legal;
316     }
317     break;
318   case ISD::ADD:
319   case ISD::SUB:
320   case ISD::MUL:
321   case ISD::MULHS:
322   case ISD::MULHU:
323   case ISD::SDIV:
324   case ISD::UDIV:
325   case ISD::SREM:
326   case ISD::UREM:
327   case ISD::SDIVREM:
328   case ISD::UDIVREM:
329   case ISD::FADD:
330   case ISD::FSUB:
331   case ISD::FMUL:
332   case ISD::FDIV:
333   case ISD::FREM:
334   case ISD::AND:
335   case ISD::OR:
336   case ISD::XOR:
337   case ISD::SHL:
338   case ISD::SRA:
339   case ISD::SRL:
340   case ISD::FSHL:
341   case ISD::FSHR:
342   case ISD::ROTL:
343   case ISD::ROTR:
344   case ISD::ABS:
345   case ISD::BSWAP:
346   case ISD::BITREVERSE:
347   case ISD::CTLZ:
348   case ISD::CTTZ:
349   case ISD::CTLZ_ZERO_UNDEF:
350   case ISD::CTTZ_ZERO_UNDEF:
351   case ISD::CTPOP:
352   case ISD::SELECT:
353   case ISD::VSELECT:
354   case ISD::SELECT_CC:
355   case ISD::ZERO_EXTEND:
356   case ISD::ANY_EXTEND:
357   case ISD::TRUNCATE:
358   case ISD::SIGN_EXTEND:
359   case ISD::FP_TO_SINT:
360   case ISD::FP_TO_UINT:
361   case ISD::FNEG:
362   case ISD::FABS:
363   case ISD::FMINNUM:
364   case ISD::FMAXNUM:
365   case ISD::FMINNUM_IEEE:
366   case ISD::FMAXNUM_IEEE:
367   case ISD::FMINIMUM:
368   case ISD::FMAXIMUM:
369   case ISD::FCOPYSIGN:
370   case ISD::FSQRT:
371   case ISD::FSIN:
372   case ISD::FCOS:
373   case ISD::FPOWI:
374   case ISD::FPOW:
375   case ISD::FLOG:
376   case ISD::FLOG2:
377   case ISD::FLOG10:
378   case ISD::FEXP:
379   case ISD::FEXP2:
380   case ISD::FCEIL:
381   case ISD::FTRUNC:
382   case ISD::FRINT:
383   case ISD::FNEARBYINT:
384   case ISD::FROUND:
385   case ISD::FROUNDEVEN:
386   case ISD::FFLOOR:
387   case ISD::FP_ROUND:
388   case ISD::FP_EXTEND:
389   case ISD::FMA:
390   case ISD::SIGN_EXTEND_INREG:
391   case ISD::ANY_EXTEND_VECTOR_INREG:
392   case ISD::SIGN_EXTEND_VECTOR_INREG:
393   case ISD::ZERO_EXTEND_VECTOR_INREG:
394   case ISD::SMIN:
395   case ISD::SMAX:
396   case ISD::UMIN:
397   case ISD::UMAX:
398   case ISD::SMUL_LOHI:
399   case ISD::UMUL_LOHI:
400   case ISD::SADDO:
401   case ISD::UADDO:
402   case ISD::SSUBO:
403   case ISD::USUBO:
404   case ISD::SMULO:
405   case ISD::UMULO:
406   case ISD::FCANONICALIZE:
407   case ISD::SADDSAT:
408   case ISD::UADDSAT:
409   case ISD::SSUBSAT:
410   case ISD::USUBSAT:
411   case ISD::SSHLSAT:
412   case ISD::USHLSAT:
413   case ISD::FP_TO_SINT_SAT:
414   case ISD::FP_TO_UINT_SAT:
415   case ISD::MGATHER:
416     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
417     break;
418   case ISD::SMULFIX:
419   case ISD::SMULFIXSAT:
420   case ISD::UMULFIX:
421   case ISD::UMULFIXSAT:
422   case ISD::SDIVFIX:
423   case ISD::SDIVFIXSAT:
424   case ISD::UDIVFIX:
425   case ISD::UDIVFIXSAT: {
426     unsigned Scale = Node->getConstantOperandVal(2);
427     Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
428                                               Node->getValueType(0), Scale);
429     break;
430   }
431   case ISD::SINT_TO_FP:
432   case ISD::UINT_TO_FP:
433   case ISD::VECREDUCE_ADD:
434   case ISD::VECREDUCE_MUL:
435   case ISD::VECREDUCE_AND:
436   case ISD::VECREDUCE_OR:
437   case ISD::VECREDUCE_XOR:
438   case ISD::VECREDUCE_SMAX:
439   case ISD::VECREDUCE_SMIN:
440   case ISD::VECREDUCE_UMAX:
441   case ISD::VECREDUCE_UMIN:
442   case ISD::VECREDUCE_FADD:
443   case ISD::VECREDUCE_FMUL:
444   case ISD::VECREDUCE_FMAX:
445   case ISD::VECREDUCE_FMIN:
446     Action = TLI.getOperationAction(Node->getOpcode(),
447                                     Node->getOperand(0).getValueType());
448     break;
449   case ISD::VECREDUCE_SEQ_FADD:
450   case ISD::VECREDUCE_SEQ_FMUL:
451     Action = TLI.getOperationAction(Node->getOpcode(),
452                                     Node->getOperand(1).getValueType());
453     break;
454   case ISD::SETCC: {
455     MVT OpVT = Node->getOperand(0).getSimpleValueType();
456     ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
457     Action = TLI.getCondCodeAction(CCCode, OpVT);
458     if (Action == TargetLowering::Legal)
459       Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
460     break;
461   }
462 
463 #define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...)                          \
464   case ISD::VPID: {                                                            \
465     EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS))        \
466                                   : Node->getOperand(LEGALPOS).getValueType(); \
467     Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT);            \
468   } break;
469 #include "llvm/IR/VPIntrinsics.def"
470   }
471 
472   LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
473 
474   SmallVector<SDValue, 8> ResultVals;
475   switch (Action) {
476   default: llvm_unreachable("This action is not supported yet!");
477   case TargetLowering::Promote:
478     assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
479            "This action is not supported yet!");
480     LLVM_DEBUG(dbgs() << "Promoting\n");
481     Promote(Node, ResultVals);
482     assert(!ResultVals.empty() && "No results for promotion?");
483     break;
484   case TargetLowering::Legal:
485     LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
486     break;
487   case TargetLowering::Custom:
488     LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
489     if (LowerOperationWrapper(Node, ResultVals))
490       break;
491     LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
492     LLVM_FALLTHROUGH;
493   case TargetLowering::Expand:
494     LLVM_DEBUG(dbgs() << "Expanding\n");
495     Expand(Node, ResultVals);
496     break;
497   }
498 
499   if (ResultVals.empty())
500     return TranslateLegalizeResults(Op, Node);
501 
502   Changed = true;
503   return RecursivelyLegalizeResults(Op, ResultVals);
504 }
505 
506 // FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
507 // merge them somehow?
508 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
509                                             SmallVectorImpl<SDValue> &Results) {
510   SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
511 
512   if (!Res.getNode())
513     return false;
514 
515   if (Res == SDValue(Node, 0))
516     return true;
517 
518   // If the original node has one result, take the return value from
519   // LowerOperation as is. It might not be result number 0.
520   if (Node->getNumValues() == 1) {
521     Results.push_back(Res);
522     return true;
523   }
524 
525   // If the original node has multiple results, then the return node should
526   // have the same number of results.
527   assert((Node->getNumValues() == Res->getNumValues()) &&
528          "Lowering returned the wrong number of results!");
529 
530   // Places new result values base on N result number.
531   for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
532     Results.push_back(Res.getValue(I));
533 
534   return true;
535 }
536 
537 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
538   // For a few operations there is a specific concept for promotion based on
539   // the operand's type.
540   switch (Node->getOpcode()) {
541   case ISD::SINT_TO_FP:
542   case ISD::UINT_TO_FP:
543   case ISD::STRICT_SINT_TO_FP:
544   case ISD::STRICT_UINT_TO_FP:
545     // "Promote" the operation by extending the operand.
546     PromoteINT_TO_FP(Node, Results);
547     return;
548   case ISD::FP_TO_UINT:
549   case ISD::FP_TO_SINT:
550   case ISD::STRICT_FP_TO_UINT:
551   case ISD::STRICT_FP_TO_SINT:
552     // Promote the operation by extending the operand.
553     PromoteFP_TO_INT(Node, Results);
554     return;
555   case ISD::FP_ROUND:
556   case ISD::FP_EXTEND:
557     // These operations are used to do promotion so they can't be promoted
558     // themselves.
559     llvm_unreachable("Don't know how to promote this operation!");
560   }
561 
562   // There are currently two cases of vector promotion:
563   // 1) Bitcasting a vector of integers to a different type to a vector of the
564   //    same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
565   // 2) Extending a vector of floats to a vector of the same number of larger
566   //    floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
567   assert(Node->getNumValues() == 1 &&
568          "Can't promote a vector with multiple results!");
569   MVT VT = Node->getSimpleValueType(0);
570   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
571   SDLoc dl(Node);
572   SmallVector<SDValue, 4> Operands(Node->getNumOperands());
573 
574   for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
575     if (Node->getOperand(j).getValueType().isVector())
576       if (Node->getOperand(j)
577               .getValueType()
578               .getVectorElementType()
579               .isFloatingPoint() &&
580           NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
581         Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
582       else
583         Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
584     else
585       Operands[j] = Node->getOperand(j);
586   }
587 
588   SDValue Res =
589       DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
590 
591   if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
592       (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
593        NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
594     Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res, DAG.getIntPtrConstant(0, dl));
595   else
596     Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
597 
598   Results.push_back(Res);
599 }
600 
601 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
602                                        SmallVectorImpl<SDValue> &Results) {
603   // INT_TO_FP operations may require the input operand be promoted even
604   // when the type is otherwise legal.
605   bool IsStrict = Node->isStrictFPOpcode();
606   MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
607   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
608   assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
609          "Vectors have different number of elements!");
610 
611   SDLoc dl(Node);
612   SmallVector<SDValue, 4> Operands(Node->getNumOperands());
613 
614   unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
615                   Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
616                      ? ISD::ZERO_EXTEND
617                      : ISD::SIGN_EXTEND;
618   for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
619     if (Node->getOperand(j).getValueType().isVector())
620       Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
621     else
622       Operands[j] = Node->getOperand(j);
623   }
624 
625   if (IsStrict) {
626     SDValue Res = DAG.getNode(Node->getOpcode(), dl,
627                               {Node->getValueType(0), MVT::Other}, Operands);
628     Results.push_back(Res);
629     Results.push_back(Res.getValue(1));
630     return;
631   }
632 
633   SDValue Res =
634       DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
635   Results.push_back(Res);
636 }
637 
638 // For FP_TO_INT we promote the result type to a vector type with wider
639 // elements and then truncate the result.  This is different from the default
640 // PromoteVector which uses bitcast to promote thus assumning that the
641 // promoted vector type has the same overall size.
642 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
643                                        SmallVectorImpl<SDValue> &Results) {
644   MVT VT = Node->getSimpleValueType(0);
645   MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
646   bool IsStrict = Node->isStrictFPOpcode();
647   assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
648          "Vectors have different number of elements!");
649 
650   unsigned NewOpc = Node->getOpcode();
651   // Change FP_TO_UINT to FP_TO_SINT if possible.
652   // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
653   if (NewOpc == ISD::FP_TO_UINT &&
654       TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
655     NewOpc = ISD::FP_TO_SINT;
656 
657   if (NewOpc == ISD::STRICT_FP_TO_UINT &&
658       TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
659     NewOpc = ISD::STRICT_FP_TO_SINT;
660 
661   SDLoc dl(Node);
662   SDValue Promoted, Chain;
663   if (IsStrict) {
664     Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
665                            {Node->getOperand(0), Node->getOperand(1)});
666     Chain = Promoted.getValue(1);
667   } else
668     Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
669 
670   // Assert that the converted value fits in the original type.  If it doesn't
671   // (eg: because the value being converted is too big), then the result of the
672   // original operation was undefined anyway, so the assert is still correct.
673   if (Node->getOpcode() == ISD::FP_TO_UINT ||
674       Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
675     NewOpc = ISD::AssertZext;
676   else
677     NewOpc = ISD::AssertSext;
678 
679   Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
680                          DAG.getValueType(VT.getScalarType()));
681   Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
682   Results.push_back(Promoted);
683   if (IsStrict)
684     Results.push_back(Chain);
685 }
686 
687 std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
688   LoadSDNode *LD = cast<LoadSDNode>(N);
689   return TLI.scalarizeVectorLoad(LD, DAG);
690 }
691 
692 SDValue VectorLegalizer::ExpandStore(SDNode *N) {
693   StoreSDNode *ST = cast<StoreSDNode>(N);
694   SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
695   return TF;
696 }
697 
698 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
699   switch (Node->getOpcode()) {
700   case ISD::LOAD: {
701     std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
702     Results.push_back(Tmp.first);
703     Results.push_back(Tmp.second);
704     return;
705   }
706   case ISD::STORE:
707     Results.push_back(ExpandStore(Node));
708     return;
709   case ISD::MERGE_VALUES:
710     for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
711       Results.push_back(Node->getOperand(i));
712     return;
713   case ISD::SIGN_EXTEND_INREG:
714     Results.push_back(ExpandSEXTINREG(Node));
715     return;
716   case ISD::ANY_EXTEND_VECTOR_INREG:
717     Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
718     return;
719   case ISD::SIGN_EXTEND_VECTOR_INREG:
720     Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
721     return;
722   case ISD::ZERO_EXTEND_VECTOR_INREG:
723     Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
724     return;
725   case ISD::BSWAP:
726     Results.push_back(ExpandBSWAP(Node));
727     return;
728   case ISD::VSELECT:
729     Results.push_back(ExpandVSELECT(Node));
730     return;
731   case ISD::VP_SELECT:
732     Results.push_back(ExpandVP_SELECT(Node));
733     return;
734   case ISD::SELECT:
735     Results.push_back(ExpandSELECT(Node));
736     return;
737   case ISD::FP_TO_UINT:
738     ExpandFP_TO_UINT(Node, Results);
739     return;
740   case ISD::UINT_TO_FP:
741     ExpandUINT_TO_FLOAT(Node, Results);
742     return;
743   case ISD::FNEG:
744     Results.push_back(ExpandFNEG(Node));
745     return;
746   case ISD::FSUB:
747     ExpandFSUB(Node, Results);
748     return;
749   case ISD::SETCC:
750     ExpandSETCC(Node, Results);
751     return;
752   case ISD::ABS:
753     if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
754       Results.push_back(Expanded);
755       return;
756     }
757     break;
758   case ISD::BITREVERSE:
759     ExpandBITREVERSE(Node, Results);
760     return;
761   case ISD::CTPOP:
762     if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
763       Results.push_back(Expanded);
764       return;
765     }
766     break;
767   case ISD::CTLZ:
768   case ISD::CTLZ_ZERO_UNDEF:
769     if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
770       Results.push_back(Expanded);
771       return;
772     }
773     break;
774   case ISD::CTTZ:
775   case ISD::CTTZ_ZERO_UNDEF:
776     if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
777       Results.push_back(Expanded);
778       return;
779     }
780     break;
781   case ISD::FSHL:
782   case ISD::FSHR:
783     if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
784       Results.push_back(Expanded);
785       return;
786     }
787     break;
788   case ISD::ROTL:
789   case ISD::ROTR:
790     if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
791       Results.push_back(Expanded);
792       return;
793     }
794     break;
795   case ISD::FMINNUM:
796   case ISD::FMAXNUM:
797     if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
798       Results.push_back(Expanded);
799       return;
800     }
801     break;
802   case ISD::SMIN:
803   case ISD::SMAX:
804   case ISD::UMIN:
805   case ISD::UMAX:
806     if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
807       Results.push_back(Expanded);
808       return;
809     }
810     break;
811   case ISD::UADDO:
812   case ISD::USUBO:
813     ExpandUADDSUBO(Node, Results);
814     return;
815   case ISD::SADDO:
816   case ISD::SSUBO:
817     ExpandSADDSUBO(Node, Results);
818     return;
819   case ISD::UMULO:
820   case ISD::SMULO:
821     ExpandMULO(Node, Results);
822     return;
823   case ISD::USUBSAT:
824   case ISD::SSUBSAT:
825   case ISD::UADDSAT:
826   case ISD::SADDSAT:
827     if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
828       Results.push_back(Expanded);
829       return;
830     }
831     break;
832   case ISD::SMULFIX:
833   case ISD::UMULFIX:
834     if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
835       Results.push_back(Expanded);
836       return;
837     }
838     break;
839   case ISD::SMULFIXSAT:
840   case ISD::UMULFIXSAT:
841     // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
842     // why. Maybe it results in worse codegen compared to the unroll for some
843     // targets? This should probably be investigated. And if we still prefer to
844     // unroll an explanation could be helpful.
845     break;
846   case ISD::SDIVFIX:
847   case ISD::UDIVFIX:
848     ExpandFixedPointDiv(Node, Results);
849     return;
850   case ISD::SDIVFIXSAT:
851   case ISD::UDIVFIXSAT:
852     break;
853 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
854   case ISD::STRICT_##DAGN:
855 #include "llvm/IR/ConstrainedOps.def"
856     ExpandStrictFPOp(Node, Results);
857     return;
858   case ISD::VECREDUCE_ADD:
859   case ISD::VECREDUCE_MUL:
860   case ISD::VECREDUCE_AND:
861   case ISD::VECREDUCE_OR:
862   case ISD::VECREDUCE_XOR:
863   case ISD::VECREDUCE_SMAX:
864   case ISD::VECREDUCE_SMIN:
865   case ISD::VECREDUCE_UMAX:
866   case ISD::VECREDUCE_UMIN:
867   case ISD::VECREDUCE_FADD:
868   case ISD::VECREDUCE_FMUL:
869   case ISD::VECREDUCE_FMAX:
870   case ISD::VECREDUCE_FMIN:
871     Results.push_back(TLI.expandVecReduce(Node, DAG));
872     return;
873   case ISD::VECREDUCE_SEQ_FADD:
874   case ISD::VECREDUCE_SEQ_FMUL:
875     Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
876     return;
877   case ISD::SREM:
878   case ISD::UREM:
879     ExpandREM(Node, Results);
880     return;
881   case ISD::VP_MERGE:
882     Results.push_back(ExpandVP_MERGE(Node));
883     return;
884   }
885 
886   Results.push_back(DAG.UnrollVectorOp(Node));
887 }
888 
889 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
890   // Lower a select instruction where the condition is a scalar and the
891   // operands are vectors. Lower this select to VSELECT and implement it
892   // using XOR AND OR. The selector bit is broadcasted.
893   EVT VT = Node->getValueType(0);
894   SDLoc DL(Node);
895 
896   SDValue Mask = Node->getOperand(0);
897   SDValue Op1 = Node->getOperand(1);
898   SDValue Op2 = Node->getOperand(2);
899 
900   assert(VT.isVector() && !Mask.getValueType().isVector()
901          && Op1.getValueType() == Op2.getValueType() && "Invalid type");
902 
903   // If we can't even use the basic vector operations of
904   // AND,OR,XOR, we will have to scalarize the op.
905   // Notice that the operation may be 'promoted' which means that it is
906   // 'bitcasted' to another type which is handled.
907   // Also, we need to be able to construct a splat vector using either
908   // BUILD_VECTOR or SPLAT_VECTOR.
909   // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
910   // BUILD_VECTOR?
911   if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
912       TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
913       TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
914       TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
915                                                       : ISD::SPLAT_VECTOR,
916                              VT) == TargetLowering::Expand)
917     return DAG.UnrollVectorOp(Node);
918 
919   // Generate a mask operand.
920   EVT MaskTy = VT.changeVectorElementTypeToInteger();
921 
922   // What is the size of each element in the vector mask.
923   EVT BitTy = MaskTy.getScalarType();
924 
925   Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
926                        DAG.getConstant(0, DL, BitTy));
927 
928   // Broadcast the mask so that the entire vector is all one or all zero.
929   if (VT.isFixedLengthVector())
930     Mask = DAG.getSplatBuildVector(MaskTy, DL, Mask);
931   else
932     Mask = DAG.getSplatVector(MaskTy, DL, Mask);
933 
934   // Bitcast the operands to be the same type as the mask.
935   // This is needed when we select between FP types because
936   // the mask is a vector of integers.
937   Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
938   Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
939 
940   SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
941 
942   Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
943   Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
944   SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
945   return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
946 }
947 
948 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
949   EVT VT = Node->getValueType(0);
950 
951   // Make sure that the SRA and SHL instructions are available.
952   if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
953       TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
954     return DAG.UnrollVectorOp(Node);
955 
956   SDLoc DL(Node);
957   EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
958 
959   unsigned BW = VT.getScalarSizeInBits();
960   unsigned OrigBW = OrigTy.getScalarSizeInBits();
961   SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
962 
963   SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
964   return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
965 }
966 
967 // Generically expand a vector anyext in register to a shuffle of the relevant
968 // lanes into the appropriate locations, with other lanes left undef.
969 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
970   SDLoc DL(Node);
971   EVT VT = Node->getValueType(0);
972   int NumElements = VT.getVectorNumElements();
973   SDValue Src = Node->getOperand(0);
974   EVT SrcVT = Src.getValueType();
975   int NumSrcElements = SrcVT.getVectorNumElements();
976 
977   // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
978   // into a larger vector type.
979   if (SrcVT.bitsLE(VT)) {
980     assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
981            "ANY_EXTEND_VECTOR_INREG vector size mismatch");
982     NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
983     SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
984                              NumSrcElements);
985     Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
986                       Src, DAG.getVectorIdxConstant(0, DL));
987   }
988 
989   // Build a base mask of undef shuffles.
990   SmallVector<int, 16> ShuffleMask;
991   ShuffleMask.resize(NumSrcElements, -1);
992 
993   // Place the extended lanes into the correct locations.
994   int ExtLaneScale = NumSrcElements / NumElements;
995   int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
996   for (int i = 0; i < NumElements; ++i)
997     ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
998 
999   return DAG.getNode(
1000       ISD::BITCAST, DL, VT,
1001       DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1002 }
1003 
1004 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1005   SDLoc DL(Node);
1006   EVT VT = Node->getValueType(0);
1007   SDValue Src = Node->getOperand(0);
1008   EVT SrcVT = Src.getValueType();
1009 
1010   // First build an any-extend node which can be legalized above when we
1011   // recurse through it.
1012   SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1013 
1014   // Now we need sign extend. Do this by shifting the elements. Even if these
1015   // aren't legal operations, they have a better chance of being legalized
1016   // without full scalarization than the sign extension does.
1017   unsigned EltWidth = VT.getScalarSizeInBits();
1018   unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1019   SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1020   return DAG.getNode(ISD::SRA, DL, VT,
1021                      DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1022                      ShiftAmount);
1023 }
1024 
1025 // Generically expand a vector zext in register to a shuffle of the relevant
1026 // lanes into the appropriate locations, a blend of zero into the high bits,
1027 // and a bitcast to the wider element type.
1028 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1029   SDLoc DL(Node);
1030   EVT VT = Node->getValueType(0);
1031   int NumElements = VT.getVectorNumElements();
1032   SDValue Src = Node->getOperand(0);
1033   EVT SrcVT = Src.getValueType();
1034   int NumSrcElements = SrcVT.getVectorNumElements();
1035 
1036   // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1037   // into a larger vector type.
1038   if (SrcVT.bitsLE(VT)) {
1039     assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1040            "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1041     NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1042     SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1043                              NumSrcElements);
1044     Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1045                       Src, DAG.getVectorIdxConstant(0, DL));
1046   }
1047 
1048   // Build up a zero vector to blend into this one.
1049   SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1050 
1051   // Shuffle the incoming lanes into the correct position, and pull all other
1052   // lanes from the zero vector.
1053   SmallVector<int, 16> ShuffleMask;
1054   ShuffleMask.reserve(NumSrcElements);
1055   for (int i = 0; i < NumSrcElements; ++i)
1056     ShuffleMask.push_back(i);
1057 
1058   int ExtLaneScale = NumSrcElements / NumElements;
1059   int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1060   for (int i = 0; i < NumElements; ++i)
1061     ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1062 
1063   return DAG.getNode(ISD::BITCAST, DL, VT,
1064                      DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1065 }
1066 
1067 static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1068   int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1069   for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1070     for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1071       ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1072 }
1073 
1074 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1075   EVT VT = Node->getValueType(0);
1076 
1077   // Scalable vectors can't use shuffle expansion.
1078   if (VT.isScalableVector())
1079     return TLI.expandBSWAP(Node, DAG);
1080 
1081   // Generate a byte wise shuffle mask for the BSWAP.
1082   SmallVector<int, 16> ShuffleMask;
1083   createBSWAPShuffleMask(VT, ShuffleMask);
1084   EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1085 
1086   // Only emit a shuffle if the mask is legal.
1087   if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1088     SDLoc DL(Node);
1089     SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1090     Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1091     return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1092   }
1093 
1094   // If we have the appropriate vector bit operations, it is better to use them
1095   // than unrolling and expanding each component.
1096   if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1097       TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1098       TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1099       TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1100     return TLI.expandBSWAP(Node, DAG);
1101 
1102   // Otherwise unroll.
1103   return DAG.UnrollVectorOp(Node);
1104 }
1105 
1106 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1107                                        SmallVectorImpl<SDValue> &Results) {
1108   EVT VT = Node->getValueType(0);
1109 
1110   // We can't unroll or use shuffles for scalable vectors.
1111   if (VT.isScalableVector()) {
1112     Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1113     return;
1114   }
1115 
1116   // If we have the scalar operation, it's probably cheaper to unroll it.
1117   if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1118     SDValue Tmp = DAG.UnrollVectorOp(Node);
1119     Results.push_back(Tmp);
1120     return;
1121   }
1122 
1123   // If the vector element width is a whole number of bytes, test if its legal
1124   // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1125   // vector. This greatly reduces the number of bit shifts necessary.
1126   unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1127   if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1128     SmallVector<int, 16> BSWAPMask;
1129     createBSWAPShuffleMask(VT, BSWAPMask);
1130 
1131     EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1132     if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1133         (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1134          (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1135           TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1136           TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1137           TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1138       SDLoc DL(Node);
1139       SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1140       Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1141                                 BSWAPMask);
1142       Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1143       Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1144       Results.push_back(Op);
1145       return;
1146     }
1147   }
1148 
1149   // If we have the appropriate vector bit operations, it is better to use them
1150   // than unrolling and expanding each component.
1151   if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1152       TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1153       TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1154       TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1155     Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1156     return;
1157   }
1158 
1159   // Otherwise unroll.
1160   SDValue Tmp = DAG.UnrollVectorOp(Node);
1161   Results.push_back(Tmp);
1162 }
1163 
1164 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1165   // Implement VSELECT in terms of XOR, AND, OR
1166   // on platforms which do not support blend natively.
1167   SDLoc DL(Node);
1168 
1169   SDValue Mask = Node->getOperand(0);
1170   SDValue Op1 = Node->getOperand(1);
1171   SDValue Op2 = Node->getOperand(2);
1172 
1173   EVT VT = Mask.getValueType();
1174 
1175   // If we can't even use the basic vector operations of
1176   // AND,OR,XOR, we will have to scalarize the op.
1177   // Notice that the operation may be 'promoted' which means that it is
1178   // 'bitcasted' to another type which is handled.
1179   if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1180       TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1181       TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1182     return DAG.UnrollVectorOp(Node);
1183 
1184   // This operation also isn't safe with AND, OR, XOR when the boolean type is
1185   // 0/1 and the select operands aren't also booleans, as we need an all-ones
1186   // vector constant to mask with.
1187   // FIXME: Sign extend 1 to all ones if that's legal on the target.
1188   auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1189   if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1190       !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1191         Op1.getValueType().getVectorElementType() == MVT::i1))
1192     return DAG.UnrollVectorOp(Node);
1193 
1194   // If the mask and the type are different sizes, unroll the vector op. This
1195   // can occur when getSetCCResultType returns something that is different in
1196   // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1197   if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1198     return DAG.UnrollVectorOp(Node);
1199 
1200   // Bitcast the operands to be the same type as the mask.
1201   // This is needed when we select between FP types because
1202   // the mask is a vector of integers.
1203   Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1204   Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1205 
1206   SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1207 
1208   Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1209   Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1210   SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1211   return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1212 }
1213 
1214 SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1215   // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1216   // do not support it natively.
1217   SDLoc DL(Node);
1218 
1219   SDValue Mask = Node->getOperand(0);
1220   SDValue Op1 = Node->getOperand(1);
1221   SDValue Op2 = Node->getOperand(2);
1222   SDValue EVL = Node->getOperand(3);
1223 
1224   EVT VT = Mask.getValueType();
1225 
1226   // If we can't even use the basic vector operations of
1227   // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1228   if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1229       TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1230       TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1231     return DAG.UnrollVectorOp(Node);
1232 
1233   // This operation also isn't safe when the operands aren't also booleans.
1234   if (Op1.getValueType().getVectorElementType() != MVT::i1)
1235     return DAG.UnrollVectorOp(Node);
1236 
1237   SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1238   SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Mask, EVL);
1239 
1240   Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Mask, EVL);
1241   Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Mask, EVL);
1242   return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Mask, EVL);
1243 }
1244 
1245 SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1246   // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1247   // indices less than the EVL/pivot are true. Combine that with the original
1248   // mask for a full-length mask. Use a full-length VSELECT to select between
1249   // the true and false values.
1250   SDLoc DL(Node);
1251 
1252   SDValue Mask = Node->getOperand(0);
1253   SDValue Op1 = Node->getOperand(1);
1254   SDValue Op2 = Node->getOperand(2);
1255   SDValue EVL = Node->getOperand(3);
1256 
1257   EVT MaskVT = Mask.getValueType();
1258   bool IsFixedLen = MaskVT.isFixedLengthVector();
1259 
1260   EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1261                                   MaskVT.getVectorElementCount());
1262 
1263   // If we can't construct the EVL mask efficiently, it's better to unroll.
1264   if ((IsFixedLen &&
1265        !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1266       (!IsFixedLen &&
1267        (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1268         !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1269     return DAG.UnrollVectorOp(Node);
1270 
1271   // If using a SETCC would result in a different type than the mask type,
1272   // unroll.
1273   if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1274                              EVLVecVT) != MaskVT)
1275     return DAG.UnrollVectorOp(Node);
1276 
1277   SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1278   SDValue SplatEVL = IsFixedLen ? DAG.getSplatBuildVector(EVLVecVT, DL, EVL)
1279                                 : DAG.getSplatVector(EVLVecVT, DL, EVL);
1280   SDValue EVLMask =
1281       DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1282 
1283   SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1284   return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1285 }
1286 
1287 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1288                                        SmallVectorImpl<SDValue> &Results) {
1289   // Attempt to expand using TargetLowering.
1290   SDValue Result, Chain;
1291   if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1292     Results.push_back(Result);
1293     if (Node->isStrictFPOpcode())
1294       Results.push_back(Chain);
1295     return;
1296   }
1297 
1298   // Otherwise go ahead and unroll.
1299   if (Node->isStrictFPOpcode()) {
1300     UnrollStrictFPOp(Node, Results);
1301     return;
1302   }
1303 
1304   Results.push_back(DAG.UnrollVectorOp(Node));
1305 }
1306 
1307 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1308                                           SmallVectorImpl<SDValue> &Results) {
1309   bool IsStrict = Node->isStrictFPOpcode();
1310   unsigned OpNo = IsStrict ? 1 : 0;
1311   SDValue Src = Node->getOperand(OpNo);
1312   EVT VT = Src.getValueType();
1313   SDLoc DL(Node);
1314 
1315   // Attempt to expand using TargetLowering.
1316   SDValue Result;
1317   SDValue Chain;
1318   if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1319     Results.push_back(Result);
1320     if (IsStrict)
1321       Results.push_back(Chain);
1322     return;
1323   }
1324 
1325   // Make sure that the SINT_TO_FP and SRL instructions are available.
1326   if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1327                          TargetLowering::Expand) ||
1328        (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1329                         TargetLowering::Expand)) ||
1330       TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1331     if (IsStrict) {
1332       UnrollStrictFPOp(Node, Results);
1333       return;
1334     }
1335 
1336     Results.push_back(DAG.UnrollVectorOp(Node));
1337     return;
1338   }
1339 
1340   unsigned BW = VT.getScalarSizeInBits();
1341   assert((BW == 64 || BW == 32) &&
1342          "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1343 
1344   SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1345 
1346   // Constants to clear the upper part of the word.
1347   // Notice that we can also use SHL+SHR, but using a constant is slightly
1348   // faster on x86.
1349   uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1350   SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1351 
1352   // Two to the power of half-word-size.
1353   SDValue TWOHW =
1354       DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1355 
1356   // Clear upper part of LO, lower HI
1357   SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1358   SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1359 
1360   if (IsStrict) {
1361     // Convert hi and lo to floats
1362     // Convert the hi part back to the upper values
1363     // TODO: Can any fast-math-flags be set on these nodes?
1364     SDValue fHI = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL,
1365                               {Node->getValueType(0), MVT::Other},
1366                               {Node->getOperand(0), HI});
1367     fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1368                       {fHI.getValue(1), fHI, TWOHW});
1369     SDValue fLO = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL,
1370                               {Node->getValueType(0), MVT::Other},
1371                               {Node->getOperand(0), LO});
1372 
1373     SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1374                              fLO.getValue(1));
1375 
1376     // Add the two halves
1377     SDValue Result =
1378         DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1379                     {TF, fHI, fLO});
1380 
1381     Results.push_back(Result);
1382     Results.push_back(Result.getValue(1));
1383     return;
1384   }
1385 
1386   // Convert hi and lo to floats
1387   // Convert the hi part back to the upper values
1388   // TODO: Can any fast-math-flags be set on these nodes?
1389   SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1390   fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1391   SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1392 
1393   // Add the two halves
1394   Results.push_back(
1395       DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1396 }
1397 
1398 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1399   if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1400     SDLoc DL(Node);
1401     SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1402     // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1403     return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1404                        Node->getOperand(0));
1405   }
1406   return DAG.UnrollVectorOp(Node);
1407 }
1408 
1409 void VectorLegalizer::ExpandFSUB(SDNode *Node,
1410                                  SmallVectorImpl<SDValue> &Results) {
1411   // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1412   // we can defer this to operation legalization where it will be lowered as
1413   // a+(-b).
1414   EVT VT = Node->getValueType(0);
1415   if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1416       TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1417     return; // Defer to LegalizeDAG
1418 
1419   SDValue Tmp = DAG.UnrollVectorOp(Node);
1420   Results.push_back(Tmp);
1421 }
1422 
1423 void VectorLegalizer::ExpandSETCC(SDNode *Node,
1424                                   SmallVectorImpl<SDValue> &Results) {
1425   bool NeedInvert = false;
1426   SDLoc dl(Node);
1427   MVT OpVT = Node->getOperand(0).getSimpleValueType();
1428   ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
1429 
1430   if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1431     Results.push_back(UnrollVSETCC(Node));
1432     return;
1433   }
1434 
1435   SDValue Chain;
1436   SDValue LHS = Node->getOperand(0);
1437   SDValue RHS = Node->getOperand(1);
1438   SDValue CC = Node->getOperand(2);
1439   bool Legalized = TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS,
1440                                              RHS, CC, NeedInvert, dl, Chain);
1441 
1442   if (Legalized) {
1443     // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1444     // condition code, create a new SETCC node.
1445     if (CC.getNode())
1446       LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1447                         Node->getFlags());
1448 
1449     // If we expanded the SETCC by inverting the condition code, then wrap
1450     // the existing SETCC in a NOT to restore the intended condition.
1451     if (NeedInvert)
1452       LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1453   } else {
1454     // Otherwise, SETCC for the given comparison type must be completely
1455     // illegal; expand it into a SELECT_CC.
1456     EVT VT = Node->getValueType(0);
1457     LHS =
1458         DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1459                     DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1460                     DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1461     LHS->setFlags(Node->getFlags());
1462   }
1463 
1464   Results.push_back(LHS);
1465 }
1466 
1467 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1468                                      SmallVectorImpl<SDValue> &Results) {
1469   SDValue Result, Overflow;
1470   TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1471   Results.push_back(Result);
1472   Results.push_back(Overflow);
1473 }
1474 
1475 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1476                                      SmallVectorImpl<SDValue> &Results) {
1477   SDValue Result, Overflow;
1478   TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1479   Results.push_back(Result);
1480   Results.push_back(Overflow);
1481 }
1482 
1483 void VectorLegalizer::ExpandMULO(SDNode *Node,
1484                                  SmallVectorImpl<SDValue> &Results) {
1485   SDValue Result, Overflow;
1486   if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1487     std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1488 
1489   Results.push_back(Result);
1490   Results.push_back(Overflow);
1491 }
1492 
1493 void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1494                                           SmallVectorImpl<SDValue> &Results) {
1495   SDNode *N = Node;
1496   if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1497           N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1498     Results.push_back(Expanded);
1499 }
1500 
1501 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1502                                        SmallVectorImpl<SDValue> &Results) {
1503   if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1504     ExpandUINT_TO_FLOAT(Node, Results);
1505     return;
1506   }
1507   if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1508     ExpandFP_TO_UINT(Node, Results);
1509     return;
1510   }
1511 
1512   UnrollStrictFPOp(Node, Results);
1513 }
1514 
1515 void VectorLegalizer::ExpandREM(SDNode *Node,
1516                                 SmallVectorImpl<SDValue> &Results) {
1517   assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1518          "Expected REM node");
1519 
1520   SDValue Result;
1521   if (!TLI.expandREM(Node, Result, DAG))
1522     Result = DAG.UnrollVectorOp(Node);
1523   Results.push_back(Result);
1524 }
1525 
1526 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1527                                        SmallVectorImpl<SDValue> &Results) {
1528   EVT VT = Node->getValueType(0);
1529   EVT EltVT = VT.getVectorElementType();
1530   unsigned NumElems = VT.getVectorNumElements();
1531   unsigned NumOpers = Node->getNumOperands();
1532   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1533 
1534   EVT TmpEltVT = EltVT;
1535   if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1536       Node->getOpcode() == ISD::STRICT_FSETCCS)
1537     TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1538                                       *DAG.getContext(), TmpEltVT);
1539 
1540   EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1541   SDValue Chain = Node->getOperand(0);
1542   SDLoc dl(Node);
1543 
1544   SmallVector<SDValue, 32> OpValues;
1545   SmallVector<SDValue, 32> OpChains;
1546   for (unsigned i = 0; i < NumElems; ++i) {
1547     SmallVector<SDValue, 4> Opers;
1548     SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1549 
1550     // The Chain is the first operand.
1551     Opers.push_back(Chain);
1552 
1553     // Now process the remaining operands.
1554     for (unsigned j = 1; j < NumOpers; ++j) {
1555       SDValue Oper = Node->getOperand(j);
1556       EVT OperVT = Oper.getValueType();
1557 
1558       if (OperVT.isVector())
1559         Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1560                            OperVT.getVectorElementType(), Oper, Idx);
1561 
1562       Opers.push_back(Oper);
1563     }
1564 
1565     SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1566     SDValue ScalarResult = ScalarOp.getValue(0);
1567     SDValue ScalarChain = ScalarOp.getValue(1);
1568 
1569     if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1570         Node->getOpcode() == ISD::STRICT_FSETCCS)
1571       ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1572                                    DAG.getAllOnesConstant(dl, EltVT),
1573                                    DAG.getConstant(0, dl, EltVT));
1574 
1575     OpValues.push_back(ScalarResult);
1576     OpChains.push_back(ScalarChain);
1577   }
1578 
1579   SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1580   SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1581 
1582   Results.push_back(Result);
1583   Results.push_back(NewChain);
1584 }
1585 
1586 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1587   EVT VT = Node->getValueType(0);
1588   unsigned NumElems = VT.getVectorNumElements();
1589   EVT EltVT = VT.getVectorElementType();
1590   SDValue LHS = Node->getOperand(0);
1591   SDValue RHS = Node->getOperand(1);
1592   SDValue CC = Node->getOperand(2);
1593   EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1594   SDLoc dl(Node);
1595   SmallVector<SDValue, 8> Ops(NumElems);
1596   for (unsigned i = 0; i < NumElems; ++i) {
1597     SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1598                                   DAG.getVectorIdxConstant(i, dl));
1599     SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1600                                   DAG.getVectorIdxConstant(i, dl));
1601     Ops[i] = DAG.getNode(ISD::SETCC, dl,
1602                          TLI.getSetCCResultType(DAG.getDataLayout(),
1603                                                 *DAG.getContext(), TmpEltVT),
1604                          LHSElem, RHSElem, CC);
1605     Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1606                            DAG.getConstant(0, dl, EltVT));
1607   }
1608   return DAG.getBuildVector(VT, dl, Ops);
1609 }
1610 
1611 bool SelectionDAG::LegalizeVectors() {
1612   return VectorLegalizer(*this).Run();
1613 }
1614