Lines Matching refs:Node
90 SDValue UnrollVSETCC(SDNode *Node);
96 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
100 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
104 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
107 SDValue ExpandSEXTINREG(SDNode *Node);
114 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
121 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
127 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
130 SDValue ExpandBSWAP(SDNode *Node);
134 SDValue ExpandVSELECT(SDNode *Node);
135 SDValue ExpandVP_SELECT(SDNode *Node);
136 SDValue ExpandVP_MERGE(SDNode *Node);
137 SDValue ExpandVP_REM(SDNode *Node);
138 SDValue ExpandSELECT(SDNode *Node);
141 SDValue ExpandFNEG(SDNode *Node);
142 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
150 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
152 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
154 bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall Call_F32,
160 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
166 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
171 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
177 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
183 void PromoteSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
185 void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
273 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops); in LegalizeOp() local
276 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) || in LegalizeOp()
277 llvm::any_of(Node->op_values(), in LegalizeOp()
280 return TranslateLegalizeResults(Op, Node); in LegalizeOp()
286 return TranslateLegalizeResults(Op, Node); in LegalizeOp()
288 LoadSDNode *LD = cast<LoadSDNode>(Node); in LegalizeOp()
296 StoreSDNode *ST = cast<StoreSDNode>(Node); in LegalizeOp()
304 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
313 ValVT = Node->getValueType(0); in LegalizeOp()
316 ValVT = Node->getOperand(1).getValueType(); in LegalizeOp()
319 MVT OpVT = Node->getOperand(1).getSimpleValueType(); in LegalizeOp()
320 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get(); in LegalizeOp()
323 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
325 Action = TLI.getOperationAction(Node->getOpcode(), ValVT); in LegalizeOp()
333 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) == in LegalizeOp()
336 if (TLI.getOperationAction(Node->getOpcode(), EltVT) in LegalizeOp()
338 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT) in LegalizeOp()
461 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
471 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
472 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
473 Node->getValueType(0), Scale); in LegalizeOp()
495 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
496 Node->getOperand(0).getValueType()); in LegalizeOp()
500 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
501 Node->getOperand(1).getValueType()); in LegalizeOp()
504 MVT OpVT = Node->getOperand(0).getSimpleValueType(); in LegalizeOp()
505 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in LegalizeOp()
508 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
514 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \ in LegalizeOp()
515 : Node->getOperand(LEGALPOS).getValueType(); \ in LegalizeOp()
517 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \ in LegalizeOp()
523 if (!Node->getValueType(0).isVector() && \ in LegalizeOp()
524 Node->getValueType(0) != MVT::Other) { \ in LegalizeOp()
528 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \ in LegalizeOp()
533 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG)); in LegalizeOp()
542 Promote(Node, ResultVals); in LegalizeOp()
550 if (LowerOperationWrapper(Node, ResultVals)) in LegalizeOp()
556 Expand(Node, ResultVals); in LegalizeOp()
561 return TranslateLegalizeResults(Op, Node); in LegalizeOp()
569 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node, in LowerOperationWrapper() argument
571 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LowerOperationWrapper()
576 if (Res == SDValue(Node, 0)) in LowerOperationWrapper()
581 if (Node->getNumValues() == 1) { in LowerOperationWrapper()
588 assert((Node->getNumValues() == Res->getNumValues()) && in LowerOperationWrapper()
592 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I) in LowerOperationWrapper()
598 void VectorLegalizer::PromoteSETCC(SDNode *Node, in PromoteSETCC() argument
600 MVT VecVT = Node->getOperand(0).getSimpleValueType(); in PromoteSETCC()
601 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteSETCC()
605 SDLoc DL(Node); in PromoteSETCC()
606 SmallVector<SDValue, 5> Operands(Node->getNumOperands()); in PromoteSETCC()
608 Operands[0] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(0)); in PromoteSETCC()
609 Operands[1] = DAG.getNode(ExtOp, DL, NewVecVT, Node->getOperand(1)); in PromoteSETCC()
610 Operands[2] = Node->getOperand(2); in PromoteSETCC()
612 if (Node->getOpcode() == ISD::VP_SETCC) { in PromoteSETCC()
613 Operands[3] = Node->getOperand(3); // mask in PromoteSETCC()
614 Operands[4] = Node->getOperand(4); // evl in PromoteSETCC()
617 SDValue Res = DAG.getNode(Node->getOpcode(), DL, Node->getSimpleValueType(0), in PromoteSETCC()
618 Operands, Node->getFlags()); in PromoteSETCC()
623 void VectorLegalizer::PromoteSTRICT(SDNode *Node, in PromoteSTRICT() argument
625 MVT VecVT = Node->getOperand(1).getSimpleValueType(); in PromoteSTRICT()
626 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteSTRICT()
630 SDLoc DL(Node); in PromoteSTRICT()
631 SmallVector<SDValue, 5> Operands(Node->getNumOperands()); in PromoteSTRICT()
634 for (unsigned j = 1; j != Node->getNumOperands(); ++j) in PromoteSTRICT()
635 if (Node->getOperand(j).getValueType().isVector() && in PromoteSTRICT()
636 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteSTRICT()
637 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand. in PromoteSTRICT()
642 {Node->getOperand(0), Node->getOperand(j)}); in PromoteSTRICT()
646 Operands[j] = Node->getOperand(j); // Skip no vector operand. in PromoteSTRICT()
648 SDVTList VTs = DAG.getVTList(NewVecVT, Node->getValueType(1)); in PromoteSTRICT()
653 DAG.getNode(Node->getOpcode(), DL, VTs, Operands, Node->getFlags()); in PromoteSTRICT()
664 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) { in Promote() argument
667 switch (Node->getOpcode()) { in Promote()
673 PromoteINT_TO_FP(Node, Results); in Promote()
680 PromoteFP_TO_INT(Node, Results); in Promote()
685 PromoteSETCC(Node, Results); in Promote()
693 PromoteSTRICT(Node, Results); in Promote()
707 assert(Node->getNumValues() == 1 && in Promote()
709 MVT VT = Node->getSimpleValueType(0); in Promote()
710 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in Promote()
711 SDLoc dl(Node); in Promote()
712 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in Promote()
714 for (unsigned j = 0; j != Node->getNumOperands(); ++j) { in Promote()
716 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) && in Promote()
717 ISD::getVPMaskIdx(Node->getOpcode()) == j; in Promote()
718 if (Node->getOperand(j).getValueType().isVector() && !SkipPromote) in Promote()
719 if (Node->getOperand(j) in Promote()
724 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j)); in Promote()
726 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j)); in Promote()
728 Operands[j] = Node->getOperand(j); in Promote()
732 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags()); in Promote()
745 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node, in PromoteINT_TO_FP() argument
749 bool IsStrict = Node->isStrictFPOpcode(); in PromoteINT_TO_FP()
750 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType(); in PromoteINT_TO_FP()
751 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in PromoteINT_TO_FP()
755 SDLoc dl(Node); in PromoteINT_TO_FP()
756 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in PromoteINT_TO_FP()
758 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteINT_TO_FP()
759 Node->getOpcode() == ISD::STRICT_UINT_TO_FP) in PromoteINT_TO_FP()
762 for (unsigned j = 0; j != Node->getNumOperands(); ++j) { in PromoteINT_TO_FP()
763 if (Node->getOperand(j).getValueType().isVector()) in PromoteINT_TO_FP()
764 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j)); in PromoteINT_TO_FP()
766 Operands[j] = Node->getOperand(j); in PromoteINT_TO_FP()
770 SDValue Res = DAG.getNode(Node->getOpcode(), dl, in PromoteINT_TO_FP()
771 {Node->getValueType(0), MVT::Other}, Operands); in PromoteINT_TO_FP()
778 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands); in PromoteINT_TO_FP()
786 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node, in PromoteFP_TO_INT() argument
788 MVT VT = Node->getSimpleValueType(0); in PromoteFP_TO_INT()
789 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in PromoteFP_TO_INT()
790 bool IsStrict = Node->isStrictFPOpcode(); in PromoteFP_TO_INT()
794 unsigned NewOpc = Node->getOpcode(); in PromoteFP_TO_INT()
805 SDLoc dl(Node); in PromoteFP_TO_INT()
809 {Node->getOperand(0), Node->getOperand(1)}); in PromoteFP_TO_INT()
812 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0)); in PromoteFP_TO_INT()
817 if (Node->getOpcode() == ISD::FP_TO_UINT || in PromoteFP_TO_INT()
818 Node->getOpcode() == ISD::STRICT_FP_TO_UINT) in PromoteFP_TO_INT()
842 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) { in Expand() argument
843 switch (Node->getOpcode()) { in Expand()
845 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node); in Expand()
851 Results.push_back(ExpandStore(Node)); in Expand()
854 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in Expand()
855 Results.push_back(Node->getOperand(i)); in Expand()
858 Results.push_back(ExpandSEXTINREG(Node)); in Expand()
861 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node)); in Expand()
864 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node)); in Expand()
867 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node)); in Expand()
870 Results.push_back(ExpandBSWAP(Node)); in Expand()
873 Results.push_back(TLI.expandVPBSWAP(Node, DAG)); in Expand()
876 Results.push_back(ExpandVSELECT(Node)); in Expand()
879 Results.push_back(ExpandVP_SELECT(Node)); in Expand()
883 if (SDValue Expanded = ExpandVP_REM(Node)) { in Expand()
889 Results.push_back(ExpandSELECT(Node)); in Expand()
892 if (Node->getValueType(0).isScalableVector()) { in Expand()
894 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0)); in Expand()
896 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0), in Expand()
897 Node->getOperand(1), Node->getOperand(4)); in Expand()
898 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC, in Expand()
899 Node->getOperand(2), in Expand()
900 Node->getOperand(3))); in Expand()
906 ExpandFP_TO_UINT(Node, Results); in Expand()
909 ExpandUINT_TO_FLOAT(Node, Results); in Expand()
912 Results.push_back(ExpandFNEG(Node)); in Expand()
915 ExpandFSUB(Node, Results); in Expand()
919 ExpandSETCC(Node, Results); in Expand()
922 if (SDValue Expanded = TLI.expandABS(Node, DAG)) { in Expand()
929 if (SDValue Expanded = TLI.expandABD(Node, DAG)) { in Expand()
938 if (SDValue Expanded = TLI.expandAVG(Node, DAG)) { in Expand()
944 ExpandBITREVERSE(Node, Results); in Expand()
947 if (SDValue Expanded = TLI.expandVPBITREVERSE(Node, DAG)) { in Expand()
953 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) { in Expand()
959 if (SDValue Expanded = TLI.expandVPCTPOP(Node, DAG)) { in Expand()
966 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) { in Expand()
973 if (SDValue Expanded = TLI.expandVPCTLZ(Node, DAG)) { in Expand()
980 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) { in Expand()
987 if (SDValue Expanded = TLI.expandVPCTTZ(Node, DAG)) { in Expand()
996 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) { in Expand()
1003 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) { in Expand()
1010 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) { in Expand()
1017 Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(Node, DAG)); in Expand()
1023 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) { in Expand()
1030 ExpandUADDSUBO(Node, Results); in Expand()
1034 ExpandSADDSUBO(Node, Results); in Expand()
1038 ExpandMULO(Node, Results); in Expand()
1044 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) { in Expand()
1051 if (SDValue Expanded = TLI.expandShlSat(Node, DAG)) { in Expand()
1059 if (Node->getValueType(0).isScalableVector()) { in Expand()
1060 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) { in Expand()
1068 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) { in Expand()
1082 ExpandFixedPointDiv(Node, Results); in Expand()
1090 ExpandStrictFPOp(Node, Results); in Expand()
1107 Results.push_back(TLI.expandVecReduce(Node, DAG)); in Expand()
1111 Results.push_back(TLI.expandVecReduceSeq(Node, DAG)); in Expand()
1115 ExpandREM(Node, Results); in Expand()
1118 Results.push_back(ExpandVP_MERGE(Node)); in Expand()
1121 if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, in Expand()
1128 Results.push_back(TLI.expandVECTOR_COMPRESS(Node, DAG)); in Expand()
1132 SDValue Unrolled = DAG.UnrollVectorOp(Node); in Expand()
1133 if (Node->getNumValues() == 1) { in Expand()
1136 assert(Node->getNumValues() == Unrolled->getNumValues() && in Expand()
1143 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) { in ExpandSELECT() argument
1147 EVT VT = Node->getValueType(0); in ExpandSELECT()
1148 SDLoc DL(Node); in ExpandSELECT()
1150 SDValue Mask = Node->getOperand(0); in ExpandSELECT()
1151 SDValue Op1 = Node->getOperand(1); in ExpandSELECT()
1152 SDValue Op2 = Node->getOperand(2); in ExpandSELECT()
1171 return DAG.UnrollVectorOp(Node); in ExpandSELECT()
1196 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandSELECT()
1199 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) { in ExpandSEXTINREG() argument
1200 EVT VT = Node->getValueType(0); in ExpandSEXTINREG()
1205 return DAG.UnrollVectorOp(Node); in ExpandSEXTINREG()
1207 SDLoc DL(Node); in ExpandSEXTINREG()
1208 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandSEXTINREG()
1214 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz); in ExpandSEXTINREG()
1220 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) { in ExpandANY_EXTEND_VECTOR_INREG() argument
1221 SDLoc DL(Node); in ExpandANY_EXTEND_VECTOR_INREG()
1222 EVT VT = Node->getValueType(0); in ExpandANY_EXTEND_VECTOR_INREG()
1224 SDValue Src = Node->getOperand(0); in ExpandANY_EXTEND_VECTOR_INREG()
1255 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) { in ExpandSIGN_EXTEND_VECTOR_INREG() argument
1256 SDLoc DL(Node); in ExpandSIGN_EXTEND_VECTOR_INREG()
1257 EVT VT = Node->getValueType(0); in ExpandSIGN_EXTEND_VECTOR_INREG()
1258 SDValue Src = Node->getOperand(0); in ExpandSIGN_EXTEND_VECTOR_INREG()
1279 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) { in ExpandZERO_EXTEND_VECTOR_INREG() argument
1280 SDLoc DL(Node); in ExpandZERO_EXTEND_VECTOR_INREG()
1281 EVT VT = Node->getValueType(0); in ExpandZERO_EXTEND_VECTOR_INREG()
1283 SDValue Src = Node->getOperand(0); in ExpandZERO_EXTEND_VECTOR_INREG()
1322 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) { in ExpandBSWAP() argument
1323 EVT VT = Node->getValueType(0); in ExpandBSWAP()
1327 return TLI.expandBSWAP(Node, DAG); in ExpandBSWAP()
1336 SDLoc DL(Node); in ExpandBSWAP()
1337 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBSWAP()
1348 return TLI.expandBSWAP(Node, DAG); in ExpandBSWAP()
1351 return DAG.UnrollVectorOp(Node); in ExpandBSWAP()
1354 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node, in ExpandBITREVERSE() argument
1356 EVT VT = Node->getValueType(0); in ExpandBITREVERSE()
1360 Results.push_back(TLI.expandBITREVERSE(Node, DAG)); in ExpandBITREVERSE()
1366 SDValue Tmp = DAG.UnrollVectorOp(Node); in ExpandBITREVERSE()
1386 SDLoc DL(Node); in ExpandBITREVERSE()
1387 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBITREVERSE()
1403 Results.push_back(TLI.expandBITREVERSE(Node, DAG)); in ExpandBITREVERSE()
1408 SDValue Tmp = DAG.UnrollVectorOp(Node); in ExpandBITREVERSE()
1412 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) { in ExpandVSELECT() argument
1415 SDLoc DL(Node); in ExpandVSELECT()
1417 SDValue Mask = Node->getOperand(0); in ExpandVSELECT()
1418 SDValue Op1 = Node->getOperand(1); in ExpandVSELECT()
1419 SDValue Op2 = Node->getOperand(2); in ExpandVSELECT()
1430 return DAG.UnrollVectorOp(Node); in ExpandVSELECT()
1440 return DAG.UnrollVectorOp(Node); in ExpandVSELECT()
1446 return DAG.UnrollVectorOp(Node); in ExpandVSELECT()
1459 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandVSELECT()
1462 SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) { in ExpandVP_SELECT() argument
1465 SDLoc DL(Node); in ExpandVP_SELECT()
1467 SDValue Mask = Node->getOperand(0); in ExpandVP_SELECT()
1468 SDValue Op1 = Node->getOperand(1); in ExpandVP_SELECT()
1469 SDValue Op2 = Node->getOperand(2); in ExpandVP_SELECT()
1470 SDValue EVL = Node->getOperand(3); in ExpandVP_SELECT()
1479 return DAG.UnrollVectorOp(Node); in ExpandVP_SELECT()
1483 return DAG.UnrollVectorOp(Node); in ExpandVP_SELECT()
1493 SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) { in ExpandVP_MERGE() argument
1498 SDLoc DL(Node); in ExpandVP_MERGE()
1500 SDValue Mask = Node->getOperand(0); in ExpandVP_MERGE()
1501 SDValue Op1 = Node->getOperand(1); in ExpandVP_MERGE()
1502 SDValue Op2 = Node->getOperand(2); in ExpandVP_MERGE()
1503 SDValue EVL = Node->getOperand(3); in ExpandVP_MERGE()
1517 return DAG.UnrollVectorOp(Node); in ExpandVP_MERGE()
1523 return DAG.UnrollVectorOp(Node); in ExpandVP_MERGE()
1531 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2); in ExpandVP_MERGE()
1534 SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) { in ExpandVP_REM() argument
1536 EVT VT = Node->getValueType(0); in ExpandVP_REM()
1538 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV; in ExpandVP_REM()
1545 SDLoc DL(Node); in ExpandVP_REM()
1547 SDValue Dividend = Node->getOperand(0); in ExpandVP_REM()
1548 SDValue Divisor = Node->getOperand(1); in ExpandVP_REM()
1549 SDValue Mask = Node->getOperand(2); in ExpandVP_REM()
1550 SDValue EVL = Node->getOperand(3); in ExpandVP_REM()
1558 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node, in ExpandFP_TO_UINT() argument
1562 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) { in ExpandFP_TO_UINT()
1564 if (Node->isStrictFPOpcode()) in ExpandFP_TO_UINT()
1570 if (Node->isStrictFPOpcode()) { in ExpandFP_TO_UINT()
1571 UnrollStrictFPOp(Node, Results); in ExpandFP_TO_UINT()
1575 Results.push_back(DAG.UnrollVectorOp(Node)); in ExpandFP_TO_UINT()
1578 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node, in ExpandUINT_TO_FLOAT() argument
1580 bool IsStrict = Node->isStrictFPOpcode(); in ExpandUINT_TO_FLOAT()
1582 SDValue Src = Node->getOperand(OpNo); in ExpandUINT_TO_FLOAT()
1584 SDLoc DL(Node); in ExpandUINT_TO_FLOAT()
1589 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) { in ExpandUINT_TO_FLOAT()
1603 UnrollStrictFPOp(Node, Results); in ExpandUINT_TO_FLOAT()
1607 Results.push_back(DAG.UnrollVectorOp(Node)); in ExpandUINT_TO_FLOAT()
1625 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0)); in ExpandUINT_TO_FLOAT()
1636 {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1637 {Node->getOperand(0), HI}); in ExpandUINT_TO_FLOAT()
1638 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1641 {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1642 {Node->getOperand(0), LO}); in ExpandUINT_TO_FLOAT()
1649 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1660 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI); in ExpandUINT_TO_FLOAT()
1661 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW); in ExpandUINT_TO_FLOAT()
1662 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO); in ExpandUINT_TO_FLOAT()
1666 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO)); in ExpandUINT_TO_FLOAT()
1669 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) { in ExpandFNEG() argument
1670 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) { in ExpandFNEG()
1671 SDLoc DL(Node); in ExpandFNEG()
1672 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0)); in ExpandFNEG()
1674 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero, in ExpandFNEG()
1675 Node->getOperand(0)); in ExpandFNEG()
1677 return DAG.UnrollVectorOp(Node); in ExpandFNEG()
1680 void VectorLegalizer::ExpandFSUB(SDNode *Node, in ExpandFSUB() argument
1685 EVT VT = Node->getValueType(0); in ExpandFSUB()
1690 SDValue Tmp = DAG.UnrollVectorOp(Node); in ExpandFSUB()
1694 void VectorLegalizer::ExpandSETCC(SDNode *Node, in ExpandSETCC() argument
1697 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandSETCC()
1698 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandSETCC()
1699 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1700 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1703 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandSETCC()
1704 SDValue LHS = Node->getOperand(0 + Offset); in ExpandSETCC()
1705 SDValue RHS = Node->getOperand(1 + Offset); in ExpandSETCC()
1706 SDValue CC = Node->getOperand(2 + Offset); in ExpandSETCC()
1713 UnrollStrictFPOp(Node, Results); in ExpandSETCC()
1716 Results.push_back(UnrollVSETCC(Node)); in ExpandSETCC()
1722 Mask = Node->getOperand(3 + Offset); in ExpandSETCC()
1723 EVL = Node->getOperand(4 + Offset); in ExpandSETCC()
1726 SDLoc dl(Node); in ExpandSETCC()
1728 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask, in ExpandSETCC()
1736 LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandSETCC()
1737 {Chain, LHS, RHS, CC}, Node->getFlags()); in ExpandSETCC()
1740 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0), in ExpandSETCC()
1741 {LHS, RHS, CC, Mask, EVL}, Node->getFlags()); in ExpandSETCC()
1743 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC, in ExpandSETCC()
1744 Node->getFlags()); in ExpandSETCC()
1761 EVT VT = Node->getValueType(0); in ExpandSETCC()
1766 LHS->setFlags(Node->getFlags()); in ExpandSETCC()
1774 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node, in ExpandUADDSUBO() argument
1777 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandUADDSUBO()
1782 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node, in ExpandSADDSUBO() argument
1785 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandSADDSUBO()
1790 void VectorLegalizer::ExpandMULO(SDNode *Node, in ExpandMULO() argument
1793 if (!TLI.expandMULO(Node, Result, Overflow, DAG)) in ExpandMULO()
1794 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node); in ExpandMULO()
1800 void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node, in ExpandFixedPointDiv() argument
1802 SDNode *N = Node; in ExpandFixedPointDiv()
1808 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node, in ExpandStrictFPOp() argument
1810 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) { in ExpandStrictFPOp()
1811 ExpandUINT_TO_FLOAT(Node, Results); in ExpandStrictFPOp()
1814 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) { in ExpandStrictFPOp()
1815 ExpandFP_TO_UINT(Node, Results); in ExpandStrictFPOp()
1819 if (Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandStrictFPOp()
1820 Node->getOpcode() == ISD::STRICT_FSETCCS) { in ExpandStrictFPOp()
1821 ExpandSETCC(Node, Results); in ExpandStrictFPOp()
1825 UnrollStrictFPOp(Node, Results); in ExpandStrictFPOp()
1828 void VectorLegalizer::ExpandREM(SDNode *Node, in ExpandREM() argument
1830 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) && in ExpandREM()
1834 if (!TLI.expandREM(Node, Result, DAG)) in ExpandREM()
1835 Result = DAG.UnrollVectorOp(Node); in ExpandREM()
1845 bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC, in tryExpandVecMathCall() argument
1849 assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!"); in tryExpandVecMathCall()
1856 EVT VT = Node->getValueType(0); in tryExpandVecMathCall()
1874 for (unsigned i = 0; i < Node->getNumOperands(); ++i) { in tryExpandVecMathCall()
1875 assert(Node->getOperand(i).getValueType() == VT && in tryExpandVecMathCall()
1892 Node->getNumOperands() + VD->isMasked()) in tryExpandVecMathCall()
1897 SDLoc DL(Node); in tryExpandVecMathCall()
1907 Entry.Node = DAG.getBoolConstant(true, DL, MaskVT, VT); in tryExpandVecMathCall()
1917 Entry.Node = Node->getOperand(OpNum++); in tryExpandVecMathCall()
1937 SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, in tryExpandVecMathCall() argument
1941 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64, in tryExpandVecMathCall()
1947 return tryExpandVecMathCall(Node, LC, Results); in tryExpandVecMathCall()
1950 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node, in UnrollStrictFPOp() argument
1952 EVT VT = Node->getValueType(0); in UnrollStrictFPOp()
1955 unsigned NumOpers = Node->getNumOperands(); in UnrollStrictFPOp()
1959 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1960 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
1965 SDValue Chain = Node->getOperand(0); in UnrollStrictFPOp()
1966 SDLoc dl(Node); in UnrollStrictFPOp()
1979 SDValue Oper = Node->getOperand(j); in UnrollStrictFPOp()
1989 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers); in UnrollStrictFPOp()
1993 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1994 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
2010 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) { in UnrollVSETCC() argument
2011 EVT VT = Node->getValueType(0); in UnrollVSETCC()
2014 SDValue LHS = Node->getOperand(0); in UnrollVSETCC()
2015 SDValue RHS = Node->getOperand(1); in UnrollVSETCC()
2016 SDValue CC = Node->getOperand(2); in UnrollVSETCC()
2018 SDLoc dl(Node); in UnrollVSETCC()