Lines Matching refs:Node
113 void LegalizeOp(SDNode *Node);
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
131 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
135 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
136 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
143 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
149 void ExpandArgFPLibCall(SDNode *Node,
154 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161 SDValue ExpandBUILD_VECTOR(SDNode *Node);
162 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
170 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171 SDValue ExpandFABS(SDNode *Node) const;
172 SDValue ExpandFNEG(SDNode *Node) const;
173 SDValue expandLdexp(SDNode *Node) const;
174 SDValue expandFrexp(SDNode *Node) const;
176 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
181 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
188 SDValue PromoteReduction(SDNode *Node);
194 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
200 bool ExpandNode(SDNode *Node);
201 void ConvertNodeToLibcall(SDNode *Node);
202 void PromoteNode(SDNode *Node);
477 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { in LegalizeStoreOps() argument
478 StoreSDNode *ST = cast<StoreSDNode>(Node); in LegalizeStoreOps()
481 SDLoc dl(Node); in LegalizeStoreOps()
513 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
514 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
515 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
525 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
548 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
606 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
622 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
623 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
624 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
649 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
655 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { in LegalizeLoadOps() argument
656 LoadSDNode *LD = cast<LoadSDNode>(Node); in LegalizeLoadOps()
660 SDLoc dl(Node); in LegalizeLoadOps()
665 MVT VT = Node->getSimpleValueType(0); in LegalizeLoadOps()
666 SDValue RVal = SDValue(Node, 0); in LegalizeLoadOps()
667 SDValue RChain = SDValue(Node, 1); in LegalizeLoadOps()
669 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { in LegalizeLoadOps()
690 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in LegalizeLoadOps()
700 if (RChain.getNode() != Node) { in LegalizeLoadOps()
701 assert(RVal.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
702 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); in LegalizeLoadOps()
703 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); in LegalizeLoadOps()
708 ReplacedNode(Node); in LegalizeLoadOps()
728 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == in LegalizeLoadOps()
742 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), in LegalizeLoadOps()
782 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
790 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
806 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
811 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
819 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
835 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
841 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), in LegalizeLoadOps()
848 Value = SDValue(Node, 0); in LegalizeLoadOps()
849 Chain = SDValue(Node, 1); in LegalizeLoadOps()
852 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeLoadOps()
869 EVT DestVT = Node->getValueType(0); in LegalizeLoadOps()
886 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); in LegalizeLoadOps()
923 Node->getValueType(0), in LegalizeLoadOps()
942 if (Chain.getNode() != Node) { in LegalizeLoadOps()
943 assert(Value.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
944 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); in LegalizeLoadOps()
945 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in LegalizeLoadOps()
950 ReplacedNode(Node); in LegalizeLoadOps()
955 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { in LegalizeOp() argument
956 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); in LegalizeOp()
959 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
960 Node->getOpcode() == ISD::Register) in LegalizeOp()
964 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in LegalizeOp()
965 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == in LegalizeOp()
969 for (const SDValue &Op : Node->op_values()) in LegalizeOp()
980 switch (Node->getOpcode()) { in LegalizeOp()
985 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
988 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
989 Node->getValueType(0)); in LegalizeOp()
992 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
993 Node->getValueType(0)); in LegalizeOp()
995 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
999 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1000 Node->getOperand(1).getValueType()); in LegalizeOp()
1011 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1012 Node->getOperand(0).getValueType()); in LegalizeOp()
1025 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1026 Node->getOperand(1).getValueType()); in LegalizeOp()
1029 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); in LegalizeOp()
1030 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); in LegalizeOp()
1034 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1035 Node->getOperand(1).getValueType()); in LegalizeOp()
1044 unsigned Opc = Node->getOpcode(); in LegalizeOp()
1055 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); in LegalizeOp()
1057 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); in LegalizeOp()
1060 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1061 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1062 Node->getValueType(0)); in LegalizeOp()
1064 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
1092 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1104 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1111 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1117 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); in LegalizeOp()
1127 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1131 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1132 Node->getOperand(0)); in LegalizeOp()
1133 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1139 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1143 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1144 Node->getOperand(0)); in LegalizeOp()
1145 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1160 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1170 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
1171 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
1172 Node->getValueType(0), Scale); in LegalizeOp()
1176 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1177 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1180 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1181 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1185 Node->getOpcode(), in LegalizeOp()
1186 cast<VPScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1190 Node->getOpcode(), in LegalizeOp()
1191 cast<VPStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1195 Node->getOpcode(), in LegalizeOp()
1196 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1215 Node->getOpcode(), Node->getOperand(0).getValueType()); in LegalizeOp()
1237 Node->getOpcode(), Node->getOperand(1).getValueType()); in LegalizeOp()
1241 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1242 Node->getOperand(0).getValueType()); in LegalizeOp()
1245 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1246 Action = TLI.getCustomOperationAction(*Node); in LegalizeOp()
1248 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1254 SDNode *NewNode = Node; in LegalizeOp()
1255 switch (Node->getOpcode()) { in LegalizeOp()
1264 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1265 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1274 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); in LegalizeOp()
1285 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1286 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1287 SDValue Op2 = Node->getOperand(2); in LegalizeOp()
1294 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); in LegalizeOp()
1300 if (NewNode != Node) { in LegalizeOp()
1301 ReplaceNode(Node, NewNode); in LegalizeOp()
1302 Node = NewNode; in LegalizeOp()
1312 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeOp()
1313 if (!(Res.getNode() != Node || Res.getResNo() != 0)) in LegalizeOp()
1316 if (Node->getNumValues() == 1) { in LegalizeOp()
1319 assert((Res.getValueType() == Node->getValueType(0) || in LegalizeOp()
1320 Node->getValueType(0) == MVT::Glue) && in LegalizeOp()
1324 ReplaceNode(SDValue(Node, 0), Res); in LegalizeOp()
1329 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { in LegalizeOp()
1332 assert((Res->getValueType(i) == Node->getValueType(i) || in LegalizeOp()
1333 Node->getValueType(i) == MVT::Glue) && in LegalizeOp()
1338 ReplaceNode(Node, ResultVals.data()); in LegalizeOp()
1344 if (ExpandNode(Node)) in LegalizeOp()
1348 ConvertNodeToLibcall(Node); in LegalizeOp()
1351 PromoteNode(Node); in LegalizeOp()
1356 switch (Node->getOpcode()) { in LegalizeOp()
1360 Node->dump( &DAG); in LegalizeOp()
1369 return LegalizeLoadOps(Node); in LegalizeOp()
1371 return LegalizeStoreOps(Node); in LegalizeOp()
1506 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { in ExpandVectorBuildThroughStack() argument
1507 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1508 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1515 EVT VT = Node->getValueType(0); in ExpandVectorBuildThroughStack()
1516 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() in ExpandVectorBuildThroughStack()
1517 : Node->getOperand(0).getValueType(); in ExpandVectorBuildThroughStack()
1518 SDLoc dl(Node); in ExpandVectorBuildThroughStack()
1531 bool Truncate = isa<BuildVectorSDNode>(Node) && in ExpandVectorBuildThroughStack()
1532 MemVT.bitsLT(Node->getOperand(0).getValueType()); in ExpandVectorBuildThroughStack()
1535 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { in ExpandVectorBuildThroughStack()
1537 if (Node->getOperand(i).isUndef()) continue; in ExpandVectorBuildThroughStack()
1546 Node->getOperand(i), Idx, in ExpandVectorBuildThroughStack()
1549 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), in ExpandVectorBuildThroughStack()
1631 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { in ExpandFCOPYSIGN()
1632 SDLoc DL(Node); in ExpandFCOPYSIGN()
1633 SDValue Mag = Node->getOperand(0); in ExpandFCOPYSIGN()
1634 SDValue Sign = Node->getOperand(1); in ExpandFCOPYSIGN()
1694 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { in ExpandFNEG()
1696 SDLoc DL(Node); in ExpandFNEG()
1698 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); in ExpandFNEG()
1710 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { in ExpandFABS()
1711 SDLoc DL(Node); in ExpandFABS()
1712 SDValue Value = Node->getOperand(0); in ExpandFABS()
1731 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, in ExpandDYNAMIC_STACKALLOC() argument
1736 SDLoc dl(Node); in ExpandDYNAMIC_STACKALLOC()
1737 EVT VT = Node->getValueType(0); in ExpandDYNAMIC_STACKALLOC()
1738 SDValue Tmp1 = SDValue(Node, 0); in ExpandDYNAMIC_STACKALLOC()
1739 SDValue Tmp2 = SDValue(Node, 1); in ExpandDYNAMIC_STACKALLOC()
1740 SDValue Tmp3 = Node->getOperand(2); in ExpandDYNAMIC_STACKALLOC()
1823 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { in ExpandSCALAR_TO_VECTOR() argument
1824 SDLoc dl(Node); in ExpandSCALAR_TO_VECTOR()
1827 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); in ExpandSCALAR_TO_VECTOR()
1833 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, in ExpandSCALAR_TO_VECTOR()
1835 Node->getValueType(0).getVectorElementType()); in ExpandSCALAR_TO_VECTOR()
1837 Node->getValueType(0), dl, Ch, StackPtr, in ExpandSCALAR_TO_VECTOR()
1842 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, in ExpandBVWithShuffles() argument
1844 unsigned NumElems = Node->getNumOperands(); in ExpandBVWithShuffles()
1845 SDLoc dl(Node); in ExpandBVWithShuffles()
1846 EVT VT = Node->getValueType(0); in ExpandBVWithShuffles()
1859 SDValue V = Node->getOperand(i); in ExpandBVWithShuffles()
1937 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { in ExpandBUILD_VECTOR() argument
1938 unsigned NumElems = Node->getNumOperands(); in ExpandBUILD_VECTOR()
1940 SDLoc dl(Node); in ExpandBUILD_VECTOR()
1941 EVT VT = Node->getValueType(0); in ExpandBUILD_VECTOR()
1942 EVT OpVT = Node->getOperand(0).getValueType(); in ExpandBUILD_VECTOR()
1951 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1980 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1983 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1995 assert(Node->getOperand(i).isUndef()); in ExpandBUILD_VECTOR()
2012 if (Node->getOperand(i).isUndef()) in ExpandBUILD_VECTOR()
2014 DefinedValues.insert(Node->getOperand(i)); in ExpandBUILD_VECTOR()
2021 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
2026 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { in ExpandBUILD_VECTOR()
2040 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) in ExpandBUILD_VECTOR()
2046 return ExpandVectorBuildThroughStack(Node); in ExpandBUILD_VECTOR()
2049 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { in ExpandSPLAT_VECTOR() argument
2050 SDLoc DL(Node); in ExpandSPLAT_VECTOR()
2051 EVT VT = Node->getValueType(0); in ExpandSPLAT_VECTOR()
2052 SDValue SplatVal = Node->getOperand(0); in ExpandSPLAT_VECTOR()
2062 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, in ExpandLibCall() argument
2072 Node->getOperationName(&DAG)); in ExpandLibCall()
2075 EVT RetVT = Node->getValueType(0); in ExpandLibCall()
2089 TLI.isInTailCallPosition(DAG, Node, TCChain) && in ExpandLibCall()
2096 CLI.setDebugLoc(SDLoc(Node)) in ExpandLibCall()
2117 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, in ExpandLibCall() argument
2121 for (const SDValue &Op : Node->op_values()) { in ExpandLibCall()
2124 Entry.Node = Op; in ExpandLibCall()
2131 return ExpandLibCall(LC, Node, std::move(Args), isSigned); in ExpandLibCall()
2135 SDNode *Node, SmallVectorImpl<SDValue> &Results) { in ExpandFrexpLibCall() argument
2136 SDLoc dl(Node); in ExpandFrexpLibCall()
2137 EVT VT = Node->getValueType(0); in ExpandFrexpLibCall()
2138 EVT ExpVT = Node->getValueType(1); in ExpandFrexpLibCall()
2140 SDValue FPOp = Node->getOperand(0); in ExpandFrexpLibCall()
2146 FPArgEntry.Node = FPOp; in ExpandFrexpLibCall()
2151 PtrArgEntry.Node = StackSlot; in ExpandFrexpLibCall()
2158 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false); in ExpandFrexpLibCall()
2175 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2181 if (Node->isStrictFPOpcode()) { in ExpandFPLibCall()
2182 EVT RetVT = Node->getValueType(0); in ExpandFPLibCall()
2183 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); in ExpandFPLibCall()
2188 SDLoc(Node), in ExpandFPLibCall()
2189 Node->getOperand(0)); in ExpandFPLibCall()
2193 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP; in ExpandFPLibCall()
2194 SDValue Tmp = ExpandLibCall(LC, Node, IsSignedArgument).first; in ExpandFPLibCall()
2200 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2207 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), in ExpandFPLibCall()
2210 ExpandFPLibCall(Node, LC, Results); in ExpandFPLibCall()
2213 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, in ExpandIntLibCall() argument
2220 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandIntLibCall()
2228 return ExpandLibCall(LC, Node, isSigned).first; in ExpandIntLibCall()
2233 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, in ExpandArgFPLibCall() argument
2240 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); in ExpandArgFPLibCall()
2244 ExpandFPLibCall(Node, LC, Results); in ExpandArgFPLibCall()
2249 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, in ExpandDivRemLibCall() argument
2251 unsigned Opcode = Node->getOpcode(); in ExpandDivRemLibCall()
2255 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandDivRemLibCall()
2269 EVT RetVT = Node->getValueType(0); in ExpandDivRemLibCall()
2274 for (const SDValue &Op : Node->op_values()) { in ExpandDivRemLibCall()
2277 Entry.Node = Op; in ExpandDivRemLibCall()
2286 Entry.Node = FIPtr; in ExpandDivRemLibCall()
2295 SDLoc dl(Node); in ExpandDivRemLibCall()
2314 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { in isSinCosLibcallAvailable() argument
2316 switch (Node->getSimpleValueType(0).SimpleTy) { in isSinCosLibcallAvailable()
2328 static bool useSinCos(SDNode *Node) { in useSinCos() argument
2329 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2332 SDValue Op0 = Node->getOperand(0); in useSinCos()
2334 if (User == Node) in useSinCos()
2345 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, in ExpandSinCosLibCall() argument
2348 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandSinCosLibCall()
2362 EVT RetVT = Node->getValueType(0); in ExpandSinCosLibCall()
2369 Entry.Node = Node->getOperand(0); in ExpandSinCosLibCall()
2377 Entry.Node = SinPtr; in ExpandSinCosLibCall()
2385 Entry.Node = CosPtr; in ExpandSinCosLibCall()
2394 SDLoc dl(Node); in ExpandSinCosLibCall()
2408 SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const { in expandLdexp()
2409 SDLoc dl(Node); in expandLdexp()
2410 EVT VT = Node->getValueType(0); in expandLdexp()
2411 SDValue X = Node->getOperand(0); in expandLdexp()
2412 SDValue N = Node->getOperand(1); in expandLdexp()
2418 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO in expandLdexp()
2520 SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const { in expandFrexp()
2521 SDLoc dl(Node); in expandFrexp()
2522 SDValue Val = Node->getOperand(0); in expandFrexp()
2524 EVT ExpVT = Node->getValueType(1); in expandFrexp()
2643 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, in ExpandLegalINT_TO_FP() argument
2645 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2646 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2647 EVT DestVT = Node->getValueType(0); in ExpandLegalINT_TO_FP()
2648 SDLoc dl(Node); in ExpandLegalINT_TO_FP()
2649 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; in ExpandLegalINT_TO_FP()
2650 SDValue Op0 = Node->getOperand(OpNo); in ExpandLegalINT_TO_FP()
2657 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2703 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2705 {Node->getOperand(0), Load, Bias}); in ExpandLegalINT_TO_FP()
2757 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2762 { Node->getOperand(0), InCvt }); in ExpandLegalINT_TO_FP()
2770 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); in ExpandLegalINT_TO_FP()
2785 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2796 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2798 { Node->getOperand(0), Op0 }); in ExpandLegalINT_TO_FP()
2847 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2973 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, in PromoteLegalFP_TO_INT_SAT() argument
2975 unsigned Opcode = Node->getOpcode(); in PromoteLegalFP_TO_INT_SAT()
2978 EVT NewOutTy = Node->getValueType(0); in PromoteLegalFP_TO_INT_SAT()
2989 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), in PromoteLegalFP_TO_INT_SAT()
2990 Node->getOperand(1)); in PromoteLegalFP_TO_INT_SAT()
2991 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
3016 SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) { in PromoteReduction() argument
3017 MVT VecVT = Node->getOperand(1).getSimpleValueType(); in PromoteReduction()
3018 MVT NewVecVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VecVT); in PromoteReduction()
3019 MVT ScalarVT = Node->getSimpleValueType(0); in PromoteReduction()
3022 SDLoc DL(Node); in PromoteReduction()
3023 SmallVector<SDValue, 4> Operands(Node->getNumOperands()); in PromoteReduction()
3027 assert(Node->getOperand(0).getValueType().isFloatingPoint() && in PromoteReduction()
3030 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
3032 for (unsigned j = 1; j != Node->getNumOperands(); ++j) in PromoteReduction()
3033 if (Node->getOperand(j).getValueType().isVector() && in PromoteReduction()
3034 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteReduction()
3035 ISD::getVPMaskIdx(Node->getOpcode()) == j)) { // Skip mask operand. in PromoteReduction()
3038 assert(Node->getOperand(j).getValueType().isFloatingPoint() && in PromoteReduction()
3041 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
3043 Operands[j] = Node->getOperand(j); // Skip VL operand. in PromoteReduction()
3046 SDValue Res = DAG.getNode(Node->getOpcode(), DL, NewScalarVT, Operands, in PromoteReduction()
3047 Node->getFlags()); in PromoteReduction()
3054 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { in ExpandNode() argument
3057 SDLoc dl(Node); in ExpandNode()
3060 switch (Node->getOpcode()) { in ExpandNode()
3062 if ((Tmp1 = TLI.expandABS(Node, DAG))) in ExpandNode()
3067 if ((Tmp1 = TLI.expandABD(Node, DAG))) in ExpandNode()
3074 if ((Tmp1 = TLI.expandAVG(Node, DAG))) in ExpandNode()
3078 if ((Tmp1 = TLI.expandCTPOP(Node, DAG))) in ExpandNode()
3083 if ((Tmp1 = TLI.expandCTLZ(Node, DAG))) in ExpandNode()
3088 if ((Tmp1 = TLI.expandCTTZ(Node, DAG))) in ExpandNode()
3092 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) in ExpandNode()
3096 if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) in ExpandNode()
3100 Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); in ExpandNode()
3105 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3108 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, in ExpandNode()
3123 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); in ExpandNode()
3124 Results.push_back(Node->getOperand(0)); in ExpandNode()
3133 Results.push_back(Node->getOperand(0)); in ExpandNode()
3139 Results.append(Node->getNumValues() - 1, in ExpandNode()
3140 DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3141 Results.push_back(Node->getOperand(0)); in ExpandNode()
3147 Results.push_back(Node->getOperand(0)); in ExpandNode()
3151 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); in ExpandNode()
3152 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3154 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3155 Node->getOperand(0), Node->getOperand(1), Zero, Zero, in ExpandNode()
3156 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
3164 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
3165 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1), in ExpandNode()
3166 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
3174 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3176 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3177 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), in ExpandNode()
3178 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); in ExpandNode()
3182 SDValue RHS = Node->getOperand(1); in ExpandNode()
3184 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); in ExpandNode()
3185 EVT OuterType = Node->getValueType(0); in ExpandNode()
3191 Node->getOperand(2), DAG.getValueType(AtomicType)); in ExpandNode()
3197 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3202 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3209 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
3217 SDLoc DL(Node); in ExpandNode()
3218 EVT VT = Node->getValueType(0); in ExpandNode()
3219 SDValue RHS = Node->getOperand(2); in ExpandNode()
3220 AtomicSDNode *AN = cast<AtomicSDNode>(Node); in ExpandNode()
3227 Node->getOperand(0), Node->getOperand(1), in ExpandNode()
3234 ExpandDYNAMIC_STACKALLOC(Node, Results); in ExpandNode()
3237 for (unsigned i = 0; i < Node->getNumValues(); i++) in ExpandNode()
3238 Results.push_back(Node->getOperand(i)); in ExpandNode()
3241 EVT VT = Node->getValueType(0); in ExpandNode()
3257 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3258 Node->getValueType(0)) in ExpandNode()
3263 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), in ExpandNode()
3264 Node->getValueType(0), dl, in ExpandNode()
3265 Node->getOperand(0)))) { in ExpandNode()
3266 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
3272 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) { in ExpandNode()
3280 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), in ExpandNode()
3281 Node->getValueType(0), dl))) in ExpandNode()
3291 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3292 Node->getValueType(0)) in ExpandNode()
3298 Node->getOperand(1), Node->getOperand(1).getValueType(), in ExpandNode()
3299 Node->getValueType(0), dl, Node->getOperand(0)))) { in ExpandNode()
3300 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
3306 SDValue Op = Node->getOperand(0); in ExpandNode()
3308 EVT DstVT = Node->getValueType(0); in ExpandNode()
3310 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op)); in ExpandNode()
3323 SDValue Op = Node->getOperand(0); in ExpandNode()
3336 if (Node->getValueType(0) != MVT::f32) in ExpandNode()
3337 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op); in ExpandNode()
3342 SDValue Op = Node->getOperand(0); in ExpandNode()
3348 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags()); in ExpandNode()
3356 if (Node->getValueType(0) == MVT::bf16) { in ExpandNode()
3360 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0)); in ExpandNode()
3366 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandNode()
3367 EVT VT = Node->getValueType(0); in ExpandNode()
3378 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
3391 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3392 Node->getOperand(0), ShiftCst); in ExpandNode()
3393 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3399 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
3401 if (Node->isStrictFPOpcode()) in ExpandNode()
3408 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { in ExpandNode()
3410 if (Node->isStrictFPOpcode()) in ExpandNode()
3415 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) in ExpandNode()
3419 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { in ExpandNode()
3420 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
3426 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) in ExpandNode()
3430 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
3432 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); in ExpandNode()
3434 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); in ExpandNode()
3441 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); in ExpandNode()
3444 Results.push_back(DAG.expandVAArg(Node)); in ExpandNode()
3448 Results.push_back(DAG.expandVACopy(Node)); in ExpandNode()
3451 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar()) in ExpandNode()
3453 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3454 Node->getOperand(0)); in ExpandNode()
3456 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); in ExpandNode()
3460 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
3463 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
3466 Results.push_back(ExpandVectorBuildThroughStack(Node)); in ExpandNode()
3469 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); in ExpandNode()
3472 Results.push_back(ExpandINSERT_VECTOR_ELT(SDValue(Node, 0))); in ExpandNode()
3476 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in ExpandNode()
3478 EVT VT = Node->getValueType(0); in ExpandNode()
3480 SDValue Op0 = Node->getOperand(0); in ExpandNode()
3481 SDValue Op1 = Node->getOperand(1); in ExpandNode()
3544 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3549 Results.push_back(TLI.expandVectorSplice(Node, DAG)); in ExpandNode()
3553 EVT OpTy = Node->getOperand(0).getValueType(); in ExpandNode()
3554 if (Node->getConstantOperandVal(1)) { in ExpandNode()
3556 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3559 Node->getOperand(0).getValueType(), in ExpandNode()
3561 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3564 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3565 Node->getOperand(0)); in ExpandNode()
3574 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, in ExpandNode()
3575 Node->getValueType(0))); in ExpandNode()
3578 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ExpandNode()
3579 Results.push_back(Node->getOperand(0)); in ExpandNode()
3586 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, in ExpandNode()
3587 Node->getOperand(1))); in ExpandNode()
3589 Results.push_back(Node->getOperand(0)); in ExpandNode()
3593 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3597 Results.push_back(ExpandFCOPYSIGN(Node)); in ExpandNode()
3600 Results.push_back(ExpandFNEG(Node)); in ExpandNode()
3603 Results.push_back(ExpandFABS(Node)); in ExpandNode()
3606 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1)); in ExpandNode()
3608 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0), in ExpandNode()
3609 Test, Node->getFlags(), SDLoc(Node), DAG)) in ExpandNode()
3619 switch (Node->getOpcode()) { in ExpandNode()
3626 Tmp1 = Node->getOperand(0); in ExpandNode()
3627 Tmp2 = Node->getOperand(1); in ExpandNode()
3634 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) in ExpandNode()
3640 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(Node, DAG)) in ExpandNode()
3646 EVT VT = Node->getValueType(0); in ExpandNode()
3650 isSinCosLibcallAvailable(Node, TLI)) in ExpandNode()
3651 && useSinCos(Node)) { in ExpandNode()
3653 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3654 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3662 EVT VT = Node->getValueType(0); in ExpandNode()
3669 if (SDValue Expanded = expandLdexp(Node)) { in ExpandNode()
3671 if (Node->getOpcode() == ISD::STRICT_FLDEXP) in ExpandNode()
3678 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0)); in ExpandNode()
3684 if (SDValue Expanded = expandFrexp(Node)) { in ExpandNode()
3694 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3699 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3701 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3706 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3710 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other}, in ExpandNode()
3711 {Node->getOperand(0), Node->getOperand(1)}); in ExpandNode()
3713 {Node->getValueType(0), MVT::Other}, in ExpandNode()
3722 SDValue Op = Node->getOperand(0); in ExpandNode()
3732 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3737 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); in ExpandNode()
3740 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), in ExpandNode()
3746 ConstantSDNode *CP = cast<ConstantSDNode>(Node); in ExpandNode()
3751 EVT VT = Node->getValueType(0); in ExpandNode()
3754 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3755 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3756 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3762 EVT VT = Node->getValueType(0); in ExpandNode()
3766 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT); in ExpandNode()
3768 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3773 if (TLI.expandREM(Node, Tmp1, DAG)) in ExpandNode()
3778 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3780 EVT VT = Node->getValueType(0); in ExpandNode()
3783 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), in ExpandNode()
3784 Node->getOperand(1)); in ExpandNode()
3792 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3793 EVT VT = Node->getValueType(0); in ExpandNode()
3796 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), in ExpandNode()
3797 Node->getOperand(1)); in ExpandNode()
3803 SDValue LHS = Node->getOperand(0); in ExpandNode()
3804 SDValue RHS = Node->getOperand(1); in ExpandNode()
3807 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3818 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, in ExpandNode()
3835 EVT VT = Node->getValueType(0); in ExpandNode()
3857 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), in ExpandNode()
3858 Node->getOperand(1))); in ExpandNode()
3868 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, in ExpandNode()
3882 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) in ExpandNode()
3887 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG)) in ExpandNode()
3894 Results.push_back(TLI.expandAddSubSat(Node, DAG)); in ExpandNode()
3898 Results.push_back(TLI.expandCMP(Node, DAG)); in ExpandNode()
3902 Results.push_back(TLI.expandShlSat(Node, DAG)); in ExpandNode()
3908 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); in ExpandNode()
3914 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), in ExpandNode()
3915 Node->getOperand(0), in ExpandNode()
3916 Node->getOperand(1), in ExpandNode()
3917 Node->getConstantOperandVal(2), in ExpandNode()
3932 SDValue LHS = Node->getOperand(0); in ExpandNode()
3933 SDValue RHS = Node->getOperand(1); in ExpandNode()
3934 SDValue Carry = Node->getOperand(2); in ExpandNode()
3936 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY; in ExpandNode()
3944 EVT CarryType = Node->getValueType(1); in ExpandNode()
3945 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); in ExpandNode()
3976 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3984 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3992 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { in ExpandNode()
3999 EVT PairTy = Node->getValueType(0); in ExpandNode()
4000 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
4001 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
4010 Tmp1 = Node->getOperand(0); in ExpandNode()
4011 Tmp2 = Node->getOperand(1); in ExpandNode()
4012 Tmp3 = Node->getOperand(2); in ExpandNode()
4022 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4026 SDValue Chain = Node->getOperand(0); in ExpandNode()
4027 SDValue Table = Node->getOperand(1); in ExpandNode()
4028 SDValue Index = Node->getOperand(2); in ExpandNode()
4071 Tmp1 = Node->getOperand(0); in ExpandNode()
4072 Tmp2 = Node->getOperand(1); in ExpandNode()
4078 Node->getOperand(2)); in ExpandNode()
4090 Node->getOperand(2)); in ExpandNode()
4098 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
4099 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
4100 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4101 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4102 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandNode()
4104 Tmp1 = Node->getOperand(0 + Offset); in ExpandNode()
4105 Tmp2 = Node->getOperand(1 + Offset); in ExpandNode()
4106 Tmp3 = Node->getOperand(2 + Offset); in ExpandNode()
4109 Mask = Node->getOperand(3 + Offset); in ExpandNode()
4110 EVL = Node->getOperand(4 + Offset); in ExpandNode()
4113 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl, in ExpandNode()
4121 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandNode()
4122 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags()); in ExpandNode()
4125 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), in ExpandNode()
4126 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags()); in ExpandNode()
4128 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1, in ExpandNode()
4129 Tmp2, Tmp3, Node->getFlags()); in ExpandNode()
4157 EVT VT = Node->getValueType(0); in ExpandNode()
4162 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4168 Tmp1 = Node->getOperand(0); // LHS in ExpandNode()
4169 Tmp2 = Node->getOperand(1); // RHS in ExpandNode()
4170 Tmp3 = Node->getOperand(2); // True in ExpandNode()
4171 Tmp4 = Node->getOperand(3); // False in ExpandNode()
4172 EVT VT = Node->getValueType(0); in ExpandNode()
4174 SDValue CC = Node->getOperand(4); in ExpandNode()
4185 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
4187 DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4, Node->getFlags())); in ExpandNode()
4201 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4211 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4230 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
4235 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4238 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4246 Tmp1 = Node->getOperand(0); // Chain in ExpandNode()
4247 Tmp2 = Node->getOperand(2); // LHS in ExpandNode()
4248 Tmp3 = Node->getOperand(3); // RHS in ExpandNode()
4249 Tmp4 = Node->getOperand(1); // CC in ExpandNode()
4262 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4263 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
4267 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
4268 Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
4274 Results.push_back(ExpandBUILD_VECTOR(Node)); in ExpandNode()
4277 Results.push_back(ExpandSPLAT_VECTOR(Node)); in ExpandNode()
4283 EVT VT = Node->getValueType(0); in ExpandNode()
4292 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4295 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4296 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, in ExpandNode()
4300 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); in ExpandNode()
4319 Results.push_back(TLI.expandVecReduce(Node, DAG)); in ExpandNode()
4323 Results.push_back(TLI.expandVPCTTZElements(Node, DAG)); in ExpandNode()
4328 Results.push_back(Node->getOperand(0)); in ExpandNode()
4345 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { in ExpandNode()
4351 switch (Node->getOpcode()) { in ExpandNode()
4353 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
4354 Node->getValueType(0)) in ExpandNode()
4360 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
4363 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
4366 EVT VT = Node->getValueType(0); in ExpandNode()
4367 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
4368 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
4369 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
4370 {Node->getOperand(0), Node->getOperand(1), Neg}, in ExpandNode()
4385 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
4386 Node->getOperand(1).getValueType()) in ExpandNode()
4400 ReplaceNode(Node, Results.data()); in ExpandNode()
4404 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { in ConvertNodeToLibcall() argument
4407 SDLoc dl(Node); in ConvertNodeToLibcall()
4409 unsigned Opc = Node->getOpcode(); in ConvertNodeToLibcall()
4418 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
4446 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); in ConvertNodeToLibcall()
4447 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); in ConvertNodeToLibcall()
4449 EVT RetVT = Node->getValueType(0); in ConvertNodeToLibcall()
4454 Ops.append(Node->op_begin() + 2, Node->op_end()); in ConvertNodeToLibcall()
4455 Ops.push_back(Node->getOperand(1)); in ConvertNodeToLibcall()
4462 Ops.append(Node->op_begin() + 1, Node->op_end()); in ConvertNodeToLibcall()
4466 SDLoc(Node), in ConvertNodeToLibcall()
4467 Node->getOperand(0)); in ConvertNodeToLibcall()
4477 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
4489 SDValue InputChain = Node->getOperand(0); in ConvertNodeToLibcall()
4490 SDValue StartVal = Node->getOperand(1); in ConvertNodeToLibcall()
4491 SDValue EndVal = Node->getOperand(2); in ConvertNodeToLibcall()
4494 SDLoc(Node), InputChain); in ConvertNodeToLibcall()
4500 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, in ConvertNodeToLibcall()
4509 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, in ConvertNodeToLibcall()
4515 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, in ConvertNodeToLibcall()
4520 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, in ConvertNodeToLibcall()
4526 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, in ConvertNodeToLibcall()
4532 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, in ConvertNodeToLibcall()
4538 ExpandFPLibCall(Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80, in ConvertNodeToLibcall()
4543 ExpandFPLibCall(Node, RTLIB::ASIN_F32, RTLIB::ASIN_F64, RTLIB::ASIN_F80, in ConvertNodeToLibcall()
4548 ExpandFPLibCall(Node, RTLIB::ACOS_F32, RTLIB::ACOS_F64, RTLIB::ACOS_F80, in ConvertNodeToLibcall()
4553 ExpandFPLibCall(Node, RTLIB::ATAN_F32, RTLIB::ATAN_F64, RTLIB::ATAN_F80, in ConvertNodeToLibcall()
4558 ExpandFPLibCall(Node, RTLIB::SINH_F32, RTLIB::SINH_F64, RTLIB::SINH_F80, in ConvertNodeToLibcall()
4563 ExpandFPLibCall(Node, RTLIB::COSH_F32, RTLIB::COSH_F64, RTLIB::COSH_F80, in ConvertNodeToLibcall()
4568 ExpandFPLibCall(Node, RTLIB::TANH_F32, RTLIB::TANH_F64, RTLIB::TANH_F80, in ConvertNodeToLibcall()
4573 ExpandSinCosLibCall(Node, Results); in ConvertNodeToLibcall()
4577 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, in ConvertNodeToLibcall()
4582 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, in ConvertNodeToLibcall()
4587 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, in ConvertNodeToLibcall()
4592 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, in ConvertNodeToLibcall()
4597 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, in ConvertNodeToLibcall()
4601 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80, in ConvertNodeToLibcall()
4606 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, in ConvertNodeToLibcall()
4612 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, in ConvertNodeToLibcall()
4618 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, in ConvertNodeToLibcall()
4624 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, in ConvertNodeToLibcall()
4630 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, in ConvertNodeToLibcall()
4638 ExpandFPLibCall(Node, RTLIB::ROUND_F32, in ConvertNodeToLibcall()
4646 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, in ConvertNodeToLibcall()
4654 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80, in ConvertNodeToLibcall()
4658 ExpandFrexpLibCall(Node, Results); in ConvertNodeToLibcall()
4663 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); in ConvertNodeToLibcall()
4667 if (Node->isStrictFPOpcode()) { in ConvertNodeToLibcall()
4669 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4670 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4671 {Node->getOperand(0), Node->getOperand(2)}); in ConvertNodeToLibcall()
4673 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4674 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4675 {Exponent.getValue(1), Node->getOperand(1), Exponent}); in ConvertNodeToLibcall()
4680 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4681 Node->getOperand(1)); in ConvertNodeToLibcall()
4682 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4683 Node->getValueType(0), in ConvertNodeToLibcall()
4684 Node->getOperand(0), Exponent)); in ConvertNodeToLibcall()
4688 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; in ConvertNodeToLibcall()
4691 Node->getOperand(1 + Offset).getValueType().getSizeInBits(); in ConvertNodeToLibcall()
4696 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ConvertNodeToLibcall()
4699 ExpandFPLibCall(Node, LC, Results); in ConvertNodeToLibcall()
4704 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, in ConvertNodeToLibcall()
4709 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, in ConvertNodeToLibcall()
4716 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, in ConvertNodeToLibcall()
4723 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, in ConvertNodeToLibcall()
4730 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, in ConvertNodeToLibcall()
4737 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, in ConvertNodeToLibcall()
4743 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, in ConvertNodeToLibcall()
4749 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, in ConvertNodeToLibcall()
4755 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, in ConvertNodeToLibcall()
4761 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, in ConvertNodeToLibcall()
4766 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4767 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first); in ConvertNodeToLibcall()
4771 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4774 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1), in ConvertNodeToLibcall()
4775 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4781 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4784 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, in ConvertNodeToLibcall()
4785 SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4793 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4795 Results.push_back(ExpandLibCall(LC, Node, false).first); in ConvertNodeToLibcall()
4800 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16); in ConvertNodeToLibcall()
4802 Results.push_back(ExpandLibCall(LC, Node, false).first); in ConvertNodeToLibcall()
4810 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4811 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4812 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4813 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); in ConvertNodeToLibcall()
4814 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4816 SDLoc dl(Node); in ConvertNodeToLibcall()
4833 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4836 NVT, Node->getOperand(IsStrict ? 1 : 0)); in ConvertNodeToLibcall()
4851 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4852 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4853 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4855 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4857 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4859 SDLoc dl(Node); in ConvertNodeToLibcall()
4876 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4895 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4896 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4897 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4898 EVT VT = Node->getValueType(0); in ConvertNodeToLibcall()
4899 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() && in ConvertNodeToLibcall()
4907 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); in ConvertNodeToLibcall()
4915 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), in ConvertNodeToLibcall()
4916 Node->getValueType(0)), in ConvertNodeToLibcall()
4917 Node, false).first); in ConvertNodeToLibcall()
4924 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16) in ConvertNodeToLibcall()
4925 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4926 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16) in ConvertNodeToLibcall()
4927 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::bf16); in ConvertNodeToLibcall()
4929 LC = RTLIB::getFPEXT(Node->getOperand(1).getValueType(), in ConvertNodeToLibcall()
4930 Node->getValueType(0)); in ConvertNodeToLibcall()
4936 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), in ConvertNodeToLibcall()
4937 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4944 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, in ConvertNodeToLibcall()
4949 Results.push_back(ExpandIntLibCall(Node, true, in ConvertNodeToLibcall()
4955 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4961 Results.push_back(ExpandIntLibCall(Node, true, in ConvertNodeToLibcall()
4967 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4975 ExpandDivRemLibCall(Node, Results); in ConvertNodeToLibcall()
4978 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4984 switch (Node->getSimpleValueType(0).SimpleTy) { in ConvertNodeToLibcall()
4988 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first); in ConvertNodeToLibcall()
4991 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first); in ConvertNodeToLibcall()
4994 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first); in ConvertNodeToLibcall()
5002 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
5008 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
5009 SDValue EnvPtr = Node->getOperand(1); in ConvertNodeToLibcall()
5015 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
5016 SDValue EnvPtr = Node->getOperand(1); in ConvertNodeToLibcall()
5024 EVT ModeVT = Node->getValueType(0); in ConvertNodeToLibcall()
5028 Node->getOperand(0), dl); in ConvertNodeToLibcall()
5039 SDValue Mode = Node->getOperand(1); in ConvertNodeToLibcall()
5044 Node->getOperand(0), dl, Mode, StackPtr, in ConvertNodeToLibcall()
5058 Node->getOperand(0), dl)); in ConvertNodeToLibcall()
5066 ReplaceNode(Node, Results.data()); in ConvertNodeToLibcall()
5083 void SelectionDAGLegalize::PromoteNode(SDNode *Node) { in PromoteNode() argument
5086 MVT OVT = Node->getSimpleValueType(0); in PromoteNode()
5087 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
5088 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
5089 Node->getOpcode() == ISD::SETCC || in PromoteNode()
5090 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
5091 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
5092 OVT = Node->getOperand(0).getSimpleValueType(); in PromoteNode()
5094 if (Node->getOpcode() == ISD::ATOMIC_STORE || in PromoteNode()
5095 Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
5096 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
5097 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
5098 Node->getOpcode() == ISD::STRICT_FSETCCS || in PromoteNode()
5099 Node->getOpcode() == ISD::VP_REDUCE_FADD || in PromoteNode()
5100 Node->getOpcode() == ISD::VP_REDUCE_FMUL || in PromoteNode()
5101 Node->getOpcode() == ISD::VP_REDUCE_FMAX || in PromoteNode()
5102 Node->getOpcode() == ISD::VP_REDUCE_FMIN || in PromoteNode()
5103 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM || in PromoteNode()
5104 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM || in PromoteNode()
5105 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD) in PromoteNode()
5106 OVT = Node->getOperand(1).getSimpleValueType(); in PromoteNode()
5107 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
5108 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
5109 OVT = Node->getOperand(2).getSimpleValueType(); in PromoteNode()
5110 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); in PromoteNode()
5111 SDLoc dl(Node); in PromoteNode()
5113 switch (Node->getOpcode()) { in PromoteNode()
5119 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
5120 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
5121 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5123 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5125 unsigned NewOpc = Node->getOpcode(); in PromoteNode()
5154 DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5163 auto CTLZResult = DAG.getNode(Node->getOpcode(), dl, NVT, LeftShiftResult); in PromoteNode()
5170 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5171 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5184 PromoteLegalFP_TO_INT(Node, dl, Results); in PromoteNode()
5188 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); in PromoteNode()
5194 PromoteLegalINT_TO_FP(Node, dl, Results); in PromoteNode()
5197 SDValue Chain = Node->getOperand(0); // Get the chain. in PromoteNode()
5198 SDValue Ptr = Node->getOperand(1); // Get the pointer. in PromoteNode()
5210 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), in PromoteNode()
5211 Node->getConstantOperandVal(3)); in PromoteNode()
5218 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); in PromoteNode()
5219 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in PromoteNode()
5224 ReplacedNode(Node); in PromoteNode()
5246 switch (Node->getOpcode()) { in PromoteNode()
5271 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5272 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5274 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5281 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
5283 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5284 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5298 if (Node->getValueType(0).isVector() || in PromoteNode()
5299 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { in PromoteNode()
5302 } else if (Node->getValueType(0).isInteger()) { in PromoteNode()
5309 Tmp1 = Node->getOperand(0); in PromoteNode()
5311 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5312 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5315 Tmp1->setFlags(Node->getFlags()); in PromoteNode()
5317 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); in PromoteNode()
5319 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5325 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in PromoteNode()
5328 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
5329 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
5338 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5339 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5341 Node->getOperand(2)); in PromoteNode()
5346 SDValue Cond = Node->getOperand(4); in PromoteNode()
5349 MVT CVT = Node->getSimpleValueType(0); in PromoteNode()
5359 Tmp1 = Node->getOperand(0); in PromoteNode()
5360 Tmp2 = Node->getOperand(1); in PromoteNode()
5362 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5363 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5366 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5367 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5370 Node->getFlags()); in PromoteNode()
5387 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
5389 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT)) in PromoteNode()
5394 if (Node->isStrictFPOpcode()) { in PromoteNode()
5395 SDValue InChain = Node->getOperand(0); in PromoteNode()
5397 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); in PromoteNode()
5399 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); in PromoteNode()
5402 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in PromoteNode()
5403 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, in PromoteNode()
5404 {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, in PromoteNode()
5405 Node->getFlags())); in PromoteNode()
5409 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5410 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5411 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5412 Tmp2, Node->getOperand(2), Node->getFlags())); in PromoteNode()
5419 cast<CondCodeSDNode>(Node->getOperand(1))->get(); in PromoteNode()
5422 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5423 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5424 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
5425 Node->getOperand(0), Node->getOperand(1), in PromoteNode()
5426 Tmp1, Tmp2, Node->getOperand(4))); in PromoteNode()
5439 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5440 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5441 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, in PromoteNode()
5442 Node->getFlags()); in PromoteNode()
5456 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5458 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
5461 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5469 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5470 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5471 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
5474 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), in PromoteNode()
5479 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5481 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
5483 {Node->getOperand(0), Node->getOperand(3)}); in PromoteNode()
5486 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5496 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5497 Tmp2 = Node->getOperand(1); in PromoteNode()
5498 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5505 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
5513 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5514 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5515 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)}); in PromoteNode()
5522 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5523 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1); in PromoteNode()
5558 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5559 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5587 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5588 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5612 for (const SDValue &Op : Node->op_values()) in PromoteNode()
5615 SDLoc SL(Node); in PromoteNode()
5645 SDValue Idx = Node->getOperand(1); in PromoteNode()
5647 SDLoc SL(Node); in PromoteNode()
5651 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5691 SDValue Val = Node->getOperand(1); in PromoteNode()
5692 SDValue Idx = Node->getOperand(2); in PromoteNode()
5694 SDLoc SL(Node); in PromoteNode()
5699 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5729 SDValue Val = Node->getOperand(0); in PromoteNode()
5730 SDLoc SL(Node); in PromoteNode()
5747 AtomicSDNode *AM = cast<AtomicSDNode>(Node); in PromoteNode()
5748 SDLoc SL(Node); in PromoteNode()
5774 AtomicSDNode *AM = cast<AtomicSDNode>(Node); in PromoteNode()
5775 SDLoc SL(Node); in PromoteNode()
5789 SDValue Scalar = Node->getOperand(0); in PromoteNode()
5794 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5799 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5812 Results.push_back(PromoteReduction(Node)); in PromoteNode()
5819 ReplaceNode(Node, Results.data()); in PromoteNode()