Lines Matching refs:ISD
341 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && in ExpandConstantFP()
357 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, in ExpandConstantFP()
394 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in ExpandINSERT_VECTOR_ELT()
412 if (!ISD::isNormalStore(ST)) in OptimizeFloatStore()
430 if (Value.getOpcode() == ISD::TargetConstantFP) in OptimizeFloatStore()
470 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in OptimizeFloatStore()
495 switch (TLI.getOperationAction(ISD::STORE, VT)) { in LegalizeStoreOps()
519 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); in LegalizeStoreOps()
522 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); in LegalizeStoreOps()
577 ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
588 ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
596 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeStoreOps()
605 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in LegalizeStoreOps()
635 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); in LegalizeStoreOps()
641 Value = DAG.getNode(ISD::TRUNCATE, dl, in LegalizeStoreOps()
662 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeLoadOps()
663 if (ExtType == ISD::NON_EXTLOAD) { in LegalizeLoadOps()
695 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); in LegalizeLoadOps()
739 ISD::LoadExtType NewExtType = in LegalizeLoadOps()
740 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; in LegalizeLoadOps()
748 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
750 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
753 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) in LegalizeLoadOps()
755 Result = DAG.getNode(ISD::AssertZext, dl, in LegalizeLoadOps()
782 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
796 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
801 ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
806 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
819 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
825 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
830 ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
835 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
870 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { in LegalizeLoadOps()
879 ISD::LoadExtType MidExtType = in LegalizeLoadOps()
880 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; in LegalizeLoadOps()
885 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); in LegalizeLoadOps()
901 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, in LegalizeLoadOps()
904 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, in LegalizeLoadOps()
918 assert(ExtType != ISD::EXTLOAD && in LegalizeLoadOps()
922 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, in LegalizeLoadOps()
927 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
928 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
959 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
960 Node->getOpcode() == ISD::Register) in LegalizeOp()
972 Op.getOpcode() == ISD::TargetConstant || in LegalizeOp()
973 Op.getOpcode() == ISD::Register) && in LegalizeOp()
981 case ISD::INTRINSIC_W_CHAIN: in LegalizeOp()
982 case ISD::INTRINSIC_WO_CHAIN: in LegalizeOp()
983 case ISD::INTRINSIC_VOID: in LegalizeOp()
984 case ISD::STACKSAVE: in LegalizeOp()
987 case ISD::GET_DYNAMIC_AREA_OFFSET: in LegalizeOp()
991 case ISD::VAARG: in LegalizeOp()
997 case ISD::SET_FPENV: in LegalizeOp()
998 case ISD::SET_FPMODE: in LegalizeOp()
1002 case ISD::FP_TO_FP16: in LegalizeOp()
1003 case ISD::FP_TO_BF16: in LegalizeOp()
1004 case ISD::SINT_TO_FP: in LegalizeOp()
1005 case ISD::UINT_TO_FP: in LegalizeOp()
1006 case ISD::EXTRACT_VECTOR_ELT: in LegalizeOp()
1007 case ISD::LROUND: in LegalizeOp()
1008 case ISD::LLROUND: in LegalizeOp()
1009 case ISD::LRINT: in LegalizeOp()
1010 case ISD::LLRINT: in LegalizeOp()
1014 case ISD::STRICT_FP_TO_FP16: in LegalizeOp()
1015 case ISD::STRICT_FP_TO_BF16: in LegalizeOp()
1016 case ISD::STRICT_SINT_TO_FP: in LegalizeOp()
1017 case ISD::STRICT_UINT_TO_FP: in LegalizeOp()
1018 case ISD::STRICT_LRINT: in LegalizeOp()
1019 case ISD::STRICT_LLRINT: in LegalizeOp()
1020 case ISD::STRICT_LROUND: in LegalizeOp()
1021 case ISD::STRICT_LLROUND: in LegalizeOp()
1028 case ISD::SIGN_EXTEND_INREG: { in LegalizeOp()
1033 case ISD::ATOMIC_STORE: in LegalizeOp()
1037 case ISD::SELECT_CC: in LegalizeOp()
1038 case ISD::STRICT_FSETCC: in LegalizeOp()
1039 case ISD::STRICT_FSETCCS: in LegalizeOp()
1040 case ISD::SETCC: in LegalizeOp()
1041 case ISD::SETCCCARRY: in LegalizeOp()
1042 case ISD::VP_SETCC: in LegalizeOp()
1043 case ISD::BR_CC: { in LegalizeOp()
1045 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4 in LegalizeOp()
1046 : Opc == ISD::STRICT_FSETCC ? 3 in LegalizeOp()
1047 : Opc == ISD::STRICT_FSETCCS ? 3 in LegalizeOp()
1048 : Opc == ISD::SETCCCARRY ? 3 in LegalizeOp()
1049 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2 in LegalizeOp()
1051 unsigned CompareOperand = Opc == ISD::BR_CC ? 2 in LegalizeOp()
1052 : Opc == ISD::STRICT_FSETCC ? 1 in LegalizeOp()
1053 : Opc == ISD::STRICT_FSETCCS ? 1 in LegalizeOp()
1056 ISD::CondCode CCCode = in LegalizeOp()
1060 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1068 case ISD::LOAD: in LegalizeOp()
1069 case ISD::STORE: in LegalizeOp()
1074 case ISD::CALLSEQ_START: in LegalizeOp()
1075 case ISD::CALLSEQ_END: in LegalizeOp()
1081 case ISD::EXTRACT_ELEMENT: in LegalizeOp()
1082 case ISD::GET_ROUNDING: in LegalizeOp()
1083 case ISD::MERGE_VALUES: in LegalizeOp()
1084 case ISD::EH_RETURN: in LegalizeOp()
1085 case ISD::FRAME_TO_ARGS_OFFSET: in LegalizeOp()
1086 case ISD::EH_DWARF_CFA: in LegalizeOp()
1087 case ISD::EH_SJLJ_SETJMP: in LegalizeOp()
1088 case ISD::EH_SJLJ_LONGJMP: in LegalizeOp()
1089 case ISD::EH_SJLJ_SETUP_DISPATCH: in LegalizeOp()
1096 case ISD::INIT_TRAMPOLINE: in LegalizeOp()
1097 case ISD::ADJUST_TRAMPOLINE: in LegalizeOp()
1098 case ISD::FRAMEADDR: in LegalizeOp()
1099 case ISD::RETURNADDR: in LegalizeOp()
1100 case ISD::ADDROFRETURNADDR: in LegalizeOp()
1101 case ISD::SPONENTRY: in LegalizeOp()
1108 case ISD::CLEAR_CACHE: in LegalizeOp()
1113 case ISD::READCYCLECOUNTER: in LegalizeOp()
1114 case ISD::READSTEADYCOUNTER: in LegalizeOp()
1119 case ISD::READ_REGISTER: in LegalizeOp()
1120 case ISD::WRITE_REGISTER: in LegalizeOp()
1126 case ISD::UBSANTRAP: in LegalizeOp()
1131 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1138 case ISD::DEBUGTRAP: in LegalizeOp()
1143 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1150 case ISD::SADDSAT: in LegalizeOp()
1151 case ISD::UADDSAT: in LegalizeOp()
1152 case ISD::SSUBSAT: in LegalizeOp()
1153 case ISD::USUBSAT: in LegalizeOp()
1154 case ISD::SSHLSAT: in LegalizeOp()
1155 case ISD::USHLSAT: in LegalizeOp()
1156 case ISD::SCMP: in LegalizeOp()
1157 case ISD::UCMP: in LegalizeOp()
1158 case ISD::FP_TO_SINT_SAT: in LegalizeOp()
1159 case ISD::FP_TO_UINT_SAT: in LegalizeOp()
1162 case ISD::SMULFIX: in LegalizeOp()
1163 case ISD::SMULFIXSAT: in LegalizeOp()
1164 case ISD::UMULFIX: in LegalizeOp()
1165 case ISD::UMULFIXSAT: in LegalizeOp()
1166 case ISD::SDIVFIX: in LegalizeOp()
1167 case ISD::SDIVFIXSAT: in LegalizeOp()
1168 case ISD::UDIVFIX: in LegalizeOp()
1169 case ISD::UDIVFIXSAT: { in LegalizeOp()
1175 case ISD::MSCATTER: in LegalizeOp()
1179 case ISD::MSTORE: in LegalizeOp()
1183 case ISD::VP_SCATTER: in LegalizeOp()
1188 case ISD::VP_STORE: in LegalizeOp()
1193 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: in LegalizeOp()
1198 case ISD::VECREDUCE_FADD: in LegalizeOp()
1199 case ISD::VECREDUCE_FMUL: in LegalizeOp()
1200 case ISD::VECREDUCE_ADD: in LegalizeOp()
1201 case ISD::VECREDUCE_MUL: in LegalizeOp()
1202 case ISD::VECREDUCE_AND: in LegalizeOp()
1203 case ISD::VECREDUCE_OR: in LegalizeOp()
1204 case ISD::VECREDUCE_XOR: in LegalizeOp()
1205 case ISD::VECREDUCE_SMAX: in LegalizeOp()
1206 case ISD::VECREDUCE_SMIN: in LegalizeOp()
1207 case ISD::VECREDUCE_UMAX: in LegalizeOp()
1208 case ISD::VECREDUCE_UMIN: in LegalizeOp()
1209 case ISD::VECREDUCE_FMAX: in LegalizeOp()
1210 case ISD::VECREDUCE_FMIN: in LegalizeOp()
1211 case ISD::VECREDUCE_FMAXIMUM: in LegalizeOp()
1212 case ISD::VECREDUCE_FMINIMUM: in LegalizeOp()
1213 case ISD::IS_FPCLASS: in LegalizeOp()
1217 case ISD::VECREDUCE_SEQ_FADD: in LegalizeOp()
1218 case ISD::VECREDUCE_SEQ_FMUL: in LegalizeOp()
1219 case ISD::VP_REDUCE_FADD: in LegalizeOp()
1220 case ISD::VP_REDUCE_FMUL: in LegalizeOp()
1221 case ISD::VP_REDUCE_ADD: in LegalizeOp()
1222 case ISD::VP_REDUCE_MUL: in LegalizeOp()
1223 case ISD::VP_REDUCE_AND: in LegalizeOp()
1224 case ISD::VP_REDUCE_OR: in LegalizeOp()
1225 case ISD::VP_REDUCE_XOR: in LegalizeOp()
1226 case ISD::VP_REDUCE_SMAX: in LegalizeOp()
1227 case ISD::VP_REDUCE_SMIN: in LegalizeOp()
1228 case ISD::VP_REDUCE_UMAX: in LegalizeOp()
1229 case ISD::VP_REDUCE_UMIN: in LegalizeOp()
1230 case ISD::VP_REDUCE_FMAX: in LegalizeOp()
1231 case ISD::VP_REDUCE_FMIN: in LegalizeOp()
1232 case ISD::VP_REDUCE_FMAXIMUM: in LegalizeOp()
1233 case ISD::VP_REDUCE_FMINIMUM: in LegalizeOp()
1234 case ISD::VP_REDUCE_SEQ_FADD: in LegalizeOp()
1235 case ISD::VP_REDUCE_SEQ_FMUL: in LegalizeOp()
1239 case ISD::VP_CTTZ_ELTS: in LegalizeOp()
1240 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF: in LegalizeOp()
1245 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1257 case ISD::SHL: in LegalizeOp()
1258 case ISD::SRL: in LegalizeOp()
1259 case ISD::SRA: in LegalizeOp()
1260 case ISD::ROTL: in LegalizeOp()
1261 case ISD::ROTR: { in LegalizeOp()
1278 case ISD::FSHL: in LegalizeOp()
1279 case ISD::FSHR: in LegalizeOp()
1280 case ISD::SRL_PARTS: in LegalizeOp()
1281 case ISD::SRA_PARTS: in LegalizeOp()
1282 case ISD::SHL_PARTS: { in LegalizeOp()
1365 case ISD::CALLSEQ_START: in LegalizeOp()
1366 case ISD::CALLSEQ_END: in LegalizeOp()
1368 case ISD::LOAD: in LegalizeOp()
1370 case ISD::STORE: in LegalizeOp()
1442 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, in ExpandExtractFromVectorThroughStack()
1507 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1508 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1555 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandVectorBuildThroughStack()
1575 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); in getSignAsIntValue()
1610 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, in getSignAsIntValue()
1622 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); in modifySignAsInt()
1642 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, in ExpandFCOPYSIGN()
1647 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && in ExpandFCOPYSIGN()
1648 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { in ExpandFCOPYSIGN()
1649 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); in ExpandFCOPYSIGN()
1650 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); in ExpandFCOPYSIGN()
1652 DAG.getConstant(0, DL, IntVT), ISD::SETNE); in ExpandFCOPYSIGN()
1661 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, in ExpandFCOPYSIGN()
1669 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1674 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1677 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1681 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1689 DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit, Flags); in ExpandFCOPYSIGN()
1704 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); in ExpandFNEG()
1716 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { in ExpandFABS()
1718 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); in ExpandFABS()
1726 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, in ExpandFABS()
1754 ISD::ADD : ISD::SUB; in ExpandDYNAMIC_STACKALLOC()
1759 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, in ExpandDYNAMIC_STACKALLOC()
1789 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) in EmitStackConvert()
1819 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, in EmitStackConvert()
1865 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); in ExpandBVWithShuffles()
1973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
2028 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); in ExpandBUILD_VECTOR()
2031 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); in ExpandBUILD_VECTOR()
2167 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ExpandFrexpLibCall()
2193 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP; in ExpandFPLibCall()
2252 bool isSigned = Opcode == ISD::SDIVREM; in ExpandDivRemLibCall()
2329 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2330 ? ISD::FCOS : ISD::FSIN; in useSinCos()
2337 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) in useSinCos()
2418 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO in expandLdexp()
2451 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT); in expandLdexp()
2453 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW); in expandLdexp()
2455 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal); in expandLdexp()
2457 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW); in expandLdexp()
2460 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT); in expandLdexp()
2463 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal); in expandLdexp()
2464 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal); in expandLdexp()
2467 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0); in expandLdexp()
2469 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0); in expandLdexp()
2472 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT); in expandLdexp()
2477 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW); in expandLdexp()
2481 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal); in expandLdexp()
2483 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW); in expandLdexp()
2486 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal); in expandLdexp()
2487 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal); in expandLdexp()
2491 ISD::SETULT); in expandLdexp()
2494 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0); in expandLdexp()
2496 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0); in expandLdexp()
2501 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big, in expandLdexp()
2502 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X)); in expandLdexp()
2505 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big, in expandLdexp()
2506 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N)); in expandLdexp()
2508 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW); in expandLdexp()
2514 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy, in expandLdexp()
2516 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt); in expandLdexp()
2517 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP); in expandLdexp()
2578 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK); in expandFrexp()
2583 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val); in expandFrexp()
2585 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask); in expandFrexp()
2588 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt); in expandFrexp()
2590 NegSmallestNormalizedInt, ISD::SETULE); in expandFrexp()
2593 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT); in expandFrexp()
2598 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp); in expandFrexp()
2600 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt); in expandFrexp()
2603 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask); in expandFrexp()
2606 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs); in expandFrexp()
2612 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt); in expandFrexp()
2615 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp); in expandFrexp()
2618 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero); in expandFrexp()
2621 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask); in expandFrexp()
2624 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf); in expandFrexp()
2625 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or); in expandFrexp()
2628 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias); in expandFrexp()
2631 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract); in expandFrexp()
2634 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp); in expandFrexp()
2645 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2646 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2657 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2658 : ISD::FP_EXTEND, in ExpandLegalINT_TO_FP()
2670 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, in ExpandLegalINT_TO_FP()
2690 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in ExpandLegalINT_TO_FP()
2704 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, in ExpandLegalINT_TO_FP()
2717 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); in ExpandLegalINT_TO_FP()
2747 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2751 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); in ExpandLegalINT_TO_FP()
2753 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); in ExpandLegalINT_TO_FP()
2754 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); in ExpandLegalINT_TO_FP()
2761 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2763 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2775 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); in ExpandLegalINT_TO_FP()
2776 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); in ExpandLegalINT_TO_FP()
2777 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2785 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2797 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2800 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2803 DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2829 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); in ExpandLegalINT_TO_FP()
2839 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, in ExpandLegalINT_TO_FP()
2848 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2854 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); in ExpandLegalINT_TO_FP()
2865 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || in PromoteLegalINT_TO_FP()
2866 N->getOpcode() == ISD::STRICT_SINT_TO_FP; in PromoteLegalINT_TO_FP()
2869 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; in PromoteLegalINT_TO_FP()
2870 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; in PromoteLegalINT_TO_FP()
2905 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2914 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2926 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in PromoteLegalFP_TO_INT()
2927 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in PromoteLegalFP_TO_INT()
2942 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; in PromoteLegalFP_TO_INT()
2947 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; in PromoteLegalFP_TO_INT()
2964 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); in PromoteLegalFP_TO_INT()
2991 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
3002 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) { in ExpandPARITY()
3003 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); in ExpandPARITY()
3007 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, in ExpandPARITY()
3009 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); in ExpandPARITY()
3013 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); in ExpandPARITY()
3030 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
3034 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteReduction()
3035 ISD::getVPMaskIdx(Node->getOpcode()) == j)) { // Skip mask operand. in PromoteReduction()
3041 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
3050 return DAG.getNode(ISD::FP_ROUND, DL, ScalarVT, Res, in PromoteReduction()
3061 case ISD::ABS: in ExpandNode()
3065 case ISD::ABDS: in ExpandNode()
3066 case ISD::ABDU: in ExpandNode()
3070 case ISD::AVGCEILS: in ExpandNode()
3071 case ISD::AVGCEILU: in ExpandNode()
3072 case ISD::AVGFLOORS: in ExpandNode()
3073 case ISD::AVGFLOORU: in ExpandNode()
3077 case ISD::CTPOP: in ExpandNode()
3081 case ISD::CTLZ: in ExpandNode()
3082 case ISD::CTLZ_ZERO_UNDEF: in ExpandNode()
3086 case ISD::CTTZ: in ExpandNode()
3087 case ISD::CTTZ_ZERO_UNDEF: in ExpandNode()
3091 case ISD::BITREVERSE: in ExpandNode()
3095 case ISD::BSWAP: in ExpandNode()
3099 case ISD::PARITY: in ExpandNode()
3102 case ISD::FRAMEADDR: in ExpandNode()
3103 case ISD::RETURNADDR: in ExpandNode()
3104 case ISD::FRAME_TO_ARGS_OFFSET: in ExpandNode()
3107 case ISD::EH_DWARF_CFA: { in ExpandNode()
3110 SDValue Offset = DAG.getNode(ISD::ADD, dl, in ExpandNode()
3112 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, in ExpandNode()
3116 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), in ExpandNode()
3118 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), in ExpandNode()
3122 case ISD::GET_ROUNDING: in ExpandNode()
3126 case ISD::EH_RETURN: in ExpandNode()
3127 case ISD::EH_LABEL: in ExpandNode()
3128 case ISD::PREFETCH: in ExpandNode()
3129 case ISD::VAEND: in ExpandNode()
3130 case ISD::EH_SJLJ_LONGJMP: in ExpandNode()
3135 case ISD::READCYCLECOUNTER: in ExpandNode()
3136 case ISD::READSTEADYCOUNTER: in ExpandNode()
3143 case ISD::EH_SJLJ_SETJMP: in ExpandNode()
3149 case ISD::ATOMIC_LOAD: { in ExpandNode()
3154 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3161 case ISD::ATOMIC_STORE: { in ExpandNode()
3164 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
3170 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { in ExpandNode()
3176 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3187 case ISD::SIGN_EXTEND: in ExpandNode()
3188 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, in ExpandNode()
3190 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, in ExpandNode()
3194 case ISD::ZERO_EXTEND: in ExpandNode()
3195 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, in ExpandNode()
3200 case ISD::ANY_EXTEND: in ExpandNode()
3209 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
3216 case ISD::ATOMIC_LOAD_SUB: { in ExpandNode()
3221 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG && in ExpandNode()
3225 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS); in ExpandNode()
3226 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(), in ExpandNode()
3233 case ISD::DYNAMIC_STACKALLOC: in ExpandNode()
3236 case ISD::MERGE_VALUES: in ExpandNode()
3240 case ISD::UNDEF: { in ExpandNode()
3250 case ISD::STRICT_FP_ROUND: in ExpandNode()
3271 case ISD::FP_ROUND: { in ExpandNode()
3279 case ISD::BITCAST: in ExpandNode()
3284 case ISD::STRICT_FP_EXTEND: in ExpandNode()
3305 case ISD::FP_EXTEND: { in ExpandNode()
3310 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op)); in ExpandNode()
3318 case ISD::BF16_TO_FP: { in ExpandNode()
3325 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, in ExpandNode()
3326 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op)); in ExpandNode()
3331 ISD::SHL, dl, MVT::i32, Op, in ExpandNode()
3334 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op); in ExpandNode()
3337 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op); in ExpandNode()
3341 case ISD::FP_TO_BF16: { in ExpandNode()
3344 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3348 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags()); in ExpandNode()
3351 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op), in ExpandNode()
3357 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16, in ExpandNode()
3358 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op)); in ExpandNode()
3365 case ISD::SIGN_EXTEND_INREG: { in ExpandNode()
3378 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
3380 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); in ExpandNode()
3391 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3393 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3397 case ISD::UINT_TO_FP: in ExpandNode()
3398 case ISD::STRICT_UINT_TO_FP: in ExpandNode()
3406 case ISD::SINT_TO_FP: in ExpandNode()
3407 case ISD::STRICT_SINT_TO_FP: in ExpandNode()
3414 case ISD::FP_TO_SINT: in ExpandNode()
3418 case ISD::STRICT_FP_TO_SINT: in ExpandNode()
3425 case ISD::FP_TO_UINT: in ExpandNode()
3429 case ISD::STRICT_FP_TO_UINT: in ExpandNode()
3439 case ISD::FP_TO_SINT_SAT: in ExpandNode()
3440 case ISD::FP_TO_UINT_SAT: in ExpandNode()
3443 case ISD::VAARG: in ExpandNode()
3447 case ISD::VACOPY: in ExpandNode()
3450 case ISD::EXTRACT_VECTOR_ELT: in ExpandNode()
3453 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3459 case ISD::EXTRACT_SUBVECTOR: in ExpandNode()
3462 case ISD::INSERT_SUBVECTOR: in ExpandNode()
3465 case ISD::CONCAT_VECTORS: in ExpandNode()
3468 case ISD::SCALAR_TO_VECTOR: in ExpandNode()
3471 case ISD::INSERT_VECTOR_ELT: in ExpandNode()
3474 case ISD::VECTOR_SHUFFLE: { in ExpandNode()
3500 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); in ExpandNode()
3501 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); in ExpandNode()
3534 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in ExpandNode()
3538 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, in ExpandNode()
3544 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3548 case ISD::VECTOR_SPLICE: { in ExpandNode()
3552 case ISD::EXTRACT_ELEMENT: { in ExpandNode()
3556 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3561 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3564 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3570 case ISD::STACKSAVE: in ExpandNode()
3582 case ISD::STACKRESTORE: in ExpandNode()
3592 case ISD::GET_DYNAMIC_AREA_OFFSET: in ExpandNode()
3596 case ISD::FCOPYSIGN: in ExpandNode()
3599 case ISD::FNEG: in ExpandNode()
3602 case ISD::FABS: in ExpandNode()
3605 case ISD::IS_FPCLASS: { in ExpandNode()
3613 case ISD::SMIN: in ExpandNode()
3614 case ISD::SMAX: in ExpandNode()
3615 case ISD::UMIN: in ExpandNode()
3616 case ISD::UMAX: { in ExpandNode()
3618 ISD::CondCode Pred; in ExpandNode()
3621 case ISD::SMAX: Pred = ISD::SETGT; break; in ExpandNode()
3622 case ISD::SMIN: Pred = ISD::SETLT; break; in ExpandNode()
3623 case ISD::UMAX: Pred = ISD::SETUGT; break; in ExpandNode()
3624 case ISD::UMIN: Pred = ISD::SETULT; break; in ExpandNode()
3632 case ISD::FMINNUM: in ExpandNode()
3633 case ISD::FMAXNUM: { in ExpandNode()
3638 case ISD::FMINIMUM: in ExpandNode()
3639 case ISD::FMAXIMUM: { in ExpandNode()
3644 case ISD::FSIN: in ExpandNode()
3645 case ISD::FCOS: { in ExpandNode()
3649 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || in ExpandNode()
3653 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3654 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3660 case ISD::FLDEXP: in ExpandNode()
3661 case ISD::STRICT_FLDEXP: { in ExpandNode()
3671 if (Node->getOpcode() == ISD::STRICT_FLDEXP) in ExpandNode()
3677 case ISD::FFREXP: { in ExpandNode()
3690 case ISD::FMAD: in ExpandNode()
3693 case ISD::FP16_TO_FP: in ExpandNode()
3699 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3701 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3704 case ISD::STRICT_BF16_TO_FP: in ExpandNode()
3705 case ISD::STRICT_FP16_TO_FP: in ExpandNode()
3712 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in ExpandNode()
3719 case ISD::FP_TO_FP16: in ExpandNode()
3725 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { in ExpandNode()
3729 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3732 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3736 case ISD::ConstantFP: { in ExpandNode()
3745 case ISD::Constant: { in ExpandNode()
3750 case ISD::FSUB: { in ExpandNode()
3752 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && in ExpandNode()
3753 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { in ExpandNode()
3755 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3756 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3761 case ISD::SUB: { in ExpandNode()
3763 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in ExpandNode()
3764 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && in ExpandNode()
3767 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); in ExpandNode()
3768 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3771 case ISD::UREM: in ExpandNode()
3772 case ISD::SREM: in ExpandNode()
3776 case ISD::UDIV: in ExpandNode()
3777 case ISD::SDIV: { in ExpandNode()
3778 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3779 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in ExpandNode()
3789 case ISD::MULHU: in ExpandNode()
3790 case ISD::MULHS: { in ExpandNode()
3792 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3801 case ISD::UMUL_LOHI: in ExpandNode()
3802 case ISD::SMUL_LOHI: { in ExpandNode()
3807 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3810 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); in ExpandNode()
3822 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); in ExpandNode()
3823 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); in ExpandNode()
3827 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3828 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3834 case ISD::MUL: { in ExpandNode()
3842 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); in ExpandNode()
3843 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); in ExpandNode()
3844 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); in ExpandNode()
3845 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); in ExpandNode()
3848 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3850 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3852 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3854 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3864 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && in ExpandNode()
3865 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && in ExpandNode()
3866 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandNode()
3867 TLI.isOperationLegalOrCustom(ISD::OR, VT) && in ExpandNode()
3870 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in ExpandNode()
3871 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); in ExpandNode()
3875 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3876 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3880 case ISD::FSHL: in ExpandNode()
3881 case ISD::FSHR: in ExpandNode()
3885 case ISD::ROTL: in ExpandNode()
3886 case ISD::ROTR: in ExpandNode()
3890 case ISD::SADDSAT: in ExpandNode()
3891 case ISD::UADDSAT: in ExpandNode()
3892 case ISD::SSUBSAT: in ExpandNode()
3893 case ISD::USUBSAT: in ExpandNode()
3896 case ISD::SCMP: in ExpandNode()
3897 case ISD::UCMP: in ExpandNode()
3900 case ISD::SSHLSAT: in ExpandNode()
3901 case ISD::USHLSAT: in ExpandNode()
3904 case ISD::SMULFIX: in ExpandNode()
3905 case ISD::SMULFIXSAT: in ExpandNode()
3906 case ISD::UMULFIX: in ExpandNode()
3907 case ISD::UMULFIXSAT: in ExpandNode()
3910 case ISD::SDIVFIX: in ExpandNode()
3911 case ISD::SDIVFIXSAT: in ExpandNode()
3912 case ISD::UDIVFIX: in ExpandNode()
3913 case ISD::UDIVFIXSAT: in ExpandNode()
3930 case ISD::UADDO_CARRY: in ExpandNode()
3931 case ISD::USUBO_CARRY: { in ExpandNode()
3936 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY; in ExpandNode()
3939 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; in ExpandNode()
3946 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; in ExpandNode()
3952 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); in ExpandNode()
3961 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) in ExpandNode()
3962 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); in ExpandNode()
3963 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, in ExpandNode()
3967 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); in ExpandNode()
3973 case ISD::SADDO: in ExpandNode()
3974 case ISD::SSUBO: { in ExpandNode()
3981 case ISD::UADDO: in ExpandNode()
3982 case ISD::USUBO: { in ExpandNode()
3989 case ISD::UMULO: in ExpandNode()
3990 case ISD::SMULO: { in ExpandNode()
3998 case ISD::BUILD_PAIR: { in ExpandNode()
4000 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
4001 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
4003 ISD::SHL, dl, PairTy, Tmp2, in ExpandNode()
4006 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); in ExpandNode()
4009 case ISD::SELECT: in ExpandNode()
4013 if (Tmp1.getOpcode() == ISD::SETCC) { in ExpandNode()
4020 Tmp2, Tmp3, ISD::SETNE); in ExpandNode()
4025 case ISD::BR_JT: { in ExpandNode()
4043 ISD::SHL, dl, Index.getValueType(), Index, in ExpandNode()
4046 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, in ExpandNode()
4048 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), in ExpandNode()
4053 ISD::SEXTLOAD, dl, PTy, Chain, Addr, in ExpandNode()
4060 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, in ExpandNode()
4068 case ISD::BRCOND: in ExpandNode()
4073 if (Tmp2.getOpcode() == ISD::SETCC && in ExpandNode()
4074 TLI.isOperationLegalOrCustom(ISD::BR_CC, in ExpandNode()
4076 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2), in ExpandNode()
4082 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1)))) in ExpandNode()
4085 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, in ExpandNode()
4087 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, in ExpandNode()
4088 DAG.getCondCode(ISD::SETNE), Tmp3, in ExpandNode()
4094 case ISD::SETCC: in ExpandNode()
4095 case ISD::VP_SETCC: in ExpandNode()
4096 case ISD::STRICT_FSETCC: in ExpandNode()
4097 case ISD::STRICT_FSETCCS: { in ExpandNode()
4098 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
4099 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
4100 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4101 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4159 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, in ExpandNode()
4166 case ISD::SELECT_CC: { in ExpandNode()
4175 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); in ExpandNode()
4181 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && in ExpandNode()
4185 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
4196 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType()); in ExpandNode()
4205 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); in ExpandNode()
4230 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
4234 CC = DAG.getCondCode(ISD::SETNE); in ExpandNode()
4235 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4243 case ISD::BR_CC: { in ExpandNode()
4262 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4266 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); in ExpandNode()
4267 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
4273 case ISD::BUILD_VECTOR: in ExpandNode()
4276 case ISD::SPLAT_VECTOR: in ExpandNode()
4279 case ISD::SRA: in ExpandNode()
4280 case ISD::SRL: in ExpandNode()
4281 case ISD::SHL: { in ExpandNode()
4291 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
4294 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
4304 case ISD::VECREDUCE_FADD: in ExpandNode()
4305 case ISD::VECREDUCE_FMUL: in ExpandNode()
4306 case ISD::VECREDUCE_ADD: in ExpandNode()
4307 case ISD::VECREDUCE_MUL: in ExpandNode()
4308 case ISD::VECREDUCE_AND: in ExpandNode()
4309 case ISD::VECREDUCE_OR: in ExpandNode()
4310 case ISD::VECREDUCE_XOR: in ExpandNode()
4311 case ISD::VECREDUCE_SMAX: in ExpandNode()
4312 case ISD::VECREDUCE_SMIN: in ExpandNode()
4313 case ISD::VECREDUCE_UMAX: in ExpandNode()
4314 case ISD::VECREDUCE_UMIN: in ExpandNode()
4315 case ISD::VECREDUCE_FMAX: in ExpandNode()
4316 case ISD::VECREDUCE_FMIN: in ExpandNode()
4317 case ISD::VECREDUCE_FMAXIMUM: in ExpandNode()
4318 case ISD::VECREDUCE_FMINIMUM: in ExpandNode()
4321 case ISD::VP_CTTZ_ELTS: in ExpandNode()
4322 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF: in ExpandNode()
4325 case ISD::CLEAR_CACHE: in ExpandNode()
4330 case ISD::GLOBAL_OFFSET_TABLE: in ExpandNode()
4331 case ISD::GlobalAddress: in ExpandNode()
4332 case ISD::GlobalTLSAddress: in ExpandNode()
4333 case ISD::ExternalSymbol: in ExpandNode()
4334 case ISD::ConstantPool: in ExpandNode()
4335 case ISD::JumpTable: in ExpandNode()
4336 case ISD::INTRINSIC_W_CHAIN: in ExpandNode()
4337 case ISD::INTRINSIC_WO_CHAIN: in ExpandNode()
4338 case ISD::INTRINSIC_VOID: in ExpandNode()
4358 case ISD::STRICT_FSUB: { in ExpandNode()
4360 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
4363 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
4368 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
4369 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
4377 case ISD::STRICT_SINT_TO_FP: in ExpandNode()
4378 case ISD::STRICT_UINT_TO_FP: in ExpandNode()
4379 case ISD::STRICT_LRINT: in ExpandNode()
4380 case ISD::STRICT_LLRINT: in ExpandNode()
4381 case ISD::STRICT_LROUND: in ExpandNode()
4382 case ISD::STRICT_LLROUND: in ExpandNode()
4411 case ISD::ATOMIC_FENCE: { in ConvertNodeToLibcall()
4433 case ISD::ATOMIC_SWAP: in ConvertNodeToLibcall()
4434 case ISD::ATOMIC_LOAD_ADD: in ConvertNodeToLibcall()
4435 case ISD::ATOMIC_LOAD_SUB: in ConvertNodeToLibcall()
4436 case ISD::ATOMIC_LOAD_AND: in ConvertNodeToLibcall()
4437 case ISD::ATOMIC_LOAD_CLR: in ConvertNodeToLibcall()
4438 case ISD::ATOMIC_LOAD_OR: in ConvertNodeToLibcall()
4439 case ISD::ATOMIC_LOAD_XOR: in ConvertNodeToLibcall()
4440 case ISD::ATOMIC_LOAD_NAND: in ConvertNodeToLibcall()
4441 case ISD::ATOMIC_LOAD_MIN: in ConvertNodeToLibcall()
4442 case ISD::ATOMIC_LOAD_MAX: in ConvertNodeToLibcall()
4443 case ISD::ATOMIC_LOAD_UMIN: in ConvertNodeToLibcall()
4444 case ISD::ATOMIC_LOAD_UMAX: in ConvertNodeToLibcall()
4445 case ISD::ATOMIC_CMP_SWAP: { in ConvertNodeToLibcall()
4472 case ISD::TRAP: { in ConvertNodeToLibcall()
4487 case ISD::CLEAR_CACHE: { in ConvertNodeToLibcall()
4498 case ISD::FMINNUM: in ConvertNodeToLibcall()
4499 case ISD::STRICT_FMINNUM: in ConvertNodeToLibcall()
4507 case ISD::FMAXNUM: in ConvertNodeToLibcall()
4508 case ISD::STRICT_FMAXNUM: in ConvertNodeToLibcall()
4513 case ISD::FSQRT: in ConvertNodeToLibcall()
4514 case ISD::STRICT_FSQRT: in ConvertNodeToLibcall()
4519 case ISD::FCBRT: in ConvertNodeToLibcall()
4524 case ISD::FSIN: in ConvertNodeToLibcall()
4525 case ISD::STRICT_FSIN: in ConvertNodeToLibcall()
4530 case ISD::FCOS: in ConvertNodeToLibcall()
4531 case ISD::STRICT_FCOS: in ConvertNodeToLibcall()
4536 case ISD::FTAN: in ConvertNodeToLibcall()
4537 case ISD::STRICT_FTAN: in ConvertNodeToLibcall()
4541 case ISD::FASIN: in ConvertNodeToLibcall()
4542 case ISD::STRICT_FASIN: in ConvertNodeToLibcall()
4546 case ISD::FACOS: in ConvertNodeToLibcall()
4547 case ISD::STRICT_FACOS: in ConvertNodeToLibcall()
4551 case ISD::FATAN: in ConvertNodeToLibcall()
4552 case ISD::STRICT_FATAN: in ConvertNodeToLibcall()
4556 case ISD::FSINH: in ConvertNodeToLibcall()
4557 case ISD::STRICT_FSINH: in ConvertNodeToLibcall()
4561 case ISD::FCOSH: in ConvertNodeToLibcall()
4562 case ISD::STRICT_FCOSH: in ConvertNodeToLibcall()
4566 case ISD::FTANH: in ConvertNodeToLibcall()
4567 case ISD::STRICT_FTANH: in ConvertNodeToLibcall()
4571 case ISD::FSINCOS: in ConvertNodeToLibcall()
4575 case ISD::FLOG: in ConvertNodeToLibcall()
4576 case ISD::STRICT_FLOG: in ConvertNodeToLibcall()
4580 case ISD::FLOG2: in ConvertNodeToLibcall()
4581 case ISD::STRICT_FLOG2: in ConvertNodeToLibcall()
4585 case ISD::FLOG10: in ConvertNodeToLibcall()
4586 case ISD::STRICT_FLOG10: in ConvertNodeToLibcall()
4590 case ISD::FEXP: in ConvertNodeToLibcall()
4591 case ISD::STRICT_FEXP: in ConvertNodeToLibcall()
4595 case ISD::FEXP2: in ConvertNodeToLibcall()
4596 case ISD::STRICT_FEXP2: in ConvertNodeToLibcall()
4600 case ISD::FEXP10: in ConvertNodeToLibcall()
4604 case ISD::FTRUNC: in ConvertNodeToLibcall()
4605 case ISD::STRICT_FTRUNC: in ConvertNodeToLibcall()
4610 case ISD::FFLOOR: in ConvertNodeToLibcall()
4611 case ISD::STRICT_FFLOOR: in ConvertNodeToLibcall()
4616 case ISD::FCEIL: in ConvertNodeToLibcall()
4617 case ISD::STRICT_FCEIL: in ConvertNodeToLibcall()
4622 case ISD::FRINT: in ConvertNodeToLibcall()
4623 case ISD::STRICT_FRINT: in ConvertNodeToLibcall()
4628 case ISD::FNEARBYINT: in ConvertNodeToLibcall()
4629 case ISD::STRICT_FNEARBYINT: in ConvertNodeToLibcall()
4636 case ISD::FROUND: in ConvertNodeToLibcall()
4637 case ISD::STRICT_FROUND: in ConvertNodeToLibcall()
4644 case ISD::FROUNDEVEN: in ConvertNodeToLibcall()
4645 case ISD::STRICT_FROUNDEVEN: in ConvertNodeToLibcall()
4652 case ISD::FLDEXP: in ConvertNodeToLibcall()
4653 case ISD::STRICT_FLDEXP: in ConvertNodeToLibcall()
4657 case ISD::FFREXP: { in ConvertNodeToLibcall()
4661 case ISD::FPOWI: in ConvertNodeToLibcall()
4662 case ISD::STRICT_FPOWI: { in ConvertNodeToLibcall()
4669 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4673 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4680 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4682 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4702 case ISD::FPOW: in ConvertNodeToLibcall()
4703 case ISD::STRICT_FPOW: in ConvertNodeToLibcall()
4707 case ISD::LROUND: in ConvertNodeToLibcall()
4708 case ISD::STRICT_LROUND: in ConvertNodeToLibcall()
4714 case ISD::LLROUND: in ConvertNodeToLibcall()
4715 case ISD::STRICT_LLROUND: in ConvertNodeToLibcall()
4721 case ISD::LRINT: in ConvertNodeToLibcall()
4722 case ISD::STRICT_LRINT: in ConvertNodeToLibcall()
4728 case ISD::LLRINT: in ConvertNodeToLibcall()
4729 case ISD::STRICT_LLRINT: in ConvertNodeToLibcall()
4735 case ISD::FDIV: in ConvertNodeToLibcall()
4736 case ISD::STRICT_FDIV: in ConvertNodeToLibcall()
4741 case ISD::FREM: in ConvertNodeToLibcall()
4742 case ISD::STRICT_FREM: in ConvertNodeToLibcall()
4747 case ISD::FMA: in ConvertNodeToLibcall()
4748 case ISD::STRICT_FMA: in ConvertNodeToLibcall()
4753 case ISD::FADD: in ConvertNodeToLibcall()
4754 case ISD::STRICT_FADD: in ConvertNodeToLibcall()
4759 case ISD::FMUL: in ConvertNodeToLibcall()
4760 case ISD::STRICT_FMUL: in ConvertNodeToLibcall()
4765 case ISD::FP16_TO_FP: in ConvertNodeToLibcall()
4770 case ISD::STRICT_BF16_TO_FP: in ConvertNodeToLibcall()
4780 case ISD::STRICT_FP16_TO_FP: { in ConvertNodeToLibcall()
4791 case ISD::FP_TO_FP16: { in ConvertNodeToLibcall()
4798 case ISD::FP_TO_BF16: { in ConvertNodeToLibcall()
4805 case ISD::STRICT_SINT_TO_FP: in ConvertNodeToLibcall()
4806 case ISD::STRICT_UINT_TO_FP: in ConvertNodeToLibcall()
4807 case ISD::SINT_TO_FP: in ConvertNodeToLibcall()
4808 case ISD::UINT_TO_FP: { in ConvertNodeToLibcall()
4811 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4812 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4835 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, in ConvertNodeToLibcall()
4846 case ISD::FP_TO_SINT: in ConvertNodeToLibcall()
4847 case ISD::FP_TO_UINT: in ConvertNodeToLibcall()
4848 case ISD::STRICT_FP_TO_SINT: in ConvertNodeToLibcall()
4849 case ISD::STRICT_FP_TO_UINT: { in ConvertNodeToLibcall()
4852 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4853 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4882 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); in ConvertNodeToLibcall()
4888 case ISD::FP_ROUND: in ConvertNodeToLibcall()
4889 case ISD::STRICT_FP_ROUND: { in ConvertNodeToLibcall()
4913 case ISD::FP_EXTEND: { in ConvertNodeToLibcall()
4920 case ISD::STRICT_FP_EXTEND: in ConvertNodeToLibcall()
4921 case ISD::STRICT_FP_TO_FP16: in ConvertNodeToLibcall()
4922 case ISD::STRICT_FP_TO_BF16: { in ConvertNodeToLibcall()
4924 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16) in ConvertNodeToLibcall()
4926 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16) in ConvertNodeToLibcall()
4942 case ISD::FSUB: in ConvertNodeToLibcall()
4943 case ISD::STRICT_FSUB: in ConvertNodeToLibcall()
4948 case ISD::SREM: in ConvertNodeToLibcall()
4954 case ISD::UREM: in ConvertNodeToLibcall()
4960 case ISD::SDIV: in ConvertNodeToLibcall()
4966 case ISD::UDIV: in ConvertNodeToLibcall()
4972 case ISD::SDIVREM: in ConvertNodeToLibcall()
4973 case ISD::UDIVREM: in ConvertNodeToLibcall()
4977 case ISD::MUL: in ConvertNodeToLibcall()
4983 case ISD::CTLZ_ZERO_UNDEF: in ConvertNodeToLibcall()
4998 case ISD::RESET_FPENV: { in ConvertNodeToLibcall()
5007 case ISD::GET_FPENV_MEM: { in ConvertNodeToLibcall()
5014 case ISD::SET_FPENV_MEM: { in ConvertNodeToLibcall()
5021 case ISD::GET_FPMODE: { in ConvertNodeToLibcall()
5036 case ISD::SET_FPMODE: { in ConvertNodeToLibcall()
5050 case ISD::RESET_FPMODE: { in ConvertNodeToLibcall()
5087 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
5088 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
5089 Node->getOpcode() == ISD::SETCC || in PromoteNode()
5090 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
5091 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
5094 if (Node->getOpcode() == ISD::ATOMIC_STORE || in PromoteNode()
5095 Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
5096 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
5097 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
5098 Node->getOpcode() == ISD::STRICT_FSETCCS || in PromoteNode()
5099 Node->getOpcode() == ISD::VP_REDUCE_FADD || in PromoteNode()
5100 Node->getOpcode() == ISD::VP_REDUCE_FMUL || in PromoteNode()
5101 Node->getOpcode() == ISD::VP_REDUCE_FMAX || in PromoteNode()
5102 Node->getOpcode() == ISD::VP_REDUCE_FMIN || in PromoteNode()
5103 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM || in PromoteNode()
5104 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM || in PromoteNode()
5105 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD) in PromoteNode()
5107 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
5108 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
5114 case ISD::CTTZ: in PromoteNode()
5115 case ISD::CTTZ_ZERO_UNDEF: in PromoteNode()
5116 case ISD::CTLZ: in PromoteNode()
5117 case ISD::CTPOP: { in PromoteNode()
5119 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
5120 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
5121 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5123 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5126 if (NewOpc == ISD::CTTZ) { in PromoteNode()
5132 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, in PromoteNode()
5134 NewOpc = ISD::CTTZ_ZERO_UNDEF; in PromoteNode()
5139 if (NewOpc == ISD::CTLZ) { in PromoteNode()
5141 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, in PromoteNode()
5145 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5148 case ISD::CTLZ_ZERO_UNDEF: { in PromoteNode()
5154 DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5160 DAG.getNode(ISD::SHL, dl, NVT, AnyExtendedNode, ShiftConstant); in PromoteNode()
5164 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, CTLZResult)); in PromoteNode()
5167 case ISD::BITREVERSE: in PromoteNode()
5168 case ISD::BSWAP: { in PromoteNode()
5170 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5173 ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
5177 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5180 case ISD::FP_TO_UINT: in PromoteNode()
5181 case ISD::STRICT_FP_TO_UINT: in PromoteNode()
5182 case ISD::FP_TO_SINT: in PromoteNode()
5183 case ISD::STRICT_FP_TO_SINT: in PromoteNode()
5186 case ISD::FP_TO_UINT_SAT: in PromoteNode()
5187 case ISD::FP_TO_SINT_SAT: in PromoteNode()
5190 case ISD::UINT_TO_FP: in PromoteNode()
5191 case ISD::STRICT_UINT_TO_FP: in PromoteNode()
5192 case ISD::SINT_TO_FP: in PromoteNode()
5193 case ISD::STRICT_SINT_TO_FP: in PromoteNode()
5196 case ISD::VAARG: { in PromoteNode()
5202 TruncOp = ISD::BITCAST; in PromoteNode()
5206 TruncOp = ISD::TRUNCATE; in PromoteNode()
5227 case ISD::MUL: in PromoteNode()
5228 case ISD::SDIV: in PromoteNode()
5229 case ISD::SREM: in PromoteNode()
5230 case ISD::UDIV: in PromoteNode()
5231 case ISD::UREM: in PromoteNode()
5232 case ISD::SMIN: in PromoteNode()
5233 case ISD::SMAX: in PromoteNode()
5234 case ISD::UMIN: in PromoteNode()
5235 case ISD::UMAX: in PromoteNode()
5236 case ISD::AND: in PromoteNode()
5237 case ISD::OR: in PromoteNode()
5238 case ISD::XOR: { in PromoteNode()
5241 ExtOp = ISD::BITCAST; in PromoteNode()
5242 TruncOp = ISD::BITCAST; in PromoteNode()
5248 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
5250 case ISD::SDIV: in PromoteNode()
5251 case ISD::SREM: in PromoteNode()
5252 case ISD::SMIN: in PromoteNode()
5253 case ISD::SMAX: in PromoteNode()
5254 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
5256 case ISD::UDIV: in PromoteNode()
5257 case ISD::UREM: in PromoteNode()
5258 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
5260 case ISD::UMIN: in PromoteNode()
5261 case ISD::UMAX: in PromoteNode()
5263 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
5265 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
5268 TruncOp = ISD::TRUNCATE; in PromoteNode()
5278 case ISD::UMUL_LOHI: in PromoteNode()
5279 case ISD::SMUL_LOHI: { in PromoteNode()
5281 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
5282 : ISD::SIGN_EXTEND; in PromoteNode()
5285 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); in PromoteNode()
5290 ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
5292 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5293 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
5296 case ISD::SELECT: { in PromoteNode()
5300 ExtOp = ISD::BITCAST; in PromoteNode()
5301 TruncOp = ISD::BITCAST; in PromoteNode()
5303 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
5304 TruncOp = ISD::TRUNCATE; in PromoteNode()
5306 ExtOp = ISD::FP_EXTEND; in PromoteNode()
5307 TruncOp = ISD::FP_ROUND; in PromoteNode()
5316 if (TruncOp != ISD::FP_ROUND) in PromoteNode()
5324 case ISD::VECTOR_SHUFFLE: { in PromoteNode()
5328 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
5329 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
5333 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); in PromoteNode()
5337 case ISD::VECTOR_SPLICE: { in PromoteNode()
5338 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5339 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5340 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2, in PromoteNode()
5342 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3)); in PromoteNode()
5345 case ISD::SELECT_CC: { in PromoteNode()
5347 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get(); in PromoteNode()
5352 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
5354 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
5369 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, in PromoteNode()
5373 if (ExtOp != ISD::FP_EXTEND) in PromoteNode()
5374 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1); in PromoteNode()
5376 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1, in PromoteNode()
5382 case ISD::SETCC: in PromoteNode()
5383 case ISD::STRICT_FSETCC: in PromoteNode()
5384 case ISD::STRICT_FSETCCS: { in PromoteNode()
5385 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
5387 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
5390 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
5392 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
5411 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5415 case ISD::BR_CC: { in PromoteNode()
5416 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
5418 ISD::CondCode CCCode = in PromoteNode()
5420 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
5424 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
5429 case ISD::FADD: in PromoteNode()
5430 case ISD::FSUB: in PromoteNode()
5431 case ISD::FMUL: in PromoteNode()
5432 case ISD::FDIV: in PromoteNode()
5433 case ISD::FREM: in PromoteNode()
5434 case ISD::FMINNUM: in PromoteNode()
5435 case ISD::FMAXNUM: in PromoteNode()
5436 case ISD::FMINIMUM: in PromoteNode()
5437 case ISD::FMAXIMUM: in PromoteNode()
5438 case ISD::FPOW: in PromoteNode()
5439 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5440 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5444 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3, in PromoteNode()
5447 case ISD::STRICT_FADD: in PromoteNode()
5448 case ISD::STRICT_FSUB: in PromoteNode()
5449 case ISD::STRICT_FMUL: in PromoteNode()
5450 case ISD::STRICT_FDIV: in PromoteNode()
5451 case ISD::STRICT_FMINNUM: in PromoteNode()
5452 case ISD::STRICT_FMAXNUM: in PromoteNode()
5453 case ISD::STRICT_FREM: in PromoteNode()
5454 case ISD::STRICT_FPOW: in PromoteNode()
5455 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5457 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5459 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
5463 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5468 case ISD::FMA: in PromoteNode()
5469 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5470 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5471 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
5473 DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
5477 case ISD::STRICT_FMA: in PromoteNode()
5478 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5480 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5482 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5484 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
5488 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5493 case ISD::FCOPYSIGN: in PromoteNode()
5494 case ISD::FLDEXP: in PromoteNode()
5495 case ISD::FPOWI: { in PromoteNode()
5496 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5505 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
5507 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3, in PromoteNode()
5511 case ISD::STRICT_FPOWI: in PromoteNode()
5512 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5516 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5521 case ISD::FFREXP: { in PromoteNode()
5522 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5523 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1); in PromoteNode()
5526 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5532 case ISD::FFLOOR: in PromoteNode()
5533 case ISD::FCEIL: in PromoteNode()
5534 case ISD::FRINT: in PromoteNode()
5535 case ISD::FNEARBYINT: in PromoteNode()
5536 case ISD::FROUND: in PromoteNode()
5537 case ISD::FROUNDEVEN: in PromoteNode()
5538 case ISD::FTRUNC: in PromoteNode()
5539 case ISD::FNEG: in PromoteNode()
5540 case ISD::FSQRT: in PromoteNode()
5541 case ISD::FSIN: in PromoteNode()
5542 case ISD::FCOS: in PromoteNode()
5543 case ISD::FTAN: in PromoteNode()
5544 case ISD::FASIN: in PromoteNode()
5545 case ISD::FACOS: in PromoteNode()
5546 case ISD::FATAN: in PromoteNode()
5547 case ISD::FSINH: in PromoteNode()
5548 case ISD::FCOSH: in PromoteNode()
5549 case ISD::FTANH: in PromoteNode()
5550 case ISD::FLOG: in PromoteNode()
5551 case ISD::FLOG2: in PromoteNode()
5552 case ISD::FLOG10: in PromoteNode()
5553 case ISD::FABS: in PromoteNode()
5554 case ISD::FEXP: in PromoteNode()
5555 case ISD::FEXP2: in PromoteNode()
5556 case ISD::FEXP10: in PromoteNode()
5557 case ISD::FCANONICALIZE: in PromoteNode()
5558 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5561 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5564 case ISD::STRICT_FFLOOR: in PromoteNode()
5565 case ISD::STRICT_FCEIL: in PromoteNode()
5566 case ISD::STRICT_FRINT: in PromoteNode()
5567 case ISD::STRICT_FNEARBYINT: in PromoteNode()
5568 case ISD::STRICT_FROUND: in PromoteNode()
5569 case ISD::STRICT_FROUNDEVEN: in PromoteNode()
5570 case ISD::STRICT_FTRUNC: in PromoteNode()
5571 case ISD::STRICT_FSQRT: in PromoteNode()
5572 case ISD::STRICT_FSIN: in PromoteNode()
5573 case ISD::STRICT_FCOS: in PromoteNode()
5574 case ISD::STRICT_FTAN: in PromoteNode()
5575 case ISD::STRICT_FASIN: in PromoteNode()
5576 case ISD::STRICT_FACOS: in PromoteNode()
5577 case ISD::STRICT_FATAN: in PromoteNode()
5578 case ISD::STRICT_FSINH: in PromoteNode()
5579 case ISD::STRICT_FCOSH: in PromoteNode()
5580 case ISD::STRICT_FTANH: in PromoteNode()
5581 case ISD::STRICT_FLOG: in PromoteNode()
5582 case ISD::STRICT_FLOG2: in PromoteNode()
5583 case ISD::STRICT_FLOG10: in PromoteNode()
5584 case ISD::STRICT_FEXP: in PromoteNode()
5585 case ISD::STRICT_FEXP2: in PromoteNode()
5586 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5590 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5595 case ISD::BUILD_VECTOR: { in PromoteNode()
5613 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); in PromoteNode()
5617 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS, in PromoteNode()
5619 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5623 case ISD::EXTRACT_VECTOR_ELT: { in PromoteNode()
5649 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
5651 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5656 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
5658 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
5664 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); in PromoteNode()
5667 case ISD::INSERT_VECTOR_ELT: { in PromoteNode()
5697 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
5699 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5700 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5705 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
5707 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
5710 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, in PromoteNode()
5714 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); in PromoteNode()
5717 case ISD::SCALAR_TO_VECTOR: { in PromoteNode()
5732 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5740 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); in PromoteNode()
5741 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5745 case ISD::ATOMIC_SWAP: in PromoteNode()
5746 case ISD::ATOMIC_STORE: { in PromoteNode()
5749 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); in PromoteNode()
5760 if (AM->getOpcode() == ISD::ATOMIC_STORE) in PromoteNode()
5766 if (AM->getOpcode() != ISD::ATOMIC_STORE) { in PromoteNode()
5767 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); in PromoteNode()
5773 case ISD::ATOMIC_LOAD: { in PromoteNode()
5782 DAG.getAtomic(ISD::ATOMIC_LOAD, SL, NVT, DAG.getVTList(NVT, MVT::Other), in PromoteNode()
5784 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); in PromoteNode()
5788 case ISD::SPLAT_VECTOR: { in PromoteNode()
5793 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar); in PromoteNode()
5795 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
5798 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar); in PromoteNode()
5801 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5805 case ISD::VP_REDUCE_FADD: in PromoteNode()
5806 case ISD::VP_REDUCE_FMUL: in PromoteNode()
5807 case ISD::VP_REDUCE_FMAX: in PromoteNode()
5808 case ISD::VP_REDUCE_FMIN: in PromoteNode()
5809 case ISD::VP_REDUCE_FMAXIMUM: in PromoteNode()
5810 case ISD::VP_REDUCE_FMINIMUM: in PromoteNode()
5811 case ISD::VP_REDUCE_SEQ_FADD: in PromoteNode()