Lines Matching refs:N0
424 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
430 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
436 SDValue visitUADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
438 SDValue visitSADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
447 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
449 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
460 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
462 SDValue visitORLike(SDValue N0, SDValue N1, const SDLoc &DL);
573 SDValue N0,
575 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
577 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
580 EVT VT, SDValue N0, SDValue N1,
590 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
591 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
595 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
598 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
602 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
608 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
637 SDValue buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N);
650 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
652 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
657 SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
1003 SDValue N0, N1, N2; in isOneUseSetCC() local
1004 if (isSetCCEquivalent(N, N0, N1, N2) && N->hasOneUse()) in isOneUseSetCC()
1075 SDValue N0, in reassociationCanBreakAddressingModePattern() argument
1086 if (N0.getOpcode() != ISD::ADD) in reassociationCanBreakAddressingModePattern()
1134 if (auto *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { in reassociationCanBreakAddressingModePattern()
1135 if (N0.hasOneUse()) in reassociationCanBreakAddressingModePattern()
1165 if (auto *GA = dyn_cast<GlobalAddressSDNode>(N0.getOperand(1))) in reassociationCanBreakAddressingModePattern()
1194 SDValue N0, SDValue N1, in reassociateOpsCommutative() argument
1196 EVT VT = N0.getValueType(); in reassociateOpsCommutative()
1198 if (N0.getOpcode() != Opc) in reassociateOpsCommutative()
1201 SDValue N00 = N0.getOperand(0); in reassociateOpsCommutative()
1202 SDValue N01 = N0.getOperand(1); in reassociateOpsCommutative()
1206 if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() && in reassociateOpsCommutative()
1216 if (TLI.isReassocProfitable(DAG, N0, N1)) { in reassociateOpsCommutative()
1219 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, NewFlags); in reassociateOpsCommutative()
1231 return N0; in reassociateOpsCommutative()
1242 if (TLI.isReassocProfitable(DAG, N0, N1)) { in reassociateOpsCommutative()
1277 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, Flags); in reassociateOpsCommutative()
1281 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N01, N1, Flags); in reassociateOpsCommutative()
1293 SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0, in reassociateOps() argument
1298 if (N0.getValueType().isFloatingPoint() || in reassociateOps()
1303 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1, Flags)) in reassociateOps()
1305 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0, Flags)) in reassociateOps()
1314 const SDLoc &DL, EVT VT, SDValue N0, in reassociateReduction() argument
1316 if (N0.getOpcode() == RedOpc && N1.getOpcode() == RedOpc && in reassociateReduction()
1317 N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && in reassociateReduction()
1318 N0->hasOneUse() && N1->hasOneUse() && in reassociateReduction()
1319 TLI.isOperationLegalOrCustom(Opc, N0.getOperand(0).getValueType()) && in reassociateReduction()
1320 TLI.shouldReassociateReduction(RedOpc, N0.getOperand(0).getValueType())) { in reassociateReduction()
1323 DAG.getNode(Opc, DL, N0.getOperand(0).getValueType(), in reassociateReduction()
1324 N0.getOperand(0), N1.getOperand(0))); in reassociateReduction()
1522 SDValue N0 = Op.getOperand(0); in PromoteIntBinOp() local
1523 SDValue NN0 = PromoteOperand(N0, PVT, Replace0); in PromoteIntBinOp()
1538 Replace0 &= !N0->hasOneUse(); in PromoteIntBinOp()
1539 Replace1 &= (N0 != N1) && !N1->hasOneUse(); in PromoteIntBinOp()
1546 if (Replace0 && Replace1 && N0->isPredecessorOf(N1.getNode())) { in PromoteIntBinOp()
1547 std::swap(N0, N1); in PromoteIntBinOp()
1553 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode()); in PromoteIntBinOp()
1590 SDValue N0 = Op.getOperand(0); in PromoteIntShiftOp() local
1592 N0 = SExtPromoteOperand(N0, PVT); in PromoteIntShiftOp()
1594 N0 = ZExtPromoteOperand(N0, PVT); in PromoteIntShiftOp()
1596 N0 = PromoteOperand(N0, PVT, Replace); in PromoteIntShiftOp()
1598 if (!N0.getNode()) in PromoteIntShiftOp()
1604 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1607 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode()); in PromoteIntShiftOp()
2054 SDValue N0 = N->getOperand(0); in combine() local
2058 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) { in combine()
2059 SDValue Ops[] = {N1, N0}; in combine()
2387 SDValue N0 = N->getOperand(0); in foldSelectWithIdentityConstant() local
2390 std::swap(N0, N1); in foldSelectWithIdentityConstant()
2411 SDValue F0 = DAG.getFreeze(N0); in foldSelectWithIdentityConstant()
2417 SDValue F0 = DAG.getFreeze(N0); in foldSelectWithIdentityConstant()
2565 SDValue N0 = N->getOperand(0); in foldSubToAvg() local
2566 EVT VT = N0.getValueType(); in foldSubToAvg()
2636 SDValue N0 = N->getOperand(0); in visitADDLike() local
2638 EVT VT = N0.getValueType(); in visitADDLike()
2642 if (N0.isUndef()) in visitADDLike()
2643 return N0; in visitADDLike()
2648 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1})) in visitADDLike()
2652 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitADDLike()
2654 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2656 if (areBitwiseNotOfEachother(N0, N1)) in visitADDLike()
2666 return N0; in visitADDLike()
2671 return N0; in visitADDLike()
2673 if (N0.getOpcode() == ISD::SUB) { in visitADDLike()
2674 SDValue N00 = N0.getOperand(0); in visitADDLike()
2675 SDValue N01 = N0.getOperand(1); in visitADDLike()
2679 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2683 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2690 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() && in visitADDLike()
2692 SDValue X = N0.getOperand(0); in visitADDLike()
2706 if (DAG.isADDLike(N0)) { in visitADDLike()
2707 SDValue N01 = N0.getOperand(1); in visitADDLike()
2709 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add); in visitADDLike()
2716 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) { in visitADDLike()
2717 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) in visitADDLike()
2726 auto ReassociateAddOr = [&](SDValue N0, SDValue N1) { in visitADDLike() argument
2727 if (DAG.isADDLike(N0) && N0.hasOneUse() && in visitADDLike()
2728 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) { in visitADDLike()
2731 auto TyActn = TLI.getTypeAction(*DAG.getContext(), N0.getValueType()); in visitADDLike()
2734 isMinSignedConstant(N0.getOperand(1)); in visitADDLike()
2738 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2739 N0.getOperand(1)); in visitADDLike()
2743 if (SDValue Add = ReassociateAddOr(N0, N1)) in visitADDLike()
2745 if (SDValue Add = ReassociateAddOr(N1, N0)) in visitADDLike()
2750 reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1)) in visitADDLike()
2757 if (sd_match(N0, m_Neg(m_Value(A)))) in visitADDLike()
2762 return DAG.getNode(ISD::SUB, DL, VT, N0, B); in visitADDLike()
2765 if (sd_match(N1, m_Sub(m_Value(B), m_Specific(N0)))) in visitADDLike()
2769 if (sd_match(N0, m_Sub(m_Value(B), m_Specific(N1)))) in visitADDLike()
2773 if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) && in visitADDLike()
2778 if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) && in visitADDLike()
2784 if (sd_match(N1, m_Sub(m_Value(B), m_Add(m_Specific(N0), m_Value(C))))) in visitADDLike()
2789 m_AnyOf(m_Add(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C)), in visitADDLike()
2790 m_Sub(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C))))) in visitADDLike()
2794 if (sd_match(N0, m_OneUse(m_Sub(m_Value(A), m_Value(B)))) && in visitADDLike()
2798 DAG.getNode(ISD::ADD, SDLoc(N0), VT, A, C), in visitADDLike()
2802 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2807 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT, in visitADDLike()
2809 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2810 N0.getOperand(1)); in visitADDLike()
2818 if (isBitwiseNot(N0)) in visitADDLike()
2820 N0.getOperand(0)); in visitADDLike()
2823 if (N0.getOpcode() == ISD::ADD) { in visitADDLike()
2826 if (isBitwiseNot(N0.getOperand(0))) { in visitADDLike()
2827 A = N0.getOperand(1); in visitADDLike()
2828 Xor = N0.getOperand(0); in visitADDLike()
2829 } else if (isBitwiseNot(N0.getOperand(1))) { in visitADDLike()
2830 A = N0.getOperand(0); in visitADDLike()
2831 Xor = N0.getOperand(1); in visitADDLike()
2842 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLike()
2843 N0.hasOneUse() && in visitADDLike()
2847 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT); in visitADDLike()
2848 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2853 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitADDLike()
2855 SDValue Not = DAG.getNOT(DL, N0.getOperand(1), VT); in visitADDLike()
2856 return DAG.getNode(ISD::ADD, DL, VT, Not, N0.getOperand(0)); in visitADDLike()
2864 if (sd_match(N0, m_OneUse(m_Mul(m_Add(m_Value(A), m_ConstInt(CA)), in visitADDLike()
2872 N0->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2873 N0.getOperand(0)->getFlags().hasNoUnsignedWrap()) { in visitADDLike()
2876 N0->getFlags().hasNoSignedWrap() && in visitADDLike()
2877 N0.getOperand(0)->getFlags().hasNoSignedWrap()) in visitADDLike()
2887 if (sd_match(N0, m_OneUse(m_Add( in visitADDLike()
2897 N0.getOperand(0) == B ? N0.getOperand(1) : N0.getOperand(0); in visitADDLike()
2899 N0->getFlags().hasNoUnsignedWrap() && in visitADDLike()
2904 N0->getFlags().hasNoSignedWrap() && in visitADDLike()
2919 if (SDValue Combined = visitADDLikeCommutative(N0, N1, N)) in visitADDLike()
2922 if (SDValue Combined = visitADDLikeCommutative(N1, N0, N)) in visitADDLike()
2930 SDValue N0 = N->getOperand(0); in foldAddToAvg() local
2931 EVT VT = N0.getValueType(); in foldAddToAvg()
2951 SDValue N0 = N->getOperand(0); in visitADD() local
2953 EVT VT = N0.getValueType(); in visitADD()
2971 DAG.haveNoCommonBitsSet(N0, N1)) { in visitADD()
2974 return DAG.getNode(ISD::OR, DL, VT, N0, N1, Flags); in visitADD()
2978 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) { in visitADD()
2979 const APInt &C0 = N0->getConstantOperandAPInt(0); in visitADD()
2985 if (N0.getOpcode() == ISD::ADD && in visitADD()
2986 N0.getOperand(1).getOpcode() == ISD::VSCALE && in visitADD()
2988 const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0); in visitADD()
2991 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
2995 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitADD()
2997 const APInt &C0 = N0->getConstantOperandAPInt(0); in visitADD()
3004 if (N0.getOpcode() == ISD::ADD && in visitADD()
3005 N0.getOperand(1).getOpcode() == ISD::STEP_VECTOR && in visitADD()
3007 const APInt &SV0 = N0.getOperand(1)->getConstantOperandAPInt(0); in visitADD()
3011 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
3019 SDValue N0 = N->getOperand(0); in visitADDSAT() local
3021 EVT VT = N0.getValueType(); in visitADDSAT()
3026 if (N0.isUndef() || N1.isUndef()) in visitADDSAT()
3030 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitADDSAT()
3034 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitADDSAT()
3036 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitADDSAT()
3045 return N0; in visitADDSAT()
3050 return N0; in visitADDSAT()
3053 if (DAG.willNotOverflowAdd(IsSigned, N0, N1)) in visitADDSAT()
3054 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
3111 static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, in foldAddSubMasked1() argument
3119 EVT VT = N0.getValueType(); in foldAddSubMasked1()
3132 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N10); in foldAddSubMasked1()
3136 SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1, in visitADDLikeCommutative() argument
3138 EVT VT = N0.getValueType(); in visitADDLikeCommutative()
3144 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
3147 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL)) in visitADDLikeCommutative()
3154 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLikeCommutative()
3155 N0.hasOneUse() && isOneOrOneSplat(N0.getOperand(1)) && in visitADDLikeCommutative()
3157 (Level >= AfterLegalizeDAG || (!N0->getFlags().hasNoUnsignedWrap() && in visitADDLikeCommutative()
3158 !N0->getFlags().hasNoSignedWrap()))) { in visitADDLikeCommutative()
3159 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT); in visitADDLikeCommutative()
3163 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) { in visitADDLikeCommutative()
3167 if (isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) { in visitADDLikeCommutative()
3168 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
3169 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
3173 if (isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) { in visitADDLikeCommutative()
3174 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
3175 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
3180 if (N0.getOpcode() == ISD::MUL && N0.getOperand(0) == N1 && in visitADDLikeCommutative()
3181 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true) && in visitADDLikeCommutative()
3182 N0.hasOneUse()) { in visitADDLikeCommutative()
3183 SDValue NewC = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitADDLikeCommutative()
3185 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), NewC); in visitADDLikeCommutative()
3191 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitADDLikeCommutative()
3192 N0.getOperand(0).getScalarValueSizeInBits() == 1 && in visitADDLikeCommutative()
3194 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
3204 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
3212 N0, N1.getOperand(0), N1.getOperand(2)); in visitADDLikeCommutative()
3218 DAG.getVTList(VT, Carry.getValueType()), N0, in visitADDLikeCommutative()
3225 SDValue N0 = N->getOperand(0); in visitADDC() local
3227 EVT VT = N0.getValueType(); in visitADDC()
3232 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3236 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); in visitADDC()
3239 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0); in visitADDC()
3243 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
3247 if (DAG.computeOverflowForUnsignedAdd(N0, N1) == SelectionDAG::OFK_Never) in visitADDC()
3248 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3297 SDValue N0 = N->getOperand(0); in visitADDO() local
3299 EVT VT = N0.getValueType(); in visitADDO()
3307 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3311 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitADDO()
3313 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0); in visitADDO()
3317 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT)); in visitADDO()
3320 if (DAG.willNotOverflowAdd(IsSigned, N0, N1)) in visitADDO()
3321 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3326 if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) in visitADDO()
3328 DAG.getConstant(0, DL, VT), N0.getOperand(0)); in visitADDO()
3331 if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) { in visitADDO()
3333 DAG.getConstant(0, DL, VT), N0.getOperand(0)); in visitADDO()
3338 if (SDValue Combined = visitUADDOLike(N0, N1, N)) in visitADDO()
3341 if (SDValue Combined = visitUADDOLike(N1, N0, N)) in visitADDO()
3348 SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) { in visitUADDOLike() argument
3349 EVT VT = N0.getValueType(); in visitUADDOLike()
3359 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y, in visitUADDOLike()
3366 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, in visitUADDOLike()
3373 SDValue N0 = N->getOperand(0); in visitADDE() local
3378 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); in visitADDE()
3382 N1, N0, CarryIn); in visitADDE()
3386 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); in visitADDE()
3392 SDValue N0 = N->getOperand(0); in visitUADDO_CARRY() local
3398 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); in visitUADDO_CARRY()
3401 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitUADDO_CARRY()
3407 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); in visitUADDO_CARRY()
3411 if (isNullConstant(N0) && isNullConstant(N1)) { in visitUADDO_CARRY()
3412 EVT VT = N0.getValueType(); in visitUADDO_CARRY()
3421 if (SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn, N)) in visitUADDO_CARRY()
3424 if (SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn, N)) in visitUADDO_CARRY()
3432 SDValue Ops[] = {N1, N0, CarryIn}; in visitUADDO_CARRY()
3557 SDValue N0, SDValue N1, SDNode *N) { in combineCarryDiamond() argument
3558 SDValue Carry0 = getAsCarry(TLI, N0); in combineCarryDiamond()
3628 SDValue DAGCombiner::visitUADDO_CARRYLike(SDValue N0, SDValue N1, in visitUADDO_CARRYLike() argument
3632 if (isBitwiseNot(N0)) in visitUADDO_CARRYLike()
3636 N0.getOperand(0), NotC); in visitUADDO_CARRYLike()
3645 if ((N0.getOpcode() == ISD::ADD || in visitUADDO_CARRYLike()
3646 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 && in visitUADDO_CARRYLike()
3647 N0.getValue(1) != CarryIn)) && in visitUADDO_CARRYLike()
3650 N0.getOperand(0), N0.getOperand(1), CarryIn); in visitUADDO_CARRYLike()
3659 if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, Y, CarryIn, N)) in visitUADDO_CARRYLike()
3661 if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, CarryIn, Y, N)) in visitUADDO_CARRYLike()
3668 SDValue DAGCombiner::visitSADDO_CARRYLike(SDValue N0, SDValue N1, in visitSADDO_CARRYLike() argument
3671 if (isBitwiseNot(N0)) { in visitSADDO_CARRYLike()
3674 N0.getOperand(0), NotC); in visitSADDO_CARRYLike()
3681 SDValue N0 = N->getOperand(0); in visitSADDO_CARRY() local
3687 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); in visitSADDO_CARRY()
3690 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitSADDO_CARRY()
3696 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1); in visitSADDO_CARRY()
3699 if (SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn, N)) in visitSADDO_CARRY()
3702 if (SDValue Combined = visitSADDO_CARRYLike(N1, N0, CarryIn, N)) in visitSADDO_CARRY()
3796 SDValue N0 = N->getOperand(0); in visitSUB() local
3798 EVT VT = N0.getValueType(); in visitSUB()
3810 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1)) in visitSUB()
3814 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3824 return N0; in visitSUB()
3832 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3835 if (isNullOrNullSplat(N0)) { in visitSUB()
3851 return N0; in visitSUB()
3857 return N0; in visitSUB()
3879 if (isAllOnesOrAllOnesSplat(N0)) in visitSUB()
3880 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3884 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3887 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0)) in visitSUB()
3891 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) in visitSUB()
3892 return N0.getOperand(1); in visitSUB()
3895 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) in visitSUB()
3896 return N0.getOperand(0); in visitSUB()
3899 if (N0.getOpcode() == ISD::ADD) { in visitSUB()
3900 SDValue N01 = N0.getOperand(1); in visitSUB()
3902 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3908 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11})) in visitSUB()
3913 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3914 SDValue N01 = N0.getOperand(1); in visitSUB()
3916 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3920 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3921 SDValue N00 = N0.getOperand(0); in visitSUB()
3923 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3929 if (sd_match(N0, m_Add(m_Value(A), m_Add(m_Specific(N1), m_Value(C))))) in visitSUB()
3933 if (sd_match(N0, m_Add(m_Value(A), m_Sub(m_Specific(N1), m_Value(C))))) in visitSUB()
3937 if (sd_match(N0, m_Sub(m_Value(A), m_Sub(m_Value(B), m_Specific(N1))))) in visitSUB()
3942 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3946 if (sd_match(N1, m_And(m_Specific(N0), m_Value(B))) && in visitSUB()
3948 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getNOT(DL, B, VT)); in visitSUB()
3952 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3956 if (N0.isUndef()) in visitSUB()
3957 return N0; in visitSUB()
3971 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, DL)) in visitSUB()
3986 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3992 if (!reassociationCanBreakAddressingModePattern(ISD::SUB, DL, N, N0, N1) && in visitSUB()
3993 N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSUB()
3994 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) { in visitSUB()
3995 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3996 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4001 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
4006 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSUB()
4007 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) { in visitSUB()
4008 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
4009 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4012 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSUB()
4013 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) { in visitSUB()
4014 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
4015 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
4026 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
4032 sd_match(N0, m_Xor(m_Specific(A), m_Specific(N1)))) in visitSUB()
4036 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0)) in visitSUB()
4051 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
4058 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
4064 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
4075 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
4085 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitSUB()
4089 if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(N0.getOperand(1)) && in visitSUB()
4090 N0.getResNo() == 0 && N0.hasOneUse()) in visitSUB()
4091 return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(), in visitSUB()
4092 N0.getOperand(0), N1, N0.getOperand(2)); in visitSUB()
4096 if (SDValue Carry = getAsCarry(TLI, N0)) { in visitSUB()
4108 if (ConstantSDNode *C0 = isConstOrConstSplat(N0)) { in visitSUB()
4113 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
4119 sd_match(N0, m_SMax(m_Value(A), m_Value(B))) && in visitSUB()
4125 sd_match(N0, m_UMax(m_Value(A), m_Value(B))) && in visitSUB()
4134 SDValue N0 = N->getOperand(0); in visitSUBSAT() local
4136 EVT VT = N0.getValueType(); in visitSUBSAT()
4141 if (N0.isUndef() || N1.isUndef()) in visitSUBSAT()
4145 if (N0 == N1) in visitSUBSAT()
4149 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitSUBSAT()
4159 return N0; in visitSUBSAT()
4164 return N0; in visitSUBSAT()
4167 if (DAG.willNotOverflowSub(IsSigned, N0, N1)) in visitSUBSAT()
4168 return DAG.getNode(ISD::SUB, DL, VT, N0, N1); in visitSUBSAT()
4174 SDValue N0 = N->getOperand(0); in visitSUBC() local
4176 EVT VT = N0.getValueType(); in visitSUBC()
4181 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
4185 if (N0 == N1) in visitSUBC()
4191 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4194 if (isAllOnesConstant(N0)) in visitSUBC()
4195 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
4202 SDValue N0 = N->getOperand(0); in visitSUBO() local
4204 EVT VT = N0.getValueType(); in visitSUBO()
4212 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4216 if (N0 == N1) in visitSUBO()
4223 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, in visitSUBO()
4228 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT)); in visitSUBO()
4231 if (DAG.willNotOverflowSub(IsSigned, N0, N1)) in visitSUBO()
4232 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4236 if (!IsSigned && isAllOnesOrAllOnesSplat(N0)) in visitSUBO()
4237 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
4244 SDValue N0 = N->getOperand(0); in visitSUBE() local
4250 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); in visitSUBE()
4256 SDValue N0 = N->getOperand(0); in visitUSUBO_CARRY() local
4264 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1); in visitUSUBO_CARRY()
4271 SDValue N0 = N->getOperand(0); in visitSSUBO_CARRY() local
4279 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1); in visitSSUBO_CARRY()
4288 SDValue N0 = N->getOperand(0); in visitMULFIX() local
4291 EVT VT = N0.getValueType(); in visitMULFIX()
4294 if (N0.isUndef() || N1.isUndef()) in visitMULFIX()
4298 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitMULFIX()
4300 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale); in visitMULFIX()
4310 SDValue N0 = N->getOperand(0); in visitMUL() local
4312 EVT VT = N0.getValueType(); in visitMUL()
4319 if (N0.isUndef() || N1.isUndef()) in visitMUL()
4323 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1})) in visitMUL()
4327 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitMUL()
4329 return Matcher.getNode(ISD::MUL, DL, VT, N1, N0); in visitMUL()
4359 return N0; in visitMUL()
4367 return Matcher.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0); in visitMUL()
4373 EVT ShiftVT = getShiftAmountTy(N0.getValueType()); in visitMUL()
4375 return Matcher.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
4387 Matcher.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4398 if (SDNode *LoHi = DAG.getNodeIfExists(LoHiOpc, LoHiVT, {N0, N1})) in visitMUL()
4401 if (SDNode *LoHi = DAG.getNodeIfExists(LoHiOpc, LoHiVT, {N1, N0})) in visitMUL()
4445 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
4448 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4450 : DAG.getNode(MathOp, DL, VT, Shl, N0); in visitMUL()
4458 if (sd_context_match(N0, Matcher, m_Opc(ISD::SHL))) { in visitMUL()
4459 SDValue N01 = N0.getOperand(1); in visitMUL()
4461 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3); in visitMUL()
4470 if (sd_context_match(N0, Matcher, m_OneUse(m_Opc(ISD::SHL))) && in visitMUL()
4471 isConstantOrConstantVector(N0.getOperand(1))) { in visitMUL()
4472 Sh = N0; Y = N1; in visitMUL()
4475 Sh = N1; Y = N0; in visitMUL()
4485 if (sd_context_match(N0, Matcher, m_Opc(ISD::ADD)) && in visitMUL()
4487 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) && in visitMUL()
4488 isMulAddWithConstProfitable(N, N0, N1)) in visitMUL()
4491 Matcher.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1), in visitMUL()
4492 Matcher.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1)); in visitMUL()
4496 if (!UseVP && N0.getOpcode() == ISD::VSCALE && NC1) { in visitMUL()
4497 const APInt &C0 = N0.getConstantOperandAPInt(0); in visitMUL()
4504 if (!UseVP && N0.getOpcode() == ISD::STEP_VECTOR && in visitMUL()
4506 const APInt &C0 = N0.getConstantOperandAPInt(0); in visitMUL()
4547 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4554 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags())) in visitMUL()
4561 reassociateReduction(ISD::VECREDUCE_MUL, ISD::MUL, DL, VT, N0, N1)) in visitMUL()
4660 SDValue N0 = N->getOperand(0); in simplifyDivRem() local
4674 if (DAG.isUndef(Opc, {N0, N1})) in simplifyDivRem()
4679 if (N0.isUndef()) in simplifyDivRem()
4684 ConstantSDNode *N0C = isConstOrConstSplat(N0); in simplifyDivRem()
4686 return N0; in simplifyDivRem()
4690 if (N0 == N1) in simplifyDivRem()
4700 return IsDiv ? N0 : DAG.getConstant(0, DL, VT); in simplifyDivRem()
4706 SDValue N0 = N->getOperand(0); in visitSDIV() local
4713 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4724 return DAG.getNegative(N0, DL, VT); in visitSDIV()
4728 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4740 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0)) in visitSDIV()
4741 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1); in visitSDIV()
4743 if (SDValue V = visitSDIVLike(N0, N1, N)) { in visitSDIV()
4747 { N0, N1 })) { in visitSDIV()
4749 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4784 SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) { in visitSDIVLike() argument
4800 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType()); in visitSDIVLike()
4809 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4816 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4828 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra); in visitSDIVLike()
4854 SDValue N0 = N->getOperand(0); in visitUDIV() local
4861 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4872 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4883 if (SDValue V = visitUDIVLike(N0, N1, N)) { in visitUDIV()
4887 { N0, N1 })) { in visitUDIV()
4889 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4908 SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) { in visitUDIVLike() argument
4917 EVT ShiftVT = getShiftAmountTy(N0.getValueType()); in visitUDIVLike()
4920 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4936 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4951 SDValue DAGCombiner::buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N) { in buildOptimizedSREM() argument
4953 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) { in buildOptimizedSREM()
4964 SDValue N0 = N->getOperand(0); in visitREM() local
4973 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitREM()
4980 SDValue F0 = DAG.getFreeze(N0); in visitREM()
4994 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0)) in visitREM()
4995 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
5002 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5013 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5029 if (SDValue OptimizedRem = buildOptimizedSREM(N0, N1, N)) in visitREM()
5034 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N); in visitREM()
5039 { N0, N1 })) in visitREM()
5042 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
5057 SDValue N0 = N->getOperand(0); in visitMULHS() local
5063 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
5067 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitMULHS()
5069 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0); in visitMULHS()
5088 ISD::SRA, DL, VT, N0, in visitMULHS()
5089 DAG.getShiftAmountConstant(N0.getScalarValueSizeInBits() - 1, VT, DL)); in visitMULHS()
5092 if (N0.isUndef() || N1.isUndef()) in visitMULHS()
5103 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitMULHS()
5105 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHS()
5116 SDValue N0 = N->getOperand(0); in visitMULHU() local
5122 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
5126 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitMULHU()
5128 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0); in visitMULHU()
5149 if (N0.isUndef() || N1.isUndef()) in visitMULHU()
5159 EVT ShiftVT = getShiftAmountTy(N0.getValueType()); in visitMULHU()
5161 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
5173 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitMULHU()
5175 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHU()
5193 SDValue N0 = N->getOperand(0); in visitAVG() local
5200 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitAVG()
5204 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitAVG()
5206 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0); in visitAVG()
5213 if (N0.isUndef()) in visitAVG()
5216 return N0; in visitAVG()
5219 if (N0 == N1 && Level >= AfterLegalizeTypes) in visitAVG()
5220 return N0; in visitAVG()
5255 ISD::AVGCEILU, DL, VT, N0, in visitAVG()
5257 if (DAG.isKnownNeverZero(N0)) in visitAVG()
5260 DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getAllOnesConstant(DL, VT))); in visitAVG()
5268 SDValue N0 = N->getOperand(0); in visitABD() local
5274 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitABD()
5278 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitABD()
5280 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0); in visitABD()
5287 if (N0.isUndef() || N1.isUndef()) in visitABD()
5303 DAG.SignBitIsZero(N0) && DAG.SignBitIsZero(N1)) in visitABD()
5304 return DAG.getNode(ISD::ABDU, DL, VT, N1, N0); in visitABD()
5362 SDValue N0 = N->getOperand(0); in visitSMUL_LOHI() local
5368 if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1)) in visitSMUL_LOHI()
5369 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N0, N1); in visitSMUL_LOHI()
5372 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitSMUL_LOHI()
5374 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0); in visitSMUL_LOHI()
5383 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitSMUL_LOHI()
5403 SDValue N0 = N->getOperand(0); in visitUMUL_LOHI() local
5409 if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1)) in visitUMUL_LOHI()
5410 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N0, N1); in visitUMUL_LOHI()
5413 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitUMUL_LOHI()
5415 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0); in visitUMUL_LOHI()
5426 return CombineTo(N, N0, Zero); in visitUMUL_LOHI()
5436 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitUMUL_LOHI()
5453 SDValue N0 = N->getOperand(0); in visitMULO() local
5455 EVT VT = N0.getValueType(); in visitMULO()
5461 ConstantSDNode *N0C = isConstOrConstSplat(N0); in visitMULO()
5477 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitMULO()
5479 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0); in visitMULO()
5491 N->getVTList(), N0, N0); in visitMULO()
5495 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
5502 if (DAG.willNotOverflowMul(IsSigned, N0, N1)) in visitMULO()
5503 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5514 static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2, in isSaturatingMinMax() argument
5517 auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3, in isSaturatingMinMax()
5521 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) in isSaturatingMinMax()
5536 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC); in isSaturatingMinMax()
5542 if (N0.getOpcode() == ISD::FP_TO_SINT && Opcode0 == ISD::SMAX) { in isSaturatingMinMax()
5544 EVT IntVT = N0.getValueType().getScalarType(); in isSaturatingMinMax()
5545 EVT FPVT = N0.getOperand(0).getValueType().getScalarType(); in isSaturatingMinMax()
5554 return N0; in isSaturatingMinMax()
5562 switch (N0.getOpcode()) { in isSaturatingMinMax()
5565 N00 = N02 = N0.getOperand(0); in isSaturatingMinMax()
5566 N01 = N03 = N0.getOperand(1); in isSaturatingMinMax()
5567 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT; in isSaturatingMinMax()
5570 N00 = N0.getOperand(0); in isSaturatingMinMax()
5571 N01 = N0.getOperand(1); in isSaturatingMinMax()
5572 N02 = N0.getOperand(2); in isSaturatingMinMax()
5573 N03 = N0.getOperand(3); in isSaturatingMinMax()
5574 N0CC = cast<CondCodeSDNode>(N0.getOperand(4))->get(); in isSaturatingMinMax()
5578 if (N0.getOperand(0).getOpcode() != ISD::SETCC) in isSaturatingMinMax()
5580 N00 = N0.getOperand(0).getOperand(0); in isSaturatingMinMax()
5581 N01 = N0.getOperand(0).getOperand(1); in isSaturatingMinMax()
5582 N02 = N0.getOperand(1); in isSaturatingMinMax()
5583 N03 = N0.getOperand(2); in isSaturatingMinMax()
5584 N0CC = cast<CondCodeSDNode>(N0.getOperand(0).getOperand(2))->get(); in isSaturatingMinMax()
5617 static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, in PerformMinMaxFpToSatCombine() argument
5622 SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned, DAG); in PerformMinMaxFpToSatCombine()
5639 static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, in PerformUMinFpToSatCombine() argument
5645 if ((N0 != N2 && in PerformUMinFpToSatCombine()
5646 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) || in PerformUMinFpToSatCombine()
5647 N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT) in PerformUMinFpToSatCombine()
5660 EVT FPVT = N0.getOperand(0).getValueType(); in PerformUMinFpToSatCombine()
5670 DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0), in PerformUMinFpToSatCombine()
5672 return DAG.getZExtOrTrunc(Sat, SDLoc(N0), N3.getValueType()); in PerformUMinFpToSatCombine()
5676 SDValue N0 = N->getOperand(0); in visitIMINMAX() local
5678 EVT VT = N0.getValueType(); in visitIMINMAX()
5683 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitIMINMAX()
5687 if (N0 == N1) in visitIMINMAX()
5688 return N0; in visitIMINMAX()
5691 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitIMINMAX()
5693 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitIMINMAX()
5701 if (SDValue RMINMAX = reassociateOps(Opcode, DL, N0, N1, N->getFlags())) in visitIMINMAX()
5709 bool IsSatBroken = Opcode == ISD::UMIN && N0.getOpcode() == ISD::SMAX; in visitIMINMAX()
5710 if ((IsSatBroken || IsOpIllegal) && (N0.isUndef() || DAG.SignBitIsZero(N0)) && in visitIMINMAX()
5721 return DAG.getNode(AltOpcode, DL, VT, N0, N1); in visitIMINMAX()
5726 N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG)) in visitIMINMAX()
5729 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG)) in visitIMINMAX()
5748 SDLoc(N), VT, N0, N1)) in visitIMINMAX()
5761 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in hoistLogicOpWithSameOpcodeHands() local
5762 EVT VT = N0.getValueType(); in hoistLogicOpWithSameOpcodeHands()
5764 unsigned HandOpcode = N0.getOpcode(); in hoistLogicOpWithSameOpcodeHands()
5769 if (N0.getNumOperands() == 0) in hoistLogicOpWithSameOpcodeHands()
5776 SDValue X = N0.getOperand(0); in hoistLogicOpWithSameOpcodeHands()
5782 N0.getOperand(1) == N1.getOperand(1))) { in hoistLogicOpWithSameOpcodeHands()
5785 if (!N0.hasOneUse() && !N1.hasOneUse()) in hoistLogicOpWithSameOpcodeHands()
5804 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
5812 if (!N0.hasOneUse() && !N1.hasOneUse()) in hoistLogicOpWithSameOpcodeHands()
5834 N0.getOperand(1) == N1.getOperand(1)) { in hoistLogicOpWithSameOpcodeHands()
5836 if (!N0.hasOneUse() || !N1.hasOneUse()) in hoistLogicOpWithSameOpcodeHands()
5839 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
5845 if (!N0.hasOneUse() || !N1.hasOneUse()) in hoistLogicOpWithSameOpcodeHands()
5855 N0.getOperand(2) == N1.getOperand(2)) { in hoistLogicOpWithSameOpcodeHands()
5856 if (!N0.hasOneUse() || !N1.hasOneUse()) in hoistLogicOpWithSameOpcodeHands()
5858 SDValue X1 = N0.getOperand(1); in hoistLogicOpWithSameOpcodeHands()
5860 SDValue S = N0.getOperand(2); in hoistLogicOpWithSameOpcodeHands()
5897 auto *SVN0 = cast<ShuffleVectorSDNode>(N0); in hoistLogicOpWithSameOpcodeHands()
5912 SDValue ShOp = N0.getOperand(1); in hoistLogicOpWithSameOpcodeHands()
5917 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) { in hoistLogicOpWithSameOpcodeHands()
5919 N0.getOperand(0), N1.getOperand(0)); in hoistLogicOpWithSameOpcodeHands()
5925 ShOp = N0.getOperand(0); in hoistLogicOpWithSameOpcodeHands()
5930 if (N0.getOperand(0) == N1.getOperand(0) && ShOp.getNode()) { in hoistLogicOpWithSameOpcodeHands()
5931 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1), in hoistLogicOpWithSameOpcodeHands()
5941 SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1, in foldLogicOfSetCCs() argument
5944 if (!isSetCCEquivalent(N0, LL, LR, N0CC) || in foldLogicOfSetCCs()
5948 assert(N0.getValueType() == N1.getValueType() && in foldLogicOfSetCCs()
5957 EVT VT = N0.getValueType(); in foldLogicOfSetCCs()
5986 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
6005 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
6019 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One); in foldLogicOfSetCCs()
6027 N0.hasOneUse() && N1.hasOneUse()) { in foldLogicOfSetCCs()
6031 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR); in foldLogicOfSetCCs()
6366 SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) { in visitANDLike() argument
6371 if (N0.isUndef() || N1.isUndef()) in visitANDLike()
6374 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL)) in visitANDLike()
6380 std::swap(N0, N1); in visitANDLike()
6383 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && in visitANDLike()
6384 VT.isScalarInteger() && VT.getSizeInBits() <= 64 && N0->hasOneUse()) { in visitANDLike()
6385 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { in visitANDLike()
6397 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) { in visitANDLike()
6400 SDLoc DL0(N0); in visitANDLike()
6403 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT)); in visitANDLike()
6404 CombineTo(N0.getNode(), NewAdd); in visitANDLike()
6712 SDValue N0 = N->getOperand(0); in unfoldExtremeBitClearingToShifts() local
6716 if (!TLI.shouldFoldMaskToVariableShiftPair(N0)) in unfoldExtremeBitClearingToShifts()
6741 X = N0; in unfoldExtremeBitClearingToShifts()
6742 else if (matchMask(N0)) in unfoldExtremeBitClearingToShifts()
6940 SDValue N0 = N->getOperand(0); in visitAND() local
6946 if (N0 == N1) in visitAND()
6947 return N0; in visitAND()
6950 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, DL, VT, {N0, N1})) in visitAND()
6954 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitAND()
6956 return DAG.getNode(ISD::AND, DL, VT, N1, N0); in visitAND()
6958 if (areBitwiseNotOfEachother(N0, N1)) in visitAND()
6974 return N0; in visitAND()
6977 auto *MLoad = dyn_cast<MaskedLoadSDNode>(N0); in visitAND()
6995 bool LoadHasOtherUsers = !N0.hasOneUse(); in visitAND()
7007 return N0; in visitAND()
7022 if (SDValue RAND = reassociateOps(ISD::AND, DL, N0, N1, N->getFlags())) in visitAND()
7027 reassociateReduction(ISD::VECREDUCE_AND, ISD::AND, DL, VT, N0, N1)) in visitAND()
7034 if (N0.getOpcode() == ISD::OR && in visitAND()
7035 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset)) in visitAND()
7038 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitAND()
7039 SDValue N0Op0 = N0.getOperand(0); in visitAND()
7060 if (ISD::isExtOpcode(N0.getOpcode())) { in visitAND()
7061 unsigned ExtOpc = N0.getOpcode(); in visitAND()
7062 SDValue N0Op0 = N0.getOperand(0); in visitAND()
7067 N0->hasOneUse() && N0Op0->hasOneUse()) { in visitAND()
7083 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitAND()
7084 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() && in visitAND()
7085 N0.getOperand(0).getOpcode() == ISD::LOAD && in visitAND()
7086 N0.getOperand(0).getResNo() == 0) || in visitAND()
7087 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) { in visitAND()
7089 cast<LoadSDNode>((N0.getOpcode() == ISD::LOAD) ? N0 : N0.getOperand(0)); in visitAND()
7156 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0); in visitAND()
7184 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N)) in visitAND()
7187 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C && in visitAND()
7188 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitAND()
7189 SDValue Ext = N0.getOperand(0); in visitAND()
7202 N0.getOperand(1)); in visitAND()
7207 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) { in visitAND()
7230 if (N1C && N0.getOpcode() == ISD::LOAD && !VT.isVector()) in visitAND()
7243 if (SDValue Combined = visitANDLike(N0, N1, N)) in visitAND()
7247 if (N0.getOpcode() == N1.getOpcode()) in visitAND()
7251 if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG)) in visitAND()
7253 if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG)) in visitAND()
7263 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) { in visitAND()
7264 if (isNullOrNullSplat(N0.getOperand(0))) { in visitAND()
7265 SDValue SubRHS = N0.getOperand(1); in visitAND()
7282 if (ISD::isUNINDEXEDLoad(N0.getNode()) && in visitAND()
7283 (ISD::isEXTLoad(N0.getNode()) || in visitAND()
7284 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) { in visitAND()
7285 auto *LN0 = cast<LoadSDNode>(N0); in visitAND()
7296 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
7299 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); in visitAND()
7305 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { in visitAND()
7306 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), in visitAND()
7307 N0.getOperand(1), false)) in visitAND()
7339 if (IsAndZeroExtMask(N0, N1)) in visitAND()
7340 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitAND()
7349 if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG)) in visitAND()
7356 SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, in MatchBSwapHWordLow() argument
7370 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) in MatchBSwapHWordLow()
7371 std::swap(N0, N1); in MatchBSwapHWordLow()
7373 std::swap(N0, N1); in MatchBSwapHWordLow()
7374 if (N0.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7375 if (!N0->hasOneUse()) in MatchBSwapHWordLow()
7377 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in MatchBSwapHWordLow()
7383 N0 = N0.getOperand(0); in MatchBSwapHWordLow()
7397 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
7398 std::swap(N0, N1); in MatchBSwapHWordLow()
7399 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) in MatchBSwapHWordLow()
7401 if (!N0->hasOneUse() || !N1->hasOneUse()) in MatchBSwapHWordLow()
7404 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in MatchBSwapHWordLow()
7412 SDValue N00 = N0->getOperand(0); in MatchBSwapHWordLow()
7485 SDValue N0 = N.getOperand(0); in isBSwapHWordElement() local
7486 unsigned Opc0 = N0.getOpcode(); in isBSwapHWordElement()
7495 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in isBSwapHWordElement()
7524 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in isBSwapHWordElement()
7532 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in isBSwapHWordElement()
7557 Parts[MaskByteOffset] = N0.getOperand(0).getNode(); in isBSwapHWordElement()
7583 SelectionDAG &DAG, SDNode *N, SDValue N0, in matchBSwapHWordOrAndAnd() argument
7589 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND) in matchBSwapHWordOrAndAnd()
7592 if (!N0->hasOneUse() || !N1->hasOneUse()) in matchBSwapHWordOrAndAnd()
7594 ConstantSDNode *Mask0 = isConstOrConstSplat(N0.getOperand(1)); in matchBSwapHWordOrAndAnd()
7601 SDValue Shift0 = N0.getOperand(0); in matchBSwapHWordOrAndAnd()
7626 SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) { in MatchBSwapHWord() argument
7636 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT)) in MatchBSwapHWord()
7640 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT)) in MatchBSwapHWord()
7650 if (isBSwapHWordPair(N0, Parts)) { in MatchBSwapHWord()
7654 } else if (N0.getOpcode() == ISD::OR) { in MatchBSwapHWord()
7658 SDValue N00 = N0.getOperand(0); in MatchBSwapHWord()
7659 SDValue N01 = N0.getOperand(1); in MatchBSwapHWord()
7689 SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, const SDLoc &DL) { in visitORLike() argument
7693 if (!LegalOperations && (N0.isUndef() || N1.isUndef())) in visitORLike()
7696 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL)) in visitORLike()
7700 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND && in visitORLike()
7702 (N0->hasOneUse() || N1->hasOneUse())) { in visitORLike()
7706 getAsNonOpaqueConstant(N0.getOperand(1))) { in visitORLike()
7714 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && in visitORLike()
7716 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7717 N0.getOperand(0), N1.getOperand(0)); in visitORLike()
7726 if (N0.getOpcode() == ISD::AND && in visitORLike()
7728 N0.getOperand(0) == N1.getOperand(0) && in visitORLike()
7730 (N0->hasOneUse() || N1->hasOneUse())) { in visitORLike()
7731 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7732 N0.getOperand(1), N1.getOperand(1)); in visitORLike()
7733 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
7740 static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, in visitORCommutative() argument
7742 EVT VT = N0.getValueType(); in visitORCommutative()
7752 SDValue N0Resized = peekThroughResize(N0); in visitORCommutative()
7783 if (sd_match(N0, m_Xor(m_Value(X), m_Specific(N1)))) in visitORCommutative()
7787 if (sd_match(N0, m_Xor(m_Value(X), m_Value(Y))) && in visitORCommutative()
7792 if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG)) in visitORCommutative()
7802 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL && in visitORCommutative()
7803 N0.getOperand(0) == N1.getOperand(0) && in visitORCommutative()
7804 peekThroughZext(N0.getOperand(2)) == peekThroughZext(N1.getOperand(1))) in visitORCommutative()
7805 return N0; in visitORCommutative()
7808 if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL && in visitORCommutative()
7809 N0.getOperand(1) == N1.getOperand(0) && in visitORCommutative()
7810 peekThroughZext(N0.getOperand(2)) == peekThroughZext(N1.getOperand(1))) in visitORCommutative()
7811 return N0; in visitORCommutative()
7816 if (sd_match(N0, in visitORCommutative()
7837 SDValue N0 = N->getOperand(0); in visitOR() local
7843 if (N0 == N1) in visitOR()
7844 return N0; in visitOR()
7847 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, DL, VT, {N0, N1})) in visitOR()
7851 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitOR()
7853 return DAG.getNode(ISD::OR, DL, VT, N1, N0); in visitOR()
7862 return N0; in visitOR()
7871 auto *SV0 = dyn_cast<ShuffleVectorSDNode>(N0); in visitOR()
7874 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode()); in visitOR()
7875 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode()); in visitOR()
7915 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0); in visitOR()
7928 return N0; in visitOR()
7939 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue())) in visitOR()
7945 if (SDValue Combined = visitORLike(N0, N1, DL)) in visitOR()
7948 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N)) in visitOR()
7952 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1)) in visitOR()
7954 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1)) in visitOR()
7958 if (SDValue ROR = reassociateOps(ISD::OR, DL, N0, N1, N->getFlags())) in visitOR()
7963 reassociateReduction(ISD::VECREDUCE_OR, ISD::OR, DL, VT, N0, N1)) in visitOR()
7971 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() && in visitOR()
7972 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) { in visitOR()
7974 {N1, N0.getOperand(1)})) { in visitOR()
7975 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
7981 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N)) in visitOR()
7983 if (SDValue Combined = visitORCommutative(DAG, N1, N0, N)) in visitOR()
7987 if (N0.getOpcode() == N1.getOpcode()) in visitOR()
7992 if (SDValue Rot = MatchRotate(N0, N1, DL)) in visitOR()
8011 if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG)) in visitOR()
8329 SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, in MatchFunnelPosNeg() argument
8334 EVT VT = N0.getValueType(); in MatchFunnelPosNeg()
8344 if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, /*IsRotate*/ N0 == N1)) { in MatchFunnelPosNeg()
8345 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1, in MatchFunnelPosNeg()
8366 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
8371 if (IsBinOpImm(N0, ISD::SHL, 1) && in MatchFunnelPosNeg()
8375 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8381 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N0.getOperand(1) && in MatchFunnelPosNeg()
8385 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
9336 SDValue N0 = N->getOperand(0); in unfoldMaskedMerge() local
9338 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) && in unfoldMaskedMerge()
9339 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0)) in unfoldMaskedMerge()
9387 SDValue N0 = N->getOperand(0); in visitXOR() local
9389 EVT VT = N0.getValueType(); in visitXOR()
9393 if (N0.isUndef() && N1.isUndef()) in visitXOR()
9397 if (N0.isUndef()) in visitXOR()
9398 return N0; in visitXOR()
9403 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
9407 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitXOR()
9409 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
9418 return N0; in visitXOR()
9423 return N0; in visitXOR()
9429 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) in visitXOR()
9434 reassociateReduction(ISD::VECREDUCE_XOR, ISD::XOR, DL, VT, N0, N1)) in visitXOR()
9439 DAG.haveNoCommonBitsSet(N0, N1)) { in visitXOR()
9442 return DAG.getNode(ISD::OR, DL, VT, N0, N1, Flags); in visitXOR()
9453 unsigned N0Opcode = N0.getOpcode(); in visitXOR()
9456 isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/ true)) { in visitXOR()
9465 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC); in visitXOR()
9467 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2), in visitXOR()
9468 N0.getOperand(3), NotCC); in visitXOR()
9471 if (N0.hasOneUse()) { in visitXOR()
9475 DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, in visitXOR()
9476 N0.getOperand(0), N0Opcode == ISD::STRICT_FSETCCS); in visitXOR()
9478 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), SetCC.getValue(1)); in visitXOR()
9479 recursivelyDeleteUnusedNodes(N0.getNode()); in visitXOR()
9489 if (isOneConstant(N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitXOR()
9490 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){ in visitXOR()
9491 SDValue V = N0.getOperand(0); in visitXOR()
9492 SDLoc DL0(N0); in visitXOR()
9500 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() && in visitXOR()
9502 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1); in visitXOR()
9512 if (isAllOnesConstant(N1) && N0.hasOneUse() && in visitXOR()
9514 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1); in visitXOR()
9527 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::SUB && in visitXOR()
9528 isNullConstant(N0.getOperand(0))) { in visitXOR()
9529 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
9534 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::ADD && in visitXOR()
9535 isAllOnesOrAllOnesSplat(N0.getOperand(1))) { in visitXOR()
9536 return DAG.getNegative(N0.getOperand(0), DL, VT); in visitXOR()
9540 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) { in visitXOR()
9541 SDValue X = N0.getOperand(0); in visitXOR()
9549 SDValue A = N0Opcode == ISD::ADD ? N0 : N1; in visitXOR()
9550 SDValue S = N0Opcode == ISD::SRA ? N0 : N1; in visitXOR()
9562 if (N0 == N1) in visitXOR()
9584 isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) { in visitXOR()
9586 N0.getOperand(1)); in visitXOR()
9594 if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG)) in visitXOR()
9596 if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG)) in visitXOR()
9598 if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG)) in visitXOR()
9609 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N)) in visitXOR()
9783 SDValue N0 = N->getOperand(0); in visitRotate() local
9790 return N0; in visitRotate()
9796 return N0; in visitRotate()
9810 return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt); in visitRotate()
9817 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
9827 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); in visitRotate()
9830 unsigned NextOp = N0.getOpcode(); in visitRotate()
9836 SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)); in visitRotate()
9845 {N0.getOperand(1), BitsizeC}); in visitRotate()
9853 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0), in visitRotate()
9862 SDValue N0 = N->getOperand(0); in visitSHL() local
9864 if (SDValue V = DAG.simplifyShift(N0, N1)) in visitSHL()
9868 EVT VT = N0.getValueType(); in visitSHL()
9873 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N0, N1})) in visitSHL()
9885 if (N0.getOpcode() == ISD::AND) { in visitSHL()
9886 SDValue N00 = N0->getOperand(0); in visitSHL()
9887 SDValue N01 = N0->getOperand(1); in visitSHL()
9912 return DAG.getNode(ISD::SHL, DL, VT, N0, NewOp1); in visitSHL()
9916 if (N0.getOpcode() == ISD::SHL) { in visitSHL()
9924 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSHL()
9934 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSHL()
9935 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSHL()
9936 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
9945 if ((N0.getOpcode() == ISD::ZERO_EXTEND || in visitSHL()
9946 N0.getOpcode() == ISD::ANY_EXTEND || in visitSHL()
9947 N0.getOpcode() == ISD::SIGN_EXTEND) && in visitSHL()
9948 N0.getOperand(0).getOpcode() == ISD::SHL) { in visitSHL()
9949 SDValue N0Op0 = N0.getOperand(0); in visitSHL()
9978 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0)); in visitSHL()
9988 if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitSHL()
9989 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSHL()
9990 SDValue N0Op0 = N0.getOperand(0); in visitSHL()
10006 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
10010 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) { in visitSHL()
10021 if (N0->getFlags().hasExact()) { in visitSHL()
10022 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
10025 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT); in visitSHL()
10027 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10029 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
10032 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT); in visitSHL()
10034 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Diff); in visitSHL()
10042 if (N0.getOpcode() == ISD::SRL && in visitSHL()
10043 (N0.getOperand(1) == N1 || N0.hasOneUse()) && in visitSHL()
10045 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
10048 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT); in visitSHL()
10053 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10056 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
10059 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT); in visitSHL()
10063 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10070 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) && in visitSHL()
10074 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
10081 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) && in visitSHL()
10082 N0->hasOneUse() && TLI.isDesirableToCommuteWithShift(N, Level)) { in visitSHL()
10083 SDValue N01 = N0.getOperand(1); in visitSHL()
10086 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
10090 if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint()) in visitSHL()
10092 return DAG.getNode(N0.getOpcode(), DL, VT, Shl0, Shl1, Flags); in visitSHL()
10099 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitSHL()
10100 N0.getOperand(0).getOpcode() == ISD::ADD && in visitSHL()
10101 N0.getOperand(0)->getFlags().hasNoSignedWrap() && N0->hasOneUse() && in visitSHL()
10102 N0.getOperand(0)->hasOneUse() && in visitSHL()
10104 SDValue Add = N0.getOperand(0); in visitSHL()
10105 SDLoc DL(N0); in visitSHL()
10106 if (SDValue ExtC = DAG.FoldConstantArithmetic(N0.getOpcode(), DL, VT, in visitSHL()
10110 SDValue ExtX = DAG.getNode(N0.getOpcode(), DL, VT, Add.getOperand(0)); in visitSHL()
10118 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) { in visitSHL()
10119 SDValue N01 = N0.getOperand(1); in visitSHL()
10122 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), Shl); in visitSHL()
10142 return DAG.getNode(ISD::MUL, DL, VT, And, N0); in visitSHL()
10149 if (N0.getOpcode() == ISD::VSCALE && N1C) { in visitSHL()
10150 const APInt &C0 = N0.getConstantOperandAPInt(0); in visitSHL()
10157 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitSHL()
10159 const APInt &C0 = N0.getConstantOperandAPInt(0); in visitSHL()
10290 SDValue N0 = N->getOperand(0); in foldBitOrderCrossLogicOp() local
10293 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && N0.hasOneUse()) { in foldBitOrderCrossLogicOp()
10294 SDValue OldLHS = N0.getOperand(0); in foldBitOrderCrossLogicOp()
10295 SDValue OldRHS = N0.getOperand(1); in foldBitOrderCrossLogicOp()
10300 return DAG.getNode(N0.getOpcode(), DL, VT, OldLHS.getOperand(0), in foldBitOrderCrossLogicOp()
10306 return DAG.getNode(N0.getOpcode(), DL, VT, OldLHS.getOperand(0), in foldBitOrderCrossLogicOp()
10312 return DAG.getNode(N0.getOpcode(), DL, VT, NewBitReorder, in foldBitOrderCrossLogicOp()
10320 SDValue N0 = N->getOperand(0); in visitSRA() local
10322 if (SDValue V = DAG.simplifyShift(N0, N1)) in visitSRA()
10326 EVT VT = N0.getValueType(); in visitSRA()
10330 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, DL, VT, {N0, N1})) in visitSRA()
10336 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits) in visitSRA()
10337 return N0; in visitSRA()
10351 if (N0.getOpcode() == ISD::SRA) { in visitSRA()
10366 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) { in visitSRA()
10377 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
10386 if (N0.getOpcode() == ISD::SHL && N1C) { in visitSRA()
10388 const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1)); in visitSRA()
10410 N0.getOperand(0), Amt); in visitSRA()
10424 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C && in visitSRA()
10425 N0.hasOneUse()) { in visitSRA()
10426 bool IsAdd = N0.getOpcode() == ISD::ADD; in visitSRA()
10427 SDValue Shl = N0.getOperand(IsAdd ? 0 : 1); in visitSRA()
10432 isConstOrConstSplat(N0.getOperand(IsAdd ? 1 : 0))) { in visitSRA()
10467 return DAG.getNode(ISD::SRA, DL, VT, N0, NewOp1); in visitSRA()
10474 if (N0.getOpcode() == ISD::TRUNCATE && in visitSRA()
10475 (N0.getOperand(0).getOpcode() == ISD::SRL || in visitSRA()
10476 N0.getOperand(0).getOpcode() == ISD::SRA) && in visitSRA()
10477 N0.getOperand(0).hasOneUse() && in visitSRA()
10478 N0.getOperand(0).getOperand(1).hasOneUse() && N1C) { in visitSRA()
10479 SDValue N0Op0 = N0.getOperand(0); in visitSRA()
10500 if (DAG.SignBitIsZero(N0)) in visitSRA()
10501 return DAG.getNode(ISD::SRL, DL, VT, N0, N1); in visitSRA()
10520 SDValue N0 = N->getOperand(0); in visitSRL() local
10522 if (SDValue V = DAG.simplifyShift(N0, N1)) in visitSRL()
10526 EVT VT = N0.getValueType(); in visitSRL()
10531 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, DL, VT, {N0, N1})) in visitSRL()
10549 if (N0.getOpcode() == ISD::SRL) { in visitSRL()
10557 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSRL()
10567 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSRL()
10568 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSRL()
10569 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
10573 if (N1C && N0.getOpcode() == ISD::TRUNCATE && in visitSRL()
10574 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSRL()
10575 SDValue InnerShift = N0.getOperand(0); in visitSRL()
10595 if (N0.hasOneUse() && InnerShift.hasOneUse() && in visitSRL()
10611 if (N0.getOpcode() == ISD::SHL && in visitSRL()
10612 (N0.getOperand(1) == N1 || N0->hasOneUse()) && in visitSRL()
10621 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSRL()
10624 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT); in visitSRL()
10629 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10632 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSRL()
10635 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT); in visitSRL()
10639 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10646 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitSRL()
10648 EVT SmallVT = N0.getOperand(0).getValueType(); in visitSRL()
10655 SDLoc DL0(N0); in visitSRL()
10657 DAG.getNode(ISD::SRL, DL0, SmallVT, N0.getOperand(0), in visitSRL()
10670 if (N0.getOpcode() == ISD::SRA) in visitSRL()
10671 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1); in visitSRL()
10676 if (N1C && N0.getOpcode() == ISD::CTLZ && in visitSRL()
10679 KnownBits Known = DAG.computeKnownBits(N0.getOperand(0)); in visitSRL()
10683 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT); in visitSRL()
10688 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT); in visitSRL()
10697 SDValue Op = N0.getOperand(0); in visitSRL()
10700 SDLoc DL(N0); in visitSRL()
10713 return DAG.getNode(ISD::SRL, DL, VT, N0, NewOp1); in visitSRL()
10775 SDValue N0 = N->getOperand(0); in visitFunnelShift() local
10787 return IsFSHL ? N0 : N1; in visitFunnelShift()
10800 return DAG.getNode(N->getOpcode(), DL, VT, N0, N1, in visitFunnelShift()
10806 return IsFSHL ? N0 : N1; in visitFunnelShift()
10812 if (IsUndefOrZero(N0)) in visitFunnelShift()
10818 ISD::SHL, DL, VT, N0, in visitFunnelShift()
10828 auto *LHS = dyn_cast<LoadSDNode>(N0); in visitFunnelShift()
10867 if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits)) in visitFunnelShift()
10870 return DAG.getNode(ISD::SHL, DL, VT, N0, N2); in visitFunnelShift()
10879 if (N0 == N1 && hasOperation(RotOpc, VT)) in visitFunnelShift()
10880 return DAG.getNode(RotOpc, DL, VT, N0, N2); in visitFunnelShift()
10890 SDValue N0 = N->getOperand(0); in visitSHLSAT() local
10892 if (SDValue V = DAG.simplifyShift(N0, N1)) in visitSHLSAT()
10896 EVT VT = N0.getValueType(); in visitSHLSAT()
10899 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1})) in visitSHLSAT()
10907 N1C->getAPIntValue().ult(DAG.ComputeNumSignBits(N0))) in visitSHLSAT()
10908 return DAG.getNode(ISD::SHL, DL, VT, N0, N1); in visitSHLSAT()
10913 DAG.computeKnownBits(N0).countMinLeadingZeros())) in visitSHLSAT()
10914 return DAG.getNode(ISD::SHL, DL, VT, N0, N1); in visitSHLSAT()
10992 SDValue N0 = N->getOperand(0); in visitABS() local
10997 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS, DL, VT, {N0})) in visitABS()
11000 if (N0.getOpcode() == ISD::ABS) in visitABS()
11001 return N0; in visitABS()
11003 if (DAG.SignBitIsZero(N0)) in visitABS()
11004 return N0; in visitABS()
11011 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitABS()
11012 EVT ExtVT = cast<VTSDNode>(N0.getOperand(1))->getVT(); in visitABS()
11019 DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0)))); in visitABS()
11027 SDValue N0 = N->getOperand(0); in visitBSWAP() local
11032 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BSWAP, DL, VT, {N0})) in visitBSWAP()
11035 if (N0.getOpcode() == ISD::BSWAP) in visitBSWAP()
11036 return N0.getOperand(0); in visitBSWAP()
11042 if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) { in visitBSWAP()
11043 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11050 if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) { in visitBSWAP()
11051 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in visitBSWAP()
11058 SDValue Res = N0.getOperand(0); in visitBSWAP()
11072 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitBSWAP()
11073 N0.hasOneUse()) { in visitBSWAP()
11074 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)); in visitBSWAP()
11077 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11078 unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL; in visitBSWAP()
11079 return DAG.getNode(InverseShift, DL, VT, NewSwap, N0.getOperand(1)); in visitBSWAP()
11090 SDValue N0 = N->getOperand(0); in visitBITREVERSE() local
11095 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BITREVERSE, DL, VT, {N0})) in visitBITREVERSE()
11099 if (N0.getOpcode() == ISD::BITREVERSE) in visitBITREVERSE()
11100 return N0.getOperand(0); in visitBITREVERSE()
11118 SDValue N0 = N->getOperand(0); in visitCTLZ() local
11123 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTLZ, DL, VT, {N0})) in visitCTLZ()
11128 if (DAG.isKnownNeverZero(N0)) in visitCTLZ()
11129 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, DL, VT, N0); in visitCTLZ()
11135 SDValue N0 = N->getOperand(0); in visitCTLZ_ZERO_UNDEF() local
11141 DAG.FoldConstantArithmetic(ISD::CTLZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTLZ_ZERO_UNDEF()
11147 SDValue N0 = N->getOperand(0); in visitCTTZ() local
11152 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTTZ, DL, VT, {N0})) in visitCTTZ()
11157 if (DAG.isKnownNeverZero(N0)) in visitCTTZ()
11158 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, DL, VT, N0); in visitCTTZ()
11164 SDValue N0 = N->getOperand(0); in visitCTTZ_ZERO_UNDEF() local
11170 DAG.FoldConstantArithmetic(ISD::CTTZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTTZ_ZERO_UNDEF()
11176 SDValue N0 = N->getOperand(0); in visitCTPOP() local
11182 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTPOP, DL, VT, {N0})) in visitCTPOP()
11187 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SHL) { in visitCTPOP()
11188 if (ConstantSDNode *AmtC = isConstOrConstSplat(N0.getOperand(1))) { in visitCTPOP()
11191 KnownBits KnownSrc = DAG.computeKnownBits(N0.getOperand(0)); in visitCTPOP()
11192 if ((N0.getOpcode() == ISD::SRL && in visitCTPOP()
11194 (N0.getOpcode() == ISD::SHL && in visitCTPOP()
11196 return DAG.getNode(ISD::CTPOP, DL, VT, N0.getOperand(0)); in visitCTPOP()
11208 TLI.isTruncateFree(N0, HalfVT) && TLI.isZExtFree(HalfVT, VT)) { in visitCTPOP()
11210 if (DAG.MaskedValueIsZero(N0, UpperBits)) { in visitCTPOP()
11212 DAG.getZExtOrTrunc(N0, DL, HalfVT)); in visitCTPOP()
11539 SDValue N0 = N->getOperand(0); in foldVSelectToSignBitSplatMask() local
11546 if (!sd_match(N0, m_OneUse(m_SetCC(m_Value(Cond0), m_Value(Cond1), in foldVSelectToSignBitSplatMask()
11596 SDValue N0 = N->getOperand(0); in visitSELECT() local
11600 EVT VT0 = N0.getValueType(); in visitSELECT()
11604 if (SDValue V = DAG.simplifySelect(N0, N1, N2)) in visitSELECT()
11611 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) { in visitSELECT()
11637 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { in visitSELECT()
11638 SDValue Cond0 = N0->getOperand(0); in visitSELECT()
11639 SDValue Cond1 = N0->getOperand(1); in visitSELECT()
11650 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { in visitSELECT()
11651 SDValue Cond0 = N0->getOperand(0); in visitSELECT()
11652 SDValue Cond1 = N0->getOperand(1); in visitSELECT()
11668 if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) { in visitSELECT()
11671 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0); in visitSELECT()
11676 if (SDValue Combined = visitANDLike(N0, N1_0, N)) { in visitSELECT()
11687 if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) { in visitSELECT()
11690 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0); in visitSELECT()
11695 if (SDValue Combined = visitORLike(N0, N2_0, DL)) in visitSELECT()
11703 if (N0.getOpcode() == ISD::SETCC) { in visitSELECT()
11704 SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1); in visitSELECT()
11705 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSELECT()
11711 if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2, Flags, TLI)) in visitSELECT()
11722 CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) && in visitSELECT()
11750 Flags = N0->getFlags(); in visitSELECT()
11752 N2, N0.getOperand(2)); in visitSELECT()
11757 if (SDValue NewSel = SimplifySelect(DL, N0, N1, N2)) in visitSELECT()
11765 if (SDValue R = combineSelectAsExtAnd(N0, N1, N2, DL, DAG)) in visitSELECT()
12274 SDValue N0 = N->getOperand(0); in visitVP_SELECT() local
12279 if (SDValue V = DAG.simplifySelect(N0, N1, N2)) in visitVP_SELECT()
12289 SDValue N0 = N->getOperand(0); in visitVSELECT() local
12295 if (SDValue V = DAG.simplifySelect(N0, N1, N2)) in visitVSELECT()
12302 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) in visitVSELECT()
12308 N0.getScalarValueSizeInBits() == N1.getScalarValueSizeInBits() && in visitVSELECT()
12309 TLI.getBooleanContents(N0.getValueType()) == in visitVSELECT()
12313 DAG.getNode(ISD::AND, DL, N0.getValueType(), N1.getOperand(1), N0)); in visitVSELECT()
12321 if (N0.getOpcode() == ISD::SETCC) { in visitVSELECT()
12322 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1); in visitVSELECT()
12323 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitVSELECT()
12354 if (N0.hasOneUse() && in visitVSELECT()
12559 if (ISD::isConstantSplatVectorAllOnes(N0.getNode())) in visitVSELECT()
12562 if (ISD::isConstantSplatVectorAllZeros(N0.getNode())) in visitVSELECT()
12570 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitVSELECT()
12589 SDValue N0 = N->getOperand(0); in visitSELECT_CC() local
12602 if (CC == ISD::SETEQ && !LegalTypes && N0.getValueType() == MVT::i1 && in visitSELECT_CC()
12604 return DAG.getSelect(DL, N2.getValueType(), N0, N3, N2); in visitSELECT_CC()
12607 if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1, in visitSELECT_CC()
12636 return SimplifySelectCC(DL, N0, N1, N2, N3, CC); in visitSELECT_CC()
12648 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in visitSETCC() local
12651 if (SDValue Combined = SimplifySetCC(VT, N0, N1, Cond, DL, !PreferSetCC)) { in visitSETCC()
12693 if (IsAndWithShift(N0, N1)) { in visitSETCC()
12694 AndOrOp = N0; in visitSETCC()
12696 } else if (IsAndWithShift(N1, N0)) { in visitSETCC()
12698 ShiftOrRotate = N0; in visitSETCC()
12699 } else if (IsRotateWithOp(N0, N1)) { in visitSETCC()
12701 AndOrOp = N0; in visitSETCC()
12703 } else if (IsRotateWithOp(N1, N0)) { in visitSETCC()
12706 ShiftOrRotate = N0; in visitSETCC()
12711 EVT OpVT = N0.getValueType(); in visitSETCC()
12825 SDValue N0 = N->getOperand(0); in tryToFoldExtendSelectLoad() local
12831 if (!(N0->getOpcode() == ISD::SELECT || N0->getOpcode() == ISD::VSELECT) || in tryToFoldExtendSelectLoad()
12832 !N0.hasOneUse()) in tryToFoldExtendSelectLoad()
12835 SDValue Op1 = N0->getOperand(1); in tryToFoldExtendSelectLoad()
12836 SDValue Op2 = N0->getOperand(2); in tryToFoldExtendSelectLoad()
12852 (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes && in tryToFoldExtendSelectLoad()
12858 return DAG.getSelect(DL, VT, N0->getOperand(0), Ext1, Ext2); in tryToFoldExtendSelectLoad()
12871 SDValue N0 = N->getOperand(0); in tryToFoldExtendOfConstant() local
12880 if (isa<ConstantSDNode>(N0)) in tryToFoldExtendOfConstant()
12881 return DAG.getNode(Opcode, DL, VT, N0); in tryToFoldExtendOfConstant()
12886 if (N0->getOpcode() == ISD::SELECT) { in tryToFoldExtendOfConstant()
12887 SDValue Op1 = N0->getOperand(1); in tryToFoldExtendOfConstant()
12888 SDValue Op2 = N0->getOperand(2); in tryToFoldExtendOfConstant()
12890 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
12903 return DAG.getSelect(DL, VT, N0->getOperand(0), in tryToFoldExtendOfConstant()
12914 ISD::isBuildVectorOfConstantSDNodes(N0.getNode()))) in tryToFoldExtendOfConstant()
12919 unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits(); in tryToFoldExtendOfConstant()
12924 SDValue Op = N0.getOperand(i); in tryToFoldExtendOfConstant()
12950 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, in ExtendUsesToFormExtLoad() argument
12955 bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType()); in ExtendUsesToFormExtLoad()
12956 for (SDNode::use_iterator UI = N0->use_begin(), UE = N0->use_end(); UI != UE; in ExtendUsesToFormExtLoad()
12961 if (UI.getUse().getResNo() != N0.getResNo()) in ExtendUsesToFormExtLoad()
12972 if (UseOp == N0) in ExtendUsesToFormExtLoad()
13032 SDValue N0 = N->getOperand(0); in CombineExtLoad() local
13034 EVT SrcVT = N0.getValueType(); in CombineExtLoad()
13057 if (N0->getOpcode() != ISD::LOAD) in CombineExtLoad()
13060 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in CombineExtLoad()
13063 !N0.hasOneUse() || !LN0->isSimple() || in CombineExtLoad()
13069 if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI)) in CombineExtLoad()
13123 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue); in CombineExtLoad()
13124 ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode()); in CombineExtLoad()
13125 CombineTo(N0.getNode(), Trunc, NewChain); in CombineExtLoad()
13139 SDValue N0 = N->getOperand(0); in CombineZExtLogicopShiftLoad() local
13140 if (!ISD::isBitwiseLogicOp(N0.getOpcode()) || in CombineZExtLogicopShiftLoad()
13141 N0.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
13142 (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
13146 SDValue N1 = N0->getOperand(0); in CombineZExtLogicopShiftLoad()
13164 if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND) in CombineZExtLogicopShiftLoad()
13167 if (!N0.hasOneUse() || !N1.hasOneUse()) in CombineZExtLogicopShiftLoad()
13184 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in CombineZExtLogicopShiftLoad()
13185 SDLoc DL0(N0); in CombineZExtLogicopShiftLoad()
13186 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift, in CombineZExtLogicopShiftLoad()
13200 recursivelyDeleteUnusedNodes(N0.getNode()); in CombineZExtLogicopShiftLoad()
13255 SDValue N0, ISD::LoadExtType ExtLoadType) { in tryToFoldExtOfExtload() argument
13256 SDNode *N0Node = N0.getNode(); in tryToFoldExtOfExtload()
13260 !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse()) in tryToFoldExtOfExtload()
13263 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in tryToFoldExtOfExtload()
13286 bool LegalOperations, SDNode *N, SDValue N0, in tryToFoldExtOfLoad() argument
13290 if (!ISD::isNON_EXTLoad(N0.getNode()) || !ISD::isUNINDEXEDLoad(N0.getNode())) in tryToFoldExtOfLoad()
13297 for (SDNode *User : N0->uses()) { in tryToFoldExtOfLoad()
13313 !cast<LoadSDNode>(N0)->isSimple()) && in tryToFoldExtOfLoad()
13314 !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType())) in tryToFoldExtOfLoad()
13319 if (!N0.hasOneUse()) in tryToFoldExtOfLoad()
13320 DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI); in tryToFoldExtOfLoad()
13326 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in tryToFoldExtOfLoad()
13328 LN0->getBasePtr(), N0.getValueType(), in tryToFoldExtOfLoad()
13330 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc); in tryToFoldExtOfLoad()
13339 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in tryToFoldExtOfLoad()
13347 bool LegalOperations, SDNode *N, SDValue N0, in tryToFoldExtOfMaskedLoad() argument
13349 if (!N0.hasOneUse()) in tryToFoldExtOfMaskedLoad()
13352 MaskedLoadSDNode *Ld = dyn_cast<MaskedLoadSDNode>(N0); in tryToFoldExtOfMaskedLoad()
13356 if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->isSimple()) && in tryToFoldExtOfMaskedLoad()
13376 SDValue N0, in tryToFoldExtOfAtomicLoad() argument
13378 auto *ALoad = dyn_cast<AtomicSDNode>(N0); in tryToFoldExtOfAtomicLoad()
13441 SDValue N0 = N->getOperand(0); in foldSextSetcc() local
13442 if (N0.getOpcode() != ISD::SETCC) in foldSextSetcc()
13445 SDValue N00 = N0.getOperand(0); in foldSextSetcc()
13446 SDValue N01 = N0.getOperand(1); in foldSextSetcc()
13447 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldSextSetcc()
13453 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags()); in foldSextSetcc()
13464 if (SVT != N0.getValueType()) { in foldSextSetcc()
13484 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
13511 if (UI.getUse().getResNo() != 0 || User == N0.getNode()) in foldSextSetcc()
13533 unsigned SetCCWidth = N0.getScalarValueSizeInBits(); in foldSextSetcc()
13549 if (!VT.isVector() && !shouldConvertSelectOfConstantsToMath(N0, VT, TLI)) { in foldSextSetcc()
13566 SDValue N0 = N->getOperand(0); in visitSIGN_EXTEND() local
13575 if (N0.isUndef()) in visitSIGN_EXTEND()
13583 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitSIGN_EXTEND()
13584 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13588 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitSIGN_EXTEND()
13589 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) in visitSIGN_EXTEND()
13591 N0.getOperand(0)); in visitSIGN_EXTEND()
13594 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSIGN_EXTEND()
13595 SDValue N00 = N0.getOperand(0); in visitSIGN_EXTEND()
13596 EVT ExtVT = cast<VTSDNode>(N0->getOperand(1))->getVT(); in visitSIGN_EXTEND()
13604 if (N0.getOpcode() == ISD::TRUNCATE) { in visitSIGN_EXTEND()
13607 if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) { in visitSIGN_EXTEND()
13608 SDNode *oye = N0.getOperand(0).getNode(); in visitSIGN_EXTEND()
13609 if (NarrowLoad.getNode() != N0.getNode()) { in visitSIGN_EXTEND()
13610 CombineTo(N0.getNode(), NarrowLoad); in visitSIGN_EXTEND()
13619 SDValue Op = N0.getOperand(0); in visitSIGN_EXTEND()
13621 unsigned MidBits = N0.getScalarValueSizeInBits(); in visitSIGN_EXTEND()
13644 N0.getValueType())) { in visitSIGN_EXTEND()
13646 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13648 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13650 DAG.getValueType(N0.getValueType())); in visitSIGN_EXTEND()
13656 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
13661 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
13672 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
13677 tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
13682 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && in visitSIGN_EXTEND()
13683 isa<LoadSDNode>(N0.getOperand(0)) && in visitSIGN_EXTEND()
13684 N0.getOperand(1).getOpcode() == ISD::Constant && in visitSIGN_EXTEND()
13685 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitSIGN_EXTEND()
13686 LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0)); in visitSIGN_EXTEND()
13691 bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitSIGN_EXTEND()
13698 APInt Mask = N0.getConstantOperandAPInt(1).sext(VT.getSizeInBits()); in visitSIGN_EXTEND()
13699 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitSIGN_EXTEND()
13701 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND); in visitSIGN_EXTEND()
13702 bool NoReplaceTruncAnd = !N0.hasOneUse(); in visitSIGN_EXTEND()
13708 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitSIGN_EXTEND()
13709 CombineTo(N0.getNode(), TruncAnd); in visitSIGN_EXTEND()
13730 if (!TLI.isSExtCheaperThanZExt(N0.getValueType(), VT) && in visitSIGN_EXTEND()
13732 DAG.SignBitIsZero(N0)) { in visitSIGN_EXTEND()
13735 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, Flags); in visitSIGN_EXTEND()
13743 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSIGN_EXTEND()
13744 isNullOrNullSplat(N0.getOperand(0)) && in visitSIGN_EXTEND()
13745 N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
13747 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT); in visitSIGN_EXTEND()
13752 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSIGN_EXTEND()
13753 isAllOnesOrAllOnesSplat(N0.getOperand(1)) && in visitSIGN_EXTEND()
13754 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
13756 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitSIGN_EXTEND()
13762 if (N0.getValueType() == MVT::i1 && isBitwiseNot(N0) && N0.hasOneUse() && in visitSIGN_EXTEND()
13766 if (SDValue NewXor = visitXOR(N0.getNode())) { in visitSIGN_EXTEND()
13769 if (NewXor.getNode() == N0.getNode()) { in visitSIGN_EXTEND()
13779 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13840 SDValue N0 = N->getOperand(0); in visitZERO_EXTEND() local
13849 if (N0.isUndef()) in visitZERO_EXTEND()
13857 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitZERO_EXTEND()
13859 if (N0.getOpcode() == ISD::ZERO_EXTEND) in visitZERO_EXTEND()
13860 Flags.setNonNeg(N0->getFlags().hasNonNeg()); in visitZERO_EXTEND()
13861 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0), Flags); in visitZERO_EXTEND()
13866 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitZERO_EXTEND()
13867 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) in visitZERO_EXTEND()
13868 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, N0.getOperand(0)); in visitZERO_EXTEND()
13875 if (isTruncateOf(DAG, N0, Op, Known)) { in visitZERO_EXTEND()
13877 (Op.getScalarValueSizeInBits() == N0.getScalarValueSizeInBits()) ? in visitZERO_EXTEND()
13880 N0.getScalarValueSizeInBits(), in visitZERO_EXTEND()
13885 DAG.salvageDebugInfo(*N0.getNode()); in visitZERO_EXTEND()
13892 if (N0.getOpcode() == ISD::TRUNCATE) { in visitZERO_EXTEND()
13895 if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) { in visitZERO_EXTEND()
13896 SDNode *oye = N0.getOperand(0).getNode(); in visitZERO_EXTEND()
13897 if (NarrowLoad.getNode() != N0.getNode()) { in visitZERO_EXTEND()
13898 CombineTo(N0.getNode(), NarrowLoad); in visitZERO_EXTEND()
13905 EVT SrcVT = N0.getOperand(0).getValueType(); in visitZERO_EXTEND()
13906 EVT MinVT = N0.getValueType(); in visitZERO_EXTEND()
13909 SDValue Op = N0.getOperand(0); in visitZERO_EXTEND()
13939 SDValue Op = N0.getOperand(0); in visitZERO_EXTEND()
13944 DAG.transferDbgValues(N0, ZExtOrTrunc); in visitZERO_EXTEND()
13950 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT); in visitZERO_EXTEND()
13955 DAG.transferDbgValues(N0, And); in visitZERO_EXTEND()
13962 if (N0.getOpcode() == ISD::AND && in visitZERO_EXTEND()
13963 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitZERO_EXTEND()
13964 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
13965 (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0), N0.getValueType()) || in visitZERO_EXTEND()
13966 !TLI.isZExtFree(N0.getValueType(), VT))) { in visitZERO_EXTEND()
13967 SDValue X = N0.getOperand(0).getOperand(0); in visitZERO_EXTEND()
13969 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
13976 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD, in visitZERO_EXTEND()
13981 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0, in visitZERO_EXTEND()
13992 tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
13999 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && !TLI.isZExtFree(N0, VT) && in visitZERO_EXTEND()
14000 isa<LoadSDNode>(N0.getOperand(0)) && in visitZERO_EXTEND()
14001 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
14002 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitZERO_EXTEND()
14003 LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0)); in visitZERO_EXTEND()
14009 if (!N0.hasOneUse()) { in visitZERO_EXTEND()
14010 if (N0.getOpcode() == ISD::AND) { in visitZERO_EXTEND()
14011 auto *AndC = cast<ConstantSDNode>(N0.getOperand(1)); in visitZERO_EXTEND()
14019 DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitZERO_EXTEND()
14026 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
14027 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
14029 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in visitZERO_EXTEND()
14030 bool NoReplaceTruncAnd = !N0.hasOneUse(); in visitZERO_EXTEND()
14036 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitZERO_EXTEND()
14037 CombineTo(N0.getNode(), TruncAnd); in visitZERO_EXTEND()
14058 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
14064 if (N0.getOpcode() == ISD::SETCC) { in visitZERO_EXTEND()
14066 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags()); in visitZERO_EXTEND()
14070 N0.getValueType().getVectorElementType() == MVT::i1) { in visitZERO_EXTEND()
14071 EVT N00VT = N0.getOperand(0).getValueType(); in visitZERO_EXTEND()
14072 if (getSetCCResultType(N00VT) == N0.getValueType()) in visitZERO_EXTEND()
14082 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
14083 N0.getOperand(1), N0.getOperand(2)); in visitZERO_EXTEND()
14084 return DAG.getZeroExtendInReg(VSetCC, DL, N0.getValueType()); in visitZERO_EXTEND()
14092 DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0), in visitZERO_EXTEND()
14093 N0.getOperand(1), N0.getOperand(2)); in visitZERO_EXTEND()
14095 N0.getValueType()); in visitZERO_EXTEND()
14099 EVT N0VT = N0.getValueType(); in visitZERO_EXTEND()
14100 EVT N00VT = N0.getOperand(0).getValueType(); in visitZERO_EXTEND()
14102 DL, N0.getOperand(0), N0.getOperand(1), in visitZERO_EXTEND()
14105 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true)) in visitZERO_EXTEND()
14110 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitZERO_EXTEND()
14111 !TLI.isZExtFree(N0, VT)) { in visitZERO_EXTEND()
14112 SDValue ShVal = N0.getOperand(0); in visitZERO_EXTEND()
14113 SDValue ShAmt = N0.getOperand(1); in visitZERO_EXTEND()
14115 if (ShVal.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) { in visitZERO_EXTEND()
14116 if (N0.getOpcode() == ISD::SHL) { in visitZERO_EXTEND()
14139 return DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
14158 if (N->getFlags().hasNonNeg() && !TLI.isZExtFree(N0.getValueType(), VT)) { in visitZERO_EXTEND()
14159 SDNode *CSENode = DAG.getNodeIfExists(ISD::SIGN_EXTEND, N->getVTList(), N0); in visitZERO_EXTEND()
14168 SDValue N0 = N->getOperand(0); in visitANY_EXTEND() local
14173 if (N0.isUndef()) in visitANY_EXTEND()
14182 if (N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::ZERO_EXTEND || in visitANY_EXTEND()
14183 N0.getOpcode() == ISD::SIGN_EXTEND) { in visitANY_EXTEND()
14185 if (N0.getOpcode() == ISD::ZERO_EXTEND) in visitANY_EXTEND()
14186 Flags.setNonNeg(N0->getFlags().hasNonNeg()); in visitANY_EXTEND()
14187 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Flags); in visitANY_EXTEND()
14193 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitANY_EXTEND()
14194 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG || in visitANY_EXTEND()
14195 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) in visitANY_EXTEND()
14196 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); in visitANY_EXTEND()
14200 if (N0.getOpcode() == ISD::TRUNCATE) { in visitANY_EXTEND()
14201 if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) { in visitANY_EXTEND()
14202 SDNode *oye = N0.getOperand(0).getNode(); in visitANY_EXTEND()
14203 if (NarrowLoad.getNode() != N0.getNode()) { in visitANY_EXTEND()
14204 CombineTo(N0.getNode(), NarrowLoad); in visitANY_EXTEND()
14213 if (N0.getOpcode() == ISD::TRUNCATE) in visitANY_EXTEND()
14214 return DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT); in visitANY_EXTEND()
14218 if (N0.getOpcode() == ISD::AND && in visitANY_EXTEND()
14219 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitANY_EXTEND()
14220 N0.getOperand(1).getOpcode() == ISD::Constant && in visitANY_EXTEND()
14221 !TLI.isTruncateFree(N0.getOperand(0).getOperand(0), N0.getValueType())) { in visitANY_EXTEND()
14222 SDValue X = DAG.getAnyExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitANY_EXTEND()
14223 SDValue Y = DAG.getNode(ISD::ANY_EXTEND, DL, VT, N0.getOperand(1)); in visitANY_EXTEND()
14234 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitANY_EXTEND()
14237 } else if (ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
14238 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitANY_EXTEND()
14239 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
14242 if (!N0.hasOneUse()) in visitANY_EXTEND()
14244 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
14246 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in visitANY_EXTEND()
14248 LN0->getBasePtr(), N0.getValueType(), in visitANY_EXTEND()
14250 ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND); in visitANY_EXTEND()
14252 bool NoReplaceTrunc = N0.hasOneUse(); in visitANY_EXTEND()
14259 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in visitANY_EXTEND()
14269 if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
14270 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { in visitANY_EXTEND()
14271 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in visitANY_EXTEND()
14285 if (N0.getOpcode() == ISD::SETCC) { in visitANY_EXTEND()
14287 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags()); in visitANY_EXTEND()
14295 EVT N00VT = N0.getOperand(0).getValueType(); in visitANY_EXTEND()
14296 if (getSetCCResultType(N00VT) == N0.getValueType()) in visitANY_EXTEND()
14305 return DAG.getSetCC(DL, VT, N0.getOperand(0), N0.getOperand(1), in visitANY_EXTEND()
14306 cast<CondCodeSDNode>(N0.getOperand(2))->get()); in visitANY_EXTEND()
14313 DL, MatchingVectorType, N0.getOperand(0), N0.getOperand(1), in visitANY_EXTEND()
14314 cast<CondCodeSDNode>(N0.getOperand(2))->get()); in visitANY_EXTEND()
14320 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT), in visitANY_EXTEND()
14322 cast<CondCodeSDNode>(N0.getOperand(2))->get(), true)) in visitANY_EXTEND()
14337 SDValue N0 = N->getOperand(0); in visitAssertExt() local
14342 if (N0.getOpcode() == Opcode && in visitAssertExt()
14343 AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT()) in visitAssertExt()
14344 return N0; in visitAssertExt()
14346 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
14347 N0.getOperand(0).getOpcode() == Opcode) { in visitAssertExt()
14354 SDValue BigA = N0.getOperand(0); in visitAssertExt()
14366 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
14367 N0.getOperand(0).getOpcode() == ISD::AssertSext && in visitAssertExt()
14369 SDValue BigA = N0.getOperand(0); in visitAssertExt()
14386 SDValue N0 = N->getOperand(0); in visitAssertAlign() local
14390 if (auto *AAN = dyn_cast<AssertAlignSDNode>(N0)) in visitAssertAlign()
14391 return DAG.getAssertAlign(DL, N0.getOperand(0), in visitAssertAlign()
14397 switch (N0.getOpcode()) { in visitAssertAlign()
14403 SDValue LHS = N0.getOperand(0); in visitAssertAlign()
14404 SDValue RHS = N0.getOperand(1); in visitAssertAlign()
14412 return DAG.getNode(N0.getOpcode(), DL, N0.getValueType(), LHS, RHS); in visitAssertAlign()
14428 SDValue N0 = N->getOperand(0); in reduceLoadWidth() local
14457 auto *LN = dyn_cast<LoadSDNode>(N0); in reduceLoadWidth()
14503 if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) { in reduceLoadWidth()
14504 SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0; in reduceLoadWidth()
14577 N0 = SRL.getOperand(0); in reduceLoadWidth()
14586 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in reduceLoadWidth()
14587 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) { in reduceLoadWidth()
14588 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { in reduceLoadWidth()
14590 N0 = N0.getOperand(0); in reduceLoadWidth()
14595 if (!isa<LoadSDNode>(N0)) in reduceLoadWidth()
14598 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in reduceLoadWidth()
14640 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1)); in reduceLoadWidth()
14671 SDValue N0 = N->getOperand(0); in visitSIGN_EXTEND_INREG() local
14679 if (N0.isUndef()) in visitSIGN_EXTEND_INREG()
14683 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) in visitSIGN_EXTEND_INREG()
14684 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
14687 if (ExtVTBits >= DAG.ComputeMaxSignificantBits(N0)) in visitSIGN_EXTEND_INREG()
14688 return N0; in visitSIGN_EXTEND_INREG()
14691 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitSIGN_EXTEND_INREG()
14692 ExtVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT())) in visitSIGN_EXTEND_INREG()
14693 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14700 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitSIGN_EXTEND_INREG()
14701 SDValue N00 = N0.getOperand(0); in visitSIGN_EXTEND_INREG()
14712 if (ISD::isExtVecInRegOpcode(N0.getOpcode())) { in visitSIGN_EXTEND_INREG()
14713 SDValue N00 = N0.getOperand(0); in visitSIGN_EXTEND_INREG()
14715 unsigned DstElts = N0.getValueType().getVectorMinNumElements(); in visitSIGN_EXTEND_INREG()
14717 bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in visitSIGN_EXTEND_INREG()
14729 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in visitSIGN_EXTEND_INREG()
14730 SDValue N00 = N0.getOperand(0); in visitSIGN_EXTEND_INREG()
14737 if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, ExtVTBits - 1))) in visitSIGN_EXTEND_INREG()
14738 return DAG.getZeroExtendInReg(N0, SDLoc(N), ExtVT); in visitSIGN_EXTEND_INREG()
14753 if (N0.getOpcode() == ISD::SRL) { in visitSIGN_EXTEND_INREG()
14754 if (auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1))) in visitSIGN_EXTEND_INREG()
14758 unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0)); in visitSIGN_EXTEND_INREG()
14760 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14761 N0.getOperand(1)); in visitSIGN_EXTEND_INREG()
14769 if (ISD::isEXTLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14770 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14771 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() && in visitSIGN_EXTEND_INREG()
14772 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() && in visitSIGN_EXTEND_INREG()
14773 N0.hasOneUse()) || in visitSIGN_EXTEND_INREG()
14775 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in visitSIGN_EXTEND_INREG()
14781 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); in visitSIGN_EXTEND_INREG()
14787 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14788 N0.hasOneUse() && in visitSIGN_EXTEND_INREG()
14789 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() && in visitSIGN_EXTEND_INREG()
14790 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) && in visitSIGN_EXTEND_INREG()
14792 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in visitSIGN_EXTEND_INREG()
14798 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); in visitSIGN_EXTEND_INREG()
14804 if (MaskedLoadSDNode *Ld = dyn_cast<MaskedLoadSDNode>(N0)) { in visitSIGN_EXTEND_INREG()
14805 if (ExtVT == Ld->getMemoryVT() && N0.hasOneUse() && in visitSIGN_EXTEND_INREG()
14813 CombineTo(N0.getNode(), ExtMaskedLoad, ExtMaskedLoad.getValue(1)); in visitSIGN_EXTEND_INREG()
14819 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) { in visitSIGN_EXTEND_INREG()
14831 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); in visitSIGN_EXTEND_INREG()
14838 if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) { in visitSIGN_EXTEND_INREG()
14839 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14840 N0.getOperand(1), false)) in visitSIGN_EXTEND_INREG()
14848 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && in visitSIGN_EXTEND_INREG()
14849 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitSIGN_EXTEND_INREG()
14850 SDValue InnerExt = N0.getOperand(0); in visitSIGN_EXTEND_INREG()
14860 N0.getOperand(1)); in visitSIGN_EXTEND_INREG()
14900 SDValue N0 = N->getOperand(0); in visitEXTEND_VECTOR_INREG() local
14904 if (N0.isUndef()) { in visitEXTEND_VECTOR_INREG()
14926 SDValue N0 = N->getOperand(0); in visitTRUNCATE() local
14928 EVT SrcVT = N0.getValueType(); in visitTRUNCATE()
14933 if (N0.isUndef()) in visitTRUNCATE()
14937 if (N0.getOpcode() == ISD::TRUNCATE) in visitTRUNCATE()
14938 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14941 if (SDValue C = DAG.FoldConstantArithmetic(ISD::TRUNCATE, DL, VT, {N0})) in visitTRUNCATE()
14945 if (N0.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
14946 N0.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
14947 N0.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
14949 if (N0.getOperand(0).getValueType().bitsLT(VT)) in visitTRUNCATE()
14950 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14952 if (N0.getOperand(0).getValueType().bitsGT(VT)) in visitTRUNCATE()
14953 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14956 return N0.getOperand(0); in visitTRUNCATE()
14961 if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitTRUNCATE()
14962 N0.hasOneUse()) { in visitTRUNCATE()
14963 SDValue X = N0.getOperand(0); in visitTRUNCATE()
14964 SDValue ExtVal = N0.getOperand(1); in visitTRUNCATE()
14986 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitTRUNCATE()
14987 LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) { in visitTRUNCATE()
14988 EVT VecTy = N0.getOperand(0).getValueType(); in visitTRUNCATE()
14989 EVT ExTy = N0.getValueType(); in visitTRUNCATE()
14999 SDValue EltNo = N0->getOperand(1); in visitTRUNCATE()
15004 DAG.getBitcast(NVT, N0.getOperand(0)), in visitTRUNCATE()
15010 if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) { in visitTRUNCATE()
15013 SDLoc SL(N0); in visitTRUNCATE()
15014 SDValue Cond = N0.getOperand(0); in visitTRUNCATE()
15015 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
15016 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
15022 if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in visitTRUNCATE()
15025 SDValue Amt = N0.getOperand(1); in visitTRUNCATE()
15030 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
15039 if (SDValue V = foldSubToUSubSat(VT, N0.getNode(), DL)) in visitTRUNCATE()
15046 if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations && in visitTRUNCATE()
15047 N0.hasOneUse() && in visitTRUNCATE()
15053 for (const SDValue &Op : N0->op_values()) { in visitTRUNCATE()
15061 if (N0.getOpcode() == ISD::SPLAT_VECTOR && in visitTRUNCATE()
15066 VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0))); in visitTRUNCATE()
15074 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitTRUNCATE()
15075 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && in visitTRUNCATE()
15076 N0.getOperand(0).hasOneUse()) { in visitTRUNCATE()
15077 SDValue BuildVect = N0.getOperand(0); in visitTRUNCATE()
15101 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) { in visitTRUNCATE()
15107 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) { in visitTRUNCATE()
15108 auto *LN0 = cast<LoadSDNode>(N0); in visitTRUNCATE()
15113 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1)); in visitTRUNCATE()
15121 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) { in visitTRUNCATE()
15127 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) { in visitTRUNCATE()
15128 SDValue X = N0.getOperand(i); in visitTRUNCATE()
15166 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
15167 SDValue VecSrc = N0.getOperand(0); in visitTRUNCATE()
15186 if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) { in visitTRUNCATE()
15187 SDValue N00 = N0.getOperand(0); in visitTRUNCATE()
15193 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
15194 N00.getOperand(0), N0.getOperand(1)); in visitTRUNCATE()
15205 switch (N0.getOpcode()) { in visitTRUNCATE()
15212 if (!LegalOperations && N0.hasOneUse() && in visitTRUNCATE()
15213 (isConstantOrConstantVector(N0.getOperand(0), true) || in visitTRUNCATE()
15214 isConstantOrConstantVector(N0.getOperand(1), true))) { in visitTRUNCATE()
15218 if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) { in visitTRUNCATE()
15219 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
15220 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
15221 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR); in visitTRUNCATE()
15232 if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) || in visitTRUNCATE()
15233 TLI.isOperationLegal(N0.getOpcode(), VT)) && in visitTRUNCATE()
15234 N0.hasOneUse() && !N0->hasAnyUseOfValue(1)) { in visitTRUNCATE()
15235 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
15236 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
15237 SDVTList VTs = DAG.getVTList(VT, N0->getValueType(1)); in visitTRUNCATE()
15238 return DAG.getNode(N0.getOpcode(), DL, VTs, X, Y, N0.getOperand(2)); in visitTRUNCATE()
15245 if (!LegalOperations && N0.hasOneUse() && in visitTRUNCATE()
15246 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitTRUNCATE()
15247 N0.getOperand(0).getOperand(0).getScalarValueSizeInBits() <= in visitTRUNCATE()
15249 hasOperation(N0.getOpcode(), VT)) { in visitTRUNCATE()
15250 return getTruncatedUSUBSAT(VT, SrcVT, N0.getOperand(0), N0.getOperand(1), in visitTRUNCATE()
15309 SDValue N0 = N->getOperand(0); in foldBitcastedFPLogic() local
15310 EVT SourceVT = N0.getValueType(); in foldBitcastedFPLogic()
15322 switch (N0.getOpcode()) { in foldBitcastedFPLogic()
15356 SDValue LogicOp0 = N0.getOperand(0); in foldBitcastedFPLogic()
15357 ConstantSDNode *LogicOp1 = isConstOrConstSplat(N0.getOperand(1), true); in foldBitcastedFPLogic()
15363 if (N0.getOpcode() == ISD::OR) in foldBitcastedFPLogic()
15372 SDValue N0 = N->getOperand(0); in visitBITCAST() local
15375 if (N0.isUndef()) in visitBITCAST()
15386 (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() && in visitBITCAST()
15388 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() && in visitBITCAST()
15389 cast<BuildVectorSDNode>(N0)->isConstant()) in visitBITCAST()
15390 return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), in visitBITCAST()
15394 if (isIntOrFPConstant(N0)) { in visitBITCAST()
15399 (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() && in visitBITCAST()
15401 (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() && in visitBITCAST()
15403 SDValue C = DAG.getBitcast(VT, N0); in visitBITCAST()
15410 if (N0.getOpcode() == ISD::BITCAST) in visitBITCAST()
15411 return DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15415 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && VT.isInteger() && in visitBITCAST()
15416 !TLI.isTypeLegal(N0.getOperand(0).getValueType())) { in visitBITCAST()
15423 if (IsFreeBitcast(N0.getOperand(0)) && IsFreeBitcast(N0.getOperand(1))) in visitBITCAST()
15424 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, in visitBITCAST()
15425 DAG.getBitcast(VT, N0.getOperand(0)), in visitBITCAST()
15426 DAG.getBitcast(VT, N0.getOperand(1))); in visitBITCAST()
15431 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitBITCAST()
15433 TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) == in visitBITCAST()
15440 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) || in visitBITCAST()
15442 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in visitBITCAST()
15444 if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG, in visitBITCAST()
15449 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1)); in visitBITCAST()
15469 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) || in visitBITCAST()
15470 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) && in visitBITCAST()
15471 N0->hasOneUse() && VT.isInteger() && !VT.isVector() && in visitBITCAST()
15472 !N0.getValueType().isVector()) { in visitBITCAST()
15473 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15477 if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) { in visitBITCAST()
15480 APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64); in visitBITCAST()
15482 if (N0.getOpcode() == ISD::FNEG) { in visitBITCAST()
15486 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
15492 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit); in visitBITCAST()
15496 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15501 if (N0.getOpcode() == ISD::FNEG) in visitBITCAST()
15504 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
15520 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitBITCAST()
15521 isa<ConstantFPSDNode>(N0.getOperand(0)) && VT.isInteger() && in visitBITCAST()
15523 unsigned OrigXWidth = N0.getOperand(1).getValueSizeInBits(); in visitBITCAST()
15526 SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1)); in visitBITCAST()
15547 if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) { in visitBITCAST()
15549 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15551 SDValue X = DAG.getBitcast(VT, N0.getOperand(1)); in visitBITCAST()
15553 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
15565 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15574 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15584 if (N0.getOpcode() == ISD::BUILD_PAIR) in visitBITCAST()
15585 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT)) in visitBITCAST()
15593 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() && in visitBITCAST()
15594 VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() && in visitBITCAST()
15595 !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) { in visitBITCAST()
15596 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N0); in visitBITCAST()
15612 SDValue SV0 = PeekThroughBitcast(N0->getOperand(0)); in visitBITCAST()
15613 SDValue SV1 = PeekThroughBitcast(N0->getOperand(1)); in visitBITCAST()
15618 VT.getVectorNumElements() / N0.getValueType().getVectorNumElements(); in visitBITCAST()
15639 SDValue N0 = N->getOperand(0); in visitFREEZE() local
15641 if (DAG.isGuaranteedNotToBeUndefOrPoison(N0, /*PoisonOnly*/ false)) in visitFREEZE()
15642 return N0; in visitFREEZE()
15647 if (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL) in visitFREEZE()
15657 if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false, in visitFREEZE()
15659 N0->getNumValues() != 1 || !N0->hasOneUse()) in visitFREEZE()
15663 N0.getOpcode() == ISD::SELECT_CC || in visitFREEZE()
15664 N0.getOpcode() == ISD::SETCC || in visitFREEZE()
15665 N0.getOpcode() == ISD::BUILD_VECTOR || in visitFREEZE()
15666 N0.getOpcode() == ISD::BUILD_PAIR || in visitFREEZE()
15667 N0.getOpcode() == ISD::VECTOR_SHUFFLE || in visitFREEZE()
15668 N0.getOpcode() == ISD::CONCAT_VECTORS; in visitFREEZE()
15676 if (N0.getOpcode() == ISD::BUILD_VECTOR) { in visitFREEZE()
15677 SDLoc DL(N0); in visitFREEZE()
15678 EVT VT = N0.getValueType(); in visitFREEZE()
15679 if (llvm::ISD::isBuildVectorAllOnes(N0.getNode())) in visitFREEZE()
15681 if (llvm::ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitFREEZE()
15683 for (const SDValue &Op : N0->op_values()) in visitFREEZE()
15692 for (auto [OpNo, Op] : enumerate(N0->ops())) { in visitFREEZE()
15745 N0 = N->getOperand(0); in visitFREEZE()
15749 SmallVector<SDValue> Ops(N0->op_begin(), N0->op_end()); in visitFREEZE()
15757 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0)) { in visitFREEZE()
15759 R = DAG.getVectorShuffle(N0.getValueType(), SDLoc(N0), Ops[0], Ops[1], in visitFREEZE()
15763 R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops); in visitFREEZE()
15864 SDValue N0 = N->getOperand(0); in visitFADDForFMACombine() local
15897 if (N0 == N1) in visitFADDForFMACombine()
15920 if (Aggressive && isContractableFMUL(N0) && isContractableFMUL(N1)) { in visitFADDForFMACombine()
15921 if (N0->use_size() > N1->use_size()) in visitFADDForFMACombine()
15922 std::swap(N0, N1); in visitFADDForFMACombine()
15926 if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) { in visitFADDForFMACombine()
15927 return matcher.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0), in visitFADDForFMACombine()
15928 N0.getOperand(1), N1); in visitFADDForFMACombine()
15935 N1.getOperand(1), N0); in visitFADDForFMACombine()
15950 if (isFusedOp(N0) && N0.hasOneUse()) { in visitFADDForFMACombine()
15951 FMA = N0; in visitFADDForFMACombine()
15955 E = N0; in visitFADDForFMACombine()
15978 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
15979 SDValue N00 = N0.getOperand(0); in visitFADDForFMACombine()
16000 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFADDForFMACombine()
16016 if (isFusedOp(N0)) { in visitFADDForFMACombine()
16017 SDValue N02 = N0.getOperand(2); in visitFADDForFMACombine()
16023 return FoldFAddFMAFPExtFMul(N0.getOperand(0), N0.getOperand(1), in visitFADDForFMACombine()
16045 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
16046 SDValue N00 = N0.getOperand(0); in visitFADDForFMACombine()
16070 N0); in visitFADDForFMACombine()
16089 N0); in visitFADDForFMACombine()
16101 SDValue N0 = N->getOperand(0); in visitFSUBForFMACombine() local
16172 if (isContractableFMUL(N0) && isContractableFMUL(N1) && in visitFSUBForFMACombine()
16173 (N0->use_size() > N1->use_size())) { in visitFSUBForFMACombine()
16175 if (SDValue V = tryToFoldXSubYZ(N0, N1)) in visitFSUBForFMACombine()
16178 if (SDValue V = tryToFoldXYSubZ(N0, N1)) in visitFSUBForFMACombine()
16182 if (SDValue V = tryToFoldXYSubZ(N0, N1)) in visitFSUBForFMACombine()
16185 if (SDValue V = tryToFoldXSubYZ(N0, N1)) in visitFSUBForFMACombine()
16190 if (matcher.match(N0, ISD::FNEG) && isContractableFMUL(N0.getOperand(0)) && in visitFSUBForFMACombine()
16191 (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) { in visitFSUBForFMACombine()
16192 SDValue N00 = N0.getOperand(0).getOperand(0); in visitFSUBForFMACombine()
16193 SDValue N01 = N0.getOperand(0).getOperand(1); in visitFSUBForFMACombine()
16203 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16204 SDValue N00 = N0.getOperand(0); in visitFSUBForFMACombine()
16229 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
16239 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16240 SDValue N00 = N0.getOperand(0); in visitFSUBForFMACombine()
16263 if (matcher.match(N0, ISD::FNEG)) { in visitFSUBForFMACombine()
16264 SDValue N00 = N0.getOperand(0); in visitFSUBForFMACombine()
16299 if (CanFuse && isFusedOp(N0) && in visitFSUBForFMACombine()
16300 isContractableAndReassociableFMUL(N0.getOperand(2)) && in visitFSUBForFMACombine()
16301 N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) { in visitFSUBForFMACombine()
16303 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1), in visitFSUBForFMACombine()
16305 N0.getOperand(2).getOperand(0), in visitFSUBForFMACombine()
16306 N0.getOperand(2).getOperand(1), in visitFSUBForFMACombine()
16322 matcher.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
16327 if (isFusedOp(N0) && N0->hasOneUse()) { in visitFSUBForFMACombine()
16328 SDValue N02 = N0.getOperand(2); in visitFSUBForFMACombine()
16335 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1), in visitFSUBForFMACombine()
16351 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16352 SDValue N00 = N0.getOperand(0); in visitFSUBForFMACombine()
16389 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
16418 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
16430 SDValue N0 = N->getOperand(0); in visitFMULForFMADistributiveCombine() local
16441 SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1; in visitFMULForFMADistributiveCombine()
16480 if (SDValue FMA = FuseFADD(N0, N1)) in visitFMULForFMADistributiveCombine()
16482 if (SDValue FMA = FuseFADD(N1, N0)) in visitFMULForFMADistributiveCombine()
16513 if (SDValue FMA = FuseFSUB(N0, N1)) in visitFMULForFMADistributiveCombine()
16515 if (SDValue FMA = FuseFSUB(N1, N0)) in visitFMULForFMADistributiveCombine()
16534 SDValue N0 = N->getOperand(0); in visitFADD() local
16536 SDNode *N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0); in visitFADD()
16544 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFADD()
16548 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1})) in visitFADD()
16553 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
16564 return N0; in visitFADD()
16573 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
16578 N0, DAG, LegalOperations, ForCodeSize)) in visitFADD()
16589 if (isFMulNegTwo(N0)) { in visitFADD()
16590 SDValue B = N0.getOperand(0); in visitFADD()
16598 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
16608 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) in visitFADD()
16612 if (N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) in visitFADD()
16623 if (N1CFP && N0.getOpcode() == ISD::FADD && in visitFADD()
16624 DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1))) { in visitFADD()
16625 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
16626 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
16633 if (N0.getOpcode() == ISD::FMUL) { in visitFADD()
16635 DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0)); in visitFADD()
16637 DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1)); in visitFADD()
16640 if (CFP01 && !CFP00 && N0.getOperand(0) == N1) { in visitFADD()
16641 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16649 N0.getOperand(0) == N1.getOperand(0)) { in visitFADD()
16650 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16652 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
16663 if (CFP11 && !CFP10 && N1.getOperand(0) == N0) { in visitFADD()
16666 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
16670 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD && in visitFADD()
16671 N0.getOperand(0) == N0.getOperand(1) && in visitFADD()
16672 N1.getOperand(0) == N0.getOperand(0)) { in visitFADD()
16679 if (N0.getOpcode() == ISD::FADD) { in visitFADD()
16681 DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0)); in visitFADD()
16683 if (!CFP00 && N0.getOperand(0) == N0.getOperand(1) && in visitFADD()
16684 (N0.getOperand(0) == N1)) { in visitFADD()
16695 N1.getOperand(0) == N0) { in visitFADD()
16696 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
16702 if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && in visitFADD()
16703 N0.getOperand(0) == N0.getOperand(1) && in visitFADD()
16705 N0.getOperand(0) == N1.getOperand(0)) { in visitFADD()
16706 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
16713 VT, N0, N1, Flags)) in visitFADD()
16728 SDValue N0 = N->getOperand(1); in visitSTRICT_FADD() local
16740 {Chain, N0, NegN1}); in visitSTRICT_FADD()
16746 N0, DAG, LegalOperations, ForCodeSize)) { in visitSTRICT_FADD()
16754 SDValue N0 = N->getOperand(0); in visitFSUB() local
16756 ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, true); in visitFSUB()
16764 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFSUB()
16768 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1})) in visitFSUB()
16783 return N0; in visitFSUB()
16787 if (N0 == N1) { in visitFSUB()
16816 if (N0 == N1->getOperand(0)) in visitFSUB()
16819 if (N0 == N1->getOperand(1)) in visitFSUB()
16826 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
16944 SDValue N0 = N->getOperand(0); in visitFMUL() local
16953 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFMUL()
16957 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1})) in visitFMUL()
16961 if (DAG.isConstantFPBuildVectorOrConstantFP(N0) && in visitFMUL()
16963 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
16976 N0.getOpcode() == ISD::FMUL) { in visitFMUL()
16977 SDValue N00 = N0.getOperand(0); in visitFMUL()
16978 SDValue N01 = N0.getOperand(1); in visitFMUL()
16990 if (N0.getOpcode() == ISD::FADD && N0.hasOneUse() && in visitFMUL()
16991 N0.getOperand(0) == N0.getOperand(1)) { in visitFMUL()
16994 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
16999 VT, N0, N1, Flags)) in visitFMUL()
17005 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
17011 DAG.getConstantFP(-0.0, DL, VT), N0, Flags); in visitFMUL()
17021 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize, CostN0); in visitFMUL()
17034 (N0.getOpcode() == ISD::SELECT || N1.getOpcode() == ISD::SELECT) && in visitFMUL()
17036 SDValue Select = N0, X = N1; in visitFMUL()
17092 SDValue N0 = N->getOperand(0); in visitFMA() local
17095 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); in visitFMA()
17105 if (isa<ConstantFPSDNode>(N0) && in visitFMA()
17108 return matcher.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
17117 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize, CostN0); in visitFMA()
17139 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
17142 if (DAG.isConstantFPBuildVectorOrConstantFP(N0) && in visitFMA()
17144 return matcher.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
17150 if (matcher.match(N2, ISD::FMUL) && N0 == N2.getOperand(0) && in visitFMA()
17154 ISD::FMUL, DL, VT, N0, in visitFMA()
17159 if (matcher.match(N0, ISD::FMUL) && in visitFMA()
17161 DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1))) { in visitFMA()
17163 ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
17164 matcher.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), N2); in visitFMA()
17172 return matcher.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
17176 SDValue RHSNeg = matcher.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
17182 if (matcher.match(N0, ISD::FNEG) && in visitFMA()
17186 return matcher.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
17194 if (N1CFP && N0 == N2) { in visitFMA()
17195 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
17201 if (N1CFP && matcher.match(N2, ISD::FNEG) && N2.getOperand(0) == N0) { in visitFMA()
17202 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
17218 SDValue N0 = N->getOperand(0); in visitFMAD() local
17225 if (isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1) && in visitFMAD()
17227 return DAG.getNode(ISD::FMAD, DL, VT, N0, N1, N2); in visitFMAD()
17249 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in combineRepeatedFPDivisors() local
17250 ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, /* AllowUndefs */ true); in combineRepeatedFPDivisors()
17313 SDValue N0 = N->getOperand(0); in visitFDIV() local
17321 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFDIV()
17325 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1})) in visitFDIV()
17357 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
17366 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17373 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17381 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17411 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
17423 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
17430 if (SDValue RV = BuildDivEstimate(N0, N1, Flags)) in visitFDIV()
17437 if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0)) in visitFDIV()
17446 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize, CostN0); in visitFDIV()
17463 SDValue N0 = N->getOperand(0); in visitFREM() local
17470 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags)) in visitFREM()
17474 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, DL, VT, {N0, N1})) in visitFREM()
17488 !Flags.hasNoSignedZeros() && !DAG.cannotBeOrderedNegativeFP(N0); in visitFREM()
17489 SDValue Div = DAG.getNode(ISD::FDIV, DL, VT, N0, N1); in visitFREM()
17494 N1, N0); in visitFREM()
17497 MLA = DAG.getNode(ISD::FSUB, DL, VT, N0, Mul); in visitFREM()
17499 return NeedsCopySign ? DAG.getNode(ISD::FCOPYSIGN, DL, VT, MLA, N0) : MLA; in visitFREM()
17515 SDValue N0 = N->getOperand(0); in visitFSQRT() local
17516 if (TLI.isFsqrtCheap(N0, DAG)) in visitFSQRT()
17524 return buildSqrtEstimate(N0, Flags); in visitFSQRT()
17556 SDValue N0 = N->getOperand(0); in visitFCOPYSIGN() local
17562 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, DL, VT, {N0, N1})) in visitFCOPYSIGN()
17571 return DAG.getNode(ISD::FABS, DL, VT, N0); in visitFCOPYSIGN()
17575 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
17582 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG || in visitFCOPYSIGN()
17583 N0.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
17584 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
17588 return DAG.getNode(ISD::FABS, DL, VT, N0); in visitFCOPYSIGN()
17592 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
17597 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
17606 if (SimplifyDemandedBits(N0, in visitFCOPYSIGN()
17705 SDValue N0 = N->getOperand(0); in foldFPToIntToFP() local
17706 if (N->getOpcode() == ISD::SINT_TO_FP && N0.getOpcode() == ISD::FP_TO_SINT && in foldFPToIntToFP()
17707 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
17708 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17710 if (N->getOpcode() == ISD::UINT_TO_FP && N0.getOpcode() == ISD::FP_TO_UINT && in foldFPToIntToFP()
17711 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
17712 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17718 SDValue N0 = N->getOperand(0); in visitSINT_TO_FP() local
17720 EVT OpVT = N0.getValueType(); in visitSINT_TO_FP()
17723 if (N0.isUndef()) in visitSINT_TO_FP()
17727 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitSINT_TO_FP()
17731 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17738 if (DAG.SignBitIsZero(N0)) in visitSINT_TO_FP()
17739 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17744 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 && in visitSINT_TO_FP()
17748 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(-1.0, DL, VT), in visitSINT_TO_FP()
17754 if (N0.getOpcode() == ISD::ZERO_EXTEND && in visitSINT_TO_FP()
17755 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
17758 return DAG.getSelect(DL, VT, N0.getOperand(0), in visitSINT_TO_FP()
17770 SDValue N0 = N->getOperand(0); in visitUINT_TO_FP() local
17772 EVT OpVT = N0.getValueType(); in visitUINT_TO_FP()
17775 if (N0.isUndef()) in visitUINT_TO_FP()
17779 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && in visitUINT_TO_FP()
17783 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17790 if (DAG.SignBitIsZero(N0)) in visitUINT_TO_FP()
17791 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17795 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
17798 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(1.0, DL, VT), in visitUINT_TO_FP()
17810 SDValue N0 = N->getOperand(0); in FoldIntToFPToInt() local
17813 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP) in FoldIntToFPToInt()
17816 SDValue Src = N0.getOperand(0); in FoldIntToFPToInt()
17818 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP; in FoldIntToFPToInt()
17833 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(N0.getValueType()); in FoldIntToFPToInt()
17851 SDValue N0 = N->getOperand(0); in visitFP_TO_SINT() local
17855 if (N0.isUndef()) in visitFP_TO_SINT()
17859 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFP_TO_SINT()
17860 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
17866 SDValue N0 = N->getOperand(0); in visitFP_TO_UINT() local
17870 if (N0.isUndef()) in visitFP_TO_UINT()
17874 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFP_TO_UINT()
17875 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
17881 SDValue N0 = N->getOperand(0); in visitXRINT() local
17885 if (N0.isUndef()) in visitXRINT()
17889 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitXRINT()
17890 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0); in visitXRINT()
17896 SDValue N0 = N->getOperand(0); in visitFP_ROUND() local
17902 DAG.FoldConstantArithmetic(ISD::FP_ROUND, SDLoc(N), VT, {N0, N1})) in visitFP_ROUND()
17906 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
17907 return N0.getOperand(0); in visitFP_ROUND()
17910 if (N0.getOpcode() == ISD::FP_ROUND) { in visitFP_ROUND()
17912 const bool N0IsTrunc = N0.getConstantOperandVal(1) == 1; in visitFP_ROUND()
17925 if (N0.getOperand(0).getValueType() == MVT::f80 && VT == MVT::f16) in visitFP_ROUND()
17936 ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
17946 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitFP_ROUND()
17948 N0.getValueType())) { in visitFP_ROUND()
17949 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
17950 N0.getOperand(0), N1); in visitFP_ROUND()
17953 Tmp, N0.getOperand(1)); in visitFP_ROUND()
17963 SDValue N0 = N->getOperand(0); in visitFP_EXTEND() local
17976 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFP_EXTEND()
17977 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
17980 if (N0.getOpcode() == ISD::FP16_TO_FP && in visitFP_EXTEND()
17982 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
17986 if (N0.getOpcode() == ISD::FP_ROUND in visitFP_EXTEND()
17987 && N0.getConstantOperandVal(1) == 1) { in visitFP_EXTEND()
17988 SDValue In = N0.getOperand(0); in visitFP_EXTEND()
17992 In, N0.getOperand(1)); in visitFP_EXTEND()
17997 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitFP_EXTEND()
17998 TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
17999 LoadSDNode *LN0 = cast<LoadSDNode>(N0); in visitFP_EXTEND()
18002 LN0->getBasePtr(), N0.getValueType(), in visitFP_EXTEND()
18006 N0.getNode(), in visitFP_EXTEND()
18007 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad, in visitFP_EXTEND()
18008 DAG.getIntPtrConstant(1, SDLoc(N0), /*isTarget=*/true)), in visitFP_EXTEND()
18020 SDValue N0 = N->getOperand(0); in visitFCEIL() local
18024 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFCEIL()
18025 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
18031 SDValue N0 = N->getOperand(0); in visitFTRUNC() local
18035 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFTRUNC()
18036 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
18041 switch (N0.getOpcode()) { in visitFTRUNC()
18049 return N0; in visitFTRUNC()
18056 SDValue N0 = N->getOperand(0); in visitFFREXP() local
18059 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFFREXP()
18060 return DAG.getNode(ISD::FFREXP, SDLoc(N), N->getVTList(), N0); in visitFFREXP()
18065 SDValue N0 = N->getOperand(0); in visitFFLOOR() local
18069 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFFLOOR()
18070 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
18076 SDValue N0 = N->getOperand(0); in visitFNEG() local
18081 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFNEG()
18082 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
18085 TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize)) in visitFNEG()
18092 if (N0.getOpcode() == ISD::FSUB && in visitFNEG()
18094 N->getFlags().hasNoSignedZeros()) && N0.hasOneUse()) { in visitFNEG()
18095 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
18096 N0.getOperand(0)); in visitFNEG()
18106 SDValue N0 = N->getOperand(0); in visitFMinMax() local
18116 if (SDValue C = DAG.FoldConstantArithmetic(Opc, SDLoc(N), VT, {N0, N1})) in visitFMinMax()
18120 if (DAG.isConstantFPBuildVectorOrConstantFP(N0) && in visitFMinMax()
18122 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitFMinMax()
18157 Opc, SDLoc(N), VT, N0, N1, Flags)) in visitFMinMax()
18164 SDValue N0 = N->getOperand(0); in visitFABS() local
18168 if (DAG.isConstantFPBuildVectorOrConstantFP(N0)) in visitFABS()
18169 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
18172 if (N0.getOpcode() == ISD::FABS) in visitFABS()
18177 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN) in visitFABS()
18178 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
19927 SDValue N0 = Value.getOperand(0); in ReduceLoadOpStoreWidth() local
19928 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in ReduceLoadOpStoreWidth()
19929 Chain == SDValue(N0.getNode(), 1)) { in ReduceLoadOpStoreWidth()
19930 LoadSDNode *LD = cast<LoadSDNode>(N0); in ReduceLoadOpStoreWidth()
19987 DAG.getLoad(NewVT, SDLoc(N0), LD->getChain(), NewPtr, in ReduceLoadOpStoreWidth()
20001 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1)); in ReduceLoadOpStoreWidth()
24623 SDValue N0 = N->getOperand(0); in foldExtractSubvectorFromShuffleVector() local
24627 EVT WideVT = N0.getValueType(); in foldExtractSubvectorFromShuffleVector()
24632 auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(N0); in foldExtractSubvectorFromShuffleVector()
24977 SDValue N0 = Shuf->getOperand(0), N1 = Shuf->getOperand(1); in foldShuffleOfConcatUndefs() local
24978 if (N0.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
24980 !N0.getOperand(1).isUndef() || !N1.getOperand(1).isUndef()) in foldShuffleOfConcatUndefs()
25014 SDValue X = N0.getOperand(0), Y = N1.getOperand(0); in foldShuffleOfConcatUndefs()
25027 SDValue N0 = N->getOperand(0); in partitionShuffleOfConcats() local
25033 EVT ConcatVT = N0.getOperand(0).getValueType(); in partitionShuffleOfConcats()
25045 N0 = DAG.getVectorShuffle(ConcatVT, SDLoc(N), N0.getOperand(0), in partitionShuffleOfConcats()
25046 N0.getOperand(1), in partitionShuffleOfConcats()
25049 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
25077 if (OpIdx < (int)N0.getNumOperands()) in partitionShuffleOfConcats()
25078 Ops.push_back(N0.getOperand(OpIdx)); in partitionShuffleOfConcats()
25080 Ops.push_back(N1.getOperand(OpIdx - N0.getNumOperands())); in partitionShuffleOfConcats()
25107 SDValue N0 = SVN->getOperand(0); in combineShuffleOfScalars() local
25110 if (!N0->hasOneUse()) in combineShuffleOfScalars()
25119 bool N0AnyConst = isAnyConstantBuildVector(N0); in combineShuffleOfScalars()
25121 if (N0AnyConst && !N1AnyConst && !ISD::isBuildVectorAllZeros(N0.getNode())) in combineShuffleOfScalars()
25130 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0); in combineShuffleOfScalars()
25142 SDValue &S = (M < (int)NumElts ? N0 : N1); in combineShuffleOfScalars()
25248 SDValue N0 = SVN->getOperand(0); in combineShuffleToAnyExtendVectorInreg() local
25255 return DAG.getBitcast(VT, DAG.getNode(Opcode, SDLoc(SVN), *OutVT, N0)); in combineShuffleToAnyExtendVectorInreg()
25396 SDValue N0 = peekThroughBitcasts(SVN->getOperand(0)); in combineTruncationShuffle() local
25398 unsigned Opcode = N0.getOpcode(); in combineTruncationShuffle()
25402 SDValue N00 = N0.getOperand(0); in combineTruncationShuffle()
25407 unsigned ExtDstSizeInBits = N0.getScalarValueSizeInBits(); in combineTruncationShuffle()
25753 SDValue N0 = N->getOperand(0); in visitVECTOR_SHUFFLE() local
25756 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG"); in visitVECTOR_SHUFFLE()
25759 if (N0.isUndef() && N1.isUndef()) in visitVECTOR_SHUFFLE()
25765 if (N0 == N1) in visitVECTOR_SHUFFLE()
25766 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
25770 if (N0.isUndef()) in visitVECTOR_SHUFFLE()
25786 return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask); in visitVECTOR_SHUFFLE()
25803 if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) && in visitVECTOR_SHUFFLE()
25804 TLI.isBinOp(N0.getOpcode()) && N0->getNumValues() == 1) { in visitVECTOR_SHUFFLE()
25807 SDValue L = N0.getOperand(0), R = N0.getOperand(1); in visitVECTOR_SHUFFLE()
25814 DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR, N0->getFlags()); in visitVECTOR_SHUFFLE()
25823 N0.hasOneUse()) { in visitVECTOR_SHUFFLE()
25824 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && SplatIndex == 0) in visitVECTOR_SHUFFLE()
25825 return DAG.getSplatBuildVector(VT, SDLoc(N), N0.getOperand(0)); in visitVECTOR_SHUFFLE()
25827 if (N0.getOpcode() == ISD::INSERT_VECTOR_ELT) in visitVECTOR_SHUFFLE()
25828 if (auto *Idx = dyn_cast<ConstantSDNode>(N0.getOperand(2))) in visitVECTOR_SHUFFLE()
25830 return DAG.getSplatBuildVector(VT, SDLoc(N), N0.getOperand(1)); in visitVECTOR_SHUFFLE()
25834 if (N0.getOpcode() == ISD::BITCAST && N0.getOperand(0).hasOneUse() && in visitVECTOR_SHUFFLE()
25836 (N0.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR || in visitVECTOR_SHUFFLE()
25837 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR)) { in visitVECTOR_SHUFFLE()
25838 EVT N00VT = N0.getOperand(0).getValueType(); in visitVECTOR_SHUFFLE()
25843 SDValue Op = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), in visitVECTOR_SHUFFLE()
25853 SDNode *V = N0.getNode(); in visitVECTOR_SHUFFLE()
25874 return N0; in visitVECTOR_SHUFFLE()
25883 return N0; in visitVECTOR_SHUFFLE()
25916 if (N0.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
25920 N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()))) { in visitVECTOR_SHUFFLE()
25928 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N1.isUndef() && in visitVECTOR_SHUFFLE()
25929 N0.getNumOperands() == 2 && in visitVECTOR_SHUFFLE()
25930 N0.getOperand(0) == N0.getOperand(1)) { in visitVECTOR_SHUFFLE()
25942 SDValue UndefVec = DAG.getUNDEF(N0.getOperand(0).getValueType()); in visitVECTOR_SHUFFLE()
25944 N0.getOperand(0), UndefVec); in visitVECTOR_SHUFFLE()
26003 if (SDValue InsertN1 = ShuffleToInsert(N0, N1, Mask)) in visitVECTOR_SHUFFLE()
26005 if (N0.getOpcode() == ISD::CONCAT_VECTORS) { in visitVECTOR_SHUFFLE()
26008 if (SDValue InsertN0 = ShuffleToInsert(N1, N0, CommuteMask)) in visitVECTOR_SHUFFLE()
26034 (DemandedLHS.isZero() || DAG.MaskedVectorIsZero(N0, DemandedLHS)) && in visitVECTOR_SHUFFLE()
26056 VT, DAG.getVectorShuffle(IntVT, DL, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
26061 VT, DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
26076 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitVECTOR_SHUFFLE()
26080 SDValue BC0 = peekThroughOneUseBitcasts(N0); in visitVECTOR_SHUFFLE()
26256 N0.getOpcode() != ISD::VECTOR_SHUFFLE) { in visitVECTOR_SHUFFLE()
26264 bool HasSameOp0 = N0 == SV0; in visitVECTOR_SHUFFLE()
26266 if (HasSameOp0 || IsSV1Undef || N0 == SV1) in visitVECTOR_SHUFFLE()
26273 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
26275 cast<ShuffleVectorSDNode>(N0)->isSplat() && in visitVECTOR_SHUFFLE()
26315 unsigned SrcOpcode = N0.getOpcode(); in visitVECTOR_SHUFFLE()
26316 if (TLI.isBinOp(SrcOpcode) && N->isOnlyUserOf(N0.getNode()) && in visitVECTOR_SHUFFLE()
26320 SDValue Op00 = N0.getOperand(0); in visitVECTOR_SHUFFLE()
26321 SDValue Op01 = N0.getOperand(1); in visitVECTOR_SHUFFLE()
26336 SDValue InnerN = Commute ? N1 : N0; in visitVECTOR_SHUFFLE()
26498 SDValue N0 = N->getOperand(0); in visitINSERT_SUBVECTOR() local
26505 return N0; in visitINSERT_SUBVECTOR()
26510 if (N0.isUndef() && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26521 VT, N0, N1.getOperand(0), N2); in visitINSERT_SUBVECTOR()
26532 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && N1.getOperand(0) == N0 && in visitINSERT_SUBVECTOR()
26534 return N0; in visitINSERT_SUBVECTOR()
26538 if (N0.isUndef() && N1.getOpcode() == ISD::SPLAT_VECTOR) in visitINSERT_SUBVECTOR()
26546 if (N0.isUndef() && N1.getOpcode() == ISD::BITCAST && in visitINSERT_SUBVECTOR()
26560 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
26561 SDValue CN0 = N0.getOperand(0); in visitINSERT_SUBVECTOR()
26577 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26578 N0.getOperand(1).getValueType() == N1.getValueType() && in visitINSERT_SUBVECTOR()
26579 N0.getOperand(2) == N2) in visitINSERT_SUBVECTOR()
26580 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
26586 if (N0.isUndef() && N1.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26589 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
26595 if ((N0.isUndef() || N0.getOpcode() == ISD::BITCAST) && in visitINSERT_SUBVECTOR()
26597 SDValue N0Src = peekThroughBitcasts(N0); in visitINSERT_SUBVECTOR()
26601 if ((N0.isUndef() || N0SrcSVT == N1SrcSVT) && in visitINSERT_SUBVECTOR()
26633 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
26634 N1.getValueType() == N0.getOperand(1).getValueType()) { in visitINSERT_SUBVECTOR()
26635 unsigned OtherIdx = N0.getConstantOperandVal(2); in visitINSERT_SUBVECTOR()
26639 N0.getOperand(0), N1, N2); in visitINSERT_SUBVECTOR()
26641 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N0.getNode()), in visitINSERT_SUBVECTOR()
26642 VT, NewOp, N0.getOperand(1), N0.getOperand(2)); in visitINSERT_SUBVECTOR()
26648 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
26649 N0.getOperand(0).getValueType() == N1.getValueType() && in visitINSERT_SUBVECTOR()
26650 N0.getOperand(0).getValueType().isScalableVector() == in visitINSERT_SUBVECTOR()
26653 SmallVector<SDValue, 8> Ops(N0->op_begin(), N0->op_end()); in visitINSERT_SUBVECTOR()
26666 SDValue N0 = N->getOperand(0); in visitFP_TO_FP16() local
26669 if (N0->getOpcode() == ISD::FP16_TO_FP) in visitFP_TO_FP16()
26670 return N0->getOperand(0); in visitFP_TO_FP16()
26679 SDValue N0 = N->getOperand(0); in visitFP16_TO_FP() local
26683 if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) { in visitFP16_TO_FP()
26684 ConstantSDNode *AndConst = getAsNonOpaqueConstant(N0.getOperand(1)); in visitFP16_TO_FP()
26686 return DAG.getNode(Op, SDLoc(N), N->getValueType(0), N0.getOperand(0)); in visitFP16_TO_FP()
26694 N->getValueType(0), {N0}); in visitFP16_TO_FP()
26699 SDValue N0 = N->getOperand(0); in visitFP_TO_BF16() local
26702 if (N0->getOpcode() == ISD::BF16_TO_FP) in visitFP_TO_BF16()
26703 return N0->getOperand(0); in visitFP_TO_BF16()
26714 SDValue N0 = N->getOperand(0); in visitVECREDUCE() local
26715 EVT VT = N0.getValueType(); in visitVECREDUCE()
26722 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
26736 DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits()) in visitVECREDUCE()
26737 return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), N0); in visitVECREDUCE()
26742 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitVECREDUCE()
26743 TLI.isTypeLegal(N0.getOperand(1).getValueType())) { in visitVECREDUCE()
26744 SDValue Vec = N0.getOperand(0); in visitVECREDUCE()
26745 SDValue Subvec = N0.getOperand(1); in visitVECREDUCE()
26747 (N0.getOperand(0).isUndef() || isNullOrNullSplat(Vec))) || in visitVECREDUCE()
26749 (N0.getOperand(0).isUndef() || isAllOnesOrAllOnesSplat(Vec)))) in visitVECREDUCE()
27017 SDValue N0 = N->getOperand(0); in scalarizeBinOpOfSplats() local
27028 SDValue Src0 = DAG.getSplatSourceVector(N0, Index0); in scalarizeBinOpOfSplats()
27032 bool IsBothSplatVector = N0.getOpcode() == ISD::SPLAT_VECTOR && in scalarizeBinOpOfSplats()
27048 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getOpcode() == N1.getOpcode() && in scalarizeBinOpOfSplats()
27049 count_if(N0->ops(), [](SDValue V) { return !V.isUndef(); }) == 1 && in scalarizeBinOpOfSplats()
27069 SDValue N0 = N->getOperand(0); in SimplifyVCastOp() local
27074 SDValue Src0 = DAG.getSplatSourceVector(N0, Index0); in SimplifyVCastOp()
27076 (N0.getOpcode() == ISD::SPLAT_VECTOR || in SimplifyVCastOp()
27080 EVT SrcVT = N0.getValueType(); in SimplifyVCastOp()
27210 SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, in SimplifySelect() argument
27212 assert(N0.getOpcode() == ISD::SETCC && in SimplifySelect()
27215 SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2, in SimplifySelect()
27216 cast<CondCodeSDNode>(N0.getOperand(2))->get()); in SimplifySelect()
27225 const SDNodeFlags Flags = N0->getFlags(); in SimplifySelect()
27226 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), in SimplifySelect()
27227 N0.getValueType(), in SimplifySelect()
27436 SDValue DAGCombiner::foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, in foldSelectCCToShiftAnd() argument
27443 EVT XType = N0.getValueType(); in foldSelectCCToShiftAnd()
27454 if (!(isAllOnesConstant(N1) || (isNullConstant(N1) && N0 == N2))) in foldSelectCCToShiftAnd()
27459 if (!(isNullConstant(N1) || (isOneConstant(N1) && N0 == N2))) in foldSelectCCToShiftAnd()
27472 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
27492 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
27508 SDValue N0 = N->getOperand(0); in foldSelectOfBinops() local
27522 if (!N0->hasOneUse() || !N1->hasOneUse() || !N2->hasOneUse()) in foldSelectOfBinops()
27534 SDValue NewSel = DAG.getSelect(DL, N10.getValueType(), N0, N10, N20); in foldSelectOfBinops()
27548 SDValue NewSel = DAG.getSelect(DL, N11.getValueType(), N0, N11, N21); in foldSelectOfBinops()
27563 SDValue N0 = N->getOperand(0); in foldSignChangeInBitcast() local
27568 if (IsFree || N0.getOpcode() != ISD::BITCAST || !N0.hasOneUse()) in foldSignChangeInBitcast()
27571 SDValue Int = N0.getOperand(0); in foldSignChangeInBitcast()
27581 if (N0.getValueType().isVector()) { in foldSignChangeInBitcast()
27584 SignMask = APInt::getSignMask(N0.getScalarValueSizeInBits()); in foldSignChangeInBitcast()
27594 SDLoc DL(N0); in foldSignChangeInBitcast()
27606 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3, in convertSelectOfFPConstantsToLoadOffset() argument
27608 if (!TLI.reduceSelectOfFPConstantLoads(N0.getValueType())) in convertSelectOfFPConstantsToLoadOffset()
27647 DAG.getSetCC(DL, getSetCCResultType(N0.getValueType()), N0, N1, CC); in convertSelectOfFPConstantsToLoadOffset()
27660 SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1, in SimplifySelectCC() argument
27666 EVT CmpOpVT = N0.getValueType(); in SimplifySelectCC()
27674 if (SDValue SCC = DAG.FoldSetCC(CmpResVT, N0, N1, CC, DL)) { in SimplifySelectCC()
27684 convertSelectOfFPConstantsToLoadOffset(DL, N0, N1, N2, N3, CC)) in SimplifySelectCC()
27687 if (SDValue V = foldSelectCCToShiftAnd(DL, N0, N1, N2, N3, CC)) in SimplifySelectCC()
27696 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && in SimplifySelectCC()
27697 N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) { in SimplifySelectCC()
27698 SDValue AndLHS = N0->getOperand(0); in SimplifySelectCC()
27699 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1)); in SimplifySelectCC()
27707 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
27712 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
27741 SCC = DAG.getSetCC(DL, CmpResVT, N0, N1, CC); in SimplifySelectCC()
27744 SCC = DAG.getSetCC(SDLoc(N0), MVT::i1, N0, N1, CC); in SimplifySelectCC()
27785 N0 == Count.getOperand(0) && in SimplifySelectCC()
27787 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
27792 N0 == Count.getOperand(0) && in SimplifySelectCC()
27794 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
27807 ISD::SRA, DL, CmpOpVT, N0, in SimplifySelectCC()
27813 if (SDValue S = PerformMinMaxFpToSatCombine(N0, N1, N2, N3, CC, DAG)) in SimplifySelectCC()
27815 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N2, N3, CC, DAG)) in SimplifySelectCC()
27822 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
27827 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); in SimplifySetCC()